diff --git a/ami/exports/tests.py b/ami/exports/tests.py index 3a8e6502f..e2d11b064 100644 --- a/ami/exports/tests.py +++ b/ami/exports/tests.py @@ -688,6 +688,20 @@ def test_feature_vector_and_next_detection(self): if pgvector_is_available(): self.assertEqual(rows[detections[0].pk]["has_feature_vector"], "true") + def test_an_embedding_alone_counts_as_a_feature_vector(self): + from ami.main.models import DetectionEmbedding + from ami.ml.models import Algorithm + from ami.tests.fixtures.tracking import pgvector_is_available + + if not pgvector_is_available(): + self.skipTest("This database cannot store embeddings.") + detection = self.occurrences[0].detections.order_by("source_image__timestamp").last() + algorithm = Algorithm.objects.create(name="Embedding model", key="embedding-model") + DetectionEmbedding.objects.create(detection=detection, algorithm=algorithm, vector=[0.1] * 2048) + + rows = {int(row["detection_id"]): row for row in self._rows()} + self.assertEqual(rows[detection.pk]["has_feature_vector"], "true") + def test_query_count_is_one_pair_per_chunk(self): from django.db import connection from django.test.utils import CaptureQueriesContext diff --git a/ami/exports/tracks.py b/ami/exports/tracks.py index 76150c2ef..2d36c1ff1 100644 --- a/ami/exports/tracks.py +++ b/ami/exports/tracks.py @@ -12,9 +12,9 @@ from collections.abc import Callable, Iterator from django.db import models -from django.db.models import Exists, OuterRef, Subquery +from django.db.models import Exists, ExpressionWrapper, OuterRef, Subquery -from ami.main.models import BEST_MACHINE_PREDICTION_ORDER, Classification, Detection, Occurrence +from ami.main.models import BEST_MACHINE_PREDICTION_ORDER, Classification, Detection, DetectionEmbedding, Occurrence from ami.main.models_future.tracks import CAPTURE_ORDER TRACKS_CSV_COLUMNS: typing.Final = ( @@ -67,9 +67,11 @@ def _detections_for(occurrence_ids: list[int]) -> models.QuerySet: .annotate( label=Subquery(best_classification.values("taxon__name")[:1]), label_score=Subquery(best_classification.values("score")[:1]), - # Same notion as ClassificationQuerySet.with_has_features(), per detection. - has_feature_vector=Exists( - Classification.objects.filter(detection=OuterRef("pk"), features_2048__isnull=False) + # Same notion as DetectionQuerySet.has_vector(): an embedding or a classification vector. + has_feature_vector=ExpressionWrapper( + Exists(DetectionEmbedding.objects.filter(detection=OuterRef("pk"))) + | Exists(Classification.objects.filter(detection=OuterRef("pk"), features_2048__isnull=False)), + output_field=models.BooleanField(), ), ) .order_by("occurrence_id", *CAPTURE_ORDER) diff --git a/ami/jobs/tasks.py b/ami/jobs/tasks.py index b00213ebb..40ce3704e 100644 --- a/ami/jobs/tasks.py +++ b/ami/jobs/tasks.py @@ -13,6 +13,7 @@ from redis.exceptions import RedisError from ami.main.checks.schemas import IntegrityCheckResult +from ami.ml.exceptions import FeatureResultsStoredNothing from ami.ml.orchestration.async_job_state import AsyncJobStateManager from ami.ml.orchestration.nats_queue import ConsumerState, TaskQueueManager from ami.ml.schemas import PipelineResultsError, PipelineResultsResponse @@ -323,10 +324,20 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub try: # Save to database (this is the slow operation) detections_count, classifications_count, captures_count = 0, 0, 0 + feature_counts: dict[str, int] = {} if pipeline_result: # should never happen since otherwise we could not be processing results here assert job.pipeline is not None, "Job pipeline is None" - job.pipeline.save_results(results=pipeline_result, job_id=job.pk) + # Only a feature-only save reports boxes that matched no detection and detections + # left without a vector; asking the full save for its created rows would load + # every detection's algorithm. + feature_only = job.pipeline.is_feature_only() + saved = job.pipeline.save_results(results=pipeline_result, job_id=job.pk, return_created=feature_only) + if feature_only and saved: + feature_counts = { + "unmatched": saved.unmatched_detections or 0, + "without_vector": saved.detections_without_vector or 0, + } job.logger.info(f"Successfully saved results for job {job_id}") _, t = t( @@ -385,6 +396,7 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub counts_to_apply = ( (detections_count, classifications_count, captures_count) if is_first_processing else (0, 0, 0) ) + extra_counts = {key: count if is_first_processing else 0 for key, count in feature_counts.items()} _update_job_progress( job_id, "results", @@ -393,6 +405,7 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub detections=counts_to_apply[0], classifications=counts_to_apply[1], captures=counts_to_apply[2], + **extra_counts, ) # Ack LAST — only after the results-stage SREM and progress commit are @@ -408,6 +421,19 @@ def process_nats_pipeline_result(self, job_id: int, result_data: dict, reply_sub # Celery's autoretry_for handles the transient rather than this broad # except swallowing it. raise + except FeatureResultsStoredNothing as e: + # Redelivering would return the same response, and a later job would send the same + # detections again, so record the counts and fail the job instead of retrying. + _update_job_progress( + job_id, + "results", + 0, + complete_state=JobState.FAILURE, + unmatched=e.unmatched, + without_vector=e.without_vector, + ) + _ack_task_via_nats(reply_subject, job.logger) + _fail_job(job_id, str(e)) except Exception as e: error = f"Error processing pipeline result for job {job_id}: {e}" if not acked: @@ -502,6 +528,15 @@ async def ack_task(): return False +def _get_stage_param(job, stage: str, key: str) -> int: + """The integer value of one stage parameter, or 0 when the stage or parameter is absent.""" + try: + stage_obj = job.progress.get_stage(stage) + except ValueError: + return 0 + return next((param.value or 0 for param in stage_obj.params if param.key == key), 0) + + def _get_current_counts_from_job_progress(job, stage: str) -> tuple[int, int, int]: """ Get current detections, classifications, and captures counts from job progress. @@ -628,6 +663,9 @@ def _update_job_progress( state_params["detections"] = current_detections + new_detections state_params["classifications"] = current_classifications + new_classifications state_params["captures"] = current_captures + new_captures + for key in ("unmatched", "without_vector"): + if key in state_params: + state_params[key] = _get_stage_param(job, stage, key) + state_params[key] # Don't overwrite a stage with a stale progress value. # This guards against the race where a slower worker calls _update_job_progress diff --git a/ami/jobs/tests/test_tasks.py b/ami/jobs/tests/test_tasks.py index c2a6829a3..553af81e4 100644 --- a/ami/jobs/tests/test_tasks.py +++ b/ami/jobs/tests/test_tasks.py @@ -421,6 +421,72 @@ def fail_on_results_stage(self, processed_image_ids, stage, failed_image_ids=Non self.assertEqual(process_progress.processed, 1) self.assertEqual(results_progress.processed, 0) + def _feature_only_result(self, image: SourceImage, boxes: list[list[float]]) -> dict: + """A feature-only result: the given boxes echoed back on one image, each with a vector.""" + return PipelineResultsResponse( + pipeline="test-pipeline", + total_time=1.0, + source_images=[SourceImageResponse(id=str(image.pk), url="http://example.com/x.jpg")], + detections=[ + { + "source_image_id": str(image.pk), + "bbox": dict(zip(["x1", "y1", "x2", "y2"], box)), + "algorithm": {"name": self.detector.name, "key": self.detector.key}, + "timestamp": datetime.datetime.now(), + "embeddings": [ + {"algorithm": {"name": self.extractor.name, "key": self.extractor.key}, "features": [0.1] * 8} + ], + } + for box in boxes + ], + ).dict() + + def _make_feature_only(self) -> None: + self.detector = Algorithm.objects.create( + name="feature-detector", key="feature-detector", task_type=AlgorithmTaskType.LOCALIZATION + ) + self.extractor = Algorithm.objects.create( + name="feature-backbone", key="feature-backbone", task_type=AlgorithmTaskType.EMBEDDING + ) + self.pipeline.algorithms.set([self.detector, self.extractor]) + for image in self.images: + Detection.objects.create(source_image=image, bbox=[0, 0, 10, 10], detection_algorithm=self.detector) + + def _results_param(self, key: str): + self.job.refresh_from_db() + stage = self.job.progress.get_stage("results") + return next((param.value for param in stage.params if param.key == key), None) + + @patch("ami.jobs.tasks.TaskQueueManager") + def test_feature_only_results_count_boxes_that_match_no_detection(self, mock_manager_class): + self._setup_mock_nats(mock_manager_class) + self._make_feature_only() + for image in self.images[:2]: + process_nats_pipeline_result( + job_id=self.job.pk, + result_data=self._feature_only_result(image, [[0, 0, 10, 10], [50, 50, 60, 60]]), + reply_subject=f"reply.features.{image.pk}", + ) + self.assertEqual(self._results_param("unmatched"), 2) + self.assertEqual(self._results_param("without_vector"), 0) + self.assertNotEqual(self.job.status, JobState.FAILURE.value) + + @patch("ami.jobs.tasks._ack_task_via_nats") + @patch("ami.jobs.tasks.TaskQueueManager") + def test_feature_only_batch_matching_no_detection_fails_the_job_and_acks(self, mock_manager_class, mock_ack): + """Redelivering it would return the same boxes, so the message is acked and the job fails.""" + self._setup_mock_nats(mock_manager_class) + self._make_feature_only() + process_nats_pipeline_result( + job_id=self.job.pk, + result_data=self._feature_only_result(self.images[0], [[50, 50, 60, 60], [70, 70, 80, 80]]), + reply_subject="reply.features.none", + ) + mock_ack.assert_called_once() + self.assertEqual(self._results_param("unmatched"), 2) + self.assertEqual(self._results_param("without_vector"), 1) + self.assertEqual(self.job.status, JobState.FAILURE.value) + @patch("ami.jobs.tasks.TaskQueueManager") def test_results_counter_does_not_inflate_on_replay(self, mock_manager_class): """ diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 946e1548c..d2cc9bf79 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -1182,7 +1182,7 @@ class ClassificationNestedSerializer(ClassificationSerializer): has_features = serializers.BooleanField( read_only=True, allow_null=True, - help_text="Whether a feature embedding was stored for this classification.", + help_text="Whether this classification's algorithm stored a feature vector for its detection.", ) def get_permissions(self, instance, instance_data): @@ -1392,7 +1392,7 @@ class SourceImageSerializer(SourceImageListSerializer): ) detections_with_features = serializers.IntegerField( read_only=True, - help_text="Valid detections with at least one classification that stored a feature embedding.", + help_text="Valid detections with a stored feature vector, as an embedding or on a classification.", ) # file = serializers.ImageField(allow_empty_file=False, use_url=True) @@ -1771,6 +1771,10 @@ class OccurrenceSerializer(OccurrenceListSerializer): event = EventNestedSerializer(read_only=True) grouping_verified_by = UserNestedSerializer(read_only=True) grouping_summary = serializers.SerializerMethodField() + grouping_edited_since_verified = serializers.SerializerMethodField( + help_text="Whether the detections changed since the grouping was last confirmed. " + "False while it is confirmed, and when it never was." + ) # first_appearance = TaxonSourceImageNestedSerializer(read_only=True) class Meta: @@ -1788,12 +1792,19 @@ class Meta: "grouping_verified", "grouping_verified_at", "grouping_verified_by", + "grouping_edited_since_verified", "grouping_summary", ] read_only_fields = [ "determination_score", ] + def get_grouping_edited_since_verified(self, obj: Occurrence) -> bool: + from ami.main.models_future.history import edited_since_track_complete_review + + # Editing the detections withdraws the confirmation, so a confirmed grouping is unchanged. + return obj.grouping_verified_at is None and edited_since_track_complete_review(obj) + @extend_schema_field(OccurrenceFrameSerializer(many=True)) def get_detections(self, obj: Occurrence) -> list[dict]: frames = frames_from_prefetch(obj)[: self.detections_page_size] @@ -2307,6 +2318,52 @@ class OccurrenceGroupingSerializer(serializers.Serializer): grouping_verified_by = serializers.CharField(allow_null=True) +class HistoryUserSerializer(serializers.Serializer): + """A person in an occurrence's history: name and picture only, never an email address.""" + + id = serializers.IntegerField() + name = serializers.CharField() + image = serializers.ImageField(allow_null=True) + + +class HistoryAlgorithmSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + key = serializers.CharField() + + +class HistoryJobSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + + +class HistoryTaxonSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + rank = serializers.CharField() + + +class OccurrenceHistoryEntrySerializer(serializers.Serializer): + """One entry of an occurrence's history, newest first. ``type`` says which table it came from.""" + + type = serializers.ChoiceField(choices=["algorithm_result", "review", "identification", "prediction"]) + id = serializers.IntegerField(help_text="Primary key of the row in the table ``type`` names.") + timestamp = serializers.DateTimeField() + subtype = serializers.CharField( + allow_null=True, + help_text="For algorithm results and reviews: tracking, class_masking, size_filter or track_complete.", + ) + user = HistoryUserSerializer(allow_null=True) + algorithm = HistoryAlgorithmSerializer(allow_null=True) + job = HistoryJobSerializer(allow_null=True) + taxon = HistoryTaxonSerializer( + allow_null=True, help_text="The identified or predicted taxon, or the determination after a result." + ) + taxon_before = HistoryTaxonSerializer(allow_null=True, help_text="The determination before a result.") + score = serializers.FloatField(allow_null=True) + payload = serializers.JSONField(help_text="Details that depend on the type and subtype.") + + class OccurrencePathCaptureSerializer(serializers.Serializer): """The capture one frame of a path was measured against.""" @@ -2504,3 +2561,20 @@ class CaptureMatchesResponseSerializer(serializers.Serializer): "and then every score is null.", ) detections = CaptureMatchSerializer(many=True) + + +class SessionFeatureExtractorSerializer(serializers.Serializer): + """A feature extractor with vectors stored for a session's detections.""" + + id = serializers.IntegerField(source="algorithm.pk") + name = serializers.CharField(source="algorithm.name") + key = serializers.CharField(source="algorithm.key") + task_type = serializers.CharField(source="algorithm.task_type", allow_null=True) + embedding_dimensions = serializers.IntegerField(source="algorithm.embedding_dimensions", allow_null=True) + embeddings_count = serializers.IntegerField(help_text="Detections with a stored embedding from it.") + classification_vectors_count = serializers.IntegerField( + help_text="Classifications from it that carry a vector (data processed before embeddings were stored)." + ) + is_default = serializers.BooleanField( + help_text="Whether tracking compares this extractor's vectors when none is chosen." + ) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 4d281907a..3631c1044 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -36,6 +36,8 @@ from ami.base.views import ProjectMixin from ami.main.api.schemas import limit_doc_param, project_id_doc_param from ami.main.api.serializers import TagSerializer +from ami.main.models_future.embeddings import feature_extractors_with_vectors +from ami.main.models_future.history import occurrence_timeline from ami.main.models_future.identifications import create_identifications_batch, resolve_occurrences from ami.main.models_future.merge_candidates import ( DEFAULT_ADJACENT_CAPTURES, @@ -116,6 +118,7 @@ OccurrenceAddDetectionsSerializer, OccurrenceFrameSerializer, OccurrenceGroupingSerializer, + OccurrenceHistoryEntrySerializer, OccurrenceListSerializer, OccurrenceMergeSerializer, OccurrencePathFrameSerializer, @@ -124,6 +127,7 @@ PageSerializer, ProjectListSerializer, ProjectSerializer, + SessionFeatureExtractorSerializer, SiteSerializer, SourceImageCollectionNestedSerializer, SourceImageCollectionSerializer, @@ -545,6 +549,17 @@ def get_queryset(self) -> QuerySet: return qs + @extend_schema(parameters=[project_id_doc_param], responses=SessionFeatureExtractorSerializer(many=True)) + @action(detail=True, methods=["get"], name="feature-extractors", url_path="feature-extractors") + def feature_extractors(self, request, pk=None): + """The feature extractors with vectors for this session's detections, the default one marked. + + Tracking compares vectors from one extractor only; this lists the choices. + """ + event = self.get_object() + rows = feature_extractors_with_vectors(event.project_id, source_image__event=event) + return Response(SessionFeatureExtractorSerializer(rows, many=True).data) + @action(detail=True, methods=["get"], name="timeline") def timeline(self, request, pk=None): """ @@ -1667,10 +1682,10 @@ def get_serializer_class(self): ) # Actions that open one occurrence. They drop the determination requirement; see # get_queryset. - SINGLE_OCCURRENCE_ACTIONS = ("retrieve", "path", "detections") + SINGLE_OCCURRENCE_ACTIONS = ("retrieve", "path", "history", "detections") # Actions the project's default filters never hide an occurrence from. The session - # view selects occurrences with those filters off and draws their paths. - UNFILTERED_ACTIONS = (*TRACK_EDIT_ACTIONS, "path") + # view selects occurrences with those filters off, then draws their paths and histories. + UNFILTERED_ACTIONS = (*TRACK_EDIT_ACTIONS, "path", "history") def get_queryset(self) -> QuerySet["Occurrence"]: """Occurrences this request may see, which is wider outside the list. @@ -1695,7 +1710,9 @@ def get_queryset(self) -> QuerySet["Occurrence"]: "event", ) qs = qs.with_detections_count().with_timestamps() # type: ignore - if self.action != "detections": + if self.action not in ("history", "detections"): + # The history reads identifications itself, with the fields its entries need, + # and the detections page serializes frames, not the occurrence. qs = qs.with_identifications() # type: ignore if self.action not in self.UNFILTERED_ACTIONS: qs = qs.apply_default_filters( # type: ignore @@ -1703,8 +1720,8 @@ def get_queryset(self) -> QuerySet["Occurrence"]: ) if self.action == "list": qs = qs.with_list_prefetches() # type: ignore - elif self.action not in ("path", "detections", "merge_candidates", "capture_matches"): - # `path` and the track-edit pickers build their own values() queries and never + elif self.action not in ("path", "detections", "merge_candidates", "capture_matches", "history"): + # `path`, `history`, `detections` and the track-edit pickers build their own queries and never # serialize the occurrence, so the detail prefetch would only be waste: # measured at 249ms/4 queries against 6ms/2 for the same object without # it, on a 37-detection occurrence. @@ -1766,6 +1783,18 @@ def path(self, request: Request, pk=None) -> Response: occurrence = self.get_object() return Response(OccurrencePathFrameSerializer(occurrence_path(occurrence), many=True).data) + @extend_schema(parameters=[project_id_doc_param], responses=OccurrenceHistoryEntrySerializer(many=True)) + @action(detail=True, methods=["get"], name="history", pagination_class=None) + def history(self, request: Request, pk=None) -> Response: + """Everything that happened to this occurrence, newest first. + + Merges algorithm results, reviews of its grouping, identifications and predictions + into one list. Visible to whoever can open the occurrence itself. + """ + occurrence = self.get_object() + entries = occurrence_timeline(occurrence) + return Response(OccurrenceHistoryEntrySerializer(entries, many=True, context={"request": request}).data) + @extend_schema( parameters=[ project_id_doc_param, diff --git a/ami/main/management/commands/evaluate_tracking.py b/ami/main/management/commands/evaluate_tracking.py index 4ca6ff9a1..1475a25e1 100644 --- a/ami/main/management/commands/evaluate_tracking.py +++ b/ami/main/management/commands/evaluate_tracking.py @@ -48,7 +48,7 @@ def add_arguments(self, parser): "--feature-extraction-algorithm", type=int, default=None, - help="Algorithm ID whose embeddings to compare. Default: the only one in the session.", + help="Algorithm ID whose embeddings to compare. Default: as tracking picks it.", ) parser.add_argument("--format", choices=["text", "json"], default="text") parser.add_argument("--per-track", action="store_true", help="Include per-track scores in JSON output.") diff --git a/ami/main/migrations/0102_detection_embedding.py b/ami/main/migrations/0102_detection_embedding.py new file mode 100644 index 000000000..39ff62b4f --- /dev/null +++ b/ami/main/migrations/0102_detection_embedding.py @@ -0,0 +1,67 @@ +# Generated by Django 4.2.10 on 2026-09-15 02:01 +# +# Additive: creates an empty table. The vector column has no fixed length because +# extractors differ; each algorithm records its own length. The unique constraint's index, +# which leads with detection_id, also serves reads of the vectors for a set of detections. See #1417. + +from django.db import migrations, models +import django.db.models.deletion +import pgvector.django.vector + + +class Migration(migrations.Migration): + dependencies = [ + ("jobs", "0023_alter_job_job_type_key"), + ("ml", "0028_normalize_empty_endpoint_url_to_null"), + ("main", "0101_grant_run_post_processing_to_ml_data_manager"), + ] + + operations = [ + migrations.CreateModel( + name="DetectionEmbedding", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "vector", + pgvector.django.vector.VectorField( + help_text="Feature embedding from the model backbone. Its length is the algorithm's own." + ), + ), + ( + "algorithm", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="detection_embeddings", + to="ml.algorithm", + ), + ), + ( + "detection", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="embeddings", + to="main.detection", + ), + ), + ( + "job", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="jobs.job", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="detectionembedding", + constraint=models.UniqueConstraint( + fields=("detection", "algorithm"), name="unique_detection_embedding_per_algorithm" + ), + ), + ] diff --git a/ami/main/migrations/0103_occurrence_history.py b/ami/main/migrations/0103_occurrence_history.py new file mode 100644 index 000000000..a71c3d66d --- /dev/null +++ b/ami/main/migrations/0103_occurrence_history.py @@ -0,0 +1,73 @@ +# Generated by Django 4.2.10 on 2026-09-23 04:57 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("ml", "0028_normalize_empty_endpoint_url_to_null"), + ("jobs", "0023_alter_job_job_type_key"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("main", "0102_detection_embedding"), + ] + + operations = [ + migrations.CreateModel( + name="OccurrenceHistoryRecord", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "kind", + models.CharField( + choices=[("algorithm_result", "Algorithm Result"), ("review", "Review")], max_length=32 + ), + ), + ("subtype", models.CharField(max_length=64)), + ("timestamp", models.DateTimeField()), + ("payload", models.JSONField(default=dict)), + ( + "algorithm", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="ml.algorithm", + ), + ), + ( + "job", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="jobs.job", + ), + ), + ( + "occurrence", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="history", to="main.occurrence" + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "indexes": [models.Index(fields=["occurrence", "-timestamp"], name="occur_history_occ_time_idx")], + }, + ), + ] diff --git a/ami/main/migrations/0104_detection_embedding_vector_any_length.py b/ami/main/migrations/0104_detection_embedding_vector_any_length.py new file mode 100644 index 000000000..1f2938a02 --- /dev/null +++ b/ami/main/migrations/0104_detection_embedding_vector_any_length.py @@ -0,0 +1,18 @@ +# A database that applied an earlier draft of 0102 has a vector(2048) column, which rejects +# vectors of any other length. Dropping the length is a no-op where the column already has none. +# Django's state already matches, so this only touches the database. See #1417. + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0103_occurrence_history"), + ] + + operations = [ + migrations.RunSQL( + sql="ALTER TABLE main_detectionembedding ALTER COLUMN vector TYPE vector", + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index dbbadf7fa..b097a34e3 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -2202,16 +2202,16 @@ def with_was_processed(self): def with_detections_with_features(self): """Annotate ``detections_with_features`` and the ``detections_valid`` it is out of: - valid detections on the capture, and how many of them have a classification that - stored a feature embedding. Counted in SQL so the vectors themselves are never - loaded. Both come from the same population, so a caller can show one as a share of - the other; the cached ``detections_count`` is a different, default-filtered count. + valid detections on the capture, and how many of them carry a stored feature vector + from any algorithm (see ``DetectionQuerySet.has_vector``). Counted in SQL so the + vectors themselves are never loaded. Both come from the same population, so a caller + can show one as a share of the other; the cached ``detections_count`` is a different, + default-filtered count. """ - def count_valid(**extra): + def count_valid(detections): return models.Subquery( - Detection.objects.valid() - .filter(source_image_id=models.OuterRef("pk"), **extra) + detections.filter(source_image_id=models.OuterRef("pk")) .order_by() .values("source_image_id") .annotate(count=models.Count("id", distinct=True)) @@ -2220,8 +2220,8 @@ def count_valid(**extra): ) return self.annotate( - detections_valid=Coalesce(count_valid(), 0), - detections_with_features=Coalesce(count_valid(classifications__features_2048__isnull=False), 0), + detections_valid=Coalesce(count_valid(Detection.objects.valid()), 0), + detections_with_features=Coalesce(count_valid(Detection.objects.valid().has_vector()), 0), ) def with_thumbnails(self): @@ -3061,12 +3061,21 @@ class ClassificationQuerySet(BaseQuerySet): def with_has_features(self): """Annotate ``has_features`` and defer the embedding itself. - Read paths only need to know whether a feature vector was stored; deferring the - 2048-float column keeps it out of the row's SELECT. A select_related self-join - (``applied_to``) needs its own ``defer("applied_to__features_2048")``. + ``has_features`` is true when the classification's algorithm stored a vector for + its detection, on the classification or as a ``DetectionEmbedding``. Read paths + only need to know that; deferring the 2048-float column keeps it out of the row's + SELECT. A select_related self-join (``applied_to``) needs its own + ``defer("applied_to__features_2048")``. """ + embedded = Exists( + DetectionEmbedding.objects.filter( + detection_id=OuterRef("detection_id"), algorithm_id=OuterRef("algorithm_id") + ) + ) return self.defer("features_2048").annotate( - has_features=models.ExpressionWrapper(Q(features_2048__isnull=False), output_field=models.BooleanField()) + has_features=models.ExpressionWrapper( + Q(features_2048__isnull=False) | Q(embedded), output_field=models.BooleanField() + ) ) def find_duplicates(self, project_id: int | None = None) -> models.QuerySet: @@ -3260,6 +3269,18 @@ def null_markers(self): """ return self.filter(NULL_DETECTIONS_FILTER) + def has_vector(self, algorithm=None): + """Detections with a stored feature vector: a ``DetectionEmbedding``, or a + classification's ``features_2048``. Pass ``algorithm`` to count only that + algorithm's vectors. Tested with EXISTS, so no vector is ever loaded. + """ + embeddings = DetectionEmbedding.objects.filter(detection_id=OuterRef("pk")) + classifications = Classification.objects.filter(detection_id=OuterRef("pk"), features_2048__isnull=False) + if algorithm is not None: + embeddings = embeddings.filter(algorithm=algorithm) + classifications = classifications.filter(algorithm=algorithm) + return self.filter(Exists(embeddings) | Exists(classifications)) + class DetectionManager(models.Manager.from_queryset(DetectionQuerySet)): pass @@ -3339,6 +3360,7 @@ class Detection(BaseModel): # For type hints classifications: models.QuerySet["Classification"] + embeddings: models.QuerySet["DetectionEmbedding"] source_image_id: int detection_algorithm_id: int @@ -3481,6 +3503,38 @@ def __str__(self) -> str: return f"#{self.pk} from SourceImage #{self.source_image_id} with Algorithm #{self.detection_algorithm_id}" +@final +class DetectionEmbedding(BaseModel): + """A feature vector for one detection from one algorithm, used to compare detections by appearance. + + Kept apart from classifications so every detection can have one, including those the + moth/non-moth filter rejected, without adding a prediction that could change a + determination. Vectors are only comparable within one algorithm. See #1417. + """ + + project_accessor = "detection__source_image__project" + + # No separate index: the unique constraint's index leads with detection_id. + detection = models.ForeignKey(Detection, on_delete=models.CASCADE, related_name="embeddings", db_index=False) + algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="detection_embeddings") + # No fixed length: extractors differ. Each algorithm keeps one (Algorithm.embedding_dimensions). + vector = pgvector.django.VectorField( + help_text="Feature embedding from the model backbone. Its length is the algorithm's own.", + ) + # The job whose results stored this vector; kept when the job is deleted, since the vector stays valid. + job = models.ForeignKey("jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="+") + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["detection", "algorithm"], name="unique_detection_embedding_per_algorithm" + ), + ] + + def __str__(self) -> str: + return f"#{self.pk} vector for Detection #{self.detection_id} from Algorithm #{self.algorithm_id}" + + class OccurrenceQuerySet(BaseQuerySet): def valid(self): """ @@ -3517,15 +3571,16 @@ def with_detections_count(self): return self.annotate(detections_count=models.Count("detections", distinct=True)) def with_frames_with_vectors(self): - """Annotate ``frames_with_vectors``: detections in the occurrence with at least one - classification that stored a feature embedding. Counted in SQL so the vectors - themselves are never loaded. + """Annotate ``frames_with_vectors``: detections in the occurrence with a stored + feature vector from any algorithm (see ``DetectionQuerySet.has_vector``). Counted + in SQL so the vectors themselves are never loaded. """ subquery = ( - Detection.objects.filter(occurrence_id=OuterRef("pk"), classifications__features_2048__isnull=False) + Detection.objects.has_vector() + .filter(occurrence_id=OuterRef("pk")) .order_by() .values("occurrence_id") - .annotate(count=models.Count("id", distinct=True)) + .annotate(count=models.Count("id")) .values("count") ) return self.annotate( @@ -4071,6 +4126,74 @@ class Meta: ] +@final +class OccurrenceHistoryRecord(BaseModel): + """One dated entry in an occurrence's history: an algorithm's result or a person's review. + + Identifications and predictions keep their own tables; the history endpoint merges all + three. An algorithm result stands for the prediction change a run made, and a run writes + at most one record per occurrence, none when it changed nothing about it. Reviews are + append-only; ``Occurrence.grouping_verified_at`` and ``_by`` hold the current confirmation, + which unverifying or editing the track clears while its reviews stay. + The payload is validated against the schema for its kind and subtype (ami/main/schemas.py). + """ + + project_accessor = "occurrence__project" + + class Kind(models.TextChoices): + ALGORITHM_RESULT = "algorithm_result" + REVIEW = "review" + + occurrence = models.ForeignKey(Occurrence, on_delete=models.CASCADE, related_name="history") + kind = models.CharField(max_length=32, choices=Kind.choices) + # "tracking", "class_masking", "size_filter" or "track_complete"; see HISTORY_PAYLOAD_SCHEMAS. + subtype = models.CharField(max_length=64) + job = models.ForeignKey("jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="+") + algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.SET_NULL, null=True, blank=True, related_name="+") + user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, related_name="+") + timestamp = models.DateTimeField() + payload = models.JSONField(default=dict) + + class Meta: + indexes = [models.Index(fields=["occurrence", "-timestamp"], name="occur_history_occ_time_idx")] + + def __str__(self) -> str: + return f"#{self.pk} {self.kind}/{self.subtype} for Occurrence #{self.occurrence_id}" + + @classmethod + def build( + cls, + *, + occurrence_id: int, + kind: str, + subtype: str, + payload, + timestamp: datetime.datetime | None = None, + job=None, + algorithm=None, + user: User | None = None, + ) -> "OccurrenceHistoryRecord": + """An unsaved record with a validated payload, for writers that bulk_create, which skips save().""" + from ami.main.schemas import validate_history_payload + + return cls( + occurrence_id=occurrence_id, + kind=kind, + subtype=subtype, + payload=validate_history_payload(kind, subtype, payload), + timestamp=timestamp or timezone.now(), + job=job, + algorithm=algorithm, + user=user, + ) + + def save(self, *args, **kwargs): + from ami.main.schemas import validate_history_payload + + self.payload = validate_history_payload(self.kind, self.subtype, self.payload) + super().save(*args, **kwargs) + + def update_occurrence_determination( occurrence: Occurrence, current_determination: typing.Optional["Taxon"] = None, save=True ) -> bool: diff --git a/ami/main/models_future/embeddings.py b/ami/main/models_future/embeddings.py new file mode 100644 index 000000000..121e54dbb --- /dev/null +++ b/ami/main/models_future/embeddings.py @@ -0,0 +1,210 @@ +"""Feature vectors for detections, read one algorithm at a time. + +A detection's vector is stored in one of two places: a ``DetectionEmbedding`` row, which a +processing service can send for every detection, or the ``features_2048`` of one of its +classifications, which is all that data processed before embeddings existed has. Readers +take the embedding when there is one and otherwise the most recent classification vector +from the same algorithm. Vectors from different algorithms are not comparable, so every +reader here is keyed by algorithm. See #1417. +""" + +from __future__ import annotations + +import collections +import logging +from collections.abc import Iterable +from typing import Any + +from django.db.models import Count, Exists, F, IntegerField, OuterRef, Q, QuerySet, Value + +logger = logging.getLogger(__name__) + +_PREFER_EMBEDDING = 0 +_PREFER_CLASSIFICATION = 1 + + +def _vector_rows(detection_ids: Iterable[int], algorithm_ids: Iterable[int] | None) -> QuerySet: + """(detection_id, algorithm_id, vector, ...) rows from both stores in one query, preferred rows first.""" + from ami.main.models import Classification, DetectionEmbedding + + detection_ids = list(detection_ids) + embeddings = DetectionEmbedding.objects.filter(detection_id__in=detection_ids) + classifications = Classification.objects.filter( + detection_id__in=detection_ids, algorithm_id__isnull=False, features_2048__isnull=False + ) + if algorithm_ids is not None: + algorithm_ids = list(algorithm_ids) + embeddings = embeddings.filter(algorithm_id__in=algorithm_ids) + classifications = classifications.filter(algorithm_id__in=algorithm_ids) + + # Model fields first, then the annotations in the same order, so both SELECT lists line up + # column for column (the vector columns differ in name). UNION ALL: de-duplicating would sort the vectors. + embeddings = ( + embeddings.order_by() + .annotate(preference=Value(_PREFER_EMBEDDING, output_field=IntegerField()), recorded_at=F("updated_at")) + .values_list("detection_id", "algorithm_id", "vector", "id", "preference", "recorded_at") + ) + classifications = ( + classifications.order_by() + .annotate(preference=Value(_PREFER_CLASSIFICATION, output_field=IntegerField()), recorded_at=F("timestamp")) + .values_list("detection_id", "algorithm_id", "features_2048", "id", "preference", "recorded_at") + ) + return embeddings.union(classifications, all=True).order_by("preference", "-recorded_at", "-id") + + +def latest_vectors( + detection_ids: Iterable[int], algorithm_ids: Iterable[int] | None = None +) -> dict[tuple[int, int], Any]: + """The vector for each (detection, algorithm) pair that has one, in one query. + + Pass ``algorithm_ids`` to read only those algorithms. The caller must still compare + vectors from one algorithm only. + """ + vectors: dict[tuple[int, int], Any] = {} + for detection_id, algorithm_id, vector, *_ in _vector_rows(detection_ids, algorithm_ids): + vectors.setdefault((detection_id, algorithm_id), vector) + return vectors + + +def vectors_for_detections(detection_ids: Iterable[int], algorithm_id: int) -> dict[int, Any]: + """Each detection's vector from one algorithm, in one query. Detections without one are absent. + + All the vectors returned have one length. Should an algorithm have stored two lengths + (an embedding and an older classification vector), only the more common length is + kept, so no caller can compare vectors of different sizes. + """ + vectors = { + detection_id: vector for (detection_id, _), vector in latest_vectors(detection_ids, [algorithm_id]).items() + } + lengths = collections.Counter(len(vector) for vector in vectors.values()) + if len(lengths) > 1: + keep = max(lengths, key=lambda length: (lengths[length], length)) + logger.warning(f"Algorithm {algorithm_id} has vectors of lengths {dict(lengths)}; using only length {keep}.") + vectors = {detection_id: vector for detection_id, vector in vectors.items() if len(vector) == keep} + return vectors + + +def algorithm_ids_with_vectors(**detection_lookups: Any) -> set[int]: + """Algorithms that stored a vector for any detection matching the lookups, in one query. + + Lookups are relative to the detection, e.g. ``source_image__event=event``. + """ + from ami.main.models import Classification, DetectionEmbedding + + lookups = {f"detection__{key}": value for key, value in detection_lookups.items()} + embedded = DetectionEmbedding.objects.filter(**lookups).order_by().values_list("algorithm_id", flat=True) + classified = ( + Classification.objects.filter(**lookups, features_2048__isnull=False, algorithm_id__isnull=False) + .order_by() + .values_list("algorithm_id", flat=True) + ) + return set(embedded.union(classified)) + + +# A configured extractor is preferred only once it covers nearly as many detections as the best +# one, so a feature-extraction run still in progress does not become the default for the session. +CONFIGURED_EXTRACTOR_MIN_COVERAGE = 0.9 + + +def detections_covered(algorithm_ids: Iterable[int], **detection_lookups: Any) -> dict[int, int]: + """How many detections matching the lookups have a vector from each algorithm, in one query. + + A detection counts once per algorithm whichever store (embedding or classification) holds its vector. + """ + from ami.main.models import Classification, Detection, DetectionEmbedding + + algorithm_ids = list(algorithm_ids) + if not algorithm_ids: + return {} + # The Exists terms are annotations, not a Q inside the Count, so cachalot sees their tables + # and a newly stored vector invalidates the cached count. + flags, counts = {}, {} + for algorithm_id in algorithm_ids: + embedded, classified = f"embedded_{algorithm_id}", f"classified_{algorithm_id}" + flags[embedded] = Exists( + DetectionEmbedding.objects.filter(detection_id=OuterRef("pk"), algorithm_id=algorithm_id) + ) + flags[classified] = Exists( + Classification.objects.filter( + detection_id=OuterRef("pk"), algorithm_id=algorithm_id, features_2048__isnull=False + ) + ) + counts[f"algorithm_{algorithm_id}"] = Count("pk", filter=Q(**{embedded: True}) | Q(**{classified: True})) + totals = Detection.objects.filter(**detection_lookups).order_by().annotate(**flags).aggregate(**counts) + return {algorithm_id: totals[f"algorithm_{algorithm_id}"] for algorithm_id in algorithm_ids} + + +def default_feature_algorithm_id(project: Any, algorithm_ids: Iterable[int], **detection_lookups: Any) -> int | None: + """The extractor to compare when the caller chose none, among ``algorithm_ids``. + + The one with vectors for the most detections matching the lookups, so the choice does not + depend on which job wrote last. An extractor the project runs (a feature-extraction algorithm + in a pipeline enabled for it) wins when it covers at least CONFIGURED_EXTRACTOR_MIN_COVERAGE + of that. Ties go to the newest algorithm. At most 2 queries. + """ + from ami.ml.models import Algorithm, Pipeline + + algorithm_ids = sorted(set(algorithm_ids)) + if len(algorithm_ids) <= 1: + return algorithm_ids[0] if algorithm_ids else None + configured = set( + Algorithm.objects.filter( + pk__in=algorithm_ids, + task_type__in=Algorithm.feature_extraction_task_types, + pipelines__in=Pipeline.objects.all().enabled(project).values("pk"), + ) + .order_by() + .values_list("pk", flat=True) + ) + coverage = detections_covered(algorithm_ids, **detection_lookups) + + def rank(algorithm_id: int) -> tuple[int, int]: + return coverage[algorithm_id], algorithm_id + + best = max(algorithm_ids, key=rank) + eligible = [ + algorithm_id + for algorithm_id in configured + if coverage[algorithm_id] >= CONFIGURED_EXTRACTOR_MIN_COVERAGE * coverage[best] + ] + return max(eligible, key=rank) if eligible else best + + +def feature_extractors_with_vectors(project: Any, **detection_lookups: Any) -> list[dict[str, Any]]: + """The algorithms with vectors for the detections matching the lookups, and how many each has. + + Each row: the algorithm, ``embeddings_count`` (``DetectionEmbedding`` rows), + ``classification_vectors_count`` (classifications carrying a vector) and ``is_default``, + the one tracking compares when no extractor is chosen. Newest algorithm first. At most 5 queries. + """ + from ami.main.models import Classification, DetectionEmbedding + from ami.ml.models import Algorithm + + lookups = {f"detection__{key}": value for key, value in detection_lookups.items()} + embedded = dict( + DetectionEmbedding.objects.filter(**lookups) + .order_by() + .values("algorithm_id") + .annotate(n=Count("pk")) + .values_list("algorithm_id", "n") + ) + classified = dict( + Classification.objects.filter(**lookups, features_2048__isnull=False, algorithm_id__isnull=False) + .order_by() + .values("algorithm_id") + .annotate(n=Count("pk")) + .values_list("algorithm_id", "n") + ) + algorithm_ids = set(embedded) | set(classified) + if not algorithm_ids: + return [] + default_id = default_feature_algorithm_id(project, algorithm_ids, **detection_lookups) + return [ + { + "algorithm": algorithm, + "embeddings_count": embedded.get(algorithm.pk, 0), + "classification_vectors_count": classified.get(algorithm.pk, 0), + "is_default": algorithm.pk == default_id, + } + for algorithm in Algorithm.objects.filter(pk__in=sorted(algorithm_ids)).order_by("-pk") + ] diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py new file mode 100644 index 000000000..4f9f51501 --- /dev/null +++ b/ami/main/models_future/history.py @@ -0,0 +1,237 @@ +"""An occurrence's history: what algorithms and people did to it, newest first. + +Algorithm results and reviews are ``OccurrenceHistoryRecord`` rows. Identifications and +predictions keep their own tables, and ``occurrence_timeline`` merges all of them into one +list for the history endpoint. +""" + +from __future__ import annotations + +import dataclasses +import datetime +import typing + +from ami.main.models import Classification, Detection, Identification, Occurrence, OccurrenceHistoryRecord, Taxon, User +from ami.main.schemas import TrackCompleteReviewPayload + +if typing.TYPE_CHECKING: + from ami.jobs.models import Job + from ami.ml.models import Algorithm + +TRACK_COMPLETE = "track_complete" + + +def latest_track_complete_review(occurrence: Occurrence) -> OccurrenceHistoryRecord | None: + """The latest review written for this occurrence, ignoring reviews a merge brought in.""" + return ( + OccurrenceHistoryRecord.objects.filter( + occurrence=occurrence, + kind=OccurrenceHistoryRecord.Kind.REVIEW, + subtype=TRACK_COMPLETE, + payload__occurrence_id=occurrence.pk, + ) + .order_by("-timestamp", "-pk") + .first() + ) + + +def edited_since_track_complete_review(occurrence: Occurrence) -> bool: + """Whether the occurrence's detections differ from those its latest own review confirmed.""" + review = latest_track_complete_review(occurrence) + if review is None: + return False + current = set(Detection.objects.valid().filter(occurrence=occurrence).values_list("pk", flat=True)) + return current != set(review.payload["detection_ids"]) + + +def record_track_complete_review( + occurrence: Occurrence, user: User, timestamp: datetime.datetime +) -> OccurrenceHistoryRecord | None: + """Record that ``user`` confirmed this occurrence's detections. + + A review is written only when the detections differ from the latest review or a + different person confirms them, so withdrawing and re-confirming an unchanged track + does not repeat the same review. + """ + previous = latest_track_complete_review(occurrence) + record = _build_track_complete_review(occurrence, user.pk, timestamp, previous) + if ( + previous is not None + and previous.user_id == user.pk + and sorted(previous.payload["detection_ids"]) == record.payload["detection_ids"] + ): + return None + record.save() + return record + + +def carry_confirmation_over_split(occurrence: Occurrence, pieces: list[Occurrence]) -> None: + """Restate a confirmed occurrence's review for each piece a session split left. + + Each review keeps the original reviewer and time but lists only its piece's + detections, so a later re-confirmation of a piece compares against what it holds. + """ + if occurrence.grouping_verified_at is None: + return + OccurrenceHistoryRecord.objects.bulk_create( + _build_track_complete_review( + piece, + occurrence.grouping_verified_by_id, + occurrence.grouping_verified_at, + previous=None, + split_from_occurrence_id=occurrence.pk, + ) + for piece in [occurrence, *pieces] + ) + + +def _build_track_complete_review( + occurrence: Occurrence, + user_id: int | None, + timestamp: datetime.datetime, + previous: OccurrenceHistoryRecord | None, + split_from_occurrence_id: int | None = None, +) -> OccurrenceHistoryRecord: + """An unsaved review of the occurrence's current detections, with the change since ``previous``.""" + frames = list( + Detection.objects.valid() + .filter(occurrence=occurrence) + .order_by("source_image__timestamp", "pk") + .values_list("pk", "source_image_id", "source_image__timestamp") + ) + detection_ids = sorted(pk for pk, _, _ in frames) + previous_ids = set(previous.payload["detection_ids"]) if previous else set(detection_ids) + capture_times = [captured for _, _, captured in frames if captured is not None] + payload = TrackCompleteReviewPayload( + detection_ids=detection_ids, + frames_count=len({capture_id for _, capture_id, _ in frames}), + first_timestamp=min(capture_times, default=None), + last_timestamp=max(capture_times, default=None), + detections_added=sorted(set(detection_ids) - previous_ids), + detections_removed=sorted(previous_ids - set(detection_ids)), + occurrence_id=occurrence.pk, + split_from_occurrence_id=split_from_occurrence_id, + ) + record = OccurrenceHistoryRecord.build( + occurrence_id=occurrence.pk, + kind=OccurrenceHistoryRecord.Kind.REVIEW, + subtype=TRACK_COMPLETE, + payload=payload, + timestamp=timestamp, + ) + record.user_id = user_id + return record + + +@dataclasses.dataclass +class TimelineEntry: + """One entry of the merged history, in the shape ``OccurrenceTimelineEntrySerializer`` reads.""" + + type: str + id: int + timestamp: datetime.datetime + subtype: str | None = None + user: User | None = None + algorithm: Algorithm | None = None + job: Job | None = None + taxon: Taxon | None = None + taxon_before: Taxon | None = None + score: float | None = None + payload: dict = dataclasses.field(default_factory=dict) + + +def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: + """History records, identifications and predictions of one occurrence, merged newest first. + + Each algorithm contributes one prediction, its best. A prediction made by an algorithm + that also left a history record here is left out: the record already stands for that + change and names the taxon before and after. Costs four queries whatever the number of + entries. + """ + records = list( + OccurrenceHistoryRecord.objects.filter(occurrence=occurrence) + .select_related("user", "algorithm", "job") + .order_by("-timestamp", "-pk") + ) + taxon_ids = { + record.payload.get(key) + for record in records + for key in ("taxon_before_id", "taxon_after_id") + if record.payload.get(key) is not None + } + taxa = {taxon.pk: taxon for taxon in Taxon.objects.filter(pk__in=taxon_ids)} if taxon_ids else {} + + entries = [ + TimelineEntry( + type=record.kind, + id=record.pk, + timestamp=record.timestamp, + subtype=record.subtype, + user=record.user, + algorithm=record.algorithm, + job=record.job, + taxon=taxa.get(record.payload.get("taxon_after_id")), + taxon_before=taxa.get(record.payload.get("taxon_before_id")), + payload=record.payload, + ) + for record in records + ] + + identifications = Identification.objects.filter(occurrence=occurrence).select_related("user", "taxon") + entries.extend( + TimelineEntry( + type="identification", + id=identification.pk, + timestamp=identification.created_at, + user=identification.user, + taxon=identification.taxon, + payload={ + "comment": identification.comment, + "withdrawn": identification.withdrawn, + "agreed_with_identification_id": identification.agreed_with_identification_id, + "agreed_with_prediction_id": identification.agreed_with_prediction_id, + }, + ) + for identification in identifications + ) + + folded = {record.algorithm_id for record in records if record.algorithm_id is not None} + entries.extend( + TimelineEntry( + type="prediction", + id=prediction.pk, + timestamp=prediction.created_at, + algorithm=prediction.algorithm, + taxon=prediction.taxon, + score=prediction.score, + payload={ + "detection_id": prediction.detection_id, + "terminal": prediction.terminal, + "applied_to_id": prediction.applied_to_id, + }, + ) + for prediction in _one_prediction_per_algorithm(occurrence) + if prediction.algorithm_id not in folded + ) + + entries.sort(key=lambda entry: (entry.timestamp, entry.id), reverse=True) + return entries + + +def _one_prediction_per_algorithm(occurrence: Occurrence) -> list[Classification]: + """The best prediction of each algorithm: highest score, then terminal, then latest. + + ``Occurrence.predictions()`` keeps every classification tied for an algorithm's top + score, which would show the same prediction once per detection. + """ + + def rank(prediction: Classification) -> tuple: + score = prediction.score if prediction.score is not None else float("-inf") + return (score, prediction.terminal, prediction.created_at, prediction.pk) + + best: dict[int | None, Classification] = {} + for prediction in occurrence.predictions(): + current = best.get(prediction.algorithm_id) + if current is None or rank(prediction) > rank(current): + best[prediction.algorithm_id] = prediction + return list(best.values()) diff --git a/ami/main/models_future/merge_candidates.py b/ami/main/models_future/merge_candidates.py index 7cef580ac..4279db8c3 100644 --- a/ami/main/models_future/merge_candidates.py +++ b/ami/main/models_future/merge_candidates.py @@ -25,8 +25,11 @@ positive for ``after`` and zero for ``gap``. - ``distance``: centre-to-centre distance of the nearest pair of boxes as a fraction of the frame diagonal. -- ``similarity``: cosine similarity of the pair's feature vectors, from the same - algorithm that produced the occurrence's own vector. Null when either has none. +- ``similarity``: cosine similarity of the pair's feature vectors. Vectors from different + algorithms are not comparable, so every candidate is scored with one algorithm: the one + with vectors on the most of the occurrence's scored frames. A frame's vector is its + embedding, or failing that its classification vector from that algorithm (see + ``embeddings``). Null when either frame has no vector from it. - ``cost``: the tracking method's matching cost for the pair. Geometry only when similarity is null, which lowers the total, so a candidate without a vector can outrank one with a poor vector match. Null when either box is malformed. @@ -55,11 +58,13 @@ from __future__ import annotations +import collections import datetime from typing import TYPE_CHECKING, Any from django.db.models import Q, QuerySet +from ami.main.models_future.embeddings import algorithm_ids_with_vectors, latest_vectors, vectors_for_detections from ami.main.models_future.track_stats import bbox_corners, frame_diagonal if TYPE_CHECKING: @@ -152,13 +157,11 @@ def _pair_diagonal(track_frame: dict, frame: dict, corners_a, corners_b) -> floa return frame_diagonal(None, None, max(corners_a[2], corners_b[2]), max(corners_a[3], corners_b[3])) -def _latest_vectors(classifications) -> dict[tuple[int, int], Any]: - """Most recent feature vector per (detection, algorithm) among the rows given.""" - vectors: dict[tuple[int, int], Any] = {} - rows = classifications.order_by("-timestamp", "-pk").values_list("detection_id", "algorithm_id", "features_2048") - for detection_id, algorithm_id, vector in rows: - vectors.setdefault((detection_id, algorithm_id), vector) - return vectors +def _comparison_algorithm(track_vectors: dict[tuple[int, int], Any]) -> int | None: + """The algorithm every candidate is scored with: the one with vectors on the most of the + occurrence's scored frames, the lowest id on a tie so the choice is stable.""" + counts = collections.Counter(algorithm_id for _, algorithm_id in track_vectors) + return min(counts, key=lambda algorithm_id: (-counts[algorithm_id], algorithm_id)) if counts else None def _score_pair(track_frame: dict, frame: dict, track_vector, frame_vector) -> dict[str, float | None]: @@ -219,7 +222,7 @@ def rank_merge_candidates( place of the track's frames. Raises ``DetectionNotInOccurrence`` when it is not one of the occurrence's frames. """ - from ami.main.models import Classification, Detection, get_media_url + from ami.main.models import Detection, get_media_url config = tracking_config_for(occurrence) track = Detection.objects.valid().filter(occurrence_id=occurrence.pk) @@ -279,31 +282,17 @@ def rank_merge_candidates( for candidate, relation, _ in scored } - track_vectors = _latest_vectors( - Classification.objects.filter( - detection_id__in={track_frame["pk"] for track_frame, _ in pairs.values()}, - algorithm_id__isnull=False, - features_2048__isnull=False, - ) - ) - frame_vectors: dict[tuple[int, int], Any] = {} - if track_vectors: - frame_vectors = _latest_vectors( - Classification.objects.filter( - detection_id__in=[frame["pk"] for _, frame in pairs.values()], - algorithm_id__in={algorithm_id for _, algorithm_id in track_vectors}, - features_2048__isnull=False, - ) - ) - vector_by_track_frame = { - detection_id: (algorithm_id, vector) for (detection_id, algorithm_id), vector in track_vectors.items() - } + track_vectors = latest_vectors({track_frame["pk"] for track_frame, _ in pairs.values()}) + algorithm_id = _comparison_algorithm(track_vectors) + frame_vectors: dict[int, Any] = {} + if algorithm_id is not None: + frame_vectors = vectors_for_detections([frame["pk"] for _, frame in pairs.values()], algorithm_id) rows: list[dict[str, Any]] = [] for candidate, relation, offset in scored: track_frame, frame = pairs[candidate.pk] - algorithm_id, track_vector = vector_by_track_frame.get(track_frame["pk"], (None, None)) - frame_vector = frame_vectors.get((frame["pk"], algorithm_id)) if algorithm_id is not None else None + track_vector = track_vectors.get((track_frame["pk"], algorithm_id)) + frame_vector = frame_vectors.get(frame["pk"]) scores = _score_pair(track_frame, frame, track_vector, frame_vector) crop = frame["path"] or next((f["path"] for f in frames_by_occurrence[candidate.pk] if f["path"]), None) rows.append( @@ -407,14 +396,9 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di such a pair of captures. The query count is fixed, whatever the track's length or the number of boxes. """ - from ami.main.models import Classification, Detection, SourceImage + from ami.main.models import Detection, SourceImage from ami.ml.models import Algorithm - from ami.ml.post_processing.tracking_task import ( - image_diagonal, - latest_feature_vectors, - resolve_feature_algorithm, - select_links, - ) + from ami.ml.post_processing.tracking_task import image_diagonal, resolve_feature_algorithm, select_links config = tracking_config_for(occurrence) reference, relation = _reference_frame(occurrence.pk, capture) if capture.timestamp else (None, None) @@ -429,21 +413,13 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di if reference is not None and any(box.occurrence_id == occurrence.pk for box in boxes): relation = RELATION_SAME - # Tracking takes the one extractor with embeddings anywhere in the session, which scans + # Tracking takes the one extractor with vectors anywhere in the session, which scans # every classification (about 40 ms). The two captures being paired give the same answer # unless a session mixes extractors. detection_ids = [detection.pk for detection in detections] - extractor_ids = set( - Classification.objects.filter( - detection_id__in=detection_ids, features_2048__isnull=False, algorithm_id__isnull=False - ) - .order_by() - .values_list("algorithm_id", flat=True) - .distinct() - ) - extractors = list(Algorithm.objects.filter(pk__in=extractor_ids)) + extractors = list(Algorithm.objects.filter(pk__in=algorithm_ids_with_vectors(pk__in=detection_ids))) algorithm, _, _ = resolve_feature_algorithm(occurrence.event, config, candidates=extractors) - vectors = latest_feature_vectors(detection_ids, algorithm.pk) if algorithm is not None else {} + vectors = vectors_for_detections(detection_ids, algorithm.pk) if algorithm is not None else {} def skipped(detection_id: int) -> str | None: return SKIPPED_NO_VECTOR if config.require_features and detection_id not in vectors else None diff --git a/ami/main/models_future/tracks.py b/ami/main/models_future/tracks.py index 176fe08b5..38881b78f 100644 --- a/ami/main/models_future/tracks.py +++ b/ami/main/models_future/tracks.py @@ -52,11 +52,13 @@ Event, Identification, Occurrence, + OccurrenceHistoryRecord, SourceImage, User, update_calculated_fields_for_sessions_and_stations, update_occurrence_determination, ) +from ami.main.models_future.history import carry_confirmation_over_split, record_track_complete_review from ami.main.models_future.track_stats import refresh_track_stats # Frame order within a track: capture time, then capture, then detection. The edits, @@ -214,7 +216,8 @@ def split_at_session_boundaries(occurrence: Occurrence) -> list[Occurrence]: keeps this occurrence and its identifications; each later piece is a new occurrence holding copies of them. Unlike a manual split, every piece keeps the grouping confirmation and the chain link to the next piece, since each piece is still the - whole track within its session and the link records that they are one animal. + whole track within its session and the link records that they are one animal. The + confirmation is restated as a review of each piece's own detections. Returns the new occurrences in time order, or an empty list when nothing was split. """ # Waits without a timeout, as a tracking run does: a regroup is a background job too. @@ -248,6 +251,7 @@ def split_at_session_boundaries(occurrence: Occurrence) -> list[Occurrence]: grouping_verified_at=occurrence.grouping_verified_at, grouping_verified_by_id=occurrence.grouping_verified_by_id, ) + carry_confirmation_over_split(occurrence, pieces) _copy_identifications(occurrence, pieces) for piece in [occurrence, *pieces]: @@ -451,16 +455,16 @@ def clear_grouping_verification(*occurrences: Occurrence) -> None: def _absorb(target: Occurrence, sources: list[Occurrence]) -> None: - """Move every detection and identification off ``sources`` and delete them. + """Move every detection, identification and history record off ``sources`` and delete them. - Identifications move first. ``Identification.occurrence`` cascades on delete, so - a source removed before its identifications were reassigned would take a - person's work with it. + Identifications and history move first. Both cascade on delete, so a source removed + before they were reassigned would take a person's work and its record with it. """ source_pks = [o.pk for o in sources] if not source_pks: return Identification.objects.filter(occurrence_id__in=source_pks).update(occurrence=target) + OccurrenceHistoryRecord.objects.filter(occurrence_id__in=source_pks).update(occurrence=target) Detection.objects.filter(occurrence_id__in=source_pks).update(occurrence=target) Occurrence.objects.filter(pk__in=source_pks).delete() @@ -618,12 +622,14 @@ def verify_grouping(occurrence: Occurrence, user: User) -> Occurrence: """Record that a person confirmed this occurrence holds the right detections. This is the label the tracking methods are scored against, so it is deliberately - an explicit act — no operation in this module sets it as a side effect. + an explicit act — no operation in this module sets it as a side effect. The two + fields hold the current confirmation; the history keeps every review. """ _lock_for_edit(occurrence) occurrence.grouping_verified_at = timezone.now() occurrence.grouping_verified_by = user occurrence.save(update_fields=["grouping_verified_at", "grouping_verified_by"]) + record_track_complete_review(occurrence, user, occurrence.grouping_verified_at) return occurrence diff --git a/ami/main/schemas.py b/ami/main/schemas.py new file mode 100644 index 000000000..478cbef31 --- /dev/null +++ b/ami/main/schemas.py @@ -0,0 +1,80 @@ +"""Payload schemas for an occurrence's history records, one per subtype. + +A record's ``payload`` is JSON, so these schemas are what keeps each subtype's shape +fixed: every write validates against the schema for its kind and subtype. +""" + +import datetime + +import pydantic + + +class HistoryPayload(pydantic.BaseModel): + class Config: + extra = "forbid" + + +class DeterminationChangePayload(HistoryPayload): + # The occurrence's determination before and after the run, as taxon ids. Equal + # when the run changed predictions without moving the determination. + taxon_before_id: int | None = None + taxon_after_id: int | None = None + + +class TrackingResultPayload(DeterminationChangePayload): + # The tunables the run used, as the tracking config holds them, without its scope. + settings: dict = {} + feature_algorithm_id: int | None = None + detections_count: int + frames_linked: int + occurrences_merged: list[int] = [] + cost_mean: float | None = None + cost_max: float | None = None + + +class ClassMaskingResultPayload(DeterminationChangePayload): + taxa_list_id: int + source_algorithm_id: int + detection_ids: list[int] + + +class SizeFilterResultPayload(DeterminationChangePayload): + size_threshold: float + detection_ids: list[int] + + +class TrackCompleteReviewPayload(HistoryPayload): + detection_ids: list[int] + frames_count: int + first_timestamp: datetime.datetime | None = None + last_timestamp: datetime.datetime | None = None + # Compared with the previous track_complete review; the first review lists none. + detections_added: list[int] = [] + detections_removed: list[int] = [] + # The occurrence reviewed. A merge moves reviews onto the surviving occurrence, and + # only its own reviews are compared with a later confirmation of it. + occurrence_id: int | None = None + # Set when regrouping split a confirmed occurrence at a session boundary: the + # confirmation carries over to each piece, restated for the piece's own detections. + split_from_occurrence_id: int | None = None + + +# Keyed by (kind, subtype). A subtype missing here cannot be written. +HISTORY_PAYLOAD_SCHEMAS: dict[tuple[str, str], type[HistoryPayload]] = { + ("algorithm_result", "tracking"): TrackingResultPayload, + ("algorithm_result", "class_masking"): ClassMaskingResultPayload, + ("algorithm_result", "size_filter"): SizeFilterResultPayload, + ("review", "track_complete"): TrackCompleteReviewPayload, +} + + +def validate_history_payload(kind: str, subtype: str, payload: dict | HistoryPayload) -> dict: + """The payload as JSON-ready data, or a ValueError when it does not fit its subtype's schema.""" + schema = HISTORY_PAYLOAD_SCHEMAS.get((kind, subtype)) + if schema is None: + raise ValueError(f"No history payload schema for kind={kind!r} subtype={subtype!r}") + if isinstance(payload, HistoryPayload) and not isinstance(payload, schema): + raise ValueError(f"{type(payload).__name__} is not the payload schema for {kind}/{subtype}") + data = payload.dict() if isinstance(payload, HistoryPayload) else payload + # Round-trip through JSON so datetimes are stored as ISO strings. + return pydantic.parse_raw_as(dict, schema.parse_obj(data).json()) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py new file mode 100644 index 000000000..3959ca722 --- /dev/null +++ b/ami/main/test_occurrence_history.py @@ -0,0 +1,328 @@ +"""An occurrence's history: algorithm results and reviews, and the endpoint that reads them.""" + +import datetime + +from django.test import TestCase + +from ami.main import tests as main_tests +from ami.main.models import Classification, Detection, Identification, Occurrence, OccurrenceHistoryRecord, Taxon +from ami.ml.models import Algorithm +from ami.tests.fixtures.main import setup_test_project +from ami.users.models import User +from ami.users.roles import MLDataManager + +# Measured: two savepoints, the project, the occurrence, then history records, their taxa, +# identifications and predictions. +HISTORY_QUERIES = 8 + + +class OccurrenceHistoryPayloadTestCase(TestCase): + """A history payload always fits the schema for its kind and subtype, however it is written.""" + + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + self.occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) + + def _size_filter(self, payload: dict) -> OccurrenceHistoryRecord: + return OccurrenceHistoryRecord.build( + occurrence_id=self.occurrence.pk, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="size_filter", + payload=payload, + ) + + def test_a_valid_payload_is_stored_as_given(self): + record = self._size_filter({"size_threshold": 0.001, "detection_ids": [3, 4]}) + record.save() + record.refresh_from_db() + self.assertEqual(record.payload["detection_ids"], [3, 4]) + self.assertIsNone(record.payload["taxon_after_id"]) + + def test_a_payload_that_does_not_fit_its_schema_is_refused(self): + for label, payload in ( + ("missing field", {"size_threshold": 0.001}), + ("wrong type", {"size_threshold": "small", "detection_ids": []}), + ("unknown field", {"size_threshold": 0.001, "detection_ids": [], "note": "x"}), + ): + with self.subTest(label), self.assertRaises(ValueError): + self._size_filter(payload) + + def test_save_validates_too_and_an_unknown_subtype_is_refused(self): + record = OccurrenceHistoryRecord( + occurrence=self.occurrence, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="size_filter", + payload={"size_threshold": 0.001}, + timestamp=self.occurrence.created_at, + ) + with self.assertRaises(ValueError): + record.save() + record.subtype = "not_a_subtype" + record.payload = {} + with self.assertRaises(ValueError): + record.save() + self.assertFalse(OccurrenceHistoryRecord.objects.exists()) + + +class TrackCompleteReviewTestCase(main_tests.TrackFixtureTestCase): + """Confirming a track posts a review only when its detections or its confirming person changed.""" + + def verify(self, occurrence: Occurrence | None = None, user: User | None = None): + self.client.force_authenticate(user=user or self.curator) + occurrence = occurrence or self.occurrence + response = self.client.post(f"/api/v2/occurrences/{occurrence.pk}/verify-grouping/", format="json") + self.assertEqual(response.status_code, 200, response.data) + + def reviews(self, occurrence: Occurrence | None = None): + return OccurrenceHistoryRecord.objects.filter( + occurrence=occurrence or self.occurrence, + kind=OccurrenceHistoryRecord.Kind.REVIEW, + subtype="track_complete", + ).order_by("timestamp", "pk") + + def test_the_first_confirmation_posts_a_review_and_an_unchanged_one_does_not(self): + self.verify() + self.verify() + + review = self.reviews().get() + self.assertEqual(review.user, self.curator) + self.assertEqual(review.payload["detection_ids"], sorted(d.pk for d in self.detections)) + self.assertEqual(review.payload["frames_count"], len(self.captures)) + self.assertEqual(review.payload["first_timestamp"], self.captures[0].timestamp.isoformat()) + self.assertEqual(review.payload["last_timestamp"], self.captures[-1].timestamp.isoformat()) + self.assertEqual((review.payload["detections_added"], review.payload["detections_removed"]), ([], [])) + self.occurrence.refresh_from_db() + self.assertEqual(self.occurrence.grouping_verified_by, self.curator) + self.assertGreaterEqual(self.occurrence.grouping_verified_at, review.timestamp) + + def unverify(self): + self.client.force_authenticate(user=self.curator) + response = self.client.post(f"/api/v2/occurrences/{self.occurrence.pk}/unverify-grouping/", format="json") + self.assertEqual(response.status_code, 200, response.data) + + def test_a_confirmation_after_an_edit_records_what_changed(self): + self.verify() + self.unverify() + response = self.post("remove-detection", self.detections[-1], user=self.curator) + self.assertEqual(response.status_code, 200, response.data) + self.verify() + + first, second = self.reviews() + self.assertEqual(second.payload["detections_removed"], [self.detections[-1].pk]) + self.assertEqual(second.payload["detections_added"], []) + self.assertEqual(second.payload["frames_count"], len(self.captures) - 1) + self.assertEqual(first.payload["detection_ids"], sorted(d.pk for d in self.detections)) + + def test_withdrawing_and_reconfirming_an_unchanged_track_posts_no_new_review(self): + self.verify() + review = self.reviews().get() + for _ in range(2): + self.unverify() + self.verify() + + self.assertEqual(list(self.reviews()), [review]) + self.occurrence.refresh_from_db() + self.assertEqual(self.occurrence.grouping_verified_by, self.curator) + self.assertGreater(self.occurrence.grouping_verified_at, review.timestamp) + + def test_a_confirmation_by_someone_else_posts_a_review(self): + self.verify() + other_curator = User.objects.create_user(email="second-curator@insectai.org") # type: ignore + MLDataManager.assign_user(other_curator, self.project) + self.unverify() + self.verify(user=other_curator) + + reviews = list(self.reviews()) + self.assertEqual([r.user for r in reviews], [self.curator, other_curator]) + self.assertTrue(all(r.payload["detections_added"] == [] for r in reviews)) + self.occurrence.refresh_from_db() + self.assertEqual(self.occurrence.grouping_verified_at, reviews[-1].timestamp) + + def test_a_confirmation_after_a_merge_compares_with_this_occurrences_own_review(self): + other, other_detections = self._make_track(1, captures=self._make_captures_after(1)) + self.verify() + self.verify(other) + self.client.force_authenticate(user=self.curator) + response = self.client.post( + f"/api/v2/occurrences/{self.occurrence.pk}/merge/", {"occurrence_ids": [other.pk]}, format="json" + ) + self.assertEqual(response.status_code, 200, response.data) + self.verify() + + latest = self.reviews().last() + self.assertEqual(latest.payload["detections_added"], [d.pk for d in other_detections]) + self.assertEqual(latest.payload["detections_removed"], []) + + def test_merging_an_occurrence_keeps_its_reviews(self): + other, _ = self._make_track(1, captures=self._make_captures_after(1)) + self.verify(other) + self.client.force_authenticate(user=self.curator) + response = self.client.post( + f"/api/v2/occurrences/{self.occurrence.pk}/merge/", {"occurrence_ids": [other.pk]}, format="json" + ) + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.reviews().count(), 1) + + def edited_since_verified(self) -> bool: + self.client.force_authenticate(user=self.curator) + response = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") + self.assertEqual(response.status_code, 200, response.data) + return response.data["grouping_edited_since_verified"] + + def test_the_detail_says_whether_the_detections_changed_since_the_last_confirmation(self): + self.assertFalse(self.edited_since_verified()) + self.verify() + self.assertFalse(self.edited_since_verified()) + response = self.post("remove-detection", self.detections[-1], user=self.curator) + self.assertEqual(response.status_code, 200, response.data) + self.assertTrue(self.edited_since_verified()) + + def test_a_null_marker_on_the_occurrence_is_not_an_edit(self): + self.verify() + self.client.post(f"/api/v2/occurrences/{self.occurrence.pk}/unverify-grouping/", format="json") + Detection.objects.create(source_image=self.captures[0], occurrence=self.occurrence, bbox=None) + self.assertFalse(self.edited_since_verified()) + + +class OccurrenceHistoryEndpointTestCase(main_tests.TrackFixtureTestCase): + """GET /occurrences/{id}/history/ merges records, identifications and predictions, newest first.""" + + def setUp(self) -> None: + super().setUp() + self.tracking = Algorithm.objects.create(name="Occurrence Tracking", key="tracking-history-test") + self.other_taxon = Taxon.objects.filter(projects=self.project).exclude(pk=self.taxon.pk).first() + self.superuser = User.objects.create_superuser(email="history-super@insectai.org") # type: ignore + self.outsider = User.objects.create_user(email="history-outsider@insectai.org") # type: ignore + + def url(self, occurrence: Occurrence | None = None) -> str: + return f"/api/v2/occurrences/{(occurrence or self.occurrence).pk}/history/?project_id={self.project.pk}" + + def _add_history(self, start: datetime.datetime, rounds: int = 1) -> None: + """Per round: a tracking result, a folded tracking prediction, an identification and a review.""" + for i in range(rounds): + at = start + datetime.timedelta(hours=4 * i) + OccurrenceHistoryRecord.build( + occurrence_id=self.occurrence.pk, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="tracking", + payload={ + "detections_count": 4, + "frames_linked": 3, + "taxon_before_id": self.other_taxon.pk, + "taxon_after_id": self.taxon.pk, + }, + timestamp=at, + algorithm=self.tracking, + ).save() + Classification.objects.create( + detection=self.detections[0], taxon=self.taxon, score=0.1, algorithm=self.tracking, timestamp=at + ) + identification = Identification.objects.create( + occurrence=self.occurrence, user=self.reader, taxon=self.other_taxon, comment=f"round {i}" + ) + Identification.objects.filter(pk=identification.pk).update(created_at=at + datetime.timedelta(hours=1)) + OccurrenceHistoryRecord.build( + occurrence_id=self.occurrence.pk, + kind=OccurrenceHistoryRecord.Kind.REVIEW, + subtype="track_complete", + payload={"detection_ids": [d.pk for d in self.detections], "frames_count": 4}, + timestamp=at + datetime.timedelta(hours=2), + user=self.curator, + ).save() + + def test_entries_are_merged_newest_first_and_a_folded_prediction_is_left_out(self): + # Predictions are stamped with created_at (now), so the history rows sit in the past. + self._add_history(datetime.datetime.now() - datetime.timedelta(days=2)) + self.client.force_authenticate(user=self.reader) + response = self.client.get(self.url()) + self.assertEqual(response.status_code, 200, response.data) + + types = [entry["type"] for entry in response.data] + self.assertEqual(types[-3:], ["review", "identification", "algorithm_result"]) + self.assertEqual(set(types[:-3]), {"prediction"}) + # The fixture's detections share one tied top prediction, so it is shown once. + self.assertEqual(len(types[:-3]), 1) + self.assertIsNone(response.data[0]["algorithm"]) + timestamps = [entry["timestamp"] for entry in response.data] + self.assertEqual(timestamps, sorted(timestamps, reverse=True)) + + review, identification, result = response.data[-3:] + self.assertEqual(set(review["user"]), {"id", "name", "image"}) + self.assertEqual(review["user"]["id"], self.curator.pk) + self.assertEqual(identification["user"]["id"], self.reader.pk) + self.assertEqual(identification["taxon"]["id"], self.other_taxon.pk) + self.assertEqual(identification["payload"]["comment"], "round 0") + self.assertEqual(result["subtype"], "tracking") + self.assertEqual(result["algorithm"]["key"], self.tracking.key) + self.assertEqual(result["taxon"]["id"], self.taxon.pk) + self.assertEqual(result["taxon_before"]["id"], self.other_taxon.pk) + self.assertNotIn("email", str(response.data)) + + def test_each_algorithm_shows_one_prediction_preferring_terminal_then_latest(self): + """Tied top scores would otherwise show the same prediction once per detection.""" + classifier = Algorithm.objects.create(name="Tied classifier", key="tied-classifier-history-test") + other = Algorithm.objects.create(name="Second tied classifier", key="second-tied-classifier-history-test") + now = datetime.datetime.now() + + def classify(detection, algorithm, terminal, created_at): + classification = Classification.objects.create( + detection=detection, taxon=self.taxon, score=0.8, algorithm=algorithm, terminal=terminal, timestamp=now + ) + Classification.objects.filter(pk=classification.pk).update(created_at=created_at) + return classification + + classify(self.detections[0], classifier, False, now) + terminal = classify(self.detections[1], classifier, True, now - datetime.timedelta(hours=1)) + classify(self.detections[2], other, True, now - datetime.timedelta(hours=1)) + latest = classify(self.detections[3], other, True, now) + + self.client.force_authenticate(user=self.reader) + response = self.client.get(self.url()) + self.assertEqual(response.status_code, 200, response.data) + + predictions = [entry for entry in response.data if entry["type"] == "prediction"] + by_algorithm = { + entry["algorithm"]["key"] if entry["algorithm"] else None: entry["id"] for entry in predictions + } + self.assertEqual(len(predictions), 3) + self.assertEqual(by_algorithm[classifier.key], terminal.pk) + self.assertEqual(by_algorithm[other.key], latest.pk) + self.assertIn(None, by_algorithm) + + def test_query_count_does_not_grow_with_the_number_of_entries(self): + self.client.force_authenticate(user=self.reader) + now = datetime.datetime.now() + for rounds, total in ((1, 1), (2, 3)): + self._add_history(now - datetime.timedelta(days=3 * total), rounds=rounds) + with self.subTest(rounds=total), main_tests.cachalot_disabled(): + with self.assertNumQueries(HISTORY_QUERIES): + response = self.client.get(self.url()) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 3 * total + 1) + + def test_visible_to_whoever_can_open_the_occurrence(self): + """Member, non-member, anonymous and superuser, on a public and on a draft project.""" + expected = { + False: {"member": 200, "non-member": 200, "anonymous": 200, "superuser": 200}, + True: {"member": 200, "non-member": 404, "anonymous": 404, "superuser": 200}, + } + users = {"member": self.reader, "non-member": self.outsider, "anonymous": None, "superuser": self.superuser} + for draft, codes in expected.items(): + self.project.draft = draft + self.project.save() + for label, user in users.items(): + with self.subTest(draft=draft, user=label): + self.client.force_authenticate(user=user) + detail = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") + history = self.client.get(self.url()) + self.assertEqual(history.status_code, codes[label]) + self.assertEqual(history.status_code, detail.status_code) + + def test_an_occurrence_the_default_filters_hide_still_has_a_history(self): + """The session view lists occurrences with the default filters off, so their history must open too.""" + self.project.default_filters_score_threshold = 0.95 + self.project.save() + self.client.force_authenticate(user=self.reader) + detail = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") + self.assertEqual(detail.status_code, 404) + self.assertEqual(self.client.get(self.url()).status_code, 200) diff --git a/ami/main/tests.py b/ami/main/tests.py index e63215b6d..9ddc986c1 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -29,6 +29,7 @@ Classification, Deployment, Detection, + DetectionEmbedding, Device, Event, Identification, @@ -1538,6 +1539,26 @@ def test_every_piece_keeps_the_grouping_confirmation(self): self.assertEqual(piece.grouping_verified_at, verified_at) self.assertEqual(piece.grouping_verified_by_id, self.user.pk) + def test_each_piece_gets_a_review_of_its_own_detections(self): + from ami.main.models_future.history import latest_track_complete_review + from ami.main.models_future.tracks import verify_grouping + + self._group(gap_hours=6) + occurrence, _ = self._make_track(self.captures) + verify_grouping(occurrence, self.user) + verified_at = Occurrence.objects.get(pk=occurrence.pk).grouping_verified_at + + self._group(gap_hours=2) + + for piece in Occurrence.objects.filter(deployment=self.deployment): + review = latest_track_complete_review(piece) + self.assertEqual(review.payload["detection_ids"], sorted(self._detection_ids(piece))) + self.assertEqual((review.user_id, review.timestamp), (self.user.pk, verified_at)) + self.assertEqual(review.payload["split_from_occurrence_id"], occurrence.pk) + # Re-confirming the piece as it stands changes nothing, so it records nothing. + verify_grouping(piece, self.user) + self.assertEqual(latest_track_complete_review(piece).pk, review.pk) + def test_identifications_are_copied_to_every_piece(self): self._group(gap_hours=6) occurrence, _ = self._make_track(self.captures) @@ -10109,7 +10130,8 @@ def test_relinking_a_merge_costs_the_same_queries_however_many_frames_moved(self def test_a_multi_frame_merge_does_not_query_per_frame(self): other, _ = self._make_track(3, captures=self._make_captures_after(3)) - with self.assertNumQueries(34): + # Two of these move and cascade-delete the merged track's history records, once per merge. + with self.assertNumQueries(36): merge_occurrences(self.occurrence, [other]) self.assertFullyLinked(self.occurrence) @@ -10522,18 +10544,56 @@ def test_track_edits_ignore_the_project_default_filters(self): self.assertEqual(merged.data["detections_count"], len(self.detections) + 2) self.assertFalse(Occurrence.objects.filter(pk__in=[low_score.pk, undetermined.pk]).exists()) + def test_a_candidate_whose_only_vector_is_an_embedding_is_compared_by_appearance(self): + """A crop the moth/non-moth filter rejected has no classification vector, only an embedding.""" + extractor = Algorithm.objects.create(name="Feature extractor", key="feature-extractor") + vector = [1.0] + [0.0] * 2047 + self._give_target_vectors(vector, extractor) + candidate = self._make_occurrence([self.after_capture], bbox=[12, 12, 42, 42]) + DetectionEmbedding.objects.create(detection=candidate.detections.get(), algorithm=extractor, vector=vector) + + response = self.get_candidates() + self.assertEqual(response.status_code, 200, response.data) + similarity = {row["id"]: row["similarity"] for row in response.data["candidates"]} + self.assertEqual(similarity[candidate.pk], 1.0) + + def test_similarity_is_never_taken_between_two_algorithms(self): + """Vectors from two models are not comparable, however alike their numbers.""" + extractor = Algorithm.objects.create(name="Feature extractor", key="feature-extractor") + other = Algorithm.objects.create(name="Other feature extractor", key="other-feature-extractor") + vector = [1.0] + [0.0] * 2047 + self._give_target_vectors(vector, extractor) + same = self._make_occurrence([self.after_capture], bbox=[12, 12, 42, 42]) + different = self._make_occurrence([self.after_capture], bbox=[500, 500, 530, 530]) + DetectionEmbedding.objects.create(detection=same.detections.get(), algorithm=extractor, vector=vector) + DetectionEmbedding.objects.create(detection=different.detections.get(), algorithm=other, vector=vector) + + response = self.get_candidates() + self.assertEqual(response.status_code, 200, response.data) + similarity = {row["id"]: row["similarity"] for row in response.data["candidates"]} + self.assertEqual(similarity, {same.pk: 1.0, different.pk: None}) + def test_the_candidate_count_does_not_change_the_query_count(self): extractor = Algorithm.objects.create(name="Feature extractor", key="feature-extractor") vector = [1.0] + [0.0] * 2047 self._give_target_vectors(vector, extractor) for offset in range(3): - self._make_occurrence( - [self.after_capture], bbox=[10 + offset, 10, 40 + offset, 40], vector=vector, algorithm=extractor + # Alternate the two places a vector is stored. + candidate = self._make_occurrence( + [self.after_capture], + bbox=[10 + offset, 10, 40 + offset, 40], + vector=vector if offset % 2 else None, + algorithm=extractor, ) + if not offset % 2: + DetectionEmbedding.objects.create( + detection=candidate.detections.get(), algorithm=extractor, vector=vector + ) # The savepoint pair, the object lookup with its permission checks, then the # seven ranking queries: the track's frames, the capture ids before and after - # it, the frames in those captures, the candidates, and the two vector sides. + # it, the frames in those captures, the candidates, and the two vector sides, + # each reading embeddings and classification vectors together. with self.assertNumQueries(13): response = self.get_candidates() @@ -10788,6 +10848,20 @@ def test_a_box_without_an_embedding_is_skipped_while_features_are_required(self) self.assertAlmostEqual(row["cost"], geometry, places=4) self.assertAlmostEqual(row["likelihood"], 1 - geometry / 3, places=4) + def test_a_box_whose_only_vector_is_an_embedding_is_compared_and_linked(self): + """A processing service can store a vector for a box that received no classification + vector. The preview reads the embedding store too, so such a box is compared by + appearance and linked like one whose vector came with a classification.""" + track = self._track(self.captures[1:3], vector=self.VECTOR) + box = self._box(self.captures[3], self.NEAR_BOX) + DetectionEmbedding.objects.create(detection=box, algorithm=self.extractor, vector=self.VECTOR) + + data = self.get_matches(track, self.captures[3].pk).data + + self.assertEqual(data["feature_algorithm_id"], self.extractor.pk) + row = data["detections"][0] + self.assertEqual((row["skipped_reason"], row["would_link"], row["similarity"]), (None, True, 1.0)) + def test_a_detector_only_session_links_nothing(self): """With no embeddings at all, tracking has no extractor to compare and skips the captures while it requires features, so every box and the reference frame are marked skipped.""" @@ -10802,19 +10876,17 @@ def test_a_detector_only_session_links_nothing(self): self.assertEqual((row["skipped_reason"], row["would_link"]), ("no_vector", False)) self.assertIsNotNone(row["cost"], "The geometry is still scored") - def test_two_feature_extractors_link_nothing(self): - """Embeddings from two extractors leave tracking no single one to compare, so it skips the - captures while it requires features, as it does a session with none.""" + def test_two_feature_extractors_are_never_compared_with_each_other(self): + """With embeddings from two extractors, tracking compares the default one only (the one + covering the most boxes), so a box whose only vector is from the other does not link.""" other_extractor = Algorithm.objects.create(name="Other extractor", key="other-extractor") track = self._track(self.captures[1:3], vector=self.VECTOR) self._box(self.captures[3], self.NEAR_BOX, vector=self.VECTOR, algorithm=other_extractor) data = self.get_matches(track, self.captures[3].pk).data - self.assertIsNone(data["feature_algorithm_id"]) - self.assertEqual( - (data["detections"][0]["skipped_reason"], data["detections"][0]["would_link"]), ("no_vector", False) - ) + self.assertEqual(data["feature_algorithm_id"], self.extractor.pk) + self.assertFalse(data["detections"][0]["would_link"]) def test_the_reference_is_the_nearest_track_frame_on_another_capture(self): """A track on the second and fifth of six captures a minute apart. Each capture is @@ -11383,6 +11455,40 @@ def make_detection(vectors: list[bool], bbox: list[int] | None = [10, 10, 40, 40 listed, _ = self._get(f"/api/v2/captures/?project_id={self.project.pk}") self.assertNotIn("detections_with_features", listed.data["results"][0], "Counted on the detail only") + def test_a_frame_whose_only_vector_is_an_embedding_counts_as_having_one(self): + """A crop the moth/non-moth filter rejected carries an embedding and no classification vector.""" + extractor = Algorithm.objects.create(name="Embedding model", key="embedding-model") + occurrence = self._make_occurrence([True, False, False]) + embedded = occurrence.detections.get(source_image=self.captures[1]) + DetectionEmbedding.objects.create(detection=embedded, algorithm=extractor, vector=self.vector) + + response, queries = self._get(self._occurrence_url(occurrence)) + self.assertEqual(response.data["grouping_summary"]["frames_with_vectors"], 2) + self.assertFalse(self._reads_the_vector(queries)) + + capture, queries = self._get(f"/api/v2/captures/{self.captures[1].pk}/?project_id={self.project.pk}") + self.assertEqual(capture.data["detections_with_features"], 1) + self.assertFalse(self._reads_the_vector(queries)) + + def test_has_features_counts_an_embedding_only_for_the_classifications_own_algorithm(self): + """A vector from one model says nothing about another model's classification of the crop.""" + embedder = Algorithm.objects.create(name="Species classifier", key="species-classifier") + moth_filter = Algorithm.objects.create(name="Moth filter", key="moth-filter") + capture = self.captures[0] + detection = Detection.objects.create(source_image=capture, timestamp=capture.timestamp, bbox=[10, 10, 40, 40]) + for algorithm in (embedder, moth_filter): + detection.classifications.create( + taxon=self.taxon, score=0.9, timestamp=capture.timestamp, algorithm=algorithm + ) + DetectionEmbedding.objects.create(detection=detection, algorithm=embedder, vector=self.vector) + + flags = dict( + Classification.objects.filter(detection=detection) + .with_has_features() + .values_list("algorithm_id", "has_features") + ) + self.assertEqual(flags, {embedder.pk: True, moth_filter.pk: False}) + def test_the_capture_counts_ride_on_the_capture_row(self): """Both counts are subqueries on the capture's own SELECT: five detections, one query.""" from cachalot.api import cachalot_disabled @@ -11404,3 +11510,80 @@ def test_the_capture_counts_ride_on_the_capture_row(self): annotated = SourceImage.objects.filter(pk=capture.pk).with_detections_with_features().get() self.assertEqual(annotated.detections_valid, 5) # type: ignore[attr-defined] self.assertEqual(annotated.detections_with_features, 3) # type: ignore[attr-defined] + + +class DetectionVectorReadTestCase(TestCase): + """Reading a detection's feature vector from whichever of the two stores holds it. + + A vector is a detection embedding or, on data processed before embeddings existed, a + classification's ``features_2048``. What these pin is that a reader returns only the + requested algorithm's vectors, takes the embedding when both exist, and reads any + number of detections in one query. + """ + + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + self.capture = SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + timestamp=datetime.datetime(2024, 1, 1, 22, 0), + path="test/vectors.jpg", + ) + self.extractor = Algorithm.objects.create(name="Embedding model", key="embedding-model") + self.other = Algorithm.objects.create(name="Other embedding model", key="other-embedding-model") + + def _detection(self, x: int) -> Detection: + return Detection.objects.create( + source_image=self.capture, timestamp=self.capture.timestamp, bbox=[x, 10, x + 30, 40] + ) + + def _classification_vector(self, detection: Detection, algorithm: Algorithm, value: float) -> None: + detection.classifications.create( + score=0.9, timestamp=self.capture.timestamp, algorithm=algorithm, features_2048=[value] * 2048 + ) + + def _embedding(self, detection: Detection, algorithm: Algorithm, value: float) -> None: + DetectionEmbedding.objects.create(detection=detection, algorithm=algorithm, vector=[value] * 2048) + + @staticmethod + def _read(detections: list[Detection], algorithm: Algorithm) -> dict[int, float]: + """{detection id: first component of its vector}; the test vectors are constant.""" + from ami.main.models_future.embeddings import vectors_for_detections + + vectors = vectors_for_detections([d.pk for d in detections], algorithm.pk) + return {detection_id: float(vector[0]) for detection_id, vector in vectors.items()} + + def test_each_store_supplies_the_vectors_it_holds(self): + """Older detections have only a classification vector and newer ones an embedding.""" + old, new = self._detection(0), self._detection(100) + self._classification_vector(old, self.extractor, 0.25) + self._embedding(new, self.extractor, 0.75) + self.assertEqual(self._read([old, new], self.extractor), {old.pk: 0.25, new.pk: 0.75}) + + def test_an_embedding_is_preferred_to_a_classification_vector(self): + detection = self._detection(0) + self._classification_vector(detection, self.extractor, 0.25) + self._embedding(detection, self.extractor, 0.75) + self.assertEqual(self._read([detection], self.extractor), {detection.pk: 0.75}) + + def test_another_algorithms_vector_is_never_returned(self): + first, second = self._detection(0), self._detection(100) + self._embedding(first, self.extractor, 0.25) + self._embedding(second, self.other, 0.75) + self._classification_vector(second, self.other, 0.5) + self.assertEqual(self._read([first, second], self.extractor), {first.pk: 0.25}) + self.assertEqual(self._read([first, second], self.other), {second.pk: 0.75}) + + def test_any_number_of_detections_is_read_in_one_query(self): + from cachalot.api import cachalot_disabled + + from ami.main.models_future.embeddings import vectors_for_detections + + detections = [self._detection(x) for x in range(0, 500, 100)] + for index, detection in enumerate(detections): + store = self._embedding if index % 2 else self._classification_vector + store(detection, self.extractor, 0.5) + + with cachalot_disabled(), self.assertNumQueries(1): + vectors = vectors_for_detections([d.pk for d in detections], self.extractor.pk) + self.assertEqual(len(vectors), len(detections)) diff --git a/ami/ml/exceptions.py b/ami/ml/exceptions.py index ef94e1a59..eee96c3c7 100644 --- a/ami/ml/exceptions.py +++ b/ami/ml/exceptions.py @@ -1,2 +1,16 @@ class PipelineNotConfigured(ValueError): pass + + +class FeatureResultsStoredNothing(Exception): + """A feature-only batch stored no vector while its images still have detections without one. + + This happens when no returned box matches a stored detection, when the boxes come back + without vectors, or when the service returns no boxes at all. The same detections would + be sent again on every run, so the job fails instead of skipping the batch. + """ + + def __init__(self, message: str = "", unmatched: int = 0, without_vector: int = 0): + super().__init__(message) + self.unmatched = unmatched + self.without_vector = without_vector diff --git a/ami/ml/migrations/0029_algorithm_embedding_dimensions.py b/ami/ml/migrations/0029_algorithm_embedding_dimensions.py new file mode 100644 index 000000000..96dd5dd90 --- /dev/null +++ b/ami/ml/migrations/0029_algorithm_embedding_dimensions.py @@ -0,0 +1,55 @@ +# Additive: a nullable column, and a new task type choice (no schema change for the choice). + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ml", "0028_normalize_empty_endpoint_url_to_null"), + ] + + operations = [ + migrations.AddField( + model_name="algorithm", + name="embedding_dimensions", + field=models.PositiveIntegerField( + blank=True, + help_text=( + "For a feature extractor, the length of every vector it produces. Set from the processing " + "service's /info or from the first vector stored; vectors of any other length are refused." + ), + null=True, + ), + ), + migrations.AlterField( + model_name="algorithm", + name="task_type", + field=models.CharField( + choices=[ + ("detection", "Detection"), + ("localization", "Localization"), + ("segmentation", "Segmentation"), + ("classification", "Classification"), + ("embedding", "Embedding"), + ("feature_extraction", "Feature Extraction"), + ("tracking", "Tracking"), + ("tagging", "Tagging"), + ("regression", "Regression"), + ("captioning", "Captioning"), + ("generation", "Generation"), + ("translation", "Translation"), + ("summarization", "Summarization"), + ("question_answering", "Question Answering"), + ("depth_estimation", "Depth Estimation"), + ("pose_estimation", "Pose Estimation"), + ("size_estimation", "Size Estimation"), + ("post_processing", "Post Processing"), + ("other", "Other"), + ("unknown", "Unknown"), + ], + default="unknown", + max_length=255, + null=True, + ), + ), + ] diff --git a/ami/ml/models/algorithm.py b/ami/ml/models/algorithm.py index 605f90861..ab96eb8f7 100644 --- a/ami/ml/models/algorithm.py +++ b/ami/ml/models/algorithm.py @@ -205,6 +205,7 @@ class AlgorithmTaskType(str, enum.Enum): SEGMENTATION = "segmentation" CLASSIFICATION = "classification" EMBEDDING = "embedding" + FEATURE_EXTRACTION = "feature_extraction" TRACKING = "tracking" TAGGING = "tagging" REGRESSION = "regression" @@ -249,6 +250,15 @@ class Algorithm(BaseModel): help_text=("A URI to the weights or model details. Could be a public web URL or object store path."), ) + embedding_dimensions = models.PositiveIntegerField( + null=True, + blank=True, + help_text=( + "For a feature extractor, the length of every vector it produces. Set from the processing " + "service's /info or from the first vector stored; vectors of any other length are refused." + ), + ) + category_map = models.ForeignKey( AlgorithmCategoryMap, on_delete=models.CASCADE, @@ -275,6 +285,11 @@ class Algorithm(BaseModel): AlgorithmTaskType.CLASSIFICATION, AlgorithmTaskType.TAGGING, ] + # A pipeline with one of these and no classifier returns vectors for existing detections only. + feature_extraction_task_types = [ + AlgorithmTaskType.EMBEDDING, + AlgorithmTaskType.FEATURE_EXTRACTION, + ] def __str__(self): return f'#{self.pk} "{self.name}" ({self.key}) v{self.version}' diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index be739d84d..b8aadc7bb 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -28,6 +28,7 @@ Classification, Deployment, Detection, + DetectionEmbedding, Occurrence, Project, SourceImage, @@ -39,7 +40,7 @@ update_calculated_fields_for_events, update_occurrence_determination, ) -from ami.ml.exceptions import PipelineNotConfigured +from ami.ml.exceptions import FeatureResultsStoredNothing, PipelineNotConfigured from ami.ml.models.algorithm import Algorithm, AlgorithmCategoryMap from ami.ml.schemas import ( AlgorithmConfigResponse, @@ -69,6 +70,33 @@ COLLECT_PROGRESS_MAX_FRACTION = 0.99 +class _CollectHeartbeat: + """Throttled ``collect`` progress saves, so the reaper sees a long Collect stage moving. + + Silent unless both ``job`` and a non-zero ``total`` are given. Capped at + COLLECT_PROGRESS_MAX_FRACTION so the caller's final SUCCESS flip owns the terminal value. + ``updated_at`` is saved explicitly: auto_now only fires for fields in update_fields, + and the reaper keys off it (see ``ami/jobs/tasks.py``). + """ + + def __init__(self, job: Job | None, total: int | None) -> None: + self.job, self.total = job, total + self.processed = 0 + self.last_save = time.monotonic() + + def tick(self, batch_size: int) -> None: + self.processed += batch_size + if self.job is None or not self.total: + return + now = time.monotonic() + if now - self.last_save >= COLLECT_PROGRESS_SAVE_INTERVAL_SECONDS: + self.job.progress.update_stage( + "collect", progress=min(self.processed / self.total, COLLECT_PROGRESS_MAX_FRACTION) + ) + self.job.save(update_fields=["progress", "updated_at"]) + self.last_save = now + + def filter_processed_images( images: typing.Iterable[SourceImage], pipeline: Pipeline, @@ -106,11 +134,23 @@ def filter_processed_images( pipeline_algorithms = list(pipeline.algorithms.all()) pipeline_algorithm_ids = [a.id for a in pipeline_algorithms] + feature_extractors = feature_extractors_if_feature_only(pipeline_algorithms) + if feature_extractors: + yield from filter_images_missing_features( + images, + [algorithm.pk for algorithm in feature_extractors], + batch_size=batch_size, + heartbeat=_CollectHeartbeat(job, total), + ) + return + detection_type_keys = set(Algorithm.detection_task_types) has_detection_algorithm = any(a.task_type in detection_type_keys for a in pipeline_algorithms) if not has_detection_algorithm: task_logger.warning(f"Pipeline {pipeline} has no detection algorithms saved. Will reprocess all images.") - pipeline_classifier_ids = {a.id for a in pipeline_algorithms if a.task_type not in detection_type_keys} + # A feature extractor never classifies, so it cannot mark an image as processed. + not_classifier_keys = detection_type_keys | set(Algorithm.feature_extraction_task_types) + pipeline_classifier_ids = {a.id for a in pipeline_algorithms if a.task_type not in not_classifier_keys} if not pipeline_classifier_ids: task_logger.warning(f"Pipeline {pipeline} has no classification algorithms saved. Will reprocess all images.") # set().issubset(anything) is vacuously True, so without this short-circuit @@ -120,11 +160,7 @@ def filter_processed_images( return image_iter = iter(images) - # Track how many of the input images we've inspected so far so we can emit - # a fractional `collect` progress to the Job row. Only used when both - # `job` and `total` are passed by the caller; legacy callers stay silent. - processed_count = 0 - last_progress_save_monotonic = time.monotonic() + heartbeat = _CollectHeartbeat(job, total) while True: batch = list(itertools.islice(image_iter, batch_size)) if not batch: @@ -200,25 +236,112 @@ def filter_processed_images( f"Image {image} has existing detections classified by the pipeline: {pipeline}, skipping!" ) - # Throttled progress emit. Save only when both `job` and `total` are - # provided, the total is non-zero, and at least - # COLLECT_PROGRESS_SAVE_INTERVAL_SECONDS of wall time have passed since - # the last save. Capped at COLLECT_PROGRESS_MAX_FRACTION so the caller's - # final status=SUCCESS, progress=1 flip still owns the terminal value. - # - # `updated_at` is included in update_fields explicitly: Django only fires - # auto_now's pre_save hook for fields listed in update_fields, so without - # it the reaper's `Job.updated_at < cutoff` heuristic - # (`ami/jobs/tasks.py:929-944`) would not see this heartbeat and could - # still revoke the job mid-Collect. - processed_count += len(batch) - if job is not None and total: - now_monotonic = time.monotonic() - if now_monotonic - last_progress_save_monotonic >= COLLECT_PROGRESS_SAVE_INTERVAL_SECONDS: - fraction = min(processed_count / total, COLLECT_PROGRESS_MAX_FRACTION) - job.progress.update_stage("collect", progress=fraction) - job.save(update_fields=["progress", "updated_at"]) - last_progress_save_monotonic = now_monotonic + heartbeat.tick(len(batch)) + + +def feature_extractors_if_feature_only(algorithms: list[Algorithm]) -> list[Algorithm]: + """The feature extractors of a pipeline that only extracts features, otherwise none. + + Such a pipeline has a feature extractor and otherwise only detectors: the service names the + detector of the boxes it echoes back, but detects nothing new. Any other task type, including + a blank or unknown one, rules it out, because a feature-only save drops new detections. + """ + feature_types = set(Algorithm.feature_extraction_task_types) + allowed_types = feature_types | set(Algorithm.detection_task_types) + if any(algorithm.task_type not in allowed_types for algorithm in algorithms): + return [] + return [algorithm for algorithm in algorithms if algorithm.task_type in feature_types] + + +def feature_extraction_only(algorithms: list[Algorithm]) -> bool: + """Whether a pipeline made of these algorithms only extracts features (see ``Pipeline.is_feature_only``).""" + return bool(feature_extractors_if_feature_only(algorithms)) + + +def embeddable_detections(detections: models.QuerySet) -> models.QuerySet: + """The detections a feature-only pipeline can be sent: real boxes whose detector is known. + + Choosing images and building the request both go through this, so an image is never + queued with no boxes to embed (a worker would then run its own detector on it). + """ + return detections.valid().filter(detection_algorithm__isnull=False) + + +def detections_missing_features(detections: models.QuerySet, algorithm_ids: list[int]) -> models.QuerySet: + """Embeddable detections without a stored vector from at least one of the algorithms. + + Only ``DetectionEmbedding`` rows count: they are what a feature-only pipeline writes. + """ + missing = models.Q() + for algorithm_id in algorithm_ids: + missing |= ~models.Exists( + DetectionEmbedding.objects.filter(detection_id=models.OuterRef("pk"), algorithm_id=algorithm_id) + ) + return embeddable_detections(detections).filter(missing) + + +def filter_images_missing_features( + images: typing.Iterable[SourceImage], + algorithm_ids: list[int], + batch_size: int = FILTER_PROCESSED_BATCH_SIZE, + heartbeat: _CollectHeartbeat | None = None, +) -> typing.Iterable[SourceImage]: + """The images with at least one real detection that lacks a vector from the algorithms. + + Images without detections are skipped: a feature-only pipeline has nothing to run on there. + """ + image_iter = iter(images) + while True: + batch = list(itertools.islice(image_iter, batch_size)) + if not batch: + return + needing = set( + detections_missing_features( + Detection.objects.filter(source_image_id__in=[image.pk for image in batch]), algorithm_ids + ) + .order_by() + .values_list("source_image_id", flat=True) + .distinct() + ) + yield from (image for image in batch if image.pk in needing) + if heartbeat is not None: + heartbeat.tick(len(batch)) + + +def collect_detections_for_features( + source_image_requests: list[SourceImageRequest], + algorithm_ids: list[int], + include_existing_vectors: bool = False, +) -> list[DetectionRequest]: + """The existing detections a feature-only pipeline should embed, for all the images in one query. + + Detections that already have a vector from every one of the algorithms are left out + unless ``include_existing_vectors`` is set. + """ + request_by_image_id = {int(request.id): request for request in source_image_requests} + detections = Detection.objects.filter(source_image_id__in=list(request_by_image_id)) + detections = ( + embeddable_detections(detections) + if include_existing_vectors + else detections_missing_features(detections, algorithm_ids) + ) + detection_requests: list[DetectionRequest] = [] + for detection in detections.select_related("detection_algorithm").order_by("source_image_id", "pk"): + bbox = detection.get_bbox() + if bbox is None or detection.detection_algorithm is None: + continue + detection_requests.append( + DetectionRequest( + source_image=request_by_image_id[detection.source_image_id], + bbox=bbox, + crop_image_url=detection.url(), + algorithm=AlgorithmReference( + name=detection.detection_algorithm.name, + key=detection.detection_algorithm.key, + ), + ) + ) + return detection_requests def collect_images( @@ -348,6 +471,8 @@ def process_images( if pipeline_config.reprocess_existing_detections: reprocess_existing_detections = True + feature_algorithm_ids = [algorithm.pk for algorithm in pipeline.feature_extraction_algorithms()] + for source_image, url in zip(images, urls): if url: source_image_request = SourceImageRequest( @@ -356,10 +481,20 @@ def process_images( ) source_image_requests.append(source_image_request) - if reprocess_existing_detections: + if reprocess_existing_detections and not feature_algorithm_ids: detection_requests += collect_detections(source_image, source_image_request) - if reprocess_existing_detections: + if feature_algorithm_ids: + # A feature-only pipeline only ever runs on the boxes Antenna already has. + detection_requests = collect_detections_for_features( + source_image_requests, feature_algorithm_ids, include_existing_vectors=reprocess_all_images + ) + images_with_detections = {request.source_image.id for request in detection_requests} + source_image_requests = [request for request in source_image_requests if request.id in images_with_detections] + task_logger.info(f"Sending {len(detection_requests)} existing detections for feature extraction.") + if not source_image_requests: + return PipelineResultsResponse(pipeline=pipeline.slug, source_images=[], detections=[], total_time=0) + elif reprocess_existing_detections: task_logger.info(f"Found {len(detection_requests)} existing detections to reprocess.") else: task_logger.info("Reprocessing of existing detections is disabled, sending images without detections.") @@ -512,6 +647,17 @@ def get_or_create_algorithm_and_category_map( setattr(algo, field, new_value) algo_fields_updated.append(field) + if algorithm_config.embedding_dimensions and algo.embedding_dimensions != algorithm_config.embedding_dimensions: + if algo.embedding_dimensions is None: + algo.embedding_dimensions = algorithm_config.embedding_dimensions + algo_fields_updated.append("embedding_dimensions") + else: + # Stored vectors keep their length; a new length needs a new algorithm version. + logger.warning( + f"Algorithm {algo} reports {algorithm_config.embedding_dimensions}-dimension vectors but " + f"stores {algo.embedding_dimensions}; keeping {algo.embedding_dimensions}." + ) + if algo_fields_updated: algo.save(update_fields=algo_fields_updated) @@ -680,6 +826,193 @@ def create_detections( return existing_detections + new_detections +# A vector is roughly 40 KB of SQL text (estimate), so this keeps each INSERT to a few MB. +EMBEDDING_BATCH_SIZE = 200 +# Boxes are matched to responses at this precision, so a service that re-serialises the +# coordinates it was sent still lands its vectors on the same detections. +BOX_MATCH_DECIMALS = 3 + + +def _box_key(source_image_id, coordinates) -> tuple: + return (str(source_image_id), tuple(round(float(value), BOX_MATCH_DECIMALS) for value in coordinates)) + + +class EmbeddingDimensionMismatch(PipelineNotConfigured): + """A vector's length differs from the length its algorithm has produced before.""" + + +def _check_embedding_dimensions(algorithm: Algorithm, lengths: set[int]) -> None: + """Refuse vectors whose length differs from the algorithm's, recording it on first sight. + + Vectors of different lengths can never be compared, so each algorithm keeps one length. + """ + if algorithm.embedding_dimensions is None: + if len(lengths) > 1: + raise EmbeddingDimensionMismatch( + f"Algorithm {algorithm.key} sent vectors of several lengths in one batch: {sorted(lengths)}" + ) + (length,) = lengths + # Conditional update: the first batch to record a length wins over a concurrent one. + Algorithm.objects.filter(pk=algorithm.pk, embedding_dimensions__isnull=True).update( + embedding_dimensions=length + ) + algorithm.refresh_from_db(fields=["embedding_dimensions"]) + wrong = lengths - {algorithm.embedding_dimensions} + if wrong: + raise EmbeddingDimensionMismatch( + f"Algorithm {algorithm.key} produces {algorithm.embedding_dimensions}-dimension vectors; " + f"refusing vectors of length {sorted(wrong)}." + ) + + +# How many boxes that match no stored detection are named in the job log per batch. +UNMATCHED_BOXES_TO_LOG = 5 + + +@dataclasses.dataclass +class StoredEmbeddings: + """The vectors stored from one batch, and the returned boxes that matched no detection. + + ``without_vector`` counts the detections on the batch's images that still lack a vector + after the save: boxes the service did not return, or returned without a vector. + """ + + embeddings: list[DetectionEmbedding] + unmatched: list[DetectionResponse] + without_vector: int = 0 + + +def _describe_boxes(detection_responses: list[DetectionResponse]) -> str: + return "; ".join( + f"image {response.source_image_id} box {tuple(response.bbox.dict().values()) if response.bbox else None}" + for response in detection_responses[:UNMATCHED_BOXES_TO_LOG] + ) + + +def create_detection_embeddings( + detections: list[Detection], + detection_responses: list[DetectionResponse], + algorithms_known: dict[str, Algorithm], + logger: logging.Logger = logger, + job_id: int | None = None, +) -> StoredEmbeddings: + """ + Store the feature vectors sent with each detection, one row per (detection, algorithm). + + A vector already stored for the pair is replaced, so saving the same results twice + changes nothing. Only ``DetectionEmbedding`` rows are written, never a classification, + so no determination can change. + + Responses are matched to ``detections`` by image and box (see ``BOX_MATCH_DECIMALS``), + the key ``get_or_create_detection`` reuses detections by. A returned box with no match is + skipped and listed in the result's ``unmatched``. An algorithm key the pipeline has not + registered raises ``PipelineNotConfigured``, as it does for classifications, and a vector + whose length differs from its algorithm's raises ``EmbeddingDimensionMismatch``. + ``job_id`` records the job whose results stored each vector. + """ + by_box = { + _box_key(detection.source_image_id, detection.bbox): detection + for detection in detections + if detection.bbox is not None + } + embeddings: dict[tuple[int, int], DetectionEmbedding] = {} + lengths_by_algorithm: dict[str, set[int]] = collections.defaultdict(set) + unmatched: list[DetectionResponse] = [] + for detection_resp in detection_responses: + if detection_resp.bbox is None: + continue + detection = by_box.get(_box_key(detection_resp.source_image_id, detection_resp.bbox.dict().values())) + if detection is None: + unmatched.append(detection_resp) + continue + for embedding_resp in detection_resp.embeddings or []: + try: + algorithm = algorithms_known[embedding_resp.algorithm.key] + except KeyError as err: + raise PipelineNotConfigured( + f"Embedding algorithm {embedding_resp.algorithm.key} is not a known algorithm. " + "The processing service must declare it in the /info endpoint. " + f"Known algorithms: {list(algorithms_known.keys())}" + ) from err + lengths_by_algorithm[algorithm.key].add(len(embedding_resp.features)) + embeddings[(detection.pk, algorithm.pk)] = DetectionEmbedding( + detection=detection, algorithm=algorithm, vector=embedding_resp.features, job_id=job_id + ) + + for key, lengths in lengths_by_algorithm.items(): + _check_embedding_dimensions(algorithms_known[key], lengths) + + if unmatched: + logger.warning( + f"Skipped {len(unmatched)} returned boxes that match no stored detection, " + f"for example: {_describe_boxes(unmatched)}" + ) + DetectionEmbedding.objects.bulk_create( + list(embeddings.values()), + update_conflicts=True, + unique_fields=["detection", "algorithm"], + update_fields=["vector", "job", "updated_at"], + batch_size=EMBEDDING_BATCH_SIZE, + ) + logger.info(f"Stored {len(embeddings)} detection embeddings for {len(detections)} detections.") + return StoredEmbeddings(embeddings=list(embeddings.values()), unmatched=unmatched) + + +def save_features_for_existing_detections( + results: PipelineResultsResponse, + algorithms_known: dict[str, Algorithm], + logger: logging.Logger = logger, + job_id: int | None = None, +) -> StoredEmbeddings: + """Store the vectors a feature-only pipeline returned for detections Antenna already has. + + Writes ``DetectionEmbedding`` rows and nothing else: a box that matches no stored + detection is skipped rather than created, classifications in the response are ignored, + and no occurrence, determination or null marker is touched. Detections on the batch's + images that still lack a vector afterwards are counted in ``without_vector``. A batch + that stores no vector at all while such detections remain raises + ``FeatureResultsStoredNothing``, because they would otherwise be sent again on every run. + A response that reports an error is left to the error handling of the caller. + """ + ignored = sum(len(detection.classifications) for detection in results.detections) + if ignored: + logger.warning(f"Ignored {ignored} classifications returned by a feature-only pipeline.") + returned_boxes = [detection for detection in results.detections if detection.bbox is not None] + image_ids = {int(detection.source_image_id) for detection in returned_boxes} + detections = list( + Detection.objects.valid().filter(source_image_id__in=image_ids).only("pk", "source_image_id", "bbox") + ) + stored = create_detection_embeddings( + detections=detections, + detection_responses=results.detections, + algorithms_known=algorithms_known, + logger=logger, + job_id=job_id, + ) + batch_image_ids = image_ids | {int(image.id) for image in results.source_images} + extractor_ids = [algorithm.pk for algorithm in feature_extractors_if_feature_only(list(algorithms_known.values()))] + if batch_image_ids and extractor_ids: + stored.without_vector = detections_missing_features( + Detection.objects.filter(source_image_id__in=batch_image_ids), extractor_ids + ).count() + if stored.without_vector: + logger.warning( + f"{stored.without_vector} detections on the {len(batch_image_ids)} images of this batch " + "still have no feature vector after saving it." + ) + all_unmatched = bool(returned_boxes) and len(stored.unmatched) == len(returned_boxes) + if not stored.embeddings and (stored.without_vector or all_unmatched) and not results.errors: + raise FeatureResultsStoredNothing( + f"The feature-only pipeline returned {len(returned_boxes)} boxes and no vector was saved, " + f"while {stored.without_vector} detections on these images still have none " + f"({len(stored.unmatched)} returned boxes match no stored detection). " + f"First boxes: {_describe_boxes(returned_boxes)}", + unmatched=len(stored.unmatched), + without_vector=stored.without_vector, + ) + return stored + + def create_category_map_for_classification( classification_resp: ClassificationResponse, logger: logging.Logger = logger, @@ -980,6 +1313,10 @@ class PipelineSaveResults: classifications: list[Classification] algorithms: dict[str, Algorithm] total_time: float + # Only a feature-only save reports these: returned boxes that matched no stored detection, + # and detections on the saved images that still have no vector. + unmatched_detections: int | None = None + detections_without_vector: int | None = None def create_null_detections_for_undetected_images( @@ -1060,6 +1397,28 @@ def save_results( ) algorithms_known: dict[str, Algorithm] = {algo.key: algo for algo in pipeline.algorithms.all()} + if feature_extraction_only(list(algorithms_known.values())): + job_logger.info(f"Pipeline {pipeline} only extracts features; storing vectors for existing detections.") + stored = save_features_for_existing_detections( + results, algorithms_known, logger=job_logger, job_id=job.pk if job else None + ) + total_time = time.time() - start_time + job_logger.info( + f"Saved {len(stored.embeddings)} feature vectors from pipeline {pipeline} in {total_time:.2f} seconds" + ) + if return_created: + return PipelineSaveResults( + pipeline=pipeline, + source_images=list(source_images), + detections=[], + classifications=[], + algorithms={}, + total_time=total_time, + unmatched_detections=len(stored.unmatched), + detections_without_vector=stored.without_vector, + ) + return None + try: detection_algorithm = pipeline.algorithms.get(task_type__in=Algorithm.detection_task_types) except Algorithm.DoesNotExist: @@ -1082,6 +1441,16 @@ def save_results( logger=job_logger, ) + # Before classifications, so an unregistered embedding algorithm stops the batch at the + # same point an unregistered classification algorithm does. + create_detection_embeddings( + detections=detections, + detection_responses=results.detections, + algorithms_known=algorithms_known, + logger=job_logger, + job_id=job.pk if job else None, + ) + classifications = create_classifications( detections=detections, detection_responses=results.detections, @@ -1238,6 +1607,17 @@ class Meta: def __str__(self): return f'#{self.pk} "{self.name}" ({self.slug}) v{self.version}' + def feature_extraction_algorithms(self) -> list[Algorithm]: + """The pipeline's feature extractors when it has no classifier, otherwise none. + + Such a pipeline is run on existing detections and only stores their vectors: it + creates no detection, classification or occurrence. + """ + return feature_extractors_if_feature_only(list(self.algorithms.all())) + + def is_feature_only(self) -> bool: + return bool(self.feature_extraction_algorithms()) + def get_config(self, project_id: int | None = None) -> PipelineRequestConfigParameters: """ Get the configuration for the pipeline request. @@ -1350,8 +1730,8 @@ def process_images( reprocess_all_images=reprocess_all_images, ) - def save_results(self, results: PipelineResultsResponse, job_id: int | None = None): - return save_results(results=results, job_id=job_id) + def save_results(self, results: PipelineResultsResponse, job_id: int | None = None, return_created=False): + return save_results(results=results, job_id=job_id, return_created=return_created) def save_results_async(self, results: PipelineResultsResponse, job_id: int | None = None): # Returns an AsyncResult diff --git a/ami/ml/orchestration/jobs.py b/ami/ml/orchestration/jobs.py index a965ed7c9..db833576c 100644 --- a/ami/ml/orchestration/jobs.py +++ b/ami/ml/orchestration/jobs.py @@ -81,6 +81,32 @@ async def cleanup(): return redis_success and nats_success +def _attach_detections_for_feature_pipeline(job: "Job", tasks: list[PipelineProcessingTask]) -> set[str]: + """Give each task the existing boxes a feature-only pipeline should embed, in batches of images. + + Returns the image ids left with no boxes. Those must not be queued: a worker given an + empty list runs its own detector, and boxes that match nothing are discarded. + """ + from ami.ml.models.pipeline import FILTER_PROCESSED_BATCH_SIZE, collect_detections_for_features + from ami.ml.schemas import SourceImageRequest + + algorithm_ids = ( + [algorithm.pk for algorithm in job.pipeline.feature_extraction_algorithms()] if job.pipeline else [] + ) + if not algorithm_ids: + return set() + include_existing = job.project.feature_flags.reprocess_all_images + for start in range(0, len(tasks), FILTER_PROCESSED_BATCH_SIZE): + batch = tasks[start : start + FILTER_PROCESSED_BATCH_SIZE] # noqa: E203 + requests = [SourceImageRequest(id=task.image_id, url=task.image_url) for task in batch] + by_image: dict[str, list] = {task.image_id: [] for task in batch} + for detection_request in collect_detections_for_features(requests, algorithm_ids, include_existing): + by_image[detection_request.source_image.id].append(detection_request) + for task in batch: + task.detections = by_image[task.image_id] + return {task.image_id for task in tasks if not task.detections} + + def queue_images_to_nats(job: "Job", images: list[SourceImage]): """ Queue all images for a job to a NATS JetStream stream for the job. @@ -117,6 +143,12 @@ def queue_images_to_nats(job: "Job", images: list[SourceImage]): ) tasks.append((image.pk, task)) + nothing_to_embed = _attach_detections_for_feature_pipeline(job, [task for _, task in tasks]) + if nothing_to_embed: + job.logger.info(f"Not queuing {len(nothing_to_embed)} images that have no detections to embed") + tasks = [(pk, task) for pk, task in tasks if task.image_id not in nothing_to_embed] + image_ids = [image_id for image_id in image_ids if image_id not in nothing_to_embed] + # Store all image IDs in Redis for progress tracking state_manager = AsyncJobStateManager(job.pk) state_manager.initialize_job(image_ids) diff --git a/ami/ml/post_processing/admin/tracking_form.py b/ami/ml/post_processing/admin/tracking_form.py index 214731a72..203bfb044 100644 --- a/ami/ml/post_processing/admin/tracking_form.py +++ b/ami/ml/post_processing/admin/tracking_form.py @@ -37,12 +37,14 @@ class TrackingActionForm(BasePostProcessingActionForm): ), ) feature_extraction_algorithm_id = forms.ModelChoiceField( - queryset=Algorithm.objects.filter(task_type=AlgorithmTaskType.CLASSIFICATION.value).order_by("name"), + queryset=Algorithm.objects.filter( + task_type__in=[AlgorithmTaskType.CLASSIFICATION.value, *Algorithm.feature_extraction_task_types] + ).order_by("name"), required=False, label="Feature extractor", help_text=( - "Whose embeddings to compare. Leave blank to detect it automatically; set it when " - "more than one classifier has run on the same session." + "Whose embeddings to compare. Leave blank for the session's only extractor or, " + "when there are several, the project's default one." ), ) skip_if_human_identifications = forms.BooleanField( diff --git a/ami/ml/post_processing/admin_forms.py b/ami/ml/post_processing/admin_forms.py index 41bac52b3..013593968 100644 --- a/ami/ml/post_processing/admin_forms.py +++ b/ami/ml/post_processing/admin_forms.py @@ -13,29 +13,21 @@ from django import forms from django.db.models import QuerySet -from ami.main.models import Classification, Event +from ami.main.models import Event +from ami.main.models_future.embeddings import algorithm_ids_with_vectors from ami.ml.models import Algorithm from ami.ml.post_processing.tracking_task import DEFAULT_TRACKING_PARAMS def _feature_algorithm_choices_for_events(events: QuerySet[Event]) -> list[tuple[int, str]]: - """Algorithms that produced ``features_2048`` on the given events. + """Algorithms that stored a feature vector, as an embedding or on a classification, + for detections in the given events. Scoped to the operator's selection so the dropdown stays bounded on production-sized DBs and never reveals algorithms from other projects. """ - algorithm_ids = ( - Classification.objects.filter( - detection__source_image__event__in=events, - features_2048__isnull=False, - algorithm_id__isnull=False, - ) - .values_list("algorithm_id", flat=True) - .distinct() - ) - return [ - (a.pk, f"{a.name} (#{a.pk})") for a in Algorithm.objects.filter(pk__in=list(algorithm_ids)).order_by("name") - ] + algorithm_ids = algorithm_ids_with_vectors(source_image__event__in=events) + return [(a.pk, f"{a.name} (#{a.pk})") for a in Algorithm.objects.filter(pk__in=algorithm_ids).order_by("name")] class TrackingActionForm(forms.Form): @@ -85,9 +77,9 @@ class TrackingActionForm(forms.Form): required=False, help_text=( "Override the algorithm whose embeddings are used for matching. Leave " - "blank to auto-detect (works when only one feature-extracting algorithm " - "ran on the event). Required when multiple algorithms have produced " - "embeddings on the same event." + "blank to use the only one with vectors on the event, or, when several " + "have them, the project's default: one it runs as a feature extractor, " + "else the one stored most recently." ), ) diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index 2da2001b7..7060b8ec5 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -1,4 +1,5 @@ import logging +import typing from collections.abc import Callable import numpy as np @@ -7,10 +8,14 @@ from django.db.models import QuerySet from django.utils import timezone -from ami.main.models import Classification, Occurrence, SourceImageCollection, TaxaList +from ami.main.models import Classification, Occurrence, OccurrenceHistoryRecord, SourceImageCollection, TaxaList +from ami.main.schemas import ClassMaskingResultPayload from ami.ml.models.algorithm import Algorithm, AlgorithmTaskType from ami.ml.post_processing.base import BasePostProcessingTask +if typing.TYPE_CHECKING: + from ami.jobs.models import Job + logger = logging.getLogger(__name__) @@ -52,6 +57,7 @@ def make_classifications_filtered_by_taxa_list( task_logger: logging.Logger = logger, on_setup: Callable[[int], None] | None = None, on_batch: Callable[[dict], None] | None = None, + job: "Job | None" = None, ) -> dict[str, int]: """Re-score ``classifications`` by masking out classes absent from ``taxa_list``. @@ -74,6 +80,9 @@ def make_classifications_filtered_by_taxa_list( ``occurrences_updated`` counts only occurrences whose determination actually changed (not just any occurrence touched), matching the size-filter convention. + Every occurrence with a re-scored classification gets one history record, written + once the run is done so an occurrence spanning batches is recorded once. + Returns final counters (checked / masked / occurrences updated) for stage metrics. """ taxa_in_list = set(taxa_list.taxa.all()) @@ -113,6 +122,9 @@ def make_classifications_filtered_by_taxa_list( occurrences_to_update: set[Occurrence] = set() # Tracks occurrences whose determination actually changed across all batches. changed_occurrence_ids: set[int] = set() + # Occurrence id -> (determination before the run, re-scored detection ids). + history: dict[int, tuple[int | None, set[int]]] = {} + determinations_after: dict[int, int | None] = {} timestamp = timezone.now() masked_count = 0 @@ -189,7 +201,10 @@ def make_classifications_filtered_by_taxa_list( detection = classification.detection if detection is not None and detection.occurrence is not None: - occurrences_to_update.add(detection.occurrence) + occurrence = detection.occurrence + occurrences_to_update.add(occurrence) + _, rescored = history.setdefault(occurrence.pk, (occurrence.determination_id, set())) + rescored.add(detection.pk) # Flush every batch_size items and at the final item. The flush fires even # when nothing was accumulated so the job health-check sees a heartbeat during @@ -209,6 +224,7 @@ def make_classifications_filtered_by_taxa_list( occurrence.save(update_determination=True) if occurrence.pk is not None and occurrence.determination_id != prev: changed_occurrence_ids.add(occurrence.pk) + determinations_after[occurrence.pk] = occurrence.determination_id classifications_to_demote.clear() classifications_to_add.clear() @@ -224,6 +240,24 @@ def make_classifications_filtered_by_taxa_list( } ) + OccurrenceHistoryRecord.objects.bulk_create( + OccurrenceHistoryRecord.build( + occurrence_id=occurrence_id, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="class_masking", + payload=ClassMaskingResultPayload( + taxa_list_id=taxa_list.pk, + source_algorithm_id=algorithm.pk, + detection_ids=sorted(detection_ids), + taxon_before_id=taxon_before_id, + taxon_after_id=determinations_after.get(occurrence_id), + ), + timestamp=timestamp, + job=job, + algorithm=new_algorithm, + ) + for occurrence_id, (taxon_before_id, detection_ids) in history.items() + ) task_logger.info( f"Re-scored {masked_count} of {total} classifications; updated {len(changed_occurrence_ids)} occurrences." ) @@ -368,6 +402,7 @@ def _on_batch(m: dict) -> None: task_logger=self.logger, on_setup=_on_setup, on_batch=_on_batch, + job=self.job, ) self.report_stage_metrics(metrics) self.logger.info(f"=== Completed {self.name} ===") diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index 6a39af780..feb79ddb6 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -2,7 +2,16 @@ from django.db.models import QuerySet from django.utils import timezone -from ami.main.models import Classification, Detection, Occurrence, SourceImageCollection, Taxon, TaxonRank +from ami.main.models import ( + Classification, + Detection, + Occurrence, + OccurrenceHistoryRecord, + SourceImageCollection, + Taxon, + TaxonRank, +) +from ami.main.schemas import SizeFilterResultPayload from ami.ml.post_processing.base import BasePostProcessingTask from ami.ml.schemas import BoundingBox @@ -90,8 +99,56 @@ def run(self) -> None: updated_occurrence_ids: set[int] = set() modified_occurrences = 0 checked = 0 + # Occurrence id -> (determination before the run, flagged detection ids), written + # as one history record per occurrence once the run is done. + history: dict[int, tuple[int | None, list[int]]] = {} + determinations_after: dict[int, int | None] = {} + + def flush(i: int) -> None: + nonlocal checked, modified_detections, modified_occurrences + checked = i + modified_detections += len(detections_to_update) + + # with transaction.atomic(): + self.logger.info(f"Creating {len(classifications_to_create)} new classifications") + Classification.objects.bulk_create(classifications_to_create) + classifications_to_create.clear() + + self.logger.info(f"Marking {len(detections_to_update)} detections as {not_identifiable_taxon.name}") + for det in detections_to_update: + det.updated_at = timezone.now() + Detection.objects.bulk_update(detections_to_update, ["updated_at"]) + detections_to_update.clear() + + self.logger.info(f"Updating {len(occcurrences_to_update)} occurrences") + for occ in occcurrences_to_update: + # Count an occurrence only when flagging its detection actually + # changes the determination. Re-saving recomputes it in place, so + # an occurrence pinned to a human identification keeps its taxon + # and must not inflate the metric. + prev_determination_id = occ.determination_id + occ.save(update_determination=True) + if occ.pk is not None and occ.determination_id != prev_determination_id: + updated_occurrence_ids.add(occ.pk) + determinations_after[occ.pk] = occ.determination_id + modified_occurrences = len(updated_occurrence_ids) + occcurrences_to_update.clear() + + progress = i / total if total > 0 else 1.0 + self.update_progress(progress) + self.report_stage_metrics( + { + "detections_checked": checked, + "detections_flagged": modified_detections, + "occurrences_updated": modified_occurrences, + } + ) + i = 0 for i, det in enumerate(detections.iterator(), start=1): + # Write each full batch of 100 before the next begins, whether or not its rows were skipped. + if i > 1 and (i - 1) % 100 == 0: + flush(i - 1) bbox = det.get_bbox() if not bbox: self.logger.debug(f"Detection {det.pk}: no bbox, skipping") @@ -131,45 +188,26 @@ def run(self) -> None: detections_to_update.add(det) if det.occurrence is not None: occcurrences_to_update.add(det.occurrence) + _, flagged = history.setdefault(det.occurrence.pk, (det.occurrence.determination_id, [])) + flagged.append(det.pk) self.logger.debug(f"Marking detection {det.pk} as {not_identifiable_taxon.name}") - # Update progress every 100 detections - if i % 100 == 0 or i == total: - checked = i - modified_detections += len(detections_to_update) - - # with transaction.atomic(): - self.logger.info(f"Creating {len(classifications_to_create)} new classifications") - Classification.objects.bulk_create(classifications_to_create) - classifications_to_create.clear() - - self.logger.info(f"Marking {len(detections_to_update)} detections as {not_identifiable_taxon.name}") - for det in detections_to_update: - det.updated_at = timezone.now() - Detection.objects.bulk_update(detections_to_update, ["updated_at"]) - detections_to_update.clear() - - self.logger.info(f"Updating {len(occcurrences_to_update)} occurrences") - for occ in occcurrences_to_update: - # Count an occurrence only when flagging its detection actually - # changes the determination. Re-saving recomputes it in place, so - # an occurrence pinned to a human identification keeps its taxon - # and must not inflate the metric. - prev_determination_id = occ.determination_id - occ.save(update_determination=True) - if occ.pk is not None and occ.determination_id != prev_determination_id: - updated_occurrence_ids.add(occ.pk) - modified_occurrences = len(updated_occurrence_ids) - occcurrences_to_update.clear() - - progress = i / total if total > 0 else 1.0 - self.update_progress(progress) - self.report_stage_metrics( - { - "detections_checked": checked, - "detections_flagged": modified_detections, - "occurrences_updated": modified_occurrences, - } - ) - + flush(i) + + OccurrenceHistoryRecord.objects.bulk_create( + OccurrenceHistoryRecord.build( + occurrence_id=occurrence_id, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="size_filter", + payload=SizeFilterResultPayload( + size_threshold=threshold, + detection_ids=sorted(detection_ids), + taxon_before_id=taxon_before_id, + taxon_after_id=determinations_after.get(occurrence_id), + ), + job=self.job, + algorithm=self.algorithm, + ) + for occurrence_id, (taxon_before_id, detection_ids) in history.items() + ) self.logger.info(f"=== Completed {self.name}: {modified_detections} of {total} detections modified ===") diff --git a/ami/ml/post_processing/tests/test_class_masking.py b/ami/ml/post_processing/tests/test_class_masking.py index bf38decc9..fe8510dcb 100644 --- a/ami/ml/post_processing/tests/test_class_masking.py +++ b/ami/ml/post_processing/tests/test_class_masking.py @@ -19,6 +19,7 @@ Classification, Detection, Occurrence, + OccurrenceHistoryRecord, SourceImage, SourceImageCollection, TaxaList, @@ -359,6 +360,36 @@ def test_task_run_occurrence_scope(self): self.assertIsNotNone(new_clf) self.assertEqual(new_clf.taxon, self.species_taxa[0]) + def test_a_rescored_occurrence_gets_one_history_record_and_a_rerun_none(self): + logits = [2.0, 1.0, 5.0] + taxa_list = TaxaList.objects.create(name="History list") + taxa_list.taxa.set(self.species_taxa[:2]) + det, occ = self._detection_with_occurrence() + self._create_classification_with_logits(det, self.species_taxa[2], _softmax(logits), logits) + occ.save() + + for _ in range(2): + ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() + + record = OccurrenceHistoryRecord.objects.get(occurrence=occ, subtype="class_masking") + self.assertTrue(record.algorithm.key.startswith(f"{self.algorithm.key}_filtered_by_taxa_list_{taxa_list.pk}")) + self.assertEqual(record.payload["detection_ids"], [det.pk]) + self.assertEqual(record.payload["taxa_list_id"], taxa_list.pk) + self.assertEqual(record.payload["source_algorithm_id"], self.algorithm.pk) + self.assertEqual(record.payload["taxon_before_id"], self.species_taxa[2].pk) + self.assertEqual(record.payload["taxon_after_id"], self.species_taxa[0].pk) + + def test_an_occurrence_masking_does_not_change_gets_no_history_record(self): + logits = [2.0, 1.0, 5.0] + taxa_list = TaxaList.objects.create(name="Keeps everything") + taxa_list.taxa.set(self.species_taxa) + det, occ = self._detection_with_occurrence() + self._create_classification_with_logits(det, self.species_taxa[2], _softmax(logits), logits) + + ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() + + self.assertFalse(OccurrenceHistoryRecord.objects.filter(occurrence=occ).exists()) + # ----- batched commit + heartbeat ------------------------------------- def test_batched_commit_flushes_multiple_times(self): diff --git a/ami/ml/post_processing/tests/test_occurrence_history_writers.py b/ami/ml/post_processing/tests/test_occurrence_history_writers.py new file mode 100644 index 000000000..385c0c151 --- /dev/null +++ b/ami/ml/post_processing/tests/test_occurrence_history_writers.py @@ -0,0 +1,139 @@ +"""Tracking and the small size filter leave one history record per occurrence they change, and none otherwise. + +Class masking's records are covered in test_class_masking.py, next to its other tests. +""" + +import logging + +from django.test import TestCase + +from ami.main.models import Detection, Occurrence, OccurrenceHistoryRecord, SourceImage, Taxon +from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask +from ami.ml.post_processing.tracking_task import TrackingTask +from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project + +logger = logging.getLogger(__name__) + + +class HistoryWriterFixture(TestCase): + """Three consecutive captures in one session, each with dimensions, and a project taxon.""" + + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + create_taxa(project=self.project) + create_captures(deployment=self.deployment, num_nights=1, images_per_night=3, interval_minutes=1) + self.captures = list(SourceImage.objects.filter(deployment=self.deployment).order_by("timestamp")) + SourceImage.objects.filter(pk__in=[c.pk for c in self.captures]).update(width=1000, height=1000) + for capture in self.captures: + capture.width = capture.height = 1000 + self.event = self.captures[0].event + self.taxon = Taxon.objects.filter(projects=self.project).order_by("pk").first() + + def _singleton(self, capture: SourceImage, bbox: list[int], score: float = 0.9) -> Occurrence: + """One detection in an occurrence of its own, classified as the fixture taxon.""" + occurrence = Occurrence.objects.create(event=self.event, deployment=self.deployment, project=self.project) + detection = Detection.objects.create( + source_image=capture, bbox=bbox, timestamp=capture.timestamp, occurrence=occurrence + ) + detection.classifications.create(taxon=self.taxon, score=score, timestamp=capture.timestamp) + occurrence.save() + return occurrence + + def records(self, subtype: str): + return OccurrenceHistoryRecord.objects.filter( + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, subtype=subtype + ).order_by("pk") + + +class TrackingHistoryTestCase(HistoryWriterFixture): + def test_a_merged_track_gets_one_record_and_an_untouched_occurrence_none(self): + track = [self._singleton(capture, [100, 100, 150, 150]) for capture in self.captures] + loner = self._singleton(self.captures[1], [800, 800, 820, 820]) + + TrackingTask(logger=logger, event_ids=[self.event.pk], require_features=False, cost_threshold=0.5).run() + + record = self.records("tracking").get() + keeper = Occurrence.objects.get(detections__source_image=self.captures[0]) + self.assertEqual(record.occurrence_id, keeper.pk) + self.assertNotEqual(record.occurrence_id, loner.pk) + self.assertEqual(record.algorithm.key, "tracking") + payload = record.payload + self.assertEqual(payload["detections_count"], 3) + self.assertEqual(payload["frames_linked"], 2) + self.assertEqual(payload["occurrences_merged"], sorted(o.pk for o in track if o.pk != keeper.pk)) + self.assertIsNone(payload["feature_algorithm_id"]) + self.assertEqual(payload["settings"]["cost_threshold"], 0.5) + self.assertNotIn("event_ids", payload["settings"]) + self.assertIsNotNone(payload["cost_mean"]) + self.assertLessEqual(payload["cost_mean"], payload["cost_max"]) + self.assertEqual(payload["taxon_before_id"], self.taxon.pk) + self.assertEqual(payload["taxon_after_id"], self.taxon.pk) + + def test_a_run_that_changes_nothing_writes_nothing(self): + for capture in self.captures: + self._singleton(capture, [100, 100, 150, 150]) + TrackingTask(logger=logger, event_ids=[self.event.pk], require_features=False, cost_threshold=0.5).run() + self.assertEqual(self.records("tracking").count(), 1) + + TrackingTask( + logger=logger, + event_ids=[self.event.pk], + require_features=False, + require_fresh_event=False, + skip_if_human_identifications=False, + cost_threshold=0.5, + ).run() + self.assertEqual(self.records("tracking").count(), 1) + + def test_a_merged_occurrence_hands_its_history_to_the_keeper(self): + first, second = (self._singleton(capture, [100, 100, 150, 150]) for capture in self.captures[:2]) + earlier = OccurrenceHistoryRecord.objects.create( + occurrence=second, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="size_filter", + payload={"size_threshold": 0.001, "detection_ids": []}, + timestamp=second.created_at, + ) + TrackingTask(logger=logger, event_ids=[self.event.pk], require_features=False, cost_threshold=0.5).run() + + earlier.refresh_from_db() + self.assertEqual(earlier.occurrence_id, first.pk) + + +class SizeFilterHistoryTestCase(HistoryWriterFixture): + def test_an_occurrence_with_a_flagged_detection_gets_one_record(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + Detection.objects.create( + source_image=self.captures[1], + bbox=[0, 0, 12, 12], + timestamp=self.captures[1].timestamp, + occurrence=occurrence, + ) + + SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=0.01).run() + + record = self.records("size_filter").get() + self.assertEqual(record.occurrence_id, occurrence.pk) + self.assertEqual(record.algorithm.key, "small_size_filter") + self.assertEqual(record.payload["detection_ids"], sorted(occurrence.detections.values_list("pk", flat=True))) + self.assertEqual(record.payload["size_threshold"], 0.01) + self.assertEqual(record.payload["taxon_before_id"], self.taxon.pk) + self.assertEqual(record.payload["taxon_after_id"], Taxon.objects.get(name="Not identifiable").pk) + + def test_an_occurrence_with_nothing_flagged_gets_no_record(self): + occurrence = self._singleton(self.captures[0], [0, 0, 500, 500]) + SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=0.01).run() + self.assertFalse(self.records("size_filter").exists()) + + def test_a_skipped_last_detection_still_saves_the_batch_before_it(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + # Inserted last, so the scan reaches it last; it has no box and is skipped. + Detection.objects.create( + source_image=self.captures[1], bbox=None, timestamp=self.captures[1].timestamp, occurrence=occurrence + ) + + SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=0.01).run() + + occurrence.refresh_from_db() + self.assertEqual(occurrence.determination.name, "Not identifiable") + self.assertEqual(self.records("size_filter").get().payload["taxon_after_id"], occurrence.determination_id) diff --git a/ami/ml/post_processing/tests/test_tracking_task.py b/ami/ml/post_processing/tests/test_tracking_task.py index f79f2be2c..2de16eb09 100644 --- a/ami/ml/post_processing/tests/test_tracking_task.py +++ b/ami/ml/post_processing/tests/test_tracking_task.py @@ -15,9 +15,11 @@ from ami.main.models import ( Classification, Detection, + DetectionEmbedding, Event, Identification, Occurrence, + OccurrenceHistoryRecord, SourceImageCollection, Taxon, update_calculated_fields_for_events, @@ -40,6 +42,7 @@ assign_occurrences_by_tracking_images, assign_occurrences_from_detection_chains, event_is_fresh, + pair_detections, ) from ami.tests.fixtures.images import generate_moth_series from ami.tests.fixtures.main import create_captures, create_occurrences, create_taxa, setup_test_project @@ -330,6 +333,80 @@ def test_requiring_features_leaves_data_untouched(self): ) +class TestTrackingWithEmbeddings(TestCase): + """Tracking reads a detection's vector from its embedding when it has one. + + Every detection can carry an embedding, including those the moth/non-moth filter + rejected, which have no classification vector. What these pin is that tracking finds + and uses those vectors, that it still pairs them with the classification vectors older + data has, and that it never compares vectors from two algorithms. + """ + + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + create_captures(deployment=self.deployment, num_nights=1, images_per_night=5, interval_minutes=1) + create_taxa(self.project) + # One detection per capture, all with the same box, so geometry never separates them. + create_occurrences(deployment=self.deployment, num=5) + + self.event = self.project.events.first() + assert self.event is not None + self.source_images = list(self.event.captures.order_by("timestamp")) + _give_captures_dimensions(self.source_images) + self.extractor = Algorithm.objects.create(name="Embedding model", key="embedding-model") + self.vector = [1.0] + [0.0] * 2047 + + def _detections(self, index: int) -> list[Detection]: + return list(self.source_images[index].detections.valid()) + + def _embed(self, detections, algorithm: Algorithm) -> None: + DetectionEmbedding.objects.bulk_create( + [DetectionEmbedding(detection=d, algorithm=algorithm, vector=self.vector) for d in detections] + ) + + def _pair(self, current: list[Detection], following: list[Detection]) -> int: + """Links made between two captures when both sides need a vector from the extractor.""" + return pair_detections( + current, + following, + 4096, + 2160, + cost_threshold=0.5, + algorithm=self.extractor, + logger=logger, + require_features=True, + ) + + def test_detections_whose_only_vector_is_an_embedding_are_tracked(self): + """The run finds the embedding model by itself and, requiring vectors, links a detection + only when it read one for it.""" + self._embed(Detection.objects.valid().filter(source_image__event=self.event), self.extractor) + + TrackingTask(logger=logger, event_ids=[self.event.pk], require_features=True, cost_threshold=0.5).run() + + linked = Detection.objects.filter(source_image__event=self.event, next_detection__isnull=False).count() + self.assertEqual(linked, len(self.source_images) - 1) + + def test_vectors_from_another_algorithm_are_never_compared(self): + other = Algorithm.objects.create(name="Other embedding model", key="other-embedding-model") + current, following = self._detections(0), self._detections(1) + self._embed(current, self.extractor) + self._embed(following, other) + self.assertEqual(self._pair(current, following), 0) + + self._embed(following, self.extractor) + self.assertEqual(self._pair(current, following), 1) + + def test_an_embedding_is_compared_with_a_classification_vector_from_the_same_algorithm(self): + """A session processed partly before embeddings existed still links across the boundary.""" + current, following = self._detections(0), self._detections(1) + Classification.objects.filter(detection__in=current).update( + algorithm=self.extractor, features_2048=self.vector + ) + self._embed(following, self.extractor) + self.assertEqual(self._pair(current, following), 1) + + def _one_box_in_two_captures(test: TransactionTestCase) -> None: """One session of two captures with one detection each on the same box, each detection on an occurrence of its own, so a geometry-only run links and merges them.""" @@ -393,6 +470,7 @@ def test_a_confirmed_occurrence_is_left_as_it_is(self): self.assertFalse( Detection.objects.filter(source_image__event=self.event, next_detection__isnull=False).exists() ) + self.assertFalse(OccurrenceHistoryRecord.objects.filter(occurrence__event=self.event).exists()) self.assertEqual(params["Confirmed tracks left unchanged"], 1) self.assertEqual(params["Result"], "Tracked 1 session(s). Left 1 confirmed track(s) unchanged.") diff --git a/ami/ml/post_processing/tracking_task.py b/ami/ml/post_processing/tracking_task.py index d145ec4fb..d97807f30 100644 --- a/ami/ml/post_processing/tracking_task.py +++ b/ami/ml/post_processing/tracking_task.py @@ -1,4 +1,5 @@ import collections +import dataclasses import logging import math import typing @@ -7,7 +8,7 @@ import numpy as np import pydantic from django.db import transaction -from django.db.models import Count +from django.db.models import Count, Exists, OuterRef from django.utils import timezone from ami.main.models import ( @@ -16,16 +17,26 @@ Event, Identification, Occurrence, + OccurrenceHistoryRecord, SourceImage, SourceImageCollection, update_calculated_fields_for_sessions_and_stations, update_occurrence_determination, ) +from ami.main.models_future.embeddings import ( + algorithm_ids_with_vectors, + default_feature_algorithm_id, + vectors_for_detections, +) from ami.main.models_future.track_stats import refresh_track_stats_for_ids from ami.main.models_future.tracks import lock_sessions +from ami.main.schemas import TrackingResultPayload from ami.ml.models import Algorithm from ami.ml.post_processing.base import BasePostProcessingTask +if typing.TYPE_CHECKING: + from ami.jobs.models import Job + class TrackingConfig(pydantic.BaseModel): """Scope and tunables for a tracking run. @@ -60,8 +71,8 @@ class TrackingConfig(pydantic.BaseModel): # data is a v2 concern (see #1272 for the incremental append/prepend plan). require_fresh_event: bool = True - # Which feature extractor's embeddings to compare. Left unset, the task infers it - # when exactly one algorithm produced embeddings for the event. + # Which feature extractor's embeddings to compare. Left unset: the event's only one, + # or the project's default among several (see resolve_feature_algorithm). feature_extraction_algorithm_id: int | None = None @pydantic.root_validator(skip_on_failure=True) @@ -78,6 +89,9 @@ class Config: def cosine_similarity(v1: Iterable[float], v2: Iterable[float]) -> float: a = np.array(v1) b = np.array(v2) + if a.shape != b.shape: + # Vectors of different lengths come from different extractors and are not comparable. + raise ValueError(f"Cannot compare vectors of shapes {a.shape} and {b.shape}") sim = np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)) return float(np.clip(sim, 0.0, 1.0)) @@ -130,21 +144,13 @@ def get_unique_feature_algorithm_for_event(event: Event) -> tuple[Algorithm | No """ Return ``(unique_algorithm, all_candidates)``. - If exactly one feature-extraction algorithm produced ``features_2048`` for this - event, returns that algorithm and a single-element list. Otherwise returns - ``(None, candidates)`` so the caller can either skip with a warning or require - the operator to pass an explicit ``feature_extraction_algorithm_id``. + If exactly one feature-extraction algorithm stored vectors (embeddings or + classification ``features_2048``) for this event, returns that algorithm and a + single-element list. Otherwise returns ``(None, candidates)`` and the caller picks + one (``resolve_feature_algorithm``). """ - algo_ids = ( - Classification.objects.filter( - detection__source_image__event=event, - features_2048__isnull=False, - algorithm_id__isnull=False, - ) - .values_list("algorithm_id", flat=True) - .distinct() - ) - candidates = list(Algorithm.objects.filter(pk__in=list(algo_ids))) + algo_ids = algorithm_ids_with_vectors(source_image__event=event) + candidates = list(Algorithm.objects.filter(pk__in=algo_ids)) if len(candidates) == 1: return candidates[0], candidates return None, candidates @@ -156,9 +162,11 @@ def resolve_feature_algorithm( """The feature extractor a tracking run compares embeddings from, and whether it tracks the event. Returns ``(algorithm, should_track, note)``. ``algorithm`` is None when the run falls - back to geometry alone, and ``note`` says why a run falls back or skips; it is empty - when one extractor was configured or found. ``candidates`` are the extractors that - produced embeddings: every one in the event unless the caller passes a narrower set. + back to geometry alone. With vectors from several extractors and none configured, the + project's default is taken (see ``default_feature_algorithm_id``). ``note`` says which + extractor was picked among several, or why a run falls back or skips; it is empty when + one extractor was configured or found. ``candidates`` are the extractors that produced + embeddings: every one in the event unless the caller passes a narrower set. """ if config.feature_extraction_algorithm_id is not None: algorithm = Algorithm.objects.filter(pk=config.feature_extraction_algorithm_id).first() @@ -177,14 +185,20 @@ def resolve_feature_algorithm( return candidates[0], True, "" if candidates: + # Vectors from several extractors: compare the project's default one, never a mix. + default_id = default_feature_algorithm_id( + event.project_id, [a.pk for a in candidates], source_image__event=event + ) + algorithm = next(a for a in candidates if a.pk == default_id) candidate_names = [f"#{a.pk} {a.name}" for a in candidates] - message = ( - f"Event {event.pk}: detections classified by {len(candidates)} different " - f"feature-extraction algorithms ({candidate_names}). Pass " - "feature_extraction_algorithm_id in the job config to disambiguate." + return ( + algorithm, + True, + f"Event {event.pk}: vectors from {len(candidates)} feature extractors ({candidate_names}); " + f"comparing #{algorithm.pk} {algorithm.name}. Pass feature_extraction_algorithm_id to choose another.", ) - else: - message = f"Event {event.pk}: no detections carry feature embeddings." + + message = f"Event {event.pk}: no detections carry feature embeddings." if config.require_features: return None, False, f"{message} Skipping." @@ -219,14 +233,9 @@ def event_is_fresh(event: Event) -> tuple[bool, str]: def event_fully_processed(event: Event, logger: logging.Logger, algorithm: Algorithm) -> bool: total = event.captures.count() - processed = ( - event.captures.filter( - detections__classifications__features_2048__isnull=False, - detections__classifications__algorithm=algorithm, - ) - .distinct() - .count() - ) + processed = event.captures.filter( + Exists(Detection.objects.has_vector(algorithm).filter(source_image_id=OuterRef("pk"))) + ).count() if processed < total: logger.info(f"Event {event.pk} not fully processed: {processed}/{total} captures") return False @@ -256,8 +265,47 @@ def record_tracking_determination(occurrence: Occurrence, algorithm: Algorithm) ) +@dataclasses.dataclass +class TrackingHistory: + """What a tracking run records in the history of each occurrence it changes.""" + + settings: dict + feature_algorithm_id: int | None = None + job: "Job | None" = None + algorithm: Algorithm | None = None + # Detection id -> cost of the link this run made from it to its next detection. + link_costs: dict[int, float] = dataclasses.field(default_factory=dict) + + def record( + self, occurrence: Occurrence, chain: list[Detection], merged: Iterable[int], taxon_before_id: int | None + ) -> OccurrenceHistoryRecord: + costs = [self.link_costs[d.pk] for d in chain[:-1] if d.pk in self.link_costs] + payload = TrackingResultPayload( + settings=self.settings, + feature_algorithm_id=self.feature_algorithm_id, + detections_count=len(chain), + frames_linked=len(costs), + occurrences_merged=sorted(merged), + cost_mean=sum(costs) / len(costs) if costs else None, + cost_max=max(costs, default=None), + taxon_before_id=taxon_before_id, + taxon_after_id=occurrence.determination_id, + ) + return OccurrenceHistoryRecord.build( + occurrence_id=occurrence.pk, + kind=OccurrenceHistoryRecord.Kind.ALGORITHM_RESULT, + subtype="tracking", + payload=payload, + job=self.job, + algorithm=self.algorithm, + ) + + def assign_occurrences_from_detection_chains( - source_images: list[SourceImage], logger: logging.Logger, record_as: Algorithm | None = None + source_images: list[SourceImage], + logger: logging.Logger, + record_as: Algorithm | None = None, + history: TrackingHistory | None = None, ) -> dict[str, int]: """ Walk chains via ``Detection.next_detection`` and consolidate each chain into @@ -275,6 +323,8 @@ def assign_occurrences_from_detection_chains( confirmed detections out of linking, so only a link stored earlier leads here. - Store the track statistics of every occurrence the chains settle on, so the list can sort by them (see ``track_stats.refresh_track_stats_for_ids``). + - With ``history`` set, leave one history record on each occurrence a chain changed; + a chain that was already one occurrence gets none. Designed for fresh-event input (1:1 detection/occurrence). v2 incremental tracking can reuse this primitive for prepend/append: keeper survives, new detections fold in. @@ -288,6 +338,7 @@ def assign_occurrences_from_detection_chains( identifications_moved = 0 determinations_recorded = 0 confirmed_chains_skipped = 0 + history_records: list[OccurrenceHistoryRecord] = [] existing = Occurrence.objects.filter(detections__source_image__in=source_images).distinct().count() # A chain ends at a session boundary. A regroup that splits a track keeps the link @@ -372,6 +423,7 @@ def session_of(detection: Detection) -> int | None: identifications_moved += Identification.objects.filter(occurrence_id__in=doomed).update( occurrence=keeper ) + OccurrenceHistoryRecord.objects.filter(occurrence_id__in=doomed).update(occurrence=keeper) for occ_id in doomed: try: Occurrence.objects.filter(id=occ_id).delete() @@ -386,8 +438,12 @@ def session_of(detection: Detection) -> int | None: if record_as is not None and keeper.determination_id != previous_determination_id: if record_tracking_determination(keeper, record_as) is not None: determinations_recorded += 1 + if history is not None: + history_records.append(history.record(keeper, chain, doomed, previous_determination_id)) settled.add(keeper.pk) + OccurrenceHistoryRecord.objects.bulk_create(history_records) + # Stored once every determination is settled, since id_agreement is measured against # it, and in batches for the whole event rather than three queries per chain. stats_stored = refresh_track_stats_for_ids(settled) @@ -425,24 +481,6 @@ def nothing_tracked_summary(skip_reasons: collections.Counter[str]) -> str: return f"Nothing was tracked: {total} session(s) skipped ({reasons})." -def latest_feature_vectors(detection_ids: Iterable[int], algorithm_id: int) -> dict[int, typing.Any]: - """The most recent embedding from one algorithm for each detection given, by detection id. - - Detections without one are left out. One query for the whole batch. - """ - vectors: dict[int, typing.Any] = {} - rows = ( - Classification.objects.filter( - detection_id__in=list(detection_ids), algorithm_id=algorithm_id, features_2048__isnull=False - ) - .order_by("-timestamp", "-pk") - .values_list("detection_id", "features_2048") - ) - for detection_id, vector in rows: - vectors.setdefault(detection_id, vector) - return vectors - - def select_links( current_detections: Sequence[Detection], next_detections: Sequence[Detection], @@ -498,7 +536,7 @@ def select_transition_links( """The links tracking makes between two adjacent captures, reading embeddings but saving nothing.""" vectors: dict[int, typing.Any] = {} if algorithm is not None: - vectors = latest_feature_vectors([det.pk for det in [*current_detections, *next_detections]], algorithm.pk) + vectors = vectors_for_detections([det.pk for det in [*current_detections, *next_detections]], algorithm.pk) return select_links( current_detections, next_detections, @@ -620,6 +658,7 @@ def assign_occurrences_by_tracking_images( config: TrackingConfig, progress_cb: typing.Callable[[float], None] | None = None, record_as: Algorithm | None = None, + history: TrackingHistory | None = None, ) -> dict[str, int]: source_images = list(event.captures.order_by("timestamp")) if len(source_images) < 2: @@ -651,6 +690,8 @@ def assign_occurrences_by_tracking_images( else: save_links(proposed, logger) links += len(proposed) + if history is not None: + history.link_costs.update((det.pk, cost) for det, _, cost in proposed) if progress_cb: progress_cb((i + 1) / transitions) @@ -660,7 +701,9 @@ def assign_occurrences_by_tracking_images( "due to missing image dimensions." ) - counters = assign_occurrences_from_detection_chains(source_images, logger, record_as=record_as) + counters = assign_occurrences_from_detection_chains( + source_images, logger, record_as=record_as, history=history + ) counters["links_created"] = links counters["confirmed_tracks_left_unchanged"] = confirmed_tracks @@ -802,6 +845,12 @@ def _stage_progress(p: float, _idx=idx, _total=total) -> None: config=self.config, record_as=self.algorithm, progress_cb=_stage_progress, + history=TrackingHistory( + settings=self.config.dict(exclude={"source_image_collection_id", "event_ids"}), + feature_algorithm_id=algorithm.pk if algorithm is not None else None, + job=self.job, + algorithm=self.algorithm, + ), ) totals["events_tracked"] += 1 tracked_event_ids.append(event.pk) diff --git a/ami/ml/schemas.py b/ami/ml/schemas.py index d03643f71..fa3a9d36c 100644 --- a/ami/ml/schemas.py +++ b/ami/ml/schemas.py @@ -116,6 +116,11 @@ class AlgorithmConfigResponse(pydantic.BaseModel): description="A URI to the weight or model details, could be a public web URL or object store path.", ) category_map: AlgorithmCategoryMapResponse | None = None + embedding_dimensions: int | None = pydantic.Field( + default=None, + ge=1, + description="For a feature extractor, the length of every vector it returns.", + ) class Config: extra = "ignore" @@ -182,6 +187,24 @@ class DetectionRequest(pydantic.BaseModel): algorithm: AlgorithmReference +class EmbeddingResponse(pydantic.BaseModel): + """A feature vector for one detection and the algorithm whose backbone produced it. + + Carried on the detection rather than on a classification, so storing it can never + add a prediction that competes for the occurrence's determination. Its length is the + algorithm's own (extractors differ), and only vectors from one algorithm are comparable. + """ + + features: list[float] = pydantic.Field(description="The feature vector.") + algorithm: AlgorithmReference + + @pydantic.validator("features") + def _features_not_empty(cls, v): + if not v: + raise ValueError("features must contain at least one value") + return v + + class DetectionResponse(pydantic.BaseModel): source_image_id: str bbox: BoundingBox | None = None @@ -190,6 +213,13 @@ class DetectionResponse(pydantic.BaseModel): timestamp: datetime.datetime crop_image_url: str | None = None classifications: list[ClassificationResponse] = [] + embeddings: list[EmbeddingResponse] | None = pydantic.Field( + default=None, + description=( + "Feature vectors for this detection, at most one per algorithm, including detections " + "the moth/non-moth filter rejected. Only vectors from the same algorithm are comparable." + ), + ) class PipelineRequestConfigParameters(pydantic.BaseModel): @@ -293,9 +323,13 @@ class PipelineProcessingTask(pydantic.BaseModel): image_id: str image_url: str reply_subject: str | None = None # The NATS subject to send the result to - # TODO: Do we need these? - # detections: list[DetectionRequest] | None = None - # config: PipelineRequestConfigParameters | dict | None = None + detections: list[DetectionRequest] | None = pydantic.Field( + default=None, + description=( + "Existing detections on the image to run the pipeline on instead of detecting anew. " + "Sent for feature-only pipelines, which return these boxes with an embedding each." + ), + ) class ProcessingServiceClientInfo(pydantic.BaseModel): diff --git a/ami/ml/serializers.py b/ami/ml/serializers.py index e7e9e6aaf..07306a36b 100644 --- a/ami/ml/serializers.py +++ b/ami/ml/serializers.py @@ -43,6 +43,7 @@ class Meta: "version", "version_name", "task_type", + "embedding_dimensions", "category_map", "category_count", "created_at", diff --git a/ami/ml/test_feature_extraction.py b/ami/ml/test_feature_extraction.py new file mode 100644 index 000000000..23f65d5d2 --- /dev/null +++ b/ami/ml/test_feature_extraction.py @@ -0,0 +1,467 @@ +"""Feature-only pipelines: vectors for detections Antenna already has, and nothing else. + +A feature extractor (e.g. a BioCLIP backbone) is run on existing detections so tracking +can compare every detection by appearance. These tests pin that such a run writes only +``DetectionEmbedding`` rows, skips detections that already have a vector, keeps one vector +length per algorithm, and that readers never compare vectors of different lengths. +""" + +import datetime +from unittest import mock + +import pydantic +from django.test import TestCase +from rest_framework.test import APITestCase + +from ami.jobs.models import Job, JobDispatchMode, MLJob +from ami.main.models import ( + Classification, + Deployment, + Detection, + DetectionEmbedding, + Event, + Occurrence, + Project, + SourceImage, + SourceImageCollection, + group_images_into_events, +) +from ami.main.models_future.embeddings import ( + default_feature_algorithm_id, + feature_extractors_with_vectors, + vectors_for_detections, +) +from ami.ml.exceptions import FeatureResultsStoredNothing +from ami.ml.models import Algorithm, Pipeline, ProcessingService +from ami.ml.models.pipeline import ( + COLLECT_PROGRESS_MAX_FRACTION, + COLLECT_PROGRESS_SAVE_INTERVAL_SECONDS, + EmbeddingDimensionMismatch, + collect_detections_for_features, + collect_images, + filter_processed_images, + process_images, + save_results, +) +from ami.ml.models.project_pipeline_config import ProjectPipelineConfig +from ami.ml.orchestration.jobs import queue_images_to_nats +from ami.ml.post_processing.tracking_task import TrackingConfig, cosine_similarity, resolve_feature_algorithm +from ami.ml.schemas import EmbeddingResponse, PipelineResultsResponse, SourceImageRequest +from ami.users.models import User + +BIOCLIP_DIMENSIONS = 1024 + + +def _box(offset: float) -> list[float]: + return [offset, offset, offset + 10.5, offset + 12.25] + + +class FeatureOnlyFixture: + """A project with detected, classified and determined captures, plus a feature-only pipeline.""" + + def _set_up_project(self, images: int = 3, boxes_per_image: int = 2) -> None: + self.project = Project.objects.create(name="Feature extraction") + self.deployment = Deployment.objects.create(name="Station", project=self.project) + self.detector = Algorithm.objects.create(name="Detector", key="test-detector", task_type="localization") + self.classifier = Algorithm.objects.create( + name="Classifier", key="test-classifier", task_type="classification" + ) + self.extractor = Algorithm.objects.create(name="Backbone", key="test-backbone", task_type="embedding") + self.pipeline = Pipeline.objects.create(name="Features only", slug="features-only") + # A feature-only service also lists the detector whose boxes it echoes back. + self.pipeline.algorithms.set([self.detector, self.extractor]) + + start = datetime.datetime(2024, 6, 1, 22, 0) + self.images = [ + SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + path=f"feat/{i}.jpg", + timestamp=start + datetime.timedelta(minutes=i), + ) + for i in range(images) + ] + group_images_into_events(self.deployment) + self.images = list(SourceImage.objects.filter(pk__in=[i.pk for i in self.images]).order_by("timestamp")) + self.collection = SourceImageCollection.objects.create(project=self.project, name="Scope") + self.collection.images.set(self.images) + for image in self.images: + for n in range(boxes_per_image): + occurrence = Occurrence.objects.create( + project=self.project, deployment=self.deployment, event=image.event + ) + detection = Detection.objects.create( + source_image=image, + bbox=_box(20.0 * n), + detection_algorithm=self.detector, + occurrence=occurrence, + timestamp=image.timestamp, + ) + Classification.objects.create( + detection=detection, algorithm=self.classifier, score=0.9, timestamp=image.timestamp + ) + # A null marker must never be sent or embedded. + Detection.objects.create(source_image=self.images[0], bbox=None, detection_algorithm=self.detector) + + def _embed(self, detections, algorithm: Algorithm, length: int = BIOCLIP_DIMENSIONS) -> None: + DetectionEmbedding.objects.bulk_create( + [DetectionEmbedding(detection=d, algorithm=algorithm, vector=[0.5] * length) for d in detections] + ) + + def _response(self, boxes: list[tuple[SourceImage, list[float]]], length: int = BIOCLIP_DIMENSIONS, **extra): + """What a feature-only pipeline sends back: the boxes it was given, each with a vector.""" + return PipelineResultsResponse( + pipeline=self.pipeline.slug, + total_time=0.1, + source_images=[{"id": str(image.pk), "url": "x"} for image in {image for image, _ in boxes}], + detections=[ + { + "source_image_id": str(image.pk), + "bbox": dict(zip(["x1", "y1", "x2", "y2"], box)), + "algorithm": {"name": self.detector.name, "key": self.detector.key}, + "timestamp": datetime.datetime.now().isoformat(), + "embeddings": [ + { + "algorithm": {"name": self.extractor.name, "key": self.extractor.key}, + "features": [0.1] * length, + } + ], + **extra, + } + for image, box in boxes + ], + ) + + def _counts(self) -> tuple: + return ( + Detection.objects.count(), + Classification.objects.count(), + Occurrence.objects.count(), + sorted(Occurrence.objects.values_list("pk", "determination_id", "determination_score")), + ) + + +class TestEmbeddingSchema(TestCase): + def test_a_vector_is_read_from_the_features_key_only(self): + """The processing service sends ``features``; an empty vector or any other key is refused.""" + algorithm = {"name": "Backbone", "key": "test-backbone"} + parsed = EmbeddingResponse.parse_obj({"algorithm": algorithm, "features": [0.1] * BIOCLIP_DIMENSIONS}) + self.assertEqual(len(parsed.features), BIOCLIP_DIMENSIONS) + for payload in ({"features": []}, {"vector": [0.1] * BIOCLIP_DIMENSIONS}): + with self.assertRaises(pydantic.ValidationError): + EmbeddingResponse.parse_obj({"algorithm": algorithm, **payload}) + + +class TestFeatureOnlySave(FeatureOnlyFixture, TestCase): + def setUp(self) -> None: + self._set_up_project() + + def test_a_pipeline_is_feature_only_until_it_lists_a_classifier(self): + """The detector does not count against it; a classifier does, since its pipeline + detects and classifies anew and may embed as well.""" + self.assertEqual(self.pipeline.feature_extraction_algorithms(), [self.extractor]) + self.pipeline.algorithms.add(self.classifier) + self.assertFalse(self.pipeline.is_feature_only()) + self.pipeline.algorithms.set([self.detector]) + self.assertFalse(self.pipeline.is_feature_only()) + + def test_an_algorithm_of_unknown_task_type_rules_out_feature_only(self): + """A classifier registered without a task type must not turn its pipeline feature-only, + or saving its results would drop every new detection and classification.""" + for task_type in ("", "unknown"): + untyped = Algorithm.objects.create( + name=f"Untyped {task_type!r}", key=f"untyped-{task_type}", task_type=task_type + ) + self.pipeline.algorithms.set([self.detector, self.extractor, untyped]) + self.assertFalse(self.pipeline.is_feature_only(), task_type) + + def test_an_embeddings_only_response_writes_only_vectors(self): + """No detection, classification or occurrence is created and no determination moves, even + for a box Antenna does not have or classifications the service sent anyway.""" + before = self._counts() + known = [(image, _box(0.0)) for image in self.images] + unknown = [(self.images[0], _box(500.0))] + stray = { + "classifications": [ + { + "classification": "Moth", + "scores": [0.99], + "algorithm": {"name": self.extractor.name, "key": self.extractor.key}, + "timestamp": datetime.datetime.now().isoformat(), + } + ] + } + save_results(self._response(known + unknown, **stray)) + + self.assertEqual(self._counts(), before) + stored = DetectionEmbedding.objects.filter(algorithm=self.extractor) + self.assertEqual( + sorted(stored.values_list("detection__source_image_id", flat=True)), sorted(i.pk for i in self.images) + ) + self.extractor.refresh_from_db() + self.assertEqual(self.extractor.embedding_dimensions, BIOCLIP_DIMENSIONS) + + def test_boxes_that_match_no_detection_are_counted(self): + known = [(image, _box(0.0)) for image in self.images] + unknown = [(self.images[0], _box(500.0)), (self.images[1], _box(600.0))] + saved = save_results(self._response(known + unknown), return_created=True) + self.assertEqual(saved.unmatched_detections, 2) + self.assertEqual(DetectionEmbedding.objects.count(), len(self.images)) + + def test_a_batch_whose_boxes_all_match_nothing_fails_and_stores_nothing(self): + """Skipping it would leave the same detections without vectors, to be sent again on every run.""" + with self.assertRaises(FeatureResultsStoredNothing) as raised: + save_results(self._response([(self.images[0], _box(500.0)), (self.images[1], _box(600.0))])) + self.assertEqual(raised.exception.unmatched, 2) + self.assertFalse(DetectionEmbedding.objects.exists()) + + def test_boxes_returned_without_vectors_fail_the_batch(self): + """Matching boxes that carry no vector store nothing, so the batch must not pass as a success.""" + image = self.images[0] + with self.assertRaises(FeatureResultsStoredNothing) as raised: + save_results(self._response([(image, _box(0.0)), (image, _box(20.0))], embeddings=[])) + self.assertEqual((raised.exception.unmatched, raised.exception.without_vector), (0, 2)) + self.assertFalse(DetectionEmbedding.objects.exists()) + + def test_a_response_with_no_boxes_fails_the_batch(self): + response = self._response([]) + response.source_images = [{"id": str(self.images[0].pk), "url": "x"}] + with self.assertRaises(FeatureResultsStoredNothing) as raised: + save_results(PipelineResultsResponse.parse_obj(response.dict())) + self.assertEqual(raised.exception.without_vector, 2) + + def test_detections_the_service_did_not_return_are_counted(self): + saved = save_results(self._response([(self.images[0], _box(0.0))]), return_created=True) + self.assertEqual(saved.detections_without_vector, 1) + self.assertEqual(DetectionEmbedding.objects.count(), 1) + + def test_a_vector_of_another_length_is_refused_and_nothing_is_stored(self): + save_results(self._response([(self.images[0], _box(0.0))])) + with self.assertRaises(EmbeddingDimensionMismatch): + save_results(self._response([(self.images[1], _box(0.0))], length=512)) + self.assertEqual(DetectionEmbedding.objects.count(), 1) + + +class TestExtractFeaturesScope(FeatureOnlyFixture, TestCase): + """What an extract-features run sends: only real detections still missing a vector.""" + + def setUp(self) -> None: + self._set_up_project() + # The first image is done; one box on the second is done. + self._embed(self.images[0].detections.valid(), self.extractor) + self._embed(self.images[1].detections.valid().filter(bbox=_box(0.0)), self.extractor) + + def test_a_classifier_pipeline_that_also_embeds_skips_classified_images(self): + """Its extractor writes no classification, so counting it as a classifier would + reprocess every image and add a second set of classifications.""" + pipeline = Pipeline.objects.create(name="Classify and embed", slug="classify-and-embed") + pipeline.algorithms.set([self.detector, self.classifier, self.extractor]) + self.assertEqual(list(collect_images(collection=self.collection, pipeline=pipeline)), []) + + def test_images_whose_detections_all_have_vectors_are_skipped(self): + collected = collect_images(collection=self.collection, pipeline=self.pipeline) + self.assertEqual([image.pk for image in collected], [self.images[1].pk, self.images[2].pk]) + + def test_a_vector_from_another_algorithm_does_not_count(self): + other = Algorithm.objects.create(name="Other", key="other-backbone", task_type="embedding") + self._embed(self.images[2].detections.valid(), other) + collected = collect_images(collection=self.collection, pipeline=self.pipeline) + self.assertIn(self.images[2].pk, [image.pk for image in collected]) + + def test_the_request_carries_only_the_missing_boxes_in_one_query(self): + requests = [SourceImageRequest(id=str(image.pk), url="x") for image in self.images] + with self.assertNumQueries(1): + detections = collect_detections_for_features(requests, [self.extractor.pk]) + self.assertEqual( + sorted((int(d.source_image.id), d.bbox.x1) for d in detections), + [(self.images[1].pk, 20.0), (self.images[2].pk, 0.0), (self.images[2].pk, 20.0)], + ) + self.assertEqual({d.algorithm.key for d in detections}, {self.detector.key}) + + def test_process_images_sends_existing_boxes_and_skips_done_images(self): + sent = {} + + def post(url, json): + sent.update(json) + return mock.Mock( + ok=True, + json=lambda: {"pipeline": json["pipeline"], "total_time": 0, "source_images": [], "detections": []}, + ) + + with mock.patch.object(SourceImage, "public_url", return_value="http://example.org/i.jpg"), mock.patch( + "ami.ml.models.pipeline.create_session", return_value=mock.Mock(post=post) + ): + process_images(self.pipeline, "http://example.org/process", self.images, project_id=self.project.pk) + + self.assertEqual(sorted(int(i["id"]) for i in sent["source_images"]), [self.images[1].pk, self.images[2].pk]) + self.assertEqual(len(sent["detections"]), 3) + + +class TestExtractFeaturesQueue(FeatureOnlyFixture, TestCase): + """The async path: each queued task carries its missing boxes, and no task is queued empty.""" + + def setUp(self) -> None: + self._set_up_project() + self._embed(self.images[0].detections.valid(), self.extractor) + self._embed(self.images[1].detections.valid().filter(bbox=_box(0.0)), self.extractor) + # A box whose detector is unknown cannot be sent, so it must not make its image count as pending. + Detection.objects.create(source_image=self.images[0], bbox=_box(40.0), detection_algorithm=None) + self.job = Job.objects.create( + name="Extract features", + job_type_key=MLJob.key, + project=self.project, + pipeline=self.pipeline, + source_image_collection=self.collection, + dispatch_mode=JobDispatchMode.ASYNC_API, + ) + + @mock.patch("ami.ml.orchestration.jobs.AsyncJobStateManager") + @mock.patch("ami.ml.orchestration.jobs.TaskQueueManager") + def test_each_task_carries_its_missing_boxes_and_images_with_none_are_not_queued(self, manager_cls, state_cls): + manager = manager_cls.return_value + manager.__aenter__ = mock.AsyncMock(return_value=manager) + manager.__aexit__ = mock.AsyncMock(return_value=False) + manager.ensure_job_resources = mock.AsyncMock() + manager.publish_task = mock.AsyncMock(return_value=True) + + with mock.patch.object(SourceImage, "url", return_value="http://example.org/i.jpg"): + self.assertTrue(queue_images_to_nats(self.job, self.images)) + + published = { + int(call.kwargs["data"].image_id): sorted(d.bbox.x1 for d in call.kwargs["data"].detections) + for call in manager.publish_task.await_args_list + } + self.assertEqual(published, {self.images[1].pk: [20.0], self.images[2].pk: [0.0, 20.0]}) + state_cls.return_value.initialize_job.assert_called_once_with([str(self.images[1].pk), str(self.images[2].pk)]) + + def test_images_whose_only_pending_box_has_no_detector_are_skipped(self): + collected = collect_images(collection=self.collection, pipeline=self.pipeline) + self.assertEqual([image.pk for image in collected], [self.images[1].pk, self.images[2].pk]) + + def test_the_collect_stage_reports_progress_while_filtering(self): + clock = {"t": 0.0} + + def fake_monotonic() -> float: + clock["t"] += COLLECT_PROGRESS_SAVE_INTERVAL_SECONDS + return clock["t"] + + with mock.patch("ami.ml.models.pipeline.time.monotonic", side_effect=fake_monotonic), mock.patch.object( + Job, "save", autospec=True + ) as save: + list(filter_processed_images(self.images, self.pipeline, batch_size=1, job=self.job, total=3)) + + self.assertEqual(save.call_count, 3) + self.assertEqual(save.call_args.kwargs["update_fields"], ["progress", "updated_at"]) + self.assertEqual(self.job.progress.get_stage("collect").progress, COLLECT_PROGRESS_MAX_FRACTION) + + +class TestReadersNeverMixLengths(FeatureOnlyFixture, TestCase): + def setUp(self) -> None: + self._set_up_project(images=2, boxes_per_image=2) + + def test_one_algorithm_with_two_lengths_returns_one_length(self): + detections = list(Detection.objects.valid().order_by("pk")) + self._embed(detections[:3], self.extractor, length=BIOCLIP_DIMENSIONS) + Classification.objects.filter(detection=detections[3]).update( + algorithm=self.extractor, features_2048=[0.5] * 2048 + ) + vectors = vectors_for_detections([d.pk for d in detections], self.extractor.pk) + self.assertEqual({len(v) for v in vectors.values()}, {BIOCLIP_DIMENSIONS}) + self.assertEqual(len(vectors), 3) + + def test_cosine_similarity_refuses_vectors_of_different_lengths(self): + with self.assertRaises(ValueError): + cosine_similarity([1.0] * 1024, [1.0] * 2048) + + +class TestDefaultFeatureExtractor(FeatureOnlyFixture, TestCase): + """With vectors from several extractors and none chosen, tracking compares the one covering most boxes.""" + + def setUp(self) -> None: + self._set_up_project(images=2, boxes_per_image=2) + self.event = Event.objects.get(pk=self.images[0].event_id) + self.other = Algorithm.objects.create(name="Other backbone", key="other-backbone", task_type="embedding") + self.detections = list(Detection.objects.valid().filter(source_image__event=self.event).order_by("pk")) + self._embed(self.detections, self.other, length=2048) + self._embed(self.detections[:1], self.extractor) # stored last, on one box only + + def test_the_extractor_covering_most_detections_wins_over_a_more_recent_one(self): + algorithm, should_track, note = resolve_feature_algorithm( + self.event, TrackingConfig(event_ids=[self.event.pk], require_features=True) + ) + self.assertEqual((algorithm, should_track), (self.other, True)) + self.assertIn("2 feature extractors", note) + + def test_an_extractor_the_project_runs_wins_only_once_it_covers_nearly_as_many(self): + service = ProcessingService.objects.create(name="Service", endpoint_url=None) + service.projects.add(self.project) + service.pipelines.add(self.pipeline) + ProjectPipelineConfig.objects.create(project=self.project, pipeline=self.pipeline, enabled=True) + algorithm_ids = [self.other.pk, self.extractor.pk] + + def default() -> int | None: + return default_feature_algorithm_id(self.project.pk, algorithm_ids, source_image__event=self.event) + + self.assertEqual(default(), self.other.pk) # 1 of 4 boxes: a run still in progress + self._embed(self.detections[1:], self.extractor) + self.assertEqual(default(), self.extractor.pk) + + def test_a_chosen_extractor_is_kept(self): + config = TrackingConfig( + event_ids=[self.event.pk], require_features=True, feature_extraction_algorithm_id=self.extractor.pk + ) + self.assertEqual(resolve_feature_algorithm(self.event, config)[0], self.extractor) + + def test_listing_a_sessions_extractors_takes_a_fixed_number_of_queries(self): + with self.assertNumQueries(5): + rows = feature_extractors_with_vectors(self.project.pk, source_image__event=self.event) + self.assertEqual( + [(row["algorithm"].pk, row["embeddings_count"], row["is_default"]) for row in rows], + [(self.other.pk, 4, True), (self.extractor.pk, 1, False)], # newest algorithm first + ) + + +class TestSessionFeatureExtractorsEndpoint(FeatureOnlyFixture, APITestCase): + """Who can list a session's feature extractors: whoever can open the session.""" + + def setUp(self) -> None: + self._set_up_project(images=2, boxes_per_image=2) + self._embed(Detection.objects.valid(), self.extractor) + self.owner = User.objects.create_user(email="feat-owner@insectai.org", password="x") + self.member = User.objects.create_user(email="feat-member@insectai.org", password="x") + self.outsider = User.objects.create_user(email="feat-outsider@insectai.org", password="x") + self.superuser = User.objects.create_superuser(email="feat-admin@insectai.org", password="x") + self.project.owner = self.owner + self.project.draft = True + self.project.save() + self.project.members.add(self.member) + self.url = f"/api/v2/events/{self.images[0].event_id}/feature-extractors/?project_id={self.project.pk}" + + def _status(self, user) -> int: + self.client.force_authenticate(user) + return self.client.get(self.url).status_code + + def test_permission_matrix_on_a_draft_project(self): + self.assertEqual(self._status(self.member), 200) + self.assertEqual(self._status(self.superuser), 200) + self.assertIn(self._status(self.outsider), (403, 404)) + self.assertIn(self._status(None), (401, 403, 404)) + + def test_the_response_names_each_extractor_and_the_default(self): + self.client.force_authenticate(self.member) + body = self.client.get(self.url).json() + self.assertEqual( + body, + [ + { + "id": self.extractor.pk, + "name": self.extractor.name, + "key": self.extractor.key, + "task_type": "embedding", + "embedding_dimensions": None, + "embeddings_count": 4, + "classification_vectors_count": 0, + "is_default": True, + } + ], + ) diff --git a/ami/ml/tests.py b/ami/ml/tests.py index 70bb2e16b..6caf3e09c 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -4,7 +4,8 @@ import unittest import uuid -from django.test import TestCase +import pydantic +from django.test import SimpleTestCase, TestCase from rest_framework.test import APIRequestFactory, APITestCase from ami.base.serializers import reverse_with_params @@ -12,6 +13,7 @@ Classification, Deployment, Detection, + DetectionEmbedding, Event, Identification, Occurrence, @@ -22,8 +24,14 @@ TaxonRank, group_images_into_events, ) +from ami.ml.exceptions import PipelineNotConfigured from ami.ml.models import Algorithm, Pipeline, ProcessingService -from ami.ml.models.pipeline import collect_images, get_or_create_algorithm_and_category_map, save_results +from ami.ml.models.pipeline import ( + collect_images, + create_detection_embeddings, + get_or_create_algorithm_and_category_map, + save_results, +) from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask from ami.ml.schemas import ( AlgorithmConfigResponse, @@ -2340,3 +2348,211 @@ def test_used_lookup_is_deduplicated_in_the_database(self): len(list(lookup.order_by().distinct())), 1, "Deduplicating collapses them to the one algorithm" ) self.assertIn("Chatty Masked Classifier", self._choice_names(self.project.pk)) + + +EMBEDDING_DETECTOR = ALGORITHM_CHOICES["random-detector"] +EMBEDDING_BINARY = ALGORITHM_CHOICES["random-binary-classifier"] # labels: "Moth", "Not a moth" +EMBEDDING_SPECIES = ALGORITHM_CHOICES["random-species-classifier"] + + +def _embedding_payload(vector: list[float], algorithm=EMBEDDING_SPECIES) -> list[dict]: + return [{"algorithm": {"name": algorithm.name, "key": algorithm.key}, "features": vector}] + + +class TestEmbeddingSchema(SimpleTestCase): + """The per-detection ``embeddings`` field of the processing-service results schema.""" + + def _detection(self, **extra) -> dict: + return { + "source_image_id": "1", + "bbox": {"x1": 0.0, "y1": 0.0, "x2": 10.0, "y2": 10.0}, + "algorithm": {"name": EMBEDDING_DETECTOR.name, "key": EMBEDDING_DETECTOR.key}, + "timestamp": datetime.datetime.now().isoformat(), + **extra, + } + + def test_a_detection_carries_each_vector_with_its_algorithm(self): + """The field is optional, so a service that sends no embeddings still parses.""" + parsed = DetectionResponse.parse_obj(self._detection(embeddings=_embedding_payload([0.5] * 2048))) + self.assertEqual( + [(e.algorithm.key, len(e.features)) for e in parsed.embeddings or []], [(EMBEDDING_SPECIES.key, 2048)] + ) + self.assertIsNone(DetectionResponse.parse_obj(self._detection()).embeddings) + + def test_an_empty_vector_is_refused(self): + """Any length parses (extractors differ; each algorithm's length is checked on save), but not none.""" + self.assertEqual( + len(DetectionResponse.parse_obj(self._detection(embeddings=_embedding_payload([0.5] * 512))).embeddings), + 1, + ) + with self.assertRaises(pydantic.ValidationError): + DetectionResponse.parse_obj(self._detection(embeddings=_embedding_payload([]))) + + +class TestDetectionEmbeddings(TestCase): + """Storing the feature vector a processing service sends with each detection. + + The vector is what tracking and merge ranking compare, and it arrives for every + detection, including those the moth/non-moth filter rejected. What these pin is that + it is stored once per detection and algorithm, on the detection it was sent with, and + that storing it never adds a classification or moves a determination. + """ + + LOW = [0.25] * 2048 + HIGH = [0.75] * 2048 + + def setUp(self) -> None: + self.project = Project.objects.create(name="Detection embeddings") + self.pipeline = Pipeline.objects.create(name="Embedding test pipeline") + self.pipeline.algorithms.set( + [ + get_or_create_algorithm_and_category_map(algorithm) + for algorithm in (EMBEDDING_DETECTOR, EMBEDDING_BINARY, EMBEDDING_SPECIES) + ] + ) + self.images = 0 + + def _image(self) -> SourceImage: + self.images += 1 + return SourceImage.objects.create( + path=f"emb-{self.images}-20240101000{self.images}00.jpg", project=self.project + ) + + @staticmethod + def _classification(algorithm, label: str, score: float, terminal: bool) -> dict: + return { + "classification": label, + "scores": [score], + "algorithm": {"name": algorithm.name, "key": algorithm.key}, + "terminal": terminal, + "timestamp": datetime.datetime.now().isoformat(), + } + + def _detection(self, image: SourceImage, classifications: list[dict], embeddings=None, box: float = 0.0) -> dict: + payload = { + "source_image_id": str(image.pk), + "bbox": {"x1": box, "y1": box, "x2": box + 10.0, "y2": box + 10.0}, + "algorithm": {"name": EMBEDDING_DETECTOR.name, "key": EMBEDDING_DETECTOR.key}, + "timestamp": datetime.datetime.now().isoformat(), + "classifications": classifications, + } + if embeddings is not None: + payload["embeddings"] = embeddings + return payload + + def _rejected(self, image: SourceImage, embeddings=None, box: float = 0.0) -> dict: + """A crop the moth/non-moth filter rejected, labelled as the service labels it: non-terminal.""" + label = self._classification(EMBEDDING_BINARY, "Not a moth", 0.8, terminal=False) + return self._detection(image, [label], embeddings, box) + + def _moth(self, image: SourceImage, embeddings=None, box: float = 0.0) -> dict: + labels = [ + self._classification(EMBEDDING_BINARY, "Moth", 0.95, terminal=False), + self._classification(EMBEDDING_SPECIES, "Vanessa cardui", 0.6, terminal=True), + ] + return self._detection(image, labels, embeddings, box) + + def _save(self, *detections: dict, job_id: int | None = None) -> None: + image_ids = list(dict.fromkeys(d["source_image_id"] for d in detections)) + payload = { + "pipeline": self.pipeline.slug, + "total_time": 0.01, + "source_images": [{"id": image_id, "url": f"test/{image_id}.jpg"} for image_id in image_ids], + "detections": list(detections), + } + save_results(PipelineResultsResponse.parse_obj(payload), job_id=job_id) + + @staticmethod + def _stored(image: SourceImage) -> dict[tuple[float, str], list[float]]: + """{(box corner, algorithm key): vector} for the image's stored embeddings.""" + rows = DetectionEmbedding.objects.filter(detection__source_image=image).select_related( + "detection", "algorithm" + ) + return {(row.detection.bbox[0], row.algorithm.key): list(row.vector) for row in rows} + + def test_every_detection_stores_one_vector_per_algorithm(self): + """Including the rejected crop, which has no species classification that could carry one.""" + image = self._image() + self._save( + self._moth(image, _embedding_payload(self.LOW)), + self._rejected(image, _embedding_payload(self.HIGH), box=100.0), + ) + self.assertEqual( + self._stored(image), + {(0.0, EMBEDDING_SPECIES.key): self.LOW, (100.0, EMBEDDING_SPECIES.key): self.HIGH}, + ) + + def test_a_vector_adds_no_classification_and_moves_no_determination(self): + """The rejected crop's binary label is non-terminal, so a species classification sent in + the vector's place would become its determination. An embedding must not.""" + control, treated = self._image(), self._image() + self._save(self._rejected(control)) + self._save(self._rejected(treated, _embedding_payload(self.HIGH))) + + for image in (control, treated): + detection = Detection.objects.select_related("occurrence__determination").get(source_image=image) + self.assertEqual(detection.occurrence.determination.name, "Not a moth") + self.assertEqual(detection.classifications.count(), 1) + self.assertEqual(DetectionEmbedding.objects.filter(detection__source_image=treated).count(), 1) + + def test_saving_again_keeps_one_row_per_detection_and_algorithm(self): + """Results are re-delivered and images reprocessed; the one row holds the latest vector.""" + image = self._image() + self._save(self._rejected(image, _embedding_payload(self.LOW))) + self._save(self._rejected(image, _embedding_payload(self.LOW))) + self.assertEqual(DetectionEmbedding.objects.filter(detection__source_image=image).count(), 1) + + self._save(self._rejected(image, _embedding_payload(self.HIGH))) + self.assertEqual(self._stored(image), {(0.0, EMBEDDING_SPECIES.key): self.HIGH}) + + def test_each_vector_records_the_job_that_last_stored_it_and_outlives_that_job(self): + from ami.jobs.models import Job + + image = self._image() + first, second = ( + Job.objects.create(project=self.project, name=f"Embedding job {n}", pipeline=self.pipeline) for n in (1, 2) + ) + self._save(self._rejected(image, _embedding_payload(self.LOW)), job_id=first.pk) + self._save(self._rejected(image, _embedding_payload(self.HIGH)), job_id=second.pk) + self.assertEqual(DetectionEmbedding.objects.get(detection__source_image=image).job_id, second.pk) + + second.delete() + embedding = DetectionEmbedding.objects.get(detection__source_image=image) + self.assertIsNone(embedding.job_id) + self.assertEqual(list(embedding.vector), self.HIGH) + + def test_a_vector_lands_on_its_own_detection_when_some_detections_already_exist(self): + """Detection creation returns existing detections ahead of new ones, so pairing responses + with detections by position would swap these two vectors.""" + image = self._image() + self._save(self._rejected(image, box=0.0)) + self._save( + self._rejected(image, _embedding_payload(self.HIGH), box=100.0), + self._rejected(image, _embedding_payload(self.LOW), box=0.0), + ) + self.assertEqual( + self._stored(image), + {(0.0, EMBEDDING_SPECIES.key): self.LOW, (100.0, EMBEDDING_SPECIES.key): self.HIGH}, + ) + + def test_a_vector_from_an_unregistered_algorithm_stops_the_batch_like_a_classification(self): + """It raises where an unregistered classification algorithm does: after detections are + saved and before any classification is.""" + image = self._image() + unregistered = {"algorithm": {"name": "Unregistered", "key": "unregistered-embedder"}, "features": self.LOW} + with self.assertRaises(PipelineNotConfigured): + self._save(self._rejected(image, [unregistered])) + self.assertFalse(DetectionEmbedding.objects.exists()) + self.assertFalse(Classification.objects.filter(detection__source_image=image).exists()) + + def test_storing_vectors_takes_one_query_however_many_detections(self): + image = self._image() + responses = [self._rejected(image, _embedding_payload(self.LOW), box=float(20 * i)) for i in range(5)] + self._save(*responses) + detections = list(Detection.objects.filter(source_image=image)) + parsed = [DetectionResponse.parse_obj(response) for response in responses] + algorithms_known = {algorithm.key: algorithm for algorithm in self.pipeline.algorithms.all()} + + with self.assertNumQueries(1): + stored = create_detection_embeddings(detections, parsed, algorithms_known) + self.assertEqual(len(stored.embeddings), 5) diff --git a/docs/claude/INDEX.md b/docs/claude/INDEX.md index b606cc2b1..2171d6d53 100644 --- a/docs/claude/INDEX.md +++ b/docs/claude/INDEX.md @@ -19,6 +19,7 @@ archived. | `reference/hierarchical-rollup-query-performance.md` | Per-taxon rollup counts on `GET /api/v2/taxa/` — query patterns and pitfalls. Keywords: taxa, rollup, counts | | `reference/occurrence-tracking.md` | How occurrences are populated by tracking: data model (`next_detection`, `features_2048`), cost function, config knobs, the adjacent-processed-captures constraint, how to run it, where results show up, all six editing endpoints (split/remove/merge/add/verify/unverify), the chain-boundary and clear-on-edit invariants, permission traps, and how to generate a demo session with known ground truth (`create_demo_project`). Keywords: tracking, occurrences, pgvector, embeddings, abundance, grouping verification, test set, demo data | | `reference/tracking-evaluation.md` | Scoring tracking against human-confirmed tracks (`grouping_verified_at`): metrics over detections in confirmed tracks only (pairwise and link precision/recall, fragments, completeness, purity, merges), the `evaluate_tracking` command (re-links raw detections read-only via `propose_event_links`, rolled back), the no-Django CSV mode, limits and next steps. Keywords: tracking evaluation, benchmark, ground truth, confirmed tracks, metrics | +| `reference/detection-embeddings.md` | Feature vectors per detection (2026-09-28): `DetectionEmbedding` with no fixed dimension, `Algorithm.embedding_dimensions` enforced on save, the `features`/`vector` response contract, feature-only pipelines (task type `embedding`/`feature_extraction`) that embed existing detections via an ordinary ML job, the embeddings-only save path, one-algorithm readers, how tracking picks the default extractor, and `GET /events/{id}/feature-extractors/`. Keywords: embeddings, BioCLIP, feature extraction, pgvector, extractor, tracking appearance | | `planning/idempotent-incremental-tracking.md` | Design for tracking that is always safe to re-run: the unit of work becomes a pair of adjacent processed captures rather than a whole session, with the successor marker recorded on `SourceImage`. Covers moving adjacency, bookkeeping without a new table, identification migration on merge, surviving human edits, concurrency, cost, and the trigger. Keywords: idempotent, incremental, tracking, adjacency, re-run, sampled captures | ## Architecture notes diff --git a/docs/claude/reference/detection-embeddings.md b/docs/claude/reference/detection-embeddings.md new file mode 100644 index 000000000..a34f36d5a --- /dev/null +++ b/docs/claude/reference/detection-embeddings.md @@ -0,0 +1,106 @@ +# Detection embeddings (feature vectors) + +Reference for agents. How Antenna stores a feature vector for every detection, how a +feature-only pipeline fills them in for existing detections, and how tracking picks which +extractor's vectors to compare. See #1417 for the original design. + +## Storage + +- `DetectionEmbedding` (`ami/main/models.py`, migration `main/0102_detection_embedding.py`; + `main/0104` re-runs `ALTER COLUMN vector TYPE vector` for databases that applied an earlier + 0102 draft with `vector(2048)`, which Django state cannot see): + one row per (detection, algorithm), unique constraint `unique_detection_embedding_per_algorithm` + (its index leads with `detection_id` and serves reads). `vector` is a pgvector column with + **no fixed dimension**: extractors differ (2048 for the moth classifier backbones, 1024 for BioCLIP). + `job` FK (SET_NULL) records the job that last stored the vector. +- `Algorithm.embedding_dimensions` (`ami/ml/models/algorithm.py`, migration `ml/0029`): the one + length an algorithm's vectors have. Set from `/info` (`AlgorithmConfigResponse.embedding_dimensions`, + optional) in `get_or_create_algorithm_and_category_map`, or from the first vector stored + (`_check_embedding_dimensions` in `ami/ml/models/pipeline.py`, conditional UPDATE so concurrent + batches agree). A vector of another length raises `EmbeddingDimensionMismatch` (a + `PipelineNotConfigured`) and the batch stores nothing. +- Legacy: `Classification.features_2048` (vector(2048)) from classifiers run with `include_features`. + Readers fall back to it (see below). +- No ANN index exists; one would need a fixed dimension per index (a partial index per algorithm). + +## Contract with the processing service (`ami/ml/schemas.py`) + +- `DetectionResponse.embeddings: list[EmbeddingResponse] | None`, each + `{"algorithm": {"name", "key"}, "features": [float, ...]}`. The key `vector` is accepted as an + alias (root validator). Any non-empty length; the per-algorithm length is enforced at save time. +- The algorithm key must be declared in the pipeline's `/info`, else `PipelineNotConfigured`. +- **Feature-only pipeline**: at least one algorithm has `task_type` in + `Algorithm.feature_extraction_task_types` (`embedding` or `feature_extraction`) and none is a + classifier (`classification` / `tagging`). Detector algorithms are allowed because the service + lists the detector whose boxes it echoes back (`feature_extractors_if_feature_only()` in + `ami/ml/models/pipeline.py`, `Pipeline.feature_extraction_algorithms()` returns only the extractors). +- Request for a feature-only run (sync, `process_images`): `PipelineRequest` with + `source_images` = only the images that still have a detection to embed, and `detections` = + those detections as `DetectionRequest{source_image, bbox, crop_image_url, algorithm=}` (`collect_detections_for_features`, one query). Async (NATS): the same list on + `PipelineProcessingTask.detections` (`_attach_detections_for_feature_pipeline` in + `ami/ml/orchestration/jobs.py`). The service must return the same boxes, each with embeddings, + and no new boxes. + +## Write path + +- `save_results` (`ami/ml/models/pipeline.py`): if the pipeline is feature-only it calls + `save_features_for_existing_detections` and returns. That path matches response boxes to + existing real detections by `(source_image_id, bbox rounded to BOX_MATCH_DECIMALS=3)`, skips + unmatched boxes (warning), ignores classifications (warning), and writes only + `DetectionEmbedding` rows: no detection, classification, occurrence, determination, null + marker or calculated-field update. +- Regular pipelines: `create_detection_embeddings` runs after `create_detections` and before + classifications, same matching key. Both use `bulk_create(update_conflicts=True)` on + (detection, algorithm), so re-delivery is idempotent and the newest vector wins. + +## The extract-features job + +No new job type: an ordinary ML job (`MLJob`, key `ml`) whose pipeline is feature-only. Same +permission as any ML job. Scope is the job's capture set, deployment or single capture. +`filter_processed_images` delegates to `filter_images_missing_features`: an image is sent only +when a real detection on it lacks a `DetectionEmbedding` from one of the pipeline's algorithms +(`detections_missing_features`; classification vectors do not count). Both the image filter +and the request go through `embeddable_detections` (real box and known detector), so no image +is chosen without boxes to send. The async path also drops any task left with no boxes +(`_attach_detections_for_feature_pipeline` returns them), because a worker given an empty list +runs its own detector. The filter emits the same throttled `collect` heartbeat as regular +pipelines (`_CollectHeartbeat`). The project flag `reprocess_all_images` sends every real +detection again. + +## Read path + +`ami/main/models_future/embeddings.py`: +- `latest_vectors` / `vectors_for_detections(ids, algorithm_id)`: one UNION ALL query over + embeddings and classification vectors, embedding preferred. Always one algorithm. + `vectors_for_detections` also keeps only the most common vector length, so an algorithm with + both a 1024 embedding and a 2048 classification vector never mixes them. +- `cosine_similarity` (`ami/ml/post_processing/tracking_task.py`) raises on a shape mismatch. +- `algorithm_ids_with_vectors(**lookups)`: extractors with vectors for a set of detections. +- `feature_extractors_with_vectors(project, **lookups)`: per extractor, `embeddings_count`, + `classification_vectors_count`, `is_default`; exposed as + `GET /api/v2/events/{id}/feature-extractors/?project_id=` (`EventViewSet.feature_extractors`, + `SessionFeatureExtractorSerializer`) for the tracking form. + +## How tracking picks the extractor + +`resolve_feature_algorithm(event, config)`: +1. `config.feature_extraction_algorithm_id` if given. +2. The only extractor with vectors in the session. +3. Several: `default_feature_algorithm_id(project, ids, source_image__event=event)`: the + extractor with vectors for the most detections in the session (`detections_covered`, one + query, either store counts once); a configured one (feature-extraction algorithm in a + pipeline enabled for the project) wins if it covers at least + `CONFIGURED_EXTRACTOR_MIN_COVERAGE` (0.9) of the best; ties go to the newest algorithm id. + A half-finished backfill therefore never becomes the default. The run logs which one it compares. + Gotcha: the Exists terms in `detections_covered` must be annotations, not a `Q` inside + `Count(filter=...)`, or cachalot misses their tables and serves a stale count. +4. None: `require_features=True` skips the session; otherwise geometry-only matching. + +Merge candidates use the same resolution over the two captures being compared. + +## Tests + +`ami/ml/test_feature_extraction.py` (feature-only save, dimension checks, extract scope, default +extractor, endpoint permissions and query count); `ami/ml/tests.py` `TestDetectionEmbeddings` +(regular-pipeline embeddings). diff --git a/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts b/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts new file mode 100644 index 000000000..9e3a2cdc7 --- /dev/null +++ b/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts @@ -0,0 +1,33 @@ +import { API_ROUTES, API_URL } from 'data-services/constants' +import { ServerOccurrenceHistoryEntry } from 'data-services/models/occurrence-history' +import { useAuthorizedQuery } from '../auth/useAuthorizedQuery' + +// Under the occurrences prefix, so every mutation that invalidates occurrences refreshes it. +export const getOccurrenceHistoryQueryKey = (occurrenceId: string) => [ + API_ROUTES.OCCURRENCES, + occurrenceId, + 'history', +] + +export const useOccurrenceHistory = ({ + occurrenceId, + projectId, +}: { + occurrenceId: string + projectId?: string +}): { + entries?: ServerOccurrenceHistoryEntry[] + isLoading: boolean + error?: unknown +} => { + const params = new URLSearchParams(projectId ? { project_id: projectId } : {}) + const { data, isLoading, error } = useAuthorizedQuery< + ServerOccurrenceHistoryEntry[] + >({ + enabled: !!occurrenceId, + queryKey: getOccurrenceHistoryQueryKey(occurrenceId), + url: `${API_URL}/${API_ROUTES.OCCURRENCES}/${occurrenceId}/history/?${params}`, + }) + + return { entries: data, isLoading, error } +} diff --git a/ui/src/data-services/models/capture.ts b/ui/src/data-services/models/capture.ts index a226f05be..e95f1dd4e 100644 --- a/ui/src/data-services/models/capture.ts +++ b/ui/src/data-services/models/capture.ts @@ -88,9 +88,9 @@ export class Capture { groupingVerifiedAt: detection.occurrence?.grouping_verified_at ? new Date(detection.occurrence.grouping_verified_at) : null, - // Accounts without a display name serialize as an empty string. + // An empty string is a user without a display name; null is no user. groupingVerifiedBy: - detection.occurrence?.grouping_verified_by || null, + detection.occurrence?.grouping_verified_by ?? null, id: `${detection.id}`, label: getDetectionLabel(detection), occurrenceId: detection.occurrence diff --git a/ui/src/data-services/models/occurrence-details.ts b/ui/src/data-services/models/occurrence-details.ts index f833ac2ab..0282ccfb0 100644 --- a/ui/src/data-services/models/occurrence-details.ts +++ b/ui/src/data-services/models/occurrence-details.ts @@ -1,5 +1,6 @@ import { getFormatedTimeString } from 'utils/date/getFormatedTimeString/getFormatedTimeString' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { UserPermission } from 'utils/user/types' import { Algorithm } from './algorithm' import { Occurrence, ServerOccurrence } from './occurrence' @@ -67,7 +68,6 @@ export interface HumanIdentification extends Identification { } export interface MachinePrediction extends Identification { - algorithm: Algorithm /** Whether a feature embedding was stored; null when the API did not say. */ hasFeatures?: boolean | null score: number @@ -282,12 +282,10 @@ export class OccurrenceDetails extends Occurrence { user: i.user ? { id: `${i.user.id}`, - name: i.user.name?.length - ? i.user.name - : translate(STRING.ANONYMOUS_USER), + name: getUserLabel(i.user), image: i.user.image, } - : { name: translate(STRING.ANONYMOUS_USER) }, + : { name: getUserLabel(i.user) }, comment: i.comment, userPermissions: i.user_permissions, createdAt: i.created_at, @@ -360,6 +358,10 @@ export class OccurrenceDetails extends Occurrence { return !!this._occurrence.grouping_verified } + get groupingEditedSinceVerified(): boolean { + return !!this._occurrence.grouping_edited_since_verified + } + get groupingVerifiedAt(): Date | undefined { return this._occurrence.grouping_verified_at ? new Date(this._occurrence.grouping_verified_at) @@ -378,7 +380,7 @@ export class OccurrenceDetails extends Occurrence { return { id: `${user.id}`, image: user.image ?? undefined, - name: user.name?.length ? user.name : translate(STRING.ANONYMOUS_USER), + name: getUserLabel(user), } } diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts new file mode 100644 index 000000000..b4bcc0be9 --- /dev/null +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -0,0 +1,290 @@ +import { getOccurrenceHistoryQueryKey } from 'data-services/hooks/occurrences/useOccurrenceHistory' +import { API_ROUTES } from 'data-services/constants' +import { Algorithm } from './algorithm' +import { HumanIdentification, MachinePrediction } from './occurrence-details' +import { + getFallbackTimelineItems, + getFoldedPrediction, + getTimelineItems, + ServerOccurrenceHistoryEntry, + TrackCompleteReviewEntry, +} from './occurrence-history' +import { Taxon } from './taxa' +import { UserPermission } from 'utils/user/types' + +const NOCTUA = { id: 3, name: 'Noctua pronuba', rank: 'SPECIES' } +const XESTIA = { id: 4, name: 'Xestia c-nigrum', rank: 'SPECIES' } + +const base = { + algorithm: null, + job: null, + score: null, + subtype: null, + taxon: null, + taxon_before: null, + timestamp: '2026-04-29T22:00:00', + user: null, +} + +const identificationEntry = (id: number): ServerOccurrenceHistoryEntry => ({ + ...base, + id, + payload: { + agreed_with_identification_id: null, + agreed_with_prediction_id: null, + comment: 'Looks right', + withdrawn: false, + }, + taxon: NOCTUA, + type: 'identification', + user: { id: 9, image: null, name: '' }, +}) + +const predictionEntry = ( + id: number, + taxon: typeof NOCTUA | null = XESTIA +): ServerOccurrenceHistoryEntry => ({ + ...base, + algorithm: { id: 7, key: 'classifier', name: 'Classifier' }, + id, + payload: { applied_to_id: null, detection_id: 1, terminal: true }, + score: 0.8, + taxon, + type: 'prediction', +}) + +const review = ( + id: number, + detectionIds: number[], + occurrenceId: number | null = 100 +): TrackCompleteReviewEntry => ({ + ...base, + id, + payload: { + detection_ids: detectionIds, + detections_added: [], + detections_removed: [], + first_timestamp: null, + frames_count: detectionIds.length, + last_timestamp: null, + occurrence_id: occurrenceId, + }, + subtype: 'track_complete', + type: 'review', +}) + +const classMasking: ServerOccurrenceHistoryEntry = { + ...base, + algorithm: { id: 12, key: 'mask', name: 'Masked classifier' }, + id: 5, + payload: { + detection_ids: [1], + source_algorithm_id: 7, + taxa_list_id: 2, + taxon_after_id: 3, + taxon_before_id: 4, + }, + subtype: 'class_masking', + taxon: NOCTUA, + taxon_before: XESTIA, + type: 'algorithm_result', +} + +const ownIdentification: HumanIdentification = { + comment: '', + createdAt: '2026-04-29T22:00:00', + id: '1', + user: { name: 'Someone' }, + userPermissions: [UserPermission.Delete], +} + +const ownPrediction = ( + id: string, + algorithmId: number, + taxon = NOCTUA, + createdAt = '2026-04-29T21:00:00' +): MachinePrediction => ({ + algorithm: new Algorithm({ id: algorithmId }), + createdAt, + id, + score: 0.9, + taxon: new Taxon({ ...taxon, id: `${taxon.id}`, cover_image_url: null }), + terminal: true, + userPermissions: [UserPermission.Update], +}) + +describe('getTimelineItems', () => { + test('maps each entry type to its card, keeping the server order', () => { + const items = getTimelineItems({ + entries: [ + review(8, [1]), + classMasking, + identificationEntry(2), + predictionEntry(6), + ], + identifications: [], + predictions: [], + }) + + expect(items.map((item) => item.type)).toEqual([ + 'review', + 'algorithm_result', + 'identification', + 'prediction', + ]) + }) + + test("reuses the occurrence's own records, which carry the viewer's permissions", () => { + const prediction = ownPrediction('6', 7) + const items = getTimelineItems({ + entries: [identificationEntry(1), predictionEntry(6)], + identifications: [ownIdentification], + predictions: [prediction], + }) + + expect(items[0]).toMatchObject({ identification: ownIdentification }) + expect(items[1]).toMatchObject({ prediction }) + }) + + test('builds a read-only card when the occurrence lacks the record', () => { + const [item] = getTimelineItems({ + determinationTaxonId: '3', + entries: [identificationEntry(2)], + identifications: [], + predictions: [], + }) + + expect(item.type === 'identification' && item.identification).toMatchObject( + { + applied: true, + comment: 'Looks right', + user: { id: '9', name: 'Unnamed user' }, + userPermissions: [], + } + ) + }) + + test('drops unknown subtypes and predictions without a taxon', () => { + const unknown = { + ...classMasking, + subtype: 'something_new', + } as unknown as ServerOccurrenceHistoryEntry + + expect( + getTimelineItems({ + entries: [unknown, predictionEntry(6, null)], + identifications: [], + predictions: [], + }) + ).toEqual([]) + }) +}) + +describe('prediction cards built from the history', () => { + test('have no algorithm when the server names none', () => { + const [item] = getTimelineItems({ + entries: [{ ...predictionEntry(6), algorithm: null }], + identifications: [], + predictions: [], + }) + + expect(item.type === 'prediction' && item.prediction.algorithm).toBe( + undefined + ) + }) +}) + +describe('getFallbackTimelineItems', () => { + test('merges identifications and predictions newest first', () => { + const items = getFallbackTimelineItems({ + identifications: [ownIdentification], + predictions: [ + ownPrediction('6', 7, NOCTUA, '2026-04-29T23:00:00'), + ownPrediction('7', 12, NOCTUA, '2026-04-29T20:00:00'), + ], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-6', + 'identification-1', + 'prediction-7', + ]) + }) + + test('breaks timestamp ties on id, newest first, as the server does', () => { + const items = getFallbackTimelineItems({ + identifications: [ownIdentification], + predictions: [ownPrediction('6', 7, NOCTUA, ownIdentification.createdAt)], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-6', + 'identification-1', + ]) + }) +}) + +describe('fallback predictions', () => { + const tied = (id: string, terminal: boolean, createdAt: string) => ({ + ...ownPrediction(id, 7, NOCTUA, createdAt), + terminal, + }) + + test('keep one per algorithm when frames tie for its top score', () => { + const items = getFallbackTimelineItems({ + identifications: [], + predictions: [ + tied('6', true, '2026-04-29T21:00:00'), + tied('7', true, '2026-04-29T22:00:00'), + ownPrediction('8', 12), + ], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-7', + 'prediction-8', + ]) + }) + + test('prefer a terminal prediction over a later intermediate one', () => { + const items = getFallbackTimelineItems({ + identifications: [], + predictions: [ + tied('6', true, '2026-04-29T21:00:00'), + tied('7', false, '2026-04-29T22:00:00'), + ], + }) + + expect(items.map((item) => item.id)).toEqual(['prediction-6']) + }) +}) + +describe('getFoldedPrediction', () => { + test('finds the prediction behind an algorithm result by its algorithm', () => { + const folded = ownPrediction('6', 12) + + expect( + getFoldedPrediction(classMasking as never, [ + ownPrediction('5', 7), + folded, + ]) + ).toBe(folded) + }) + + test('finds it when the determination differs from the predicted taxon', () => { + const folded = ownPrediction('6', 12, XESTIA) + + expect( + getFoldedPrediction(classMasking as never, [ + ownPrediction('5', 7), + folded, + ]) + ).toBe(folded) + }) +}) + +describe('getOccurrenceHistoryQueryKey', () => { + test('sits under the occurrences key that mutations invalidate', () => { + expect(getOccurrenceHistoryQueryKey('100')[0]).toBe(API_ROUTES.OCCURRENCES) + }) +}) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts new file mode 100644 index 000000000..f31456ec3 --- /dev/null +++ b/ui/src/data-services/models/occurrence-history.ts @@ -0,0 +1,340 @@ +import { getUserLabel } from 'utils/user/getUserLabel' +import { Algorithm } from './algorithm' +import { HumanIdentification, MachinePrediction } from './occurrence-details' +import { Taxon } from './taxa' + +export interface ServerHistoryTaxon { + id: number + name: string + rank: string +} + +export interface ServerHistoryUser { + id: number + image: string | null + name: string +} + +export interface ServerHistoryAlgorithm { + id: number + key: string + name: string +} + +export interface ServerHistoryJob { + id: number + name: string +} + +interface ServerDeterminationChangePayload { + taxon_after_id: number | null + taxon_before_id: number | null +} + +export interface ServerTrackingPayload + extends ServerDeterminationChangePayload { + cost_max: number | null + cost_mean: number | null + detections_count: number + feature_algorithm_id: number | null + frames_linked: number + occurrences_merged: number[] + settings: Record +} + +export interface ServerClassMaskingPayload + extends ServerDeterminationChangePayload { + detection_ids: number[] + source_algorithm_id: number + taxa_list_id: number +} + +export interface ServerSizeFilterPayload + extends ServerDeterminationChangePayload { + detection_ids: number[] + size_threshold: number +} + +export interface ServerTrackCompletePayload { + detection_ids: number[] + detections_added: number[] + detections_removed: number[] + first_timestamp: string | null + frames_count: number + last_timestamp: string | null + occurrence_id?: number | null + split_from_occurrence_id?: number | null +} + +export interface ServerIdentificationPayload { + agreed_with_identification_id: number | null + agreed_with_prediction_id: number | null + comment: string + withdrawn: boolean +} + +export interface ServerPredictionPayload { + applied_to_id: number | null + detection_id: number | null + terminal: boolean | null +} + +interface ServerHistoryEntryBase { + algorithm: ServerHistoryAlgorithm | null + id: number + job: ServerHistoryJob | null + payload: Payload + score: number | null + subtype: Subtype + taxon: ServerHistoryTaxon | null + taxon_before: ServerHistoryTaxon | null + timestamp: string + type: Type + user: ServerHistoryUser | null +} + +export type TrackingResultEntry = ServerHistoryEntryBase< + 'algorithm_result', + 'tracking', + ServerTrackingPayload +> +export type ClassMaskingResultEntry = ServerHistoryEntryBase< + 'algorithm_result', + 'class_masking', + ServerClassMaskingPayload +> +export type SizeFilterResultEntry = ServerHistoryEntryBase< + 'algorithm_result', + 'size_filter', + ServerSizeFilterPayload +> +export type AlgorithmResultEntry = + | TrackingResultEntry + | ClassMaskingResultEntry + | SizeFilterResultEntry +export type TrackCompleteReviewEntry = ServerHistoryEntryBase< + 'review', + 'track_complete', + ServerTrackCompletePayload +> +export type IdentificationEntry = ServerHistoryEntryBase< + 'identification', + null, + ServerIdentificationPayload +> +export type PredictionEntry = ServerHistoryEntryBase< + 'prediction', + null, + ServerPredictionPayload +> + +export type ServerOccurrenceHistoryEntry = + | AlgorithmResultEntry + | TrackCompleteReviewEntry + | IdentificationEntry + | PredictionEntry + +export type TimelineItem = + | { + type: 'identification' + id: string + identification: HumanIdentification + } + | { type: 'prediction'; id: string; prediction: MachinePrediction } + | { type: 'algorithm_result'; id: string; entry: AlgorithmResultEntry } + | { type: 'review'; id: string; entry: TrackCompleteReviewEntry } + +const ALGORITHM_RESULT_SUBTYPES = ['tracking', 'class_masking', 'size_filter'] + +export const convertHistoryTaxon = (taxon: ServerHistoryTaxon) => + new Taxon({ ...taxon, id: `${taxon.id}`, cover_image_url: null }) + +const toIdentification = ( + entry: IdentificationEntry, + determinationTaxonId?: string +): HumanIdentification => { + const taxon = entry.taxon ? convertHistoryTaxon(entry.taxon) : undefined + + return { + applied: !!taxon && taxon.id === determinationTaxonId, + comment: entry.payload.comment, + createdAt: entry.timestamp, + id: `${entry.id}`, + overridden: entry.payload.withdrawn, + taxon, + user: entry.user + ? { + id: `${entry.user.id}`, + image: entry.user.image ?? undefined, + name: getUserLabel(entry.user), + } + : { name: getUserLabel(entry.user) }, + userPermissions: [], + } +} + +const toPrediction = ( + entry: PredictionEntry & { taxon: ServerHistoryTaxon }, + determinationTaxonId?: string +): MachinePrediction => { + const taxon = convertHistoryTaxon(entry.taxon) + + return { + algorithm: entry.algorithm ? new Algorithm(entry.algorithm) : undefined, + applied: taxon.id === determinationTaxonId, + createdAt: entry.timestamp, + id: `${entry.id}`, + overridden: taxon.id !== determinationTaxonId, + score: entry.score ?? 0, + taxon, + terminal: !!entry.payload.terminal, + userPermissions: [], + } +} + +/** + * The history as cards to render, newest first. Identifications and predictions reuse the + * occurrence's own records when it has them, since only those carry the viewer's permissions. + */ +export const getTimelineItems = ({ + determinationTaxonId, + entries, + identifications, + predictions, +}: { + determinationTaxonId?: string + entries: ServerOccurrenceHistoryEntry[] + identifications: HumanIdentification[] + predictions: MachinePrediction[] +}): TimelineItem[] => + entries.flatMap((entry): TimelineItem[] => { + const id = `${entry.type}-${entry.id}` + + switch (entry.type) { + case 'identification': { + const identification = + identifications.find((i) => i.id === `${entry.id}`) ?? + toIdentification(entry, determinationTaxonId) + + return [{ type: 'identification', id, identification }] + } + case 'prediction': { + const { taxon } = entry + const prediction = + predictions.find((p) => p.id === `${entry.id}`) ?? + (taxon + ? toPrediction({ ...entry, taxon }, determinationTaxonId) + : undefined) + + return prediction ? [{ type: 'prediction', id, prediction }] : [] + } + case 'algorithm_result': + return ALGORITHM_RESULT_SUBTYPES.includes(entry.subtype) + ? [{ type: 'algorithm_result', id, entry }] + : [] + case 'review': + return entry.subtype === 'track_complete' + ? [{ type: 'review', id, entry }] + : [] + default: + return [] + } + }) + +/** + * The best prediction of each algorithm: highest score, then terminal, then latest, then id. + * The occurrence lists every frame tied for an algorithm's top score, and the history keeps one. + */ +const onePredictionPerAlgorithm = (predictions: MachinePrediction[]) => { + const rank = (p: MachinePrediction) => [ + p.score ?? -Infinity, + p.terminal ? 1 : 0, + new Date(p.createdAt).getTime() || 0, + Number(p.id) || 0, + ] + const isBetter = (a: MachinePrediction, b: MachinePrediction) => { + const [rankA, rankB] = [rank(a), rank(b)] + const index = rankA.findIndex((value, i) => value !== rankB[i]) + + return index !== -1 && rankA[index] > rankB[index] + } + const best = new Map() + predictions.forEach((prediction) => { + const key = `${prediction.algorithm?.id}` + const current = best.get(key) + if (!current || isBetter(prediction, current)) { + best.set(key, prediction) + } + }) + + return [...best.values()] +} + +/** + * Identifications and predictions merged newest first, for while the history loads or when it is + * unavailable. Predictions and tie breaks follow the server, so cards keep their place once it arrives. + */ +export const getFallbackTimelineItems = ({ + identifications, + predictions, +}: { + identifications: HumanIdentification[] + predictions: MachinePrediction[] +}): TimelineItem[] => + [ + ...identifications.map( + (identification): TimelineItem => ({ + type: 'identification', + id: `identification-${identification.id}`, + identification, + }) + ), + ...onePredictionPerAlgorithm(predictions).map( + (prediction): TimelineItem => ({ + type: 'prediction', + id: `prediction-${prediction.id}`, + prediction, + }) + ), + ] + .map((item) => ({ + item, + time: new Date(getCreatedAt(item)).getTime(), + id: Number(getSourceId(item)), + })) + .sort((a, b) => b.time - a.time || b.id - a.id) + .map(({ item }) => item) + +const getSourceId = (item: TimelineItem) => { + switch (item.type) { + case 'identification': + return item.identification.id + case 'prediction': + return item.prediction.id + default: + return item.entry.id + } +} + +const getCreatedAt = (item: TimelineItem) => { + switch (item.type) { + case 'identification': + return item.identification.createdAt + case 'prediction': + return item.prediction.createdAt + default: + return item.entry.timestamp + } +} + +/** + * The prediction an algorithm result stands in for, so it can still be agreed with. Matched on + * algorithm alone: the result's taxon is the determination after the run, not what was predicted. + */ +export const getFoldedPrediction = ( + entry: AlgorithmResultEntry, + predictions: MachinePrediction[] +) => + entry.algorithm + ? predictions.find((p) => `${p.algorithm?.id}` === `${entry.algorithm?.id}`) + : undefined diff --git a/ui/src/data-services/models/occurrence.ts b/ui/src/data-services/models/occurrence.ts index 64750712f..d4e65ad69 100644 --- a/ui/src/data-services/models/occurrence.ts +++ b/ui/src/data-services/models/occurrence.ts @@ -1,6 +1,7 @@ import { getFormatedDateString } from 'utils/date/getFormatedDateString/getFormatedDateString' import { getFormatedTimeString } from 'utils/date/getFormatedTimeString/getFormatedTimeString' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { UserPermission } from 'utils/user/types' import { Taxon } from './taxa' import { @@ -115,11 +116,9 @@ export class Occurrence { return verifiedBy ? { id: `${verifiedBy.id}`, - name: verifiedBy.name?.length - ? verifiedBy.name - : translate(STRING.ANONYMOUS_USER), + name: getUserLabel(verifiedBy), } - : { name: translate(STRING.ANONYMOUS_USER) } + : { name: getUserLabel(verifiedBy) } } get durationLabel(): string | undefined { diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx new file mode 100644 index 000000000..ab858cd4d --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -0,0 +1,189 @@ +import { + AlgorithmResultEntry, + getFoldedPrediction, + ServerHistoryTaxon, +} from 'data-services/models/occurrence-history' +import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' +import { FilterIcon, RouteIcon, RulerIcon } from 'lucide-react' +import { IdentificationCard } from 'nova-ui-kit' +import { Link, useParams } from 'react-router-dom' +import { APP_ROUTES } from 'utils/constants' +import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' +import { STRING, translate } from 'utils/language' +import { UserInfo, UserPermission } from 'utils/user/types' +import { Agree } from '../agree/agree' +import { + HistoryStat, + HistoryStats, + HistoryTime, + HistoryTypeBadge, +} from './history-stats' + +const SUBTYPES = { + class_masking: { icon: FilterIcon, label: STRING.HISTORY_CLASS_MASKING }, + size_filter: { icon: RulerIcon, label: STRING.HISTORY_SIZE_FILTER }, + tracking: { icon: RouteIcon, label: STRING.HISTORY_TRACKING }, +} + +/** The determination row, left out when there was no determination before or after. */ +const getDeterminationStats = ( + before: ServerHistoryTaxon | null, + after: ServerHistoryTaxon | null +): HistoryStat[] => { + if (!before && !after) { + return [] + } + + const notAvailable = translate(STRING.VALUE_NOT_AVAILABLE) + const value = + before?.id === after?.id + ? translate(STRING.HISTORY_DETERMINATION_UNCHANGED, { + name: after?.name ?? notAvailable, + }) + : translate(STRING.HISTORY_DETERMINATION_CHANGED, { + after: after?.name ?? notAvailable, + before: before?.name ?? notAvailable, + }) + + return [{ label: translate(STRING.HISTORY_DETERMINATION), value }] +} + +export const AlgorithmResult = ({ + currentUser, + entry, + occurrence, + onConfirmed, +}: { + currentUser?: UserInfo + entry: AlgorithmResultEntry + occurrence: Occurrence + onConfirmed?: (occurrenceId: string) => void +}) => { + const { projectId } = useParams() + const { icon: Icon, label } = SUBTYPES[entry.subtype] + const foldedPrediction = getFoldedPrediction( + entry, + occurrence.machinePredictions + ) + const canAgree = + !!foldedPrediction && + occurrence.userPermissions.includes(UserPermission.Update) + + const stats: HistoryStat[] = foldedPrediction + ? [ + { + label: translate(STRING.HISTORY_PREDICTION), + value: `${ + foldedPrediction.taxon.name + } (${foldedPrediction.score.toFixed(2)})`, + }, + ] + : [] + switch (entry.subtype) { + case 'tracking': + stats.push( + { + label: translate(STRING.HISTORY_FRAMES_LINKED), + value: entry.payload.frames_linked, + }, + { + label: translate(STRING.HISTORY_OCCURRENCES_MERGED), + value: entry.payload.occurrences_merged.length, + } + ) + break + case 'class_masking': + stats.push( + ...getDeterminationStats(entry.taxon_before, entry.taxon), + { + label: translate(STRING.HISTORY_SPECIES_LIST), + value: ( + + {translate(STRING.HISTORY_SPECIES_LIST_ID, { + id: `${entry.payload.taxa_list_id}`, + })} + + ), + }, + { + label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), + value: entry.payload.detection_ids.length, + } + ) + break + case 'size_filter': + stats.push( + ...getDeterminationStats(entry.taxon_before, entry.taxon), + { + label: translate(STRING.HISTORY_SIZE_THRESHOLD), + value: entry.payload.size_threshold, + }, + { + label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), + value: entry.payload.detection_ids.length, + } + ) + break + } + if (entry.job) { + stats.push({ + label: translate(STRING.FIELD_LABEL_JOB), + value: ( + + {entry.job.name} + + ), + }) + } + + return ( +
+ + } + subTitle={entry.algorithm ? translate(label) : undefined} + title={entry.algorithm?.name ?? translate(label)} + titleAddon={ + + } + > + + {canAgree && foldedPrediction ? ( +
+ +
+ ) : null} + +
+ ) +} diff --git a/ui/src/pages/occurrence-details/identification-card/grouping-confirmation.tsx b/ui/src/pages/occurrence-details/identification-card/grouping-confirmation.tsx deleted file mode 100644 index 6c5a660c5..000000000 --- a/ui/src/pages/occurrence-details/identification-card/grouping-confirmation.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' -import { UserIcon } from 'lucide-react' -import { IdentificationCard } from 'nova-ui-kit' -import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' -import { STRING, translate } from 'utils/language' - -export const GroupingConfirmation = ({ - occurrence, -}: { - occurrence: Occurrence -}) => { - const user = occurrence.groupingVerifiedBy - const formattedTime = occurrence.groupingVerifiedAt - ? getFormatedDateTimeString({ date: occurrence.groupingVerifiedAt }) - : translate(STRING.VALUE_NOT_AVAILABLE) - - return ( -
- - {formattedTime} - - - ) : ( - - ) - } - subTitle={translate(STRING.TRACK_GROUPING_CONFIRMED_BY, { - date: formattedTime, - name: user?.name ?? translate(STRING.ANONYMOUS_USER), - })} - title={translate(STRING.TRACK_GROUPING_CONFIRMED)} - /> -
- ) -} diff --git a/ui/src/pages/occurrence-details/identification-card/history-stats.tsx b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx new file mode 100644 index 000000000..bdbdb7a82 --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx @@ -0,0 +1,30 @@ +import { Fragment, ReactNode } from 'react' + +export interface HistoryStat { + label: string + value: ReactNode +} + +export const HistoryStats = ({ stats }: { stats: HistoryStat[] }) => + stats.length ? ( +
+ {stats.map(({ label, value }) => ( + + {label} + {value} + + ))} +
+ ) : null + +export const HistoryTime = ({ label }: { label: string }) => ( + + {label} + +) + +export const HistoryTypeBadge = ({ label }: { label: string }) => ( + + {label} + +) diff --git a/ui/src/pages/occurrence-details/identification-card/human-identification.tsx b/ui/src/pages/occurrence-details/identification-card/human-identification.tsx index 7023f33ca..68ead0f5d 100644 --- a/ui/src/pages/occurrence-details/identification-card/human-identification.tsx +++ b/ui/src/pages/occurrence-details/identification-card/human-identification.tsx @@ -18,6 +18,7 @@ import { APP_ROUTES } from 'utils/constants' import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { UserInfo, UserPermission } from 'utils/user/types' import { Agree } from '../agree/agree' @@ -25,11 +26,13 @@ export const HumanIdentification = ({ currentUser, identification, occurrence, + onConfirmed, user, }: { currentUser?: UserInfo identification: Identification occurrence: Occurrence + onConfirmed?: (occurrenceId: string) => void user: { id?: string image?: string @@ -68,7 +71,7 @@ export const HumanIdentification = ({ ) } - title={user.name} + title={getUserLabel(user, currentUser)} > )} diff --git a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx index 467e36993..a5c1efc0d 100644 --- a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx +++ b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx @@ -28,10 +28,12 @@ export const MachinePrediction = ({ currentUser, identification, occurrence, + onConfirmed, }: { currentUser?: UserInfo identification: Identification occurrence: Occurrence + onConfirmed?: (occurrenceId: string) => void }) => { const [open, setOpen] = useState(false) const { classification, error, isLoading } = useClassificationDetails( @@ -47,6 +49,7 @@ export const MachinePrediction = ({ date: new Date(identification.createdAt), }) const showAgree = occurrence.userPermissions.includes(UserPermission.Update) + const { algorithm } = identification return (
@@ -66,16 +69,14 @@ export const MachinePrediction = ({ ? translate(STRING.TERMINAL_CLASSIFICATION) : translate(STRING.INTERMEDIATE_CLASSIFICATION) } - title={ - identification.algorithm?.name ?? translate(STRING.MACHINE_SUGGESTION) - } + title={algorithm?.name ?? translate(STRING.MACHINE_SUGGESTION)} onTitleClick={ - identification.algorithm + algorithm ? () => navigate( APP_ROUTES.ALGORITHM_DETAILS({ projectId: projectId as string, - algorithmId: identification.algorithm?.id, + algorithmId: algorithm.id, }) ) : undefined @@ -99,6 +100,7 @@ export const MachinePrediction = ({ agreeWith={{ predictionId: identification.id }} applied={identification.applied} occurrenceId={occurrence.id} + onSuccess={identification.applied ? onConfirmed : undefined} taxonId={identification.taxon.id} /> )} @@ -130,6 +132,7 @@ export const MachinePrediction = ({ agreeWith={{ predictionId: identification.id }} applied={applied} occurrenceId={occurrence.id} + onSuccess={applied ? onConfirmed : undefined} taxonId={taxon.id} /> )} diff --git a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx new file mode 100644 index 000000000..18111eb22 --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx @@ -0,0 +1,111 @@ +import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' +import { + getFallbackTimelineItems, + getTimelineItems, + ServerOccurrenceHistoryEntry, +} from 'data-services/models/occurrence-history' +import { Loader2Icon } from 'lucide-react' +import { useMemo } from 'react' +import { STRING, translate } from 'utils/language' +import { UserInfo } from 'utils/user/types' +import { AlgorithmResult } from './algorithm-result' +import { HumanIdentification } from './human-identification' +import { MachinePrediction } from './machine-prediction' +import { TrackReview } from './track-review' + +export const OccurrenceTimeline = ({ + currentUser, + entries, + error, + isLoading, + occurrence, + onConfirmed, +}: { + currentUser?: UserInfo + entries?: ServerOccurrenceHistoryEntry[] + error?: unknown + isLoading: boolean + occurrence: Occurrence + /** Runs only when a card confirms the current ID, so applying a different ID keeps the reviewer here to see the result. */ + onConfirmed?: (occurrenceId: string) => void +}) => { + const items = useMemo( + () => + entries + ? getTimelineItems({ + determinationTaxonId: occurrence.determinationTaxon?.id, + entries, + identifications: occurrence.humanIdentifications, + predictions: occurrence.machinePredictions, + }) + : getFallbackTimelineItems({ + identifications: occurrence.humanIdentifications, + predictions: occurrence.machinePredictions, + }), + [entries, occurrence] + ) + + return ( + <> + {/* The fallback cards stand in while the history loads, so the spinner only fills an empty list. */} + {isLoading && !items.length ? ( +
+ +
+ ) : null} + {error ? ( +

+ {translate(STRING.HISTORY_LOAD_ERROR)} +

+ ) : null} + {!isLoading && !items.length ? ( +

+ {translate(STRING.HISTORY_EMPTY)} +

+ ) : null} + {items.map((item) => { + switch (item.type) { + case 'identification': + return ( + + ) + case 'prediction': + return ( + + ) + case 'algorithm_result': + return ( + + ) + case 'review': + return ( + + ) + } + })} + + ) +} diff --git a/ui/src/pages/occurrence-details/identification-card/track-review.tsx b/ui/src/pages/occurrence-details/identification-card/track-review.tsx new file mode 100644 index 000000000..7460a73b8 --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/track-review.tsx @@ -0,0 +1,96 @@ +import { TrackCompleteReviewEntry } from 'data-services/models/occurrence-history' +import { UserIcon } from 'lucide-react' +import { IdentificationCard } from 'nova-ui-kit' +import { Link, useParams } from 'react-router-dom' +import { APP_ROUTES } from 'utils/constants' +import { getCompactTimespanString } from 'utils/date/getCompactTimespanString/getCompactTimespanString' +import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' +import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' +import { UserInfo } from 'utils/user/types' +import { + HistoryStat, + HistoryStats, + HistoryTime, + HistoryTypeBadge, +} from './history-stats' + +export const TrackReview = ({ + currentUser, + entry, +}: { + currentUser?: UserInfo + entry: TrackCompleteReviewEntry +}) => { + const { projectId } = useParams() + const { payload, user } = entry + const added = payload.detections_added.length + const removed = payload.detections_removed.length + + const stats: HistoryStat[] = [ + { + label: translate(STRING.TRACK_SUMMARY_FRAMES), + value: + payload.frames_count === 1 + ? translate(STRING.TRACK_FRAMES_ONE) + : translate(STRING.TRACK_FRAMES_COUNT, { + count: payload.frames_count, + }), + }, + ] + if (payload.first_timestamp && payload.last_timestamp) { + stats.push({ + label: translate(STRING.HISTORY_TIME_SPAN), + value: getCompactTimespanString({ + date1: new Date(payload.first_timestamp), + date2: new Date(payload.last_timestamp), + options: { second: true }, + }), + }) + } + + return ( +
+ + + ) : ( + + ) + } + subTitle={ + added || removed + ? translate(STRING.HISTORY_REVIEW_CHANGES, { added, removed }) + : undefined + } + title={translate(STRING.HISTORY_TRACK_COMPLETE_BY, { + name: getUserLabel(user, currentUser), + })} + titleAddon={ + + } + > + + {payload.split_from_occurrence_id ? ( +
+ + {translate(STRING.HISTORY_SPLIT_FROM, { + id: `${payload.split_from_occurrence_id}`, + })} + +
+ ) : null} + +
+ ) +} diff --git a/ui/src/pages/occurrence-details/occurrence-details.tsx b/ui/src/pages/occurrence-details/occurrence-details.tsx index fc156ee24..e3235b0ef 100644 --- a/ui/src/pages/occurrence-details/occurrence-details.tsx +++ b/ui/src/pages/occurrence-details/occurrence-details.tsx @@ -1,5 +1,6 @@ import { CopyLinkButton } from 'components/copy-link-button/copy-link-button' import { TaxonDetails } from 'components/taxon-details/taxon-details' +import { useOccurrenceHistory } from 'data-services/hooks/occurrences/useOccurrenceHistory' import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' import { SearchIcon } from 'lucide-react' import { @@ -21,13 +22,12 @@ import { STRING, translate } from 'utils/language' import { UserPermission } from 'utils/user/types' import { useProjectFeature } from 'utils/project-features/useProjectFeature' import { useUser } from 'utils/user/userContext' +import { getUserLabel } from 'utils/user/getUserLabel' import { useUserInfo } from 'utils/user/userInfoContext' import { Agree } from './agree/agree' import { IdQuickActions } from './id-quick-actions/id-quick-actions' -import { GroupingConfirmation } from './identification-card/grouping-confirmation' import { GroupingSummary } from './identification-card/grouping-summary' -import { HumanIdentification } from './identification-card/human-identification' -import { MachinePrediction } from './identification-card/machine-prediction' +import { OccurrenceTimeline } from './identification-card/occurrence-timeline' import styles from './occurrence-details.module.scss' import { StatusLabel } from './status-label/status-label' import { SuggestId } from './suggest-id/suggest-id' @@ -75,6 +75,10 @@ export const OccurrenceDetails = ({ occurrence.userPermissions ) const trackingEnabled = useProjectFeature('tracking') + const history = useOccurrenceHistory({ + occurrenceId: occurrence.id, + projectId, + }) const fields = [ { @@ -140,7 +144,10 @@ export const OccurrenceDetails = ({ content={ occurrence.determinationVerified ? translate(STRING.VERIFIED_BY, { - name: occurrence.determinationVerifiedBy?.name, + name: getUserLabel( + occurrence.determinationVerifiedBy, + userInfo + ), }) : translate(STRING.MACHINE_PREDICTION_SCORE, { score: `${occurrence.determinationScore}`, @@ -259,10 +266,6 @@ export const OccurrenceDetails = ({ )} - {trackingEnabled && occurrence.groupingVerifiedAt ? ( - - ) : null} - {trackingEnabled && occurrence.groupingSummary ? ( ) : null} - {occurrence.humanIdentifications.map((i) => ( - - ))} - - {occurrence.machinePredictions.map((p) => ( - - ))} +
@@ -310,6 +303,7 @@ export const OccurrenceDetails = ({ )} diff --git a/ui/src/pages/occurrence-details/track/grouping-actions.tsx b/ui/src/pages/occurrence-details/track/grouping-actions.tsx index 22d4295e1..990cb4386 100644 --- a/ui/src/pages/occurrence-details/track/grouping-actions.tsx +++ b/ui/src/pages/occurrence-details/track/grouping-actions.tsx @@ -18,6 +18,8 @@ import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/ import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' import { parseServerError } from 'utils/parseServerError/parseServerError' +import { getUserLabel } from 'utils/user/getUserLabel' +import { useUserInfo } from 'utils/user/userInfoContext' import { getCandidateSessionRoute } from 'components/track/candidate-session-route' import { OccurrencePicker } from 'components/track/occurrence-picker' import { TrackEditDialog } from 'components/track/track-edit-dialog' @@ -28,10 +30,13 @@ const DEFAULT_SCOPE: MergeScopeKey = 'next' export const GroupingActions = ({ canRestructure, canVerify, + editedSinceComplete, occurrence, }: { canRestructure: boolean canVerify: boolean + /** The detections differ from those in the latest "Mark complete" review. */ + editedSinceComplete?: boolean occurrence: OccurrenceDetails }) => { const { projectId } = useParams() @@ -95,6 +100,7 @@ export const GroupingActions = ({ }) : undefined + const { userInfo } = useUserInfo() const verifiedBy = occurrence.groupingVerifiedBy const verifiedAt = occurrence.groupingVerifiedAt const verifyErrorMessage = verifyError @@ -117,10 +123,14 @@ export const GroupingActions = ({ date: verifiedAt ? getFormatedDateTimeString({ date: verifiedAt }) : translate(STRING.VALUE_NOT_AVAILABLE), - name: verifiedBy?.name ?? translate(STRING.ANONYMOUS_USER), + name: getUserLabel(verifiedBy, userInfo), })} + ) : editedSinceComplete ? ( + + {translate(STRING.TRACK_EDITED_SINCE_COMPLETE)} + ) : ( {translate(STRING.TRACK_GROUPING_NOT_CONFIRMED)} diff --git a/ui/src/pages/project/summary/summary.tsx b/ui/src/pages/project/summary/summary.tsx index 1214f1a6d..71b1f9351 100644 --- a/ui/src/pages/project/summary/summary.tsx +++ b/ui/src/pages/project/summary/summary.tsx @@ -14,6 +14,7 @@ import { useState } from 'react' import { Link, useOutletContext } from 'react-router-dom' import { APP_ROUTES } from 'utils/constants' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { UserPermission } from 'utils/user/types' import { DeploymentsMap } from './deployments-map' import { ListItem } from './list-item' @@ -176,9 +177,7 @@ const MostIdentifications = ({ projectId }: { projectId: string }) => { diff --git a/ui/src/pages/session-details/capture/occurrence-toolbar.tsx b/ui/src/pages/session-details/capture/occurrence-toolbar.tsx index 974a2068c..0d5d13d5a 100644 --- a/ui/src/pages/session-details/capture/occurrence-toolbar.tsx +++ b/ui/src/pages/session-details/capture/occurrence-toolbar.tsx @@ -9,6 +9,7 @@ import { ReactNode } from 'react' import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' import { getFormatedTimeString } from 'utils/date/getFormatedTimeString/getFormatedTimeString' import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { buildDetectionLink } from '../hooks/useActiveDetection' export interface ToolbarOccurrence { @@ -260,9 +261,11 @@ export const OccurrenceToolbar = ({ date: occurrence.groupingVerifiedAt, }) : translate(STRING.VALUE_NOT_AVAILABLE), - name: - occurrence.groupingVerifiedBy ?? - translate(STRING.ANONYMOUS_USER), + name: getUserLabel( + occurrence.groupingVerifiedBy === null + ? null + : { name: occurrence.groupingVerifiedBy } + ), })} ) : null} diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 2b3f6908a..37c423d24 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -397,6 +397,28 @@ export enum STRING { TRACK_BOX_EXTEND_LABEL, TRACK_BOX_LABEL, TRACK_BOX_LABEL_VERIFIED, + HISTORY_ALGORITHM_RESULT, + HISTORY_CLASS_MASKING, + HISTORY_DETECTIONS_AFFECTED, + HISTORY_DETERMINATION, + HISTORY_DETERMINATION_CHANGED, + HISTORY_DETERMINATION_UNCHANGED, + HISTORY_EMPTY, + HISTORY_FRAMES_LINKED, + HISTORY_LOAD_ERROR, + HISTORY_OCCURRENCES_MERGED, + HISTORY_PREDICTION, + HISTORY_REVIEW, + HISTORY_REVIEW_CHANGES, + HISTORY_SIZE_FILTER, + HISTORY_SIZE_THRESHOLD, + HISTORY_SPECIES_LIST, + HISTORY_SPECIES_LIST_ID, + HISTORY_SPLIT_FROM, + HISTORY_TIME_SPAN, + HISTORY_TRACK_COMPLETE_BY, + HISTORY_TRACKING, + TRACK_EDITED_SINCE_COMPLETE, TRACK_CANDIDATE_IN_GAP, TRACK_CLEARS_CONFIRMATION, TRACK_COLUMN_COST, @@ -586,6 +608,7 @@ export enum STRING { UNIDENTIFIED, UNKNOWN_ERROR, UNKNOWN, + UNNAMED_USER, UPDATING_DATA, UPLOAD_CAPTURES, USER_INFO, @@ -1064,6 +1087,30 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.TRACK_COLUMN_MATCH]: 'Match', [STRING.TRACK_COLUMN_MATCH_HELP]: 'The same cost read on a 0 to 100 scale. A pair with no feature vector is scored on one term fewer, so its percentage is not comparable with a pair that has one. Sort by Cost for the order tracking itself uses.', + [STRING.HISTORY_ALGORITHM_RESULT]: 'Algorithm result', + [STRING.HISTORY_CLASS_MASKING]: 'Class masking', + [STRING.HISTORY_DETECTIONS_AFFECTED]: 'Detections changed', + [STRING.HISTORY_DETERMINATION]: 'Determination', + [STRING.HISTORY_DETERMINATION_CHANGED]: '{{before}} → {{after}}', + [STRING.HISTORY_DETERMINATION_UNCHANGED]: '{{name}} (unchanged)', + [STRING.HISTORY_EMPTY]: 'Nothing has happened to this occurrence yet.', + [STRING.HISTORY_FRAMES_LINKED]: 'Frames linked', + [STRING.HISTORY_LOAD_ERROR]: + 'Could not load the full history. Showing identifications and predictions only.', + [STRING.HISTORY_OCCURRENCES_MERGED]: 'Occurrences merged', + [STRING.HISTORY_PREDICTION]: 'Prediction', + [STRING.HISTORY_REVIEW]: 'Review', + [STRING.HISTORY_REVIEW_CHANGES]: + '+{{added}} added, −{{removed}} removed since the previous review', + [STRING.HISTORY_SIZE_FILTER]: 'Size filter', + [STRING.HISTORY_SIZE_THRESHOLD]: 'Size threshold', + [STRING.HISTORY_SPECIES_LIST]: 'Species list', + [STRING.HISTORY_SPECIES_LIST_ID]: 'Species list #{{id}}', + [STRING.HISTORY_SPLIT_FROM]: 'Split from occurrence #{{id}}', + [STRING.HISTORY_TIME_SPAN]: 'Time span', + [STRING.HISTORY_TRACK_COMPLETE_BY]: 'Track marked complete by {{name}}', + [STRING.HISTORY_TRACKING]: 'Tracking', + [STRING.TRACK_EDITED_SINCE_COMPLETE]: 'Edited since marked complete', [STRING.TRACK_COLUMN_OVERLAP]: 'Overlap', [STRING.TRACK_COLUMN_SIMILARITY]: 'Similarity', [STRING.TRACK_COLUMN_SIZE]: 'Size match', @@ -1074,7 +1121,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.TRACK_CONFIRM_GROUPING_DESCRIPTION]: 'Records that you checked this track holds every frame of this animal, and no frames of any other. This is a separate judgement from the species it was identified as, and any later change to its frames clears it.', [STRING.TRACK_CONFIRM_GROUPING_RESULT]: 'Track marked complete and accurate.', - [STRING.TRACK_CONFIRM_GROUPING]: 'Mark track as complete and accurate', + [STRING.TRACK_CONFIRM_GROUPING]: 'Mark complete', [STRING.TRACK_GROUPING_FILTER_NO]: 'Not marked', [STRING.TRACK_GROUPING_FILTER_TOOLTIP]: 'Whether a person has marked this track complete and accurate: it holds every frame of the animal and no frames of any other. A separate judgement from the species it was identified as.', @@ -1256,6 +1303,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.UNIDENTIFIED]: 'Unidentified', [STRING.UNKNOWN_ERROR]: 'Unknown error', [STRING.UNKNOWN]: 'Unknown', + [STRING.UNNAMED_USER]: 'Unnamed user', [STRING.UPDATING_DATA]: 'Updating data', [STRING.TRACK_WHEN_EARLIER]: '{{time}} earlier', [STRING.TRACK_WHEN_LATER]: '{{time}} later', diff --git a/ui/src/utils/user/getUserLabel.test.ts b/ui/src/utils/user/getUserLabel.test.ts new file mode 100644 index 000000000..0e6c6225d --- /dev/null +++ b/ui/src/utils/user/getUserLabel.test.ts @@ -0,0 +1,30 @@ +import { getUserLabel } from './getUserLabel' + +describe('getUserLabel', () => { + test('names the current user "You" when they have no display name', () => { + expect(getUserLabel({ id: 4, name: '' }, { id: '4', name: '' })).toBe('You') + expect( + getUserLabel( + { id: 4, name: 'Unnamed user' }, + { id: '4', name: undefined } + ) + ).toBe('You') + }) + + test('uses the display name of a user who has one, including the current user', () => { + expect(getUserLabel({ id: 4, name: 'Ada' }, { id: '4', name: 'Ada' })).toBe( + 'Ada' + ) + expect(getUserLabel({ id: 5, name: 'Ada' }, { id: '4' })).toBe('Ada') + }) + + test('labels another user without a display name as unnamed, not anonymous', () => { + expect(getUserLabel({ id: 5, name: '' }, { id: '4' })).toBe('Unnamed user') + expect(getUserLabel({ id: 5, name: null })).toBe('Unnamed user') + }) + + test('keeps "Anonymous user" for actions with no user', () => { + expect(getUserLabel(undefined, { id: '4' })).toBe('Anonymous user') + expect(getUserLabel(null)).toBe('Anonymous user') + }) +}) diff --git a/ui/src/utils/user/getUserLabel.ts b/ui/src/utils/user/getUserLabel.ts new file mode 100644 index 000000000..ed1ac3196 --- /dev/null +++ b/ui/src/utils/user/getUserLabel.ts @@ -0,0 +1,24 @@ +import { STRING, translate } from 'utils/language' + +/** + * The name to show for the author of an action. "Anonymous" is kept for actions with no user, so + * a signed-in user who has not set a display name reads as "You" to themselves and as unnamed to + * others. The viewer's own name is read from their profile, since `user.name` may already be a label. + */ +export const getUserLabel = ( + user?: { id?: string | number; name?: string | null } | null, + currentUser?: { id: string; name?: string } +) => { + if (!user) { + return translate(STRING.ANONYMOUS_USER) + } + + const isCurrentUser = + !!currentUser && user.id !== undefined && `${user.id}` === currentUser.id + + if (isCurrentUser && !currentUser.name?.length) { + return translate(STRING.YOU) + } + + return user.name?.length ? user.name : translate(STRING.UNNAMED_USER) +}