Add tracking settings that avoid merging different insects, calibrate them per image model, and tune them against confirmed tracks

ami/main/api/views.py

@@ -2021,17 +2021,18 @@ def merge_candidates(self, request: Request, pk=None) -> Response:
             field=serializers.IntegerField(required=False, min_value=1),
             data=request.query_params,
         )
+        config = tracking_config_for(occurrence)
         try:
             candidates = rank_merge_candidates(
                 occurrence,
                 self.get_queryset().prefetch_related(None),
                 minutes=minutes,
                 captures=captures,
                 detection_id=detection_id,
+                config=config,
             )
         except DetectionNotInOccurrence:
             raise api_exceptions.ValidationError({"detection": "Not a detection of this occurrence."})
-        config = tracking_config_for(occurrence)
         return Response(
             MergeCandidatesResponseSerializer(
                 {

ami/main/models_future/merge_candidates.py

@@ -62,14 +62,15 @@
 import datetime
 from typing import TYPE_CHECKING, Any
 
-from django.db.models import Q, QuerySet
+import pydantic
+from django.db.models import F, Q, QuerySet
 
 from ami.main.models_future.embeddings import algorithm_ids_with_vectors, latest_vectors, vectors_for_detections
 from ami.main.models_future.track_stats import bbox_corners, frame_diagonal
 
 if TYPE_CHECKING:
     from ami.main.models import Occurrence, SourceImage
-    from ami.ml.post_processing.tracking_task import TrackingConfig
+    from ami.ml.post_processing.tracking_task import LinkOptions, TrackingConfig
 
 DEFAULT_WINDOW_MINUTES = 5
 MAX_WINDOW_MINUTES = 30
@@ -157,29 +158,38 @@ def _pair_diagonal(track_frame: dict, frame: dict, corners_a, corners_b) -> floa
     return frame_diagonal(None, None, max(corners_a[2], corners_b[2]), max(corners_a[3], corners_b[3]))
 
 
-def _comparison_algorithm(track_vectors: dict[tuple[int, int], Any]) -> int | None:
-    """The algorithm every candidate is scored with: the one with vectors on the most of the
-    occurrence's scored frames, the lowest id on a tie so the choice is stable."""
+def _comparison_algorithm(track_vectors: dict[tuple[int, int], Any], preferred: int | None = None) -> int | None:
+    """The algorithm every candidate is scored with: the tracking run's feature extractor when the
+    track has vectors from it, otherwise the one with vectors on the most of the occurrence's
+    scored frames, the lowest id on a tie so the choice is stable."""
     counts = collections.Counter(algorithm_id for _, algorithm_id in track_vectors)
+    if preferred is not None and counts.get(preferred):
+        return preferred
     return min(counts, key=lambda algorithm_id: (-counts[algorithm_id], algorithm_id)) if counts else None
 
 
-def _score_pair(track_frame: dict, frame: dict, track_vector, frame_vector) -> dict[str, float | None]:
+def _score_pair(
+    track_frame: dict, frame: dict, track_vector, frame_vector, options: LinkOptions
+) -> dict[str, float | None]:
     """The cost terms for one pair of frames, all None for a box that cannot be read."""
     corners_a = bbox_corners(track_frame["bbox"])
     corners_b = bbox_corners(frame["bbox"])
     if corners_a is None or corners_b is None:
         return dict.fromkeys(_PAIR_SCORE_FIELDS)
     diagonal = _pair_diagonal(track_frame, frame, corners_a, corners_b)
-    return _pair_scores(corners_a, corners_b, track_vector, frame_vector, diagonal)
+    return _pair_scores(corners_a, corners_b, track_vector, frame_vector, diagonal, options)
 
 
 def _would_link(scores: dict[str, float | None], config: TrackingConfig) -> bool:
-    """Whether the pair passes the tracker's pairing rule: a cost under its threshold, and a
-    vector on both frames when it requires them. A preview of the rule, not of a run: the
-    matcher only pairs adjacent captures and claims each box once."""
+    """Whether the pair passes the tracker's pairing rule: a cost under its threshold, a
+    vector on both frames when it requires them, and embeddings no less alike than its
+    appearance gate. A preview of the rule, not of a run: the matcher only pairs adjacent
+    captures, claims each box once, and applies the species gate and crowd scaling."""
     if scores["cost"] is None or (config.require_features and scores["similarity"] is None):
         return False
+    gate = config.appearance_min_similarity
+    if gate is not None and scores["similarity"] is not None and scores["similarity"] < gate:
+        return False
     return scores["cost"] < config.cost_threshold
 
 
@@ -195,6 +205,7 @@ def rank_merge_candidates(
     captures: int = DEFAULT_ADJACENT_CAPTURES,
     limit: int = MAX_CANDIDATES,
     detection_id: int | None = None,
+    config: TrackingConfig | None = None,
 ) -> list[dict[str, Any]]:
     """Candidates for merging with ``occurrence``, lowest cost first, at most ``limit``.
 
@@ -224,7 +235,8 @@ def rank_merge_candidates(
     """
     from ami.main.models import Detection, get_media_url
 
-    config = tracking_config_for(occurrence)
+    config = config or tracking_config_for(occurrence)
+    options = config.link_options()
     track = Detection.objects.valid().filter(occurrence_id=occurrence.pk)
     if detection_id is None:
         target_frames = _timed_frames(track)
@@ -283,7 +295,7 @@ def rank_merge_candidates(
     }
 
     track_vectors = latest_vectors({track_frame["pk"] for track_frame, _ in pairs.values()})
-    algorithm_id = _comparison_algorithm(track_vectors)
+    algorithm_id = _comparison_algorithm(track_vectors, config.feature_extraction_algorithm_id)
     frame_vectors: dict[int, Any] = {}
     if algorithm_id is not None:
         frame_vectors = vectors_for_detections([frame["pk"] for _, frame in pairs.values()], algorithm_id)
@@ -293,7 +305,7 @@ def rank_merge_candidates(
         track_frame, frame = pairs[candidate.pk]
         track_vector = track_vectors.get((track_frame["pk"], algorithm_id))
         frame_vector = frame_vectors.get(frame["pk"])
-        scores = _score_pair(track_frame, frame, track_vector, frame_vector)
+        scores = _score_pair(track_frame, frame, track_vector, frame_vector, options)
         crop = frame["path"] or next((f["path"] for f in frames_by_occurrence[candidate.pk] if f["path"]), None)
         rows.append(
             {
@@ -329,11 +341,33 @@ def rank_merge_candidates(
 
 
 def tracking_config_for(occurrence: Occurrence) -> TrackingConfig:
-    """The settings tracking runs with on the occurrence's session: the threshold and the
-    feature requirement the previews here judge pairs by."""
+    """The tracking settings the previews here judge pairs by: those of the latest successful
+    tracking run over the occurrence's session, so the scores shown match how the session was
+    tracked, or the defaults when no run covered it."""
+    from ami.jobs.models import Job, JobState
+    from ami.main.models import SourceImageCollection
     from ami.ml.post_processing.tracking_task import TrackingConfig
 
-    return TrackingConfig(event_ids=[occurrence.event_id])
+    event_id = occurrence.event_id
+    runs = Job.objects.filter(
+        project_id=occurrence.project_id,
+        job_type_key="post_processing",
+        status=JobState.SUCCESS.name,
+        params__task="tracking",
+    ).order_by(F("finished_at").desc(nulls_last=True), "-pk")
+    for job in runs.only("params")[:50]:
+        config = (job.params or {}).get("config") or {}
+        covers = event_id in (config.get("event_ids") or [])
+        if not covers and config.get("source_image_collection_id"):
+            covers = SourceImageCollection.objects.filter(
+                pk=config["source_image_collection_id"], images__event_id=event_id
+            ).exists()
+        if covers:
+            try:
+                return TrackingConfig(**{**config, "event_ids": [event_id], "source_image_collection_id": None})
+            except pydantic.ValidationError:
+                break  # a run stored with settings this version no longer reads
+    return TrackingConfig(event_ids=[event_id])
 
 
 def _likelihood(cost: float, similarity: float | None) -> float:
@@ -344,20 +378,32 @@ def _likelihood(cost: float, similarity: float | None) -> float:
     return round(min(max(1 - cost / terms, 0.0), 1.0), _ROUND_TO)
 
 
-def _pair_scores(bbox_a, bbox_b, vector_a, vector_b, diag: float) -> dict[str, float | None]:
-    """The tracking cost between two boxes, each of its terms, and the likelihood."""
-    from ami.ml.post_processing.tracking_task import box_ratio, cosine_similarity, distance_ratio, iou, total_cost
+def _pair_scores(
+    bbox_a,
+    bbox_b,
+    vector_a,
+    vector_b,
+    diag: float,
+    options: LinkOptions | None = None,
+    distance_multiplier: float = 1.0,
+) -> dict[str, float | None]:
+    """The tracking cost between two boxes, each of its raw terms, and the likelihood.
+
+    The cost is the tracker's own ``weighted_cost`` under the given link options (from
+    ``tracking_config_for``: the session's latest tracking run, else the defaults), so non-default weights, appearance
+    calibration or move rule would change it here as they do in a run.
+    """
+    from ami.ml.post_processing.tracking_task import DEFAULT_LINK_OPTIONS, pair_terms, weighted_cost
 
-    cost = total_cost(vector_a, vector_b, bbox_a, bbox_b, diag)
-    similarity = None
-    if vector_a is not None and vector_b is not None:
-        similarity = round(cosine_similarity(vector_a, vector_b), _ROUND_TO)
+    terms = pair_terms(vector_a, vector_b, bbox_a, bbox_b, diag)
+    cost = weighted_cost(terms, options or DEFAULT_LINK_OPTIONS, distance_multiplier)
+    similarity = None if terms.appearance is None else round(1 - terms.appearance, _ROUND_TO)
     return {
         "likelihood": _likelihood(cost, similarity),
         "cost": round(cost, _ROUND_TO),
-        "distance": round(distance_ratio(bbox_a, bbox_b, diag), _ROUND_TO),
-        "iou": round(iou(bbox_a, bbox_b), _ROUND_TO),
-        "size_ratio": round(box_ratio(bbox_a, bbox_b), _ROUND_TO),
+        "distance": round(terms.distance, _ROUND_TO),
+        "iou": round(terms.iou, _ROUND_TO),
+        "size_ratio": round(terms.size_ratio, _ROUND_TO),
         "similarity": similarity,
     }
 
@@ -387,9 +433,10 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di
 
     The reference is the track frame nearest in time on another capture. The tracker's own
     matcher runs between every box on the reference frame's capture and every box on
-    ``capture``, with the tracker's default settings, feature algorithm and image diagonal,
-    and ``would_link`` marks the box it links the reference frame to. Tracking only pairs
-    adjacent captures, so across a longer gap this previews its pairing rule, not a run.
+    ``capture``, with the settings ``tracking_config_for`` returns (the session's latest tracking run), the
+    session's feature algorithm and the image diagonal, and ``would_link`` marks the box it
+    links the reference frame to. Tracking only pairs adjacent captures, so across a longer
+    gap this previews its pairing rule, not a run.
 
     The track's own box is returned unscored. So is every box when the track has no other
     frame, or when the earlier of the two captures has no dimensions, since tracking skips
@@ -398,9 +445,15 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di
     """
     from ami.main.models import Detection, SourceImage
     from ami.ml.models import Algorithm
-    from ami.ml.post_processing.tracking_task import image_diagonal, resolve_feature_algorithm, select_links
+    from ami.ml.post_processing.tracking_task import (
+        activity_multiplier,
+        image_diagonal,
+        resolve_feature_algorithm,
+        select_links,
+    )
 
     config = tracking_config_for(occurrence)
+    options = config.link_options()
     reference, relation = _reference_frame(occurrence.pk, capture) if capture.timestamp else (None, None)
     reference_capture_id = reference["source_image_id"] if reference is not None else None
     detections = list(
@@ -424,7 +477,7 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di
     def skipped(detection_id: int) -> str | None:
         return SKIPPED_NO_VECTOR if config.require_features and detection_id not in vectors else None
 
-    diag, linked, reference_detection = None, set(), None
+    diag, linked, reference_detection, multiplier = None, set(), None, 1.0
     if reference is not None:
         reference_detection = next(d for d in detections if d.pk == reference["pk"])
         reference_boxes = [d for d in detections if d.source_image_id == reference_capture_id]
@@ -437,8 +490,11 @@ def skipped(detection_id: int) -> str | None:
         if width and height:
             diag = image_diagonal(width, height)
             current, following = (boxes, reference_boxes) if capture_first else (reference_boxes, boxes)
-            links = select_links(current, following, vectors, diag, config.cost_threshold, config.require_features)
+            links = select_links(
+                current, following, vectors, diag, config.cost_threshold, config.require_features, options
+            )
             linked = {frozenset((det.pk, nxt.pk)) for det, nxt, _ in links}
+            multiplier = activity_multiplier(max(len(current), len(following)), options)
 
     # Captures from the reference frame's to this one, signed: 1 is the adjacent capture the
     # tracker would pair, so a larger count means the preview spans captures it never compares.
@@ -480,6 +536,8 @@ def skipped(detection_id: int) -> str | None:
                         vectors.get(reference["pk"]),
                         vectors.get(box.pk),
                         diag,
+                        options,
+                        multiplier,
                     )
                 )
         rows.append(row)