diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 1a0b0bf68..03c670ff8 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -2021,6 +2021,7 @@ def merge_candidates(self, request: Request, pk=None) -> Response: field=serializers.IntegerField(required=False, min_value=1), data=request.query_params, ) + config = tracking_config_for(occurrence) try: candidates = rank_merge_candidates( occurrence, @@ -2028,10 +2029,10 @@ def merge_candidates(self, request: Request, pk=None) -> Response: minutes=minutes, captures=captures, detection_id=detection_id, + config=config, ) except DetectionNotInOccurrence: raise api_exceptions.ValidationError({"detection": "Not a detection of this occurrence."}) - config = tracking_config_for(occurrence) return Response( MergeCandidatesResponseSerializer( { diff --git a/ami/main/management/commands/evaluate_tracking.py b/ami/main/management/commands/evaluate_tracking.py index 21d2ed419..732e2d6a1 100644 --- a/ami/main/management/commands/evaluate_tracking.py +++ b/ami/main/management/commands/evaluate_tracking.py @@ -3,22 +3,109 @@ Ground truth is every occurrence whose grouping a person confirmed. Predictions come from re-running the tracking link step over the raw detections of those sessions, as if none were -linked yet, with the tunables given here. Nothing is written: the command runs inside a -transaction that is always rolled back. See docs/claude/reference/tracking-evaluation.md. +linked yet, with the tunables given here. ``--sweep`` scores many settings in one run. Nothing +is written: the command runs inside a transaction that is always rolled back. See +docs/claude/reference/tracking-evaluation.md. """ +import csv +import itertools import json import logging +import pathlib +import numpy as np +import pydantic from django.core.management.base import BaseCommand, CommandError from django.db import transaction from ami.main.models import Detection, Event, Project -from ami.ml.post_processing.tracking_evaluation import TrackingEvaluation, evaluate_tracks, tracks_from_links -from ami.ml.post_processing.tracking_task import TrackingConfig, propose_event_links, resolve_feature_algorithm +from ami.ml.post_processing.tracking_evaluation import ( + TrackingEvaluation, + evaluate_tracks, + format_sweep_markdown, + summarise_session, + sweep_row, + tracks_from_links, +) +from ami.ml.post_processing.tracking_task import ( + AmbiguousSpeciesLabels, + TrackingConfig, + event_transition_pairs, + labels_conflict, + links_from_transition_pairs, + resolve_feature_algorithm, + resolve_label_algorithm, + top_labels, +) logger = logging.getLogger(__name__) +# Fields that name the sessions to track; the command sets them itself. +SCOPE_FIELDS = {"event_ids", "source_image_collection_id"} +# A link between two detections whose labels both score at least this and name unrelated +# taxa is counted as a likely wrong link, over the whole session, confirmed or not. +CONFLICT_MIN_SCORE = 0.5 +# Guards that decide whether a run writes, not how it links; scoring ignores them. +WRITE_GUARD_FIELDS = {"skip_if_human_identifications", "require_completely_processed_session", "require_fresh_event"} + + +def _json_argument(value: str): + """Inline JSON, or the path of a file holding it.""" + text = value + if not value.lstrip().startswith(("{", "[")): + path = pathlib.Path(value) + if not path.is_file(): + raise CommandError(f"Not JSON and not a file: {value[:80]}") + text = path.read_text() + try: + return json.loads(text) + except json.JSONDecodeError as error: + raise CommandError(f"Not valid JSON: {value[:80]} ({error})") + + +def expand_sweep(spec) -> list[dict]: + """The settings a sweep runs, in order. + + ``spec`` is a list of settings, or ``{"base": {...}, "grid": {field: [values]}, "configs": [...]}``: + every combination of the grid values, each on top of ``base``, followed by each entry of + ``configs`` on top of ``base``. A grid value that is an object is merged in whole, so one + grid axis can set several fields together (for example a gate mode and its penalty). + """ + if isinstance(spec, list): + spec = {"configs": spec} + if not isinstance(spec, dict): + raise CommandError("A sweep is a list of settings or an object with base, grid and configs.") + unknown = set(spec) - {"base", "grid", "configs"} + if unknown: + raise CommandError(f"Unknown sweep keys: {sorted(unknown)}") + base = spec.get("base") or {} + grid = spec.get("grid") or {} + settings = [] + if grid: + names = list(grid) + for values in itertools.product(*(grid[name] for name in names)): + setting = dict(base) + for name, value in zip(names, values): + setting.update(value if isinstance(value, dict) else {name: value}) + settings.append(setting) + settings.extend({**base, **extra} for extra in spec.get("configs") or []) + if not settings: + raise CommandError("The sweep names no settings.") + return settings + + +def load_vectors_file(path: str) -> dict[int, np.ndarray]: + """Embeddings from an ``.npz`` file with arrays ``detection_ids`` (n) and ``vectors`` (n x d).""" + try: + with np.load(path) as data: + ids, vectors = data["detection_ids"], data["vectors"] + except (OSError, KeyError, ValueError) as error: + raise CommandError(f"Cannot read vectors from {path}: {error}") + if len(ids) != len(vectors): + raise CommandError(f"{path}: {len(ids)} detection ids but {len(vectors)} vectors") + return {int(detection_id): vector for detection_id, vector in zip(ids, vectors)} + class Command(BaseCommand): help = "Score tracking against the confirmed tracks of a project, without writing anything." @@ -43,24 +130,68 @@ def add_arguments(self, parser): default=None, help="Algorithm ID whose embeddings to compare. Default: as tracking picks it.", ) + parser.add_argument( + "--config", + default=None, + help='JSON object (inline or a file) of further TrackingConfig settings, e.g. \'{"species_gate": ' + '"forbid"}\'. Overrides the options above.', + ) + parser.add_argument( + "--sweep", + default=None, + help="JSON (inline or a file): a list of settings, or {base, grid, configs}. Scores each setting.", + ) + parser.add_argument( + "--vectors-file", + default=None, + help="An .npz of detection_ids and vectors to compare instead of the stored embeddings.", + ) + parser.add_argument("--output-dir", default=None, help="With --sweep, write sweep.json and sweep.md here.") + parser.add_argument("--per-track-csv", default=None, help="Write one row per scored track to this file.") parser.add_argument("--format", choices=["text", "json"], default="text") parser.add_argument("--per-track", action="store_true", help="Include per-track scores in JSON output.") def handle(self, *args, **options): + base = { + "cost_threshold": options["cost_threshold"], + "require_features": options["require_features"], + "feature_extraction_algorithm_id": options["feature_extraction_algorithm"], + } + if options["config"]: + extra = _json_argument(options["config"]) + if not isinstance(extra, dict): + raise CommandError("--config takes a JSON object.") + base.update(extra) + settings = [base] + if options["sweep"]: + settings = [{**base, **setting} for setting in expand_sweep(_json_argument(options["sweep"]))] + for setting in settings: + scope = SCOPE_FIELDS & set(setting) + if scope: + raise CommandError(f"The command chooses the sessions; remove {sorted(scope)} from the settings.") + vectors = load_vectors_file(options["vectors_file"]) if options["vectors_file"] else None + # Proposing links only reads, but the rollback guarantees a scoring run never changes data. with transaction.atomic(): try: - report = self._evaluate(options) + reports = self._evaluate(options, settings, vectors) finally: transaction.set_rollback(True) + if options["per_track_csv"]: + self._write_per_track_csv(options["per_track_csv"], reports) + + if options["sweep"]: + self._write_sweep(options, settings, reports) + return + + (report,) = reports if options["format"] == "json": - public = {key: value for key, value in report.items() if not key.startswith("_")} - self.stdout.write(json.dumps(public, indent=2, default=str)) + self.stdout.write(json.dumps(self._public(report, options["per_track"]), indent=2, default=str)) else: self.stdout.write(self._as_text(report)) - def _evaluate(self, options) -> dict: + def _evaluate(self, options, settings: list[dict], vectors) -> list[dict]: project = Project.objects.filter(pk=options["project"]).first() if project is None: raise CommandError(f"Project {options['project']} not found.") @@ -72,7 +203,15 @@ def _evaluate(self, options) -> dict: ) if options["events"]: detections = detections.filter(source_image__event_id__in=options["events"]) - rows = list(detections.values_list("pk", "occurrence_id", "source_image__timestamp", "source_image__event_id")) + rows = list( + detections.values_list( + "pk", + "occurrence_id", + "source_image__timestamp", + "source_image__event_id", + "occurrence__determination_id", + ) + ) event_ids = sorted({row[3] for row in rows if row[3] is not None}) missing = sorted(set(options["events"] or []) - set(event_ids)) @@ -81,91 +220,255 @@ def _evaluate(self, options) -> dict: if not event_ids: raise CommandError(f"Project {project.pk} has no confirmed tracks in the sessions asked for.") - config = TrackingConfig( - event_ids=event_ids, - cost_threshold=options["cost_threshold"], - require_features=options["require_features"], - feature_extraction_algorithm_id=options["feature_extraction_algorithm"], - ) + configs = [] + for setting in settings: + try: + configs.append(TrackingConfig(event_ids=event_ids, **setting)) + except pydantic.ValidationError as error: + raise CommandError(f"Invalid tracking settings {setting}: {error}") + + events = list(Event.objects.filter(pk__in=event_ids).order_by("pk")) + truth_by_event: dict[int, dict] = {} + for event in events: + event_rows = [row for row in rows if row[3] == event.pk] + missing_times = sorted(row[0] for row in event_rows if row[2] is None) + if missing_times: + raise CommandError( + f"Session {event.pk} has {len(missing_times)} confirmed detection(s) on captures without " + f"a timestamp, e.g. detection {missing_times[:3]}; they cannot be put in capture order." + ) + truth_by_event[event.pk] = { + "truth": {pk: occurrence_id for pk, occurrence_id, _, _, _ in event_rows}, + "times": {pk: timestamp for pk, _, timestamp, _, _ in event_rows}, + "taxa": {occurrence_id: taxon_id for _, occurrence_id, _, _, taxon_id in event_rows}, + } + + # Reading and scoring the pairs is the slow part, and it is the same for every setting + # that compares the same embeddings, so it is done once per session and extractor. + pairs_cache: dict[tuple, tuple] = {} + labels_cache: dict[tuple, dict] = {} + reports = [] + for index, config in enumerate(configs): + if len(configs) > 1: + logger.info(f"Scoring setting {index + 1}/{len(configs)}: {settings[index]}") + reports.append( + self._evaluate_config(project, events, config, truth_by_event, vectors, pairs_cache, labels_cache) + ) + return reports + def _evaluate_config(self, project, events, config, truth_by_event, vectors, pairs_cache, labels_cache) -> dict: ground_truth: dict[int, int] = {} timestamps: dict[int, object] = {} + taxa: dict[int, object] = {} predictions: dict[int, int] = {} + all_labels: dict[int, tuple] = {} per_event: list[dict] = [] skipped: list[dict] = [] - for event in Event.objects.filter(pk__in=event_ids).order_by("pk"): - event_rows = [row for row in rows if row[3] == event.pk] - algorithm, should_track, note = resolve_feature_algorithm(event, config) + for event in events: + truth = truth_by_event[event.pk] + if vectors is not None: + algorithm, should_track, note = None, True, "Embeddings from the vectors file." + else: + algorithm, should_track, note = resolve_feature_algorithm(event, config) if not should_track: - skipped.append({"event_id": event.pk, "reason": note, "confirmed_detections": len(event_rows)}) + skipped.append({"event_id": event.pk, "reason": note, "confirmed_detections": len(truth["truth"])}) continue - links = propose_event_links(event, algorithm, config, logger) - event_detection_ids = ( - Detection.objects.valid().filter(source_image__event=event).values_list("pk", flat=True) - ) - event_predictions = tracks_from_links(event_detection_ids, [(a, b) for a, b, _ in links]) - event_truth = {pk: occurrence_id for pk, occurrence_id, _, _ in event_rows} - event_times = {pk: timestamp for pk, _, timestamp, _ in event_rows} + key = (event.pk, algorithm.pk if algorithm else None) + if key not in pairs_cache: + pairs_cache[key] = event_transition_pairs(event, algorithm, vectors) + transitions, event_detection_ids = pairs_cache[key] + # Labels come from one classifier per session, as in a tracking run, so scores + # compared against one threshold share a scale. + try: + label_algorithm_id = resolve_label_algorithm(event_detection_ids, config.species_label_algorithm_id) + except AmbiguousSpeciesLabels as error: + raise CommandError(f"Session {event.pk}: {error} Pass it with --config.") from error + label_key = (event.pk, label_algorithm_id) + if label_key not in labels_cache: + labels_cache[label_key] = top_labels(event_detection_ids, label_algorithm_id) + labels = labels_cache[label_key] - missing_times = sorted(pk for pk, timestamp in event_times.items() if timestamp is None) - if missing_times: - raise CommandError( - f"Session {event.pk} has {len(missing_times)} confirmed detection(s) on captures without " - f"a timestamp, e.g. detection {missing_times[:3]}; they cannot be put in capture order." - ) + links = links_from_transition_pairs(transitions, config, labels) + event_predictions = tracks_from_links(event_detection_ids, [(a, b) for a, b, _ in links]) + session_labels = {pk: (label.taxon_id, label.score) for pk, label in labels.items()} + session = summarise_session(event_predictions, session_labels) + session["links_with_conflicting_labels"] = sum( + labels_conflict(labels.get(a), labels.get(b), CONFLICT_MIN_SCORE) for a, b, _ in links + ) - ground_truth.update(event_truth) - timestamps.update(event_times) + ground_truth.update(truth["truth"]) + timestamps.update(truth["times"]) + taxa.update(truth["taxa"]) predictions.update(event_predictions) + all_labels.update(session_labels) per_event.append( { "event_id": event.pk, "feature_extraction_algorithm_id": algorithm.pk if algorithm else None, "note": note, "links_proposed": len(links), - "evaluation": evaluate_tracks(event_truth, event_predictions, event_times), + "evaluation": evaluate_tracks(truth["truth"], event_predictions, truth["times"], truth["taxa"]), + "session": session, } ) overall: TrackingEvaluation | None = None if ground_truth: - overall = evaluate_tracks(ground_truth, predictions, timestamps) - - per_track = options["per_track"] + overall = evaluate_tracks(ground_truth, predictions, timestamps, taxa) + overall_session = None + if predictions: + overall_session = summarise_session(predictions, all_labels) + overall_session["links_with_conflicting_labels"] = sum( + entry["session"]["links_with_conflicting_labels"] for entry in per_event + ) return { "project_id": project.pk, "config": { - "cost_threshold": config.cost_threshold, - "require_features": config.require_features, - "feature_extraction_algorithm_id": config.feature_extraction_algorithm_id, + key: value + for key, value in config.dict().items() + if key not in SCOPE_FIELDS and key not in WRITE_GUARD_FIELDS }, + "events": per_event, + "skipped_events": skipped, + "overall": overall, + "overall_session": overall_session, + "links_proposed": sum(entry["links_proposed"] for entry in per_event), + } + + @staticmethod + def _public(report: dict, per_track: bool) -> dict: + overall = report["overall"] + return { + "project_id": report["project_id"], + "config": report["config"], "events": [ - {**entry, "evaluation": entry["evaluation"].to_dict(include_tracks=per_track)} for entry in per_event + {**entry, "evaluation": entry["evaluation"].to_dict(include_tracks=per_track)} + for entry in report["events"] ], - "skipped_events": skipped, + "skipped_events": report["skipped_events"], "overall": overall.to_dict(include_tracks=per_track) if overall else None, - "_overall_lines": overall.summary_lines() if overall else [], - "_event_lines": {entry["event_id"]: entry["evaluation"].summary_lines() for entry in per_event}, + "overall_session": report["overall_session"], } + def _write_sweep(self, options, settings: list[dict], reports: list[dict]) -> None: + rows = [] + for index, (setting, report) in enumerate(zip(settings, reports), start=1): + for entry in report["events"]: + rows.append( + sweep_row( + index, + setting, + entry["event_id"], + entry["evaluation"], + entry["session"], + entry["links_proposed"], + ) + ) + if report["overall"] is not None: + rows.append( + sweep_row( + index, + setting, + "overall", + report["overall"], + report["overall_session"], + report["links_proposed"], + ) + ) + document = { + "project_id": options["project"], + "vectors_file": bool(options["vectors_file"]), + "runs": [ + {"index": index, "settings": setting, **self._public(report, options["per_track"])} + for index, (setting, report) in enumerate(zip(settings, reports), start=1) + ], + "rows": rows, + } + markdown = format_sweep_markdown(rows) + if options["output_dir"]: + directory = pathlib.Path(options["output_dir"]) + directory.mkdir(parents=True, exist_ok=True) + (directory / "sweep.json").write_text(json.dumps(document, indent=2, default=str)) + (directory / "sweep.md").write_text(markdown) + if options["format"] == "json": + self.stdout.write(json.dumps(document, indent=2, default=str)) + else: + self.stdout.write(markdown) + + @staticmethod + def _write_per_track_csv(path: str, reports: list[dict]) -> None: + fields = [ + "run", + "event_id", + "kind", + "track_id", + "length", + "fragments", + "completeness", + "exactly_recovered", + "ground_truth_tracks", + "purity", + ] + # Written before the sweep files, so the folder may not exist yet. + pathlib.Path(path).parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fields) + writer.writeheader() + for run, report in enumerate(reports, start=1): + for entry in report["events"]: + evaluation = entry["evaluation"] + for score in evaluation.ground_truth_track_scores: + writer.writerow( + { + "run": run, + "event_id": entry["event_id"], + "kind": "confirmed", + "track_id": score.track_id, + "length": score.length, + "fragments": score.fragments, + "completeness": round(score.completeness, 4), + "exactly_recovered": score.exactly_recovered, + } + ) + for score in evaluation.predicted_track_scores: + writer.writerow( + { + "run": run, + "event_id": entry["event_id"], + "kind": "predicted", + "track_id": score.first_detection_id, + "length": score.length, + "ground_truth_tracks": score.ground_truth_tracks, + "purity": round(score.purity, 4), + } + ) + def _as_text(self, report: dict) -> str: config = report["config"] + changed = { + key: value + for key, value in config.items() + if key not in {"cost_threshold", "require_features", "feature_extraction_algorithm_id"} + and value != TrackingConfig.__fields__[key].default + } lines = [ f"Project {report['project_id']}: cost_threshold={config['cost_threshold']} " f"require_features={config['require_features']} " - f"feature_extraction_algorithm_id={config['feature_extraction_algorithm_id']}", + f"feature_extraction_algorithm_id={config['feature_extraction_algorithm_id']}" + + (f" {changed}" if changed else ""), ] for entry in report["events"]: lines.append("") lines.append(f"Session {entry['event_id']} ({entry['links_proposed']} links proposed)") if entry["note"]: lines.append(f" Note: {entry['note']}") - lines.extend(f" {line}" for line in report["_event_lines"][entry["event_id"]]) + lines.extend(f" {line}" for line in entry["evaluation"].summary_lines()) for entry in report["skipped_events"]: lines.append("") lines.append(f"Session {entry['event_id']} skipped: {entry['reason']}") lines.append("") lines.append("Overall") - lines.extend(f" {line}" for line in report["_overall_lines"] or ["No session could be scored."]) + overall = report["overall"] + lines.extend(f" {line}" for line in (overall.summary_lines() if overall else ["No session could be scored."])) return "\n".join(lines) diff --git a/ami/main/models_future/merge_candidates.py b/ami/main/models_future/merge_candidates.py index 4279db8c3..c6b426d3f 100644 --- a/ami/main/models_future/merge_candidates.py +++ b/ami/main/models_future/merge_candidates.py @@ -62,14 +62,15 @@ import datetime from typing import TYPE_CHECKING, Any -from django.db.models import Q, QuerySet +import pydantic +from django.db.models import F, Q, QuerySet from ami.main.models_future.embeddings import algorithm_ids_with_vectors, latest_vectors, vectors_for_detections from ami.main.models_future.track_stats import bbox_corners, frame_diagonal if TYPE_CHECKING: from ami.main.models import Occurrence, SourceImage - from ami.ml.post_processing.tracking_task import TrackingConfig + from ami.ml.post_processing.tracking_task import LinkOptions, TrackingConfig DEFAULT_WINDOW_MINUTES = 5 MAX_WINDOW_MINUTES = 30 @@ -157,29 +158,38 @@ def _pair_diagonal(track_frame: dict, frame: dict, corners_a, corners_b) -> floa return frame_diagonal(None, None, max(corners_a[2], corners_b[2]), max(corners_a[3], corners_b[3])) -def _comparison_algorithm(track_vectors: dict[tuple[int, int], Any]) -> int | None: - """The algorithm every candidate is scored with: the one with vectors on the most of the - occurrence's scored frames, the lowest id on a tie so the choice is stable.""" +def _comparison_algorithm(track_vectors: dict[tuple[int, int], Any], preferred: int | None = None) -> int | None: + """The algorithm every candidate is scored with: the tracking run's feature extractor when the + track has vectors from it, otherwise the one with vectors on the most of the occurrence's + scored frames, the lowest id on a tie so the choice is stable.""" counts = collections.Counter(algorithm_id for _, algorithm_id in track_vectors) + if preferred is not None and counts.get(preferred): + return preferred return min(counts, key=lambda algorithm_id: (-counts[algorithm_id], algorithm_id)) if counts else None -def _score_pair(track_frame: dict, frame: dict, track_vector, frame_vector) -> dict[str, float | None]: +def _score_pair( + track_frame: dict, frame: dict, track_vector, frame_vector, options: LinkOptions +) -> dict[str, float | None]: """The cost terms for one pair of frames, all None for a box that cannot be read.""" corners_a = bbox_corners(track_frame["bbox"]) corners_b = bbox_corners(frame["bbox"]) if corners_a is None or corners_b is None: return dict.fromkeys(_PAIR_SCORE_FIELDS) diagonal = _pair_diagonal(track_frame, frame, corners_a, corners_b) - return _pair_scores(corners_a, corners_b, track_vector, frame_vector, diagonal) + return _pair_scores(corners_a, corners_b, track_vector, frame_vector, diagonal, options) def _would_link(scores: dict[str, float | None], config: TrackingConfig) -> bool: - """Whether the pair passes the tracker's pairing rule: a cost under its threshold, and a - vector on both frames when it requires them. A preview of the rule, not of a run: the - matcher only pairs adjacent captures and claims each box once.""" + """Whether the pair passes the tracker's pairing rule: a cost under its threshold, a + vector on both frames when it requires them, and embeddings no less alike than its + appearance gate. A preview of the rule, not of a run: the matcher only pairs adjacent + captures, claims each box once, and applies the species gate and crowd scaling.""" if scores["cost"] is None or (config.require_features and scores["similarity"] is None): return False + gate = config.appearance_min_similarity + if gate is not None and scores["similarity"] is not None and scores["similarity"] < gate: + return False return scores["cost"] < config.cost_threshold @@ -195,6 +205,7 @@ def rank_merge_candidates( captures: int = DEFAULT_ADJACENT_CAPTURES, limit: int = MAX_CANDIDATES, detection_id: int | None = None, + config: TrackingConfig | None = None, ) -> list[dict[str, Any]]: """Candidates for merging with ``occurrence``, lowest cost first, at most ``limit``. @@ -224,7 +235,8 @@ def rank_merge_candidates( """ from ami.main.models import Detection, get_media_url - config = tracking_config_for(occurrence) + config = config or tracking_config_for(occurrence) + options = config.link_options() track = Detection.objects.valid().filter(occurrence_id=occurrence.pk) if detection_id is None: target_frames = _timed_frames(track) @@ -283,7 +295,7 @@ def rank_merge_candidates( } track_vectors = latest_vectors({track_frame["pk"] for track_frame, _ in pairs.values()}) - algorithm_id = _comparison_algorithm(track_vectors) + algorithm_id = _comparison_algorithm(track_vectors, config.feature_extraction_algorithm_id) frame_vectors: dict[int, Any] = {} if algorithm_id is not None: frame_vectors = vectors_for_detections([frame["pk"] for _, frame in pairs.values()], algorithm_id) @@ -293,7 +305,7 @@ def rank_merge_candidates( track_frame, frame = pairs[candidate.pk] track_vector = track_vectors.get((track_frame["pk"], algorithm_id)) frame_vector = frame_vectors.get(frame["pk"]) - scores = _score_pair(track_frame, frame, track_vector, frame_vector) + scores = _score_pair(track_frame, frame, track_vector, frame_vector, options) crop = frame["path"] or next((f["path"] for f in frames_by_occurrence[candidate.pk] if f["path"]), None) rows.append( { @@ -329,11 +341,33 @@ def rank_merge_candidates( def tracking_config_for(occurrence: Occurrence) -> TrackingConfig: - """The settings tracking runs with on the occurrence's session: the threshold and the - feature requirement the previews here judge pairs by.""" + """The tracking settings the previews here judge pairs by: those of the latest successful + tracking run over the occurrence's session, so the scores shown match how the session was + tracked, or the defaults when no run covered it.""" + from ami.jobs.models import Job, JobState + from ami.main.models import SourceImageCollection from ami.ml.post_processing.tracking_task import TrackingConfig - return TrackingConfig(event_ids=[occurrence.event_id]) + event_id = occurrence.event_id + runs = Job.objects.filter( + project_id=occurrence.project_id, + job_type_key="post_processing", + status=JobState.SUCCESS.name, + params__task="tracking", + ).order_by(F("finished_at").desc(nulls_last=True), "-pk") + for job in runs.only("params")[:50]: + config = (job.params or {}).get("config") or {} + covers = event_id in (config.get("event_ids") or []) + if not covers and config.get("source_image_collection_id"): + covers = SourceImageCollection.objects.filter( + pk=config["source_image_collection_id"], images__event_id=event_id + ).exists() + if covers: + try: + return TrackingConfig(**{**config, "event_ids": [event_id], "source_image_collection_id": None}) + except pydantic.ValidationError: + break # a run stored with settings this version no longer reads + return TrackingConfig(event_ids=[event_id]) def _likelihood(cost: float, similarity: float | None) -> float: @@ -344,20 +378,32 @@ def _likelihood(cost: float, similarity: float | None) -> float: return round(min(max(1 - cost / terms, 0.0), 1.0), _ROUND_TO) -def _pair_scores(bbox_a, bbox_b, vector_a, vector_b, diag: float) -> dict[str, float | None]: - """The tracking cost between two boxes, each of its terms, and the likelihood.""" - from ami.ml.post_processing.tracking_task import box_ratio, cosine_similarity, distance_ratio, iou, total_cost +def _pair_scores( + bbox_a, + bbox_b, + vector_a, + vector_b, + diag: float, + options: LinkOptions | None = None, + distance_multiplier: float = 1.0, +) -> dict[str, float | None]: + """The tracking cost between two boxes, each of its raw terms, and the likelihood. + + The cost is the tracker's own ``weighted_cost`` under the given link options (from + ``tracking_config_for``: the session's latest tracking run, else the defaults), so non-default weights, appearance + calibration or move rule would change it here as they do in a run. + """ + from ami.ml.post_processing.tracking_task import DEFAULT_LINK_OPTIONS, pair_terms, weighted_cost - cost = total_cost(vector_a, vector_b, bbox_a, bbox_b, diag) - similarity = None - if vector_a is not None and vector_b is not None: - similarity = round(cosine_similarity(vector_a, vector_b), _ROUND_TO) + terms = pair_terms(vector_a, vector_b, bbox_a, bbox_b, diag) + cost = weighted_cost(terms, options or DEFAULT_LINK_OPTIONS, distance_multiplier) + similarity = None if terms.appearance is None else round(1 - terms.appearance, _ROUND_TO) return { "likelihood": _likelihood(cost, similarity), "cost": round(cost, _ROUND_TO), - "distance": round(distance_ratio(bbox_a, bbox_b, diag), _ROUND_TO), - "iou": round(iou(bbox_a, bbox_b), _ROUND_TO), - "size_ratio": round(box_ratio(bbox_a, bbox_b), _ROUND_TO), + "distance": round(terms.distance, _ROUND_TO), + "iou": round(terms.iou, _ROUND_TO), + "size_ratio": round(terms.size_ratio, _ROUND_TO), "similarity": similarity, } @@ -387,9 +433,10 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di The reference is the track frame nearest in time on another capture. The tracker's own matcher runs between every box on the reference frame's capture and every box on - ``capture``, with the tracker's default settings, feature algorithm and image diagonal, - and ``would_link`` marks the box it links the reference frame to. Tracking only pairs - adjacent captures, so across a longer gap this previews its pairing rule, not a run. + ``capture``, with the settings ``tracking_config_for`` returns (the session's latest tracking run), the + session's feature algorithm and the image diagonal, and ``would_link`` marks the box it + links the reference frame to. Tracking only pairs adjacent captures, so across a longer + gap this previews its pairing rule, not a run. The track's own box is returned unscored. So is every box when the track has no other frame, or when the earlier of the two captures has no dimensions, since tracking skips @@ -398,9 +445,15 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di """ from ami.main.models import Detection, SourceImage from ami.ml.models import Algorithm - from ami.ml.post_processing.tracking_task import image_diagonal, resolve_feature_algorithm, select_links + from ami.ml.post_processing.tracking_task import ( + activity_multiplier, + image_diagonal, + resolve_feature_algorithm, + select_links, + ) config = tracking_config_for(occurrence) + options = config.link_options() reference, relation = _reference_frame(occurrence.pk, capture) if capture.timestamp else (None, None) reference_capture_id = reference["source_image_id"] if reference is not None else None detections = list( @@ -424,7 +477,7 @@ def match_capture_detections(occurrence: Occurrence, capture: SourceImage) -> di def skipped(detection_id: int) -> str | None: return SKIPPED_NO_VECTOR if config.require_features and detection_id not in vectors else None - diag, linked, reference_detection = None, set(), None + diag, linked, reference_detection, multiplier = None, set(), None, 1.0 if reference is not None: reference_detection = next(d for d in detections if d.pk == reference["pk"]) reference_boxes = [d for d in detections if d.source_image_id == reference_capture_id] @@ -437,8 +490,11 @@ def skipped(detection_id: int) -> str | None: if width and height: diag = image_diagonal(width, height) current, following = (boxes, reference_boxes) if capture_first else (reference_boxes, boxes) - links = select_links(current, following, vectors, diag, config.cost_threshold, config.require_features) + links = select_links( + current, following, vectors, diag, config.cost_threshold, config.require_features, options + ) linked = {frozenset((det.pk, nxt.pk)) for det, nxt, _ in links} + multiplier = activity_multiplier(max(len(current), len(following)), options) # Captures from the reference frame's to this one, signed: 1 is the adjacent capture the # tracker would pair, so a larger count means the preview spans captures it never compares. @@ -480,6 +536,8 @@ def skipped(detection_id: int) -> str | None: vectors.get(reference["pk"]), vectors.get(box.pk), diag, + options, + multiplier, ) ) rows.append(row) diff --git a/ami/main/tests.py b/ami/main/tests.py index 9843ce29c..5a8bc8fd0 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -23,7 +23,7 @@ from ami.base.permissions import add_m2m_object_permissions from ami.exports.models import DataExport -from ami.jobs.models import VALID_JOB_TYPES, Job +from ami.jobs.models import VALID_JOB_TYPES, Job, JobState from ami.main.api.serializers import MAX_BULK_IDENTIFICATIONS from ami.main.models import ( Classification, @@ -65,7 +65,9 @@ image_diagonal, iou, pair_detections, + pair_terms, total_cost, + weighted_cost, ) from ami.tests.fixtures.main import ( create_captures, @@ -10213,6 +10215,39 @@ def test_candidates_carry_every_term_of_the_cost_and_the_trackers_decision(self) self.assertLess(by_id[far.pk]["likelihood"], by_id[same_box.pk]["likelihood"]) self.assertFalse(by_id[far.pk]["would_link"]) + def test_candidates_are_scored_with_the_configured_link_options(self): + """The picker's cost is the tracker's scored cost under the settings from + ``tracking_config_for`` (mocked here to pin the settings), so an + appearance calibration changes the cost and the appearance gate stops a pair from + linking here as it does in a run.""" + extractor = Algorithm.objects.create(name="Feature extractor", key="feature-extractor") + vector, unlike_vector = [1.0] + [0.0] * 2047, [0.6, 0.8] + [0.0] * 2046 + self._give_target_vectors(vector, extractor) + track_box = [10, 10, 40, 40] + alike = self._make_occurrence([self.after_capture], bbox=track_box, vector=vector, algorithm=extractor) + unlike = self._make_occurrence([self.after_capture], bbox=track_box, vector=unlike_vector, algorithm=extractor) + config = TrackingConfig( + event_ids=[self.event.pk], + appearance_similarity_floor=0.5, + appearance_similarity_ceiling=0.9, + appearance_min_similarity=0.8, + ) + + with mock.patch("ami.main.api.views.tracking_config_for", return_value=config): + rows = self.get_candidates().data["candidates"] + + by_id = {row["id"]: row for row in rows} + diagonal = image_diagonal(self.FRAME_SIZE, self.FRAME_SIZE) + expected = weighted_cost( + pair_terms(vector, unlike_vector, track_box, track_box, diagonal), config.link_options() + ) + self.assertAlmostEqual(by_id[unlike.pk]["cost"], expected, places=4) + self.assertAlmostEqual(expected, 0.75, places=4, msg="Similarity 0.6 maps to 0.75 between 0.5 and 0.9") + self.assertNotAlmostEqual(expected, total_cost(vector, unlike_vector, track_box, track_box, diagonal)) + self.assertEqual([row["id"] for row in rows], [alike.pk, unlike.pk]) + self.assertTrue(by_id[alike.pk]["would_link"]) + self.assertFalse(by_id[unlike.pk]["would_link"], "The appearance gate forbids this pair") + def test_adjacent_captures_are_searched_by_default(self): """The default search is the one capture on either side of the track, since that is where the frame continuing it sits; `captures` widens it by count and `minutes` by time.""" @@ -10446,7 +10481,8 @@ def test_the_candidate_count_does_not_change_the_query_count(self): # seven ranking queries: the track's frames, the capture ids before and after # it, the frames in those captures, the candidates, and the two vector sides, # each reading embeddings and classification vectors together. - with self.assertNumQueries(13): + # Includes one query for the session's latest tracking run, whose settings the preview uses. + with self.assertNumQueries(14): response = self.get_candidates() self.assertEqual(response.status_code, 200, response.data) @@ -10500,7 +10536,8 @@ def test_a_detection_ranking_has_a_fixed_query_count(self): # Uncached: the savepoint pair, the object lookup with its identifications and # permission checks, then the same seven ranking queries as for the whole track, # the frame lookup doubling as the check that the detection is the occurrence's. - with cachalot_disabled(), self.assertNumQueries(15): + # Includes one query for the session's latest tracking run, whose settings the preview uses. + with cachalot_disabled(), self.assertNumQueries(16): response = self.get_candidates(f"&detection={stray.pk}") self.assertEqual(response.status_code, 200, response.data) @@ -10508,6 +10545,52 @@ def test_a_detection_ranking_has_a_fixed_query_count(self): self.assertTrue(all(row["similarity"] == 1.0 for row in response.data["candidates"])) +class TrackingConfigForPreviewsTestCase(TrackFixtureTestCase): + """The merge and capture-match previews judge pairs with the settings the session was tracked with. + + Scores shown against the defaults disagree with the tracks on screen when the session was + tracked with other settings (another threshold, another feature extractor), so the previews + read the latest successful tracking run over the session and fall back to the defaults only + when no run covered it. + """ + + def _tracking_job(self, status: str, **config) -> Job: + return Job.objects.create( + project=self.project, + name="tracking run", + job_type_key="post_processing", + status=status, + params={"task": "tracking", "config": {"event_ids": [self.event.pk], **config}}, + ) + + def test_defaults_when_no_run_covered_the_session(self): + from ami.main.models_future.merge_candidates import tracking_config_for + from ami.ml.post_processing.tracking_task import TrackingConfig + + self.assertEqual( + tracking_config_for(self.occurrence).cost_threshold, TrackingConfig(event_ids=[self.event.pk]).cost_threshold + ) + + def test_uses_the_latest_successful_run_over_the_session(self): + from ami.main.models_future.merge_candidates import tracking_config_for + + self._tracking_job(JobState.SUCCESS.name, cost_threshold=0.5) + self._tracking_job(JobState.SUCCESS.name, cost_threshold=0.9, require_features=False) + self._tracking_job(JobState.FAILURE.name, cost_threshold=1.5) + config = tracking_config_for(self.occurrence) + self.assertEqual((config.cost_threshold, config.require_features), (0.9, False)) + self.assertEqual(config.event_ids, [self.event.pk]) + + def test_prefers_the_runs_feature_extractor_over_the_most_common_one(self): + from ami.main.models_future.merge_candidates import _comparison_algorithm + + vectors = {(1, 12): [0.0], (2, 12): [0.0], (3, 12): [0.0], (1, 60): [0.0]} + self.assertEqual(_comparison_algorithm(vectors), 12) + self.assertEqual(_comparison_algorithm(vectors, preferred=60), 60) + # A preferred extractor with no vector on the track falls back to the most common one. + self.assertEqual(_comparison_algorithm(vectors, preferred=99), 12) + + class CaptureMatchesTestCase(APITestCase): """Previewing, box by box, what tracking would link on one capture of a track. @@ -10526,6 +10609,8 @@ class CaptureMatchesTestCase(APITestCase): NEAR_BOX = [11, 11, 41, 41] OFFSET_BOX = [12, 12, 42, 42] FAR_BOX = [500, 500, 530, 530] + # Cosine similarity 0.6 with VECTOR. + UNLIKE_VECTOR = [0.6, 0.8] + [0.0] * 2046 def setUp(self) -> None: self.project, self.deployment = setup_test_project(reuse=False) @@ -10659,6 +10744,37 @@ def test_a_box_matched_better_by_another_box_is_not_linked(self): self.assertFalse(row["would_link"]) self.assertGreater(row["likelihood"], 0.5) + def test_the_preview_uses_the_configured_link_options(self): + """The preview runs the matcher and scores each box with the settings from + ``tracking_config_for`` (mocked here to pin the settings): a box + that would link by default is not linked when the appearance gate forbids it, and its + cost is the calibrated one.""" + track = self._track(self.captures[1:3], vector=self.VECTOR) + capture = self.captures[3] + unlike = self._box(capture, self.NEAR_BOX, vector=self.UNLIKE_VECTOR) + config = TrackingConfig( + event_ids=[self.event.pk], + cost_threshold=1.0, + appearance_similarity_floor=0.5, + appearance_similarity_ceiling=0.9, + appearance_min_similarity=0.8, + ) + + with mock.patch("ami.main.models_future.merge_candidates.tracking_config_for", return_value=config): + row = self.get_matches(track, capture.pk).data["detections"][0] + without_gate = config.copy(update={"appearance_min_similarity": None}) + with mock.patch("ami.main.models_future.merge_candidates.tracking_config_for", return_value=without_gate): + ungated = self.get_matches(track, capture.pk).data["detections"][0] + + expected = weighted_cost( + pair_terms(self.VECTOR, self.UNLIKE_VECTOR, self.TRACK_BOX, self.NEAR_BOX, self.diagonal), + config.link_options(), + ) + self.assertEqual(row["detection_id"], unlike.pk) + self.assertAlmostEqual(row["cost"], expected, places=4) + self.assertFalse(row["would_link"], "The appearance gate forbids this pair") + self.assertTrue(ungated["would_link"], "Without the gate the same box links under the threshold") + def test_the_preview_links_what_a_tracking_pass_saves(self): """The preview and a tracking pass share one matcher, so on the same two captures the box marked as linked is the one pair_detections links the reference frame to.""" @@ -10834,7 +10950,8 @@ def test_the_query_count_does_not_grow_with_the_track_or_the_capture(self): # boxes on both captures, their extractors, that extractor, its vectors, and the count # of captures between the reference frame's capture and this one. for occurrence, capture, boxes in ((short, self.captures[3], 2), (long, dense, 12)): - with cachalot_disabled(), self.assertNumQueries(14): + # Includes one query for the session's latest tracking run, whose settings the preview uses. + with cachalot_disabled(), self.assertNumQueries(15): response = self.get_matches(occurrence, capture.pk) self.assertEqual(response.status_code, 200, response.data) self.assertEqual(len(response.data["detections"]), boxes) diff --git a/ami/ml/post_processing/tests/test_tracking_cost_terms.py b/ami/ml/post_processing/tests/test_tracking_cost_terms.py new file mode 100644 index 000000000..12cfad588 --- /dev/null +++ b/ami/ml/post_processing/tests/test_tracking_cost_terms.py @@ -0,0 +1,507 @@ +import io +import json +import logging +import pathlib +import random +import tempfile + +import numpy as np +import pydantic +from django.core.management import CommandError, call_command +from django.test import SimpleTestCase, TestCase +from django.utils import timezone + +from ami.main.management.commands.evaluate_tracking import expand_sweep +from ami.main.models import Classification, Detection, Event, Occurrence, Taxon +from ami.main.models_future.embeddings import vectors_for_detections +from ami.ml.models.algorithm import Algorithm +from ami.ml.post_processing.registry import staff_only_config_fields +from ami.ml.post_processing.tracking_evaluation import evaluate_tracks, format_sweep_markdown, summarise_session +from ami.ml.post_processing.tracking_task import ( + DEFAULT_LINK_OPTIONS, + AmbiguousSpeciesLabels, + LinkOptions, + PairTerms, + TopLabel, + TrackingConfig, + TrackingTask, + activity_multiplier, + appearance_term, + choose_links, + event_transition_pairs, + image_diagonal, + labels_conflict, + links_from_transition_pairs, + pair_terms, + propose_event_links, + resolve_feature_algorithm, + resolve_label_algorithm, + shift_in_box_sizes, + top_labels, + total_cost, + weighted_cost, +) +from ami.tests.fixtures.main import create_taxa, setup_test_project +from ami.tests.fixtures.tracking import create_tracking_session + +logger = logging.getLogger(__name__) + + +def _box(x: float, y: float, size: float = 40) -> list[float]: + return [x, y, x + size, y + size] + + +class TestCostTerms(SimpleTestCase): + def test_default_weights_give_exactly_the_plain_cost(self): + rng = random.Random(7) + for _ in range(200): + b1 = _box(rng.uniform(0, 500), rng.uniform(0, 500), rng.uniform(10, 90)) + b2 = _box(rng.uniform(0, 500), rng.uniform(0, 500), rng.uniform(10, 90)) + f1 = [rng.random() for _ in range(16)] if rng.random() > 0.2 else None + f2 = [rng.random() for _ in range(16)] + self.assertEqual(weighted_cost(pair_terms(f1, f2, b1, b2, 800)), total_cost(f1, f2, b1, b2, 800)) + + def test_weights_scale_their_own_term(self): + terms = PairTerms(appearance=0.1, iou=0.5, size_ratio=0.8, distance=0.05) + base = weighted_cost(terms) + self.assertAlmostEqual(weighted_cost(terms, LinkOptions(appearance_weight=3.0)), base + 0.2) + self.assertAlmostEqual(weighted_cost(terms, LinkOptions(iou_weight=0.0)), base - 0.5) + self.assertAlmostEqual(weighted_cost(terms, LinkOptions(distance_weight=2.0)), base + 0.05) + + def test_activity_multiplier_grows_only_past_the_reference_count(self): + log = LinkOptions(activity_scaling="log", activity_reference_count=5) + self.assertEqual(activity_multiplier(50, DEFAULT_LINK_OPTIONS), 1.0) + self.assertEqual(activity_multiplier(3, log), 1.0) + self.assertEqual(activity_multiplier(5, log), 1.0) + self.assertGreater(activity_multiplier(50, log), activity_multiplier(20, log)) + self.assertGreater(activity_multiplier(20, log), 1.0) + + steps = LinkOptions(activity_scaling="steps", activity_steps=((10, 1.5), (30, 3.0))) + self.assertEqual([activity_multiplier(n, steps) for n in (9, 10, 29, 30, 100)], [1.0, 1.5, 1.5, 3.0, 3.0]) + + def test_labels_conflict_only_for_confident_unrelated_taxa(self): + genus = TopLabel(taxon_id=1, score=0.9) + species = TopLabel(taxon_id=2, score=0.9, ancestor_ids=frozenset({1})) + other = TopLabel(taxon_id=3, score=0.9, ancestor_ids=frozenset({4})) + unsure = TopLabel(taxon_id=3, score=0.2) + self.assertTrue(labels_conflict(species, other, 0.5)) + self.assertFalse(labels_conflict(genus, species, 0.5), "A genus and its species are the same insect") + self.assertFalse(labels_conflict(species, species, 0.5)) + self.assertFalse(labels_conflict(species, unsure, 0.5), "A low score is not evidence") + self.assertFalse(labels_conflict(species, None, 0.5)) + + +class TestChooseLinks(SimpleTestCase): + # Detection 1 sits still but its crop looks different; detection 2 flew in and looks the same. + STILL = PairTerms(appearance=0.18, iou=0.95, size_ratio=1.0, distance=0.002) + MOVED = PairTerms(appearance=0.0, iou=0.3, size_ratio=0.95, distance=0.02) + + def test_defaults_take_the_lowest_cost_one_link_per_detection(self): + pairs = [(1, 10, self.STILL), (2, 10, self.MOVED)] + links = choose_links(pairs, cost_threshold=1.0) + self.assertEqual([(a, b) for a, b, _ in links], [(1, 10)]) + links = choose_links([(2, 10, self.MOVED), (1, 10, self.STILL)], cost_threshold=1.0) + self.assertEqual([(a, b) for a, b, _ in links], [(1, 10)]) + + def test_stationary_pass_claims_a_still_insect_before_a_cheaper_moving_pair(self): + still = PairTerms(appearance=0.3, iou=0.95, size_ratio=1.0, distance=0.002) + moved = PairTerms(appearance=0.0, iou=0.7, size_ratio=1.0, distance=0.02) + pairs = [(1, 10, still), (2, 10, moved)] + self.assertEqual([(a, b) for a, b, _ in choose_links(pairs, cost_threshold=1.0)], [(2, 10)]) + options = LinkOptions(stationary_first=True, stationary_cost_threshold=0.5) + self.assertEqual([(a, b) for a, b, _ in choose_links(pairs, 1.0, options=options)], [(1, 10)]) + # A still pair above the stationary threshold waits for the normal pass. + strict = LinkOptions(stationary_first=True, stationary_cost_threshold=0.1) + self.assertEqual([(a, b) for a, b, _ in choose_links(pairs, 1.0, options=strict)], [(2, 10)]) + + def test_stationary_pass_can_link_a_still_insect_without_an_embedding(self): + pairs = [(1, 10, PairTerms(appearance=None, iou=0.95, size_ratio=1.0, distance=0.001))] + self.assertEqual(choose_links(pairs, 1.0, require_features=True), []) + stationary = LinkOptions(stationary_first=True) + self.assertEqual(choose_links(pairs, 1.0, require_features=True, options=stationary), []) + allowed = LinkOptions(stationary_first=True, stationary_allow_missing_features=True) + self.assertEqual(len(choose_links(pairs, 1.0, require_features=True, options=allowed)), 1) + + def test_species_gate_forbids_or_penalises_confident_disagreement(self): + labels = {1: TopLabel(5, 0.9), 10: TopLabel(6, 0.9)} + pairs = [(1, 10, self.STILL)] + cost = weighted_cost(self.STILL) + self.assertEqual(len(choose_links(pairs, 1.0, labels=labels)), 1, "The gate is off by default") + forbid = LinkOptions(species_gate="forbid") + self.assertEqual(choose_links(pairs, 1.0, options=forbid, labels=labels), []) + penalty = LinkOptions(species_gate="penalty", species_gate_penalty=0.25) + ((_, _, penalised),) = choose_links(pairs, 1.0, options=penalty, labels=labels) + self.assertAlmostEqual(penalised, cost + 0.25) + self.assertEqual(choose_links(pairs, cost + 0.1, options=penalty, labels=labels), []) + + def test_activity_scaling_rejects_the_same_move_on_a_crowded_sheet(self): + moved = PairTerms(appearance=0.0, iou=0.5, size_ratio=1.0, distance=0.1) + options = LinkOptions(activity_scaling="log", activity_reference_count=5) + self.assertEqual(len(choose_links([(1, 10, moved)], 0.65, options=options, detection_count=4)), 1) + self.assertEqual(choose_links([(1, 10, moved)], 0.65, options=options, detection_count=60), []) + + +class TestAppearanceAndMoveRules(SimpleTestCase): + # Two boxes of side 40 whose centres are 80 px apart: no overlap, a shift of two box sizes. + APART = dict(iou=0.0, size_ratio=1.0, distance=0.02, shift=2.0) + + def test_calibration_maps_the_similarity_range_onto_zero_to_one(self): + options = LinkOptions(appearance_similarity_floor=0.4, appearance_similarity_ceiling=0.9) + self.assertEqual(appearance_term(1 - 0.95, options), 0.0) + self.assertAlmostEqual(appearance_term(1 - 0.65, options), 0.5) + self.assertEqual(appearance_term(1 - 0.2, options), 1.0) + self.assertEqual(appearance_term(0.123, DEFAULT_LINK_OPTIONS), 0.123) + + def test_appearance_gate_forbids_only_pairs_whose_embeddings_disagree(self): + alike = PairTerms(appearance=0.1, iou=0.9, size_ratio=1.0, distance=0.001) + unlike = PairTerms(appearance=0.6, iou=0.9, size_ratio=1.0, distance=0.001) + no_vector = PairTerms(appearance=None, iou=0.9, size_ratio=1.0, distance=0.001) + gate = LinkOptions(appearance_min_similarity=0.5) + self.assertEqual(len(choose_links([(1, 10, unlike)], 1.0)), 1, "The gate is off by default") + self.assertEqual(choose_links([(1, 10, unlike)], 1.0, options=gate), []) + self.assertEqual(len(choose_links([(1, 10, alike)], 1.0, options=gate)), 1) + self.assertEqual(len(choose_links([(1, 10, no_vector)], 1.0, require_features=False, options=gate)), 1) + + def test_move_rule_lets_a_look_alike_link_clear_of_its_old_box(self): + alike = PairTerms(appearance=0.05, **self.APART) + unlike = PairTerms(appearance=0.4, **self.APART) + self.assertEqual(choose_links([(1, 10, alike)], 1.0), [], "Disjoint boxes cannot link at 1.0 by default") + move = LinkOptions(motion_min_similarity=0.9, motion_max_shift=4.0) + ((_, _, cost),) = choose_links([(1, 10, alike)], 1.0, options=move) + self.assertAlmostEqual(cost, 0.05 + 0.5 + 0.02) + self.assertEqual(choose_links([(1, 10, unlike)], 1.0, options=move), []) + far = PairTerms(appearance=0.05, iou=0.0, size_ratio=1.0, distance=0.1, shift=8.0) + self.assertEqual(choose_links([(1, 10, far)], 1.0, options=move), [], "The shift is capped at one overlap") + without_vector = PairTerms(appearance=None, **self.APART) + self.assertEqual(choose_links([(1, 10, without_vector)], 1.0, require_features=False, options=move), []) + + def test_shift_is_measured_in_box_sizes(self): + self.assertAlmostEqual(shift_in_box_sizes(_box(0, 0, 39), _box(80, 0, 39)), 2.0) + self.assertAlmostEqual(shift_in_box_sizes(_box(0, 0, 9), _box(0, 30, 9)), 3.0) + self.assertEqual(pair_terms(None, None, _box(0, 0), _box(0, 0), 800).shift, 0.0) + + def test_a_malformed_box_does_not_stop_scoring(self): + """Pairs are scored with the shift whether or not the move rule is on, so a box with no + area, or corners the wrong way round, must score as before instead of raising.""" + for malformed in ([10, 10, 9, 20], [10, 10, 5, 20]): + terms = pair_terms(None, None, malformed, [10, 10, 20, 20], 800) + self.assertEqual(terms.shift, float("inf")) + self.assertEqual( + weighted_cost(terms), total_cost(None, None, malformed, [10, 10, 20, 20], 800), msg=malformed + ) + + +class TestTrackingConfigTerms(SimpleTestCase): + def test_defaults_leave_every_new_rule_off(self): + config = TrackingConfig(event_ids=[1]) + self.assertEqual(config.link_options(), DEFAULT_LINK_OPTIONS) + + def test_invalid_settings_are_refused(self): + for bad in ( + {"species_gate": "sometimes"}, + {"appearance_weight": -1}, + {"species_gate_min_score": 1.5}, + {"activity_scaling": "steps"}, + {"activity_scaling": "steps", "activity_steps": [[30, 2.0], [10, 1.5]]}, + {"activity_steps": [[10, 0]]}, + {"appearance_similarity_floor": 0.9, "appearance_similarity_ceiling": 0.4}, + {"appearance_min_similarity": 1.2}, + {"motion_max_shift": 0}, + ): + with self.subTest(bad=bad), self.assertRaises(pydantic.ValidationError): + TrackingConfig(event_ids=[1], **bad) + + def test_new_rules_are_staff_only(self): + config = {"species_gate": "forbid", "stationary_first": True, "appearance_weight": 2.0, "cost_threshold": 0.5} + self.assertEqual( + staff_only_config_fields("tracking", config), ["appearance_weight", "species_gate", "stationary_first"] + ) + self.assertEqual(staff_only_config_fields("tracking", {"species_gate": "off"}), []) + calibrated = { + "appearance_similarity_floor": 0.4, + "appearance_min_similarity": 0.5, + "motion_min_similarity": 0.8, + } + self.assertEqual(staff_only_config_fields("tracking", calibrated), sorted(calibrated)) + + +class TestSweepReporting(SimpleTestCase): + def test_multi_detection_scores_leave_out_single_detection_tracks(self): + truth = {1: "A", 2: "A", 3: "B", 4: "C", 5: "C"} + times = {1: 1, 2: 2, 3: 1, 4: 1, 5: 2} + result = evaluate_tracks(truth, {1: "x", 2: "x", 3: "y", 4: "z", 5: "w"}, times) + self.assertEqual(result.exactly_recovered, 2, "A and the singleton B") + self.assertEqual((result.multi_detection_exactly_recovered, result.multi_detection_tracks), (1, 2)) + self.assertEqual(result.multi_detection_mean_completeness, 0.75) + self.assertIsNone(result.cross_species_merges) + + def test_cross_species_merges_count_tracks_joining_different_determinations(self): + truth = {1: "A", 2: "B", 3: "C", 4: "D"} + times = {1: 1, 2: 2, 3: 1, 4: 2} + taxa = {"A": 10, "B": 10, "C": 10, "D": 11} + result = evaluate_tracks(truth, {1: "x", 2: "x", 3: "y", 4: "y"}, times, taxa) + self.assertEqual((result.merges, result.cross_species_merges), (2, 1)) + + def test_session_summary_counts_occurrences_and_species_before_and_after(self): + predictions = {1: 1, 2: 1, 3: 3, 4: 4} + labels = {1: ("moth-a", 0.9), 2: ("moth-b", 0.4), 3: ("moth-b", 0.8)} + summary = summarise_session(predictions, labels) + self.assertEqual((summary["occurrences_before"], summary["occurrences_after"]), (4, 3)) + self.assertEqual((summary["unique_determinations_before"], summary["unique_determinations_after"]), (2, 2)) + self.assertEqual((summary["track_length_median"], summary["track_length_max"]), (2, 2)) + self.assertEqual(summary["multi_detection_tracks"], 1) + merged = summarise_session({1: 1, 2: 1, 3: 1, 4: 4}, labels) + self.assertEqual(merged["unique_determinations_after"], 1, "The track takes its best label, moth-a") + + def test_sweep_expands_a_grid_on_a_base_and_merges_object_values(self): + settings = expand_sweep( + { + "base": {"require_features": False}, + "grid": { + "cost_threshold": [0.2, 0.4], + "gate": [{"species_gate": "off"}, {"species_gate": "penalty", "species_gate_penalty": 0.5}], + }, + "configs": [{"cost_threshold": 1.0}], + } + ) + self.assertEqual(len(settings), 5) + self.assertEqual( + settings[1], + {"require_features": False, "cost_threshold": 0.2, "species_gate": "penalty", "species_gate_penalty": 0.5}, + ) + self.assertEqual(settings[-1], {"require_features": False, "cost_threshold": 1.0}) + with self.assertRaises(CommandError): + expand_sweep({"grid": {}, "other": 1}) + + def test_markdown_shows_one_table_per_scope_and_only_the_settings_that_vary(self): + truth, times = {1: "A", 2: "A"}, {1: 1, 2: 2} + from ami.ml.post_processing.tracking_evaluation import sweep_row + + rows = [] + for run, threshold in ((1, 0.2), (2, 0.4)): + evaluation = evaluate_tracks(truth, {1: "x", 2: "x"}, times, {"A": 1}) + session = summarise_session({1: 1, 2: 1}, {1: (1, 0.9)}) + settings = {"cost_threshold": threshold, "require_features": True} + rows.append(sweep_row(run, settings, 7, evaluation, session, 1)) + rows.append(sweep_row(run, settings, "overall", evaluation, session, 1)) + markdown = format_sweep_markdown(rows) + self.assertIn("## Session 7", markdown) + self.assertLess(markdown.index("## Session 7"), markdown.index("## Overall")) + self.assertIn("| run | cost_threshold | links |", markdown) + self.assertNotIn("require_features", markdown) + self.assertIn("| 2 | 0.4 | 1 | 1.000 | 1.000 | 1.000 |", markdown) + # A near-perfect score must not print as perfect. + rows[0]["link_precision"] = 0.9996 + self.assertIn("| 1 | 0.2 | 1 | 0.999 |", format_sweep_markdown(rows)) + + +def _reference_links(event: Event, algorithm, cost_threshold: float, require_features: bool) -> set: + """The links the tracker made before the optional cost rules existed, kept here as a frozen copy.""" + captures = list(event.captures.order_by("timestamp")) + links = set() + for cur, nxt in zip(captures, captures[1:]): + if not cur.width or not cur.height: + continue + current, following = list(cur.detections.valid()), list(nxt.detections.valid()) + vectors = vectors_for_detections([d.pk for d in current + following], algorithm.pk) if algorithm else {} + diag = image_diagonal(cur.width, cur.height) + candidates = [] + for det in current: + if vectors.get(det.pk) is None and require_features: + continue + for other in following: + if vectors.get(other.pk) is None and require_features: + continue + cost = total_cost(vectors.get(det.pk), vectors.get(other.pk), det.bbox, other.bbox, diag) + if cost < cost_threshold: + candidates.append((cost, det.pk, other.pk)) + claimed_a, claimed_b = set(), set() + for cost, a, b in sorted(candidates): + if a in claimed_a or b in claimed_b: + continue + claimed_a.add(a) + claimed_b.add(b) + links.add((a, b)) + return links + + +class TestCostTermsOnATrackingSession(TestCase): + """The optional rules change nothing by default, and the sweep scores what a run would do.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.project, cls.deployment = setup_test_project(reuse=False) + cls.ground_truth = create_tracking_session( + cls.deployment, + taxa_list=create_taxa(cls.project), + num_frames=10, + num_moths=4, + num_transient_moths=2, + min_frames_per_moth=4, + motion_scale=0.4, + create_crops=False, + ) + cls.event = Event.objects.get(pk=cls.ground_truth.event_id) + + def _algorithm(self, require_features: bool = True): + algorithm, _, _ = resolve_feature_algorithm( + self.event, TrackingConfig(event_ids=[self.event.pk], require_features=require_features) + ) + return algorithm + + def test_default_settings_give_the_same_links_as_before_the_new_rules(self): + algorithm = self._algorithm() + for threshold, require_features in ((0.2, True), (0.6, True), (0.6, False), (1.5, False)): + with self.subTest(threshold=threshold, require_features=require_features): + config = TrackingConfig( + event_ids=[self.event.pk], cost_threshold=threshold, require_features=require_features + ) + proposed = {(a, b) for a, b, _ in propose_event_links(self.event, algorithm, config, logger)} + expected = _reference_links(self.event, algorithm, threshold, require_features) + self.assertEqual(proposed, expected) + self.assertTrue(expected, "The loosest setting links something, so the comparison is not vacuous") + + def test_precomputed_pairs_give_the_same_links_as_a_run_for_any_setting(self): + algorithm = self._algorithm() + transitions, _ = event_transition_pairs(self.event, algorithm) + detection_ids = list(Detection.objects.filter(source_image__event=self.event).values_list("pk", flat=True)) + labels = top_labels(detection_ids, resolve_label_algorithm(detection_ids, None)) + for extra in ( + {}, + {"species_gate": "forbid", "species_gate_min_score": 0.0}, + {"species_gate": "penalty", "species_gate_min_score": 0.0, "species_gate_penalty": 0.3}, + {"stationary_first": True, "stationary_max_shift": 0.05, "stationary_min_iou": 0.3}, + {"activity_scaling": "log", "activity_reference_count": 1, "appearance_weight": 0.5}, + { + "appearance_similarity_floor": 0.3, + "appearance_similarity_ceiling": 0.9, + "appearance_min_similarity": 0.2, + "motion_min_similarity": 0.5, + }, + ): + with self.subTest(extra=extra): + config = TrackingConfig(event_ids=[self.event.pk], cost_threshold=0.8, require_features=False, **extra) + proposed = propose_event_links(self.event, algorithm, config, logger) + self.assertEqual(links_from_transition_pairs(transitions, config, labels), proposed) + + def test_species_gate_blocks_links_between_differently_labelled_insects(self): + # Every simulated insect has its own species, so a forbid gate at any score can only + # remove links between different insects; with none of those the links are unchanged. + config = TrackingConfig(event_ids=[self.event.pk], cost_threshold=0.8, require_features=False) + gated = config.copy(update={"species_gate": "forbid", "species_gate_min_score": 0.0}) + plain = set(propose_event_links(self.event, None, config, logger)) + blocked = set(propose_event_links(self.event, None, gated, logger)) + self.assertTrue(blocked <= plain) + truth = {d: insect.identifier for insect in self.ground_truth.insects for d in insect.detection_ids} + self.assertTrue(all(truth[a] == truth[b] for a, b, _ in blocked)) + + def test_species_labels_come_from_one_classifier(self): + # A second classifier's more confident label on the same crop must not replace the + # first one's, and with no classifier chosen the gate refuses to mix the two. + detection_ids = list(Detection.objects.filter(source_image__event=self.event).values_list("pk", flat=True)) + first_id = resolve_label_algorithm(detection_ids, None) + other = Algorithm.objects.create(name="Second classifier", key="second-classifier-test") + own = Classification.objects.filter(detection_id=detection_ids[0], algorithm_id=first_id).first() + Classification.objects.create( + detection_id=detection_ids[0], + algorithm=other, + taxon=Taxon.objects.exclude(pk=own.taxon_id).first(), + score=0.99, + terminal=True, + timestamp=timezone.now(), + ) + self.assertEqual(top_labels(detection_ids, first_id)[detection_ids[0]].taxon_id, own.taxon_id) + self.assertEqual(top_labels(detection_ids, other.pk)[detection_ids[0]].score, 0.99) + with self.assertRaises(AmbiguousSpeciesLabels): + resolve_label_algorithm(detection_ids, None) + self.assertEqual(resolve_label_algorithm(detection_ids, first_id), first_id) + + gated = {"species_gate": "forbid", "species_label_algorithm_id": None} + config = TrackingConfig(event_ids=[self.event.pk], require_features=False, **gated) + with self.assertRaises(AmbiguousSpeciesLabels): + propose_event_links(self.event, None, config, logger) + TrackingTask(logger=logger, event_ids=[self.event.pk], require_features=False, **gated).run() + self.assertEqual(Occurrence.objects.filter(event=self.event).count(), len(detection_ids)) + + def _confirm_tracks(self) -> None: + TrackingTask(logger=logger, event_ids=[self.event.pk], require_features=False, cost_threshold=0.8).run() + Occurrence.objects.filter(event=self.event).update(grouping_verified_at=timezone.now()) + + def test_sweep_writes_a_table_per_session_and_changes_nothing(self): + self._confirm_tracks() + before = ( + list(Detection.objects.order_by("pk").values_list("pk", "occurrence_id", "next_detection_id")), + Occurrence.objects.count(), + Classification.objects.count(), + ) + sweep = { + "base": {"require_features": False}, + "grid": {"cost_threshold": [0.0, 0.8], "species_gate": ["off", "forbid"]}, + } + with tempfile.TemporaryDirectory() as directory: + per_track = pathlib.Path(directory, "tracks.csv") + output = io.StringIO() + call_command( + "evaluate_tracking", + "--project", + str(self.project.pk), + "--sweep", + json.dumps(sweep), + "--output-dir", + directory, + "--per-track-csv", + str(per_track), + stdout=output, + ) + document = json.loads(pathlib.Path(directory, "sweep.json").read_text()) + markdown = pathlib.Path(directory, "sweep.md").read_text() + csv_lines = per_track.read_text().splitlines() + + after = ( + list(Detection.objects.order_by("pk").values_list("pk", "occurrence_id", "next_detection_id")), + Occurrence.objects.count(), + Classification.objects.count(), + ) + self.assertEqual(after, before) + self.assertEqual(len(document["runs"]), 4) + self.assertEqual([row["scope"] for row in document["rows"]], [self.event.pk, "overall"] * 4) + nothing, everything = document["rows"][1], document["rows"][5] + self.assertEqual(nothing["links_proposed"], 0) + self.assertEqual(nothing["occurrences_after"], nothing["occurrences_before"]) + self.assertEqual(everything["link_precision"], 1.0) + self.assertEqual(everything["cross_individual_merges"], 0) + self.assertEqual(everything["cross_species_merges"], 0) + # Each simulated insect has its own species, so only the ungated run can join two labels. + self.assertEqual(document["rows"][7]["links_with_conflicting_labels"], 0) + self.assertLess(everything["occurrences_after"], everything["occurrences_before"]) + self.assertIn(f"## Session {self.event.pk}", markdown) + self.assertIn("species_gate", markdown) + self.assertEqual(csv_lines[0].split(",")[:3], ["run", "event_id", "kind"]) + self.assertGreater(len(csv_lines), 4) + + def test_vectors_file_replaces_the_stored_embeddings(self): + self._confirm_tracks() + detections = list(Detection.objects.filter(source_image__event=self.event).values_list("pk", flat=True)) + insect_of = {d: i for i, insect in enumerate(self.ground_truth.insects) for d in insect.detection_ids} + # One direction per insect, so each insect matches only itself on appearance. + vectors = np.eye(len(self.ground_truth.insects), 8)[[insect_of[d] for d in detections]] + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory, "vectors.npz") + np.savez(path, detection_ids=np.array(detections), vectors=vectors) + output = io.StringIO() + call_command( + "evaluate_tracking", + "--project", + str(self.project.pk), + "--vectors-file", + str(path), + "--cost-threshold", + "0.8", + "--format", + "json", + stdout=output, + ) + report = json.loads(output.getvalue()) + self.assertEqual(report["events"][0]["note"], "Embeddings from the vectors file.") + self.assertEqual(report["overall"]["merges"], 0) + self.assertGreater(report["overall"]["links_correct"], 0) diff --git a/ami/ml/post_processing/tracking_evaluation.py b/ami/ml/post_processing/tracking_evaluation.py index ca3d09e52..4ea49e9d4 100644 --- a/ami/ml/post_processing/tracking_evaluation.py +++ b/ami/ml/post_processing/tracking_evaluation.py @@ -17,6 +17,8 @@ import dataclasses import datetime import json +import math +import statistics import sys import typing from collections.abc import Hashable, Iterable, Mapping @@ -119,6 +121,16 @@ class TrackingEvaluation: mean_completeness: float | None mean_purity: float | None + # The same over confirmed tracks of two or more detections: a single-detection track is + # recovered by doing nothing, so it never counts here. + multi_detection_tracks: int + multi_detection_exactly_recovered: int + multi_detection_mean_completeness: float | None + + # Predicted tracks joining confirmed tracks whose determinations differ. None when the + # determinations were not given. + cross_species_merges: int | None + ground_truth_track_scores: list[GroundTruthTrackScore] predicted_track_scores: list[PredictedTrackScore] @@ -148,6 +160,9 @@ def fmt(value: float | None) -> str: f"Exactly recovered: {self.exactly_recovered}/{self.ground_truth_tracks} " f"fragmented: {self.fragmented_tracks} merges: {self.merges}", f"Mean completeness {fmt(self.mean_completeness)} mean purity {fmt(self.mean_purity)}", + f"Multi-detection tracks exactly recovered: {self.multi_detection_exactly_recovered}/" + f"{self.multi_detection_tracks} mean completeness {fmt(self.multi_detection_mean_completeness)} " + f"cross-species merges: {'n/a' if self.cross_species_merges is None else self.cross_species_merges}", ] @@ -168,6 +183,7 @@ def evaluate_tracks( ground_truth: Mapping[DetectionId, TrackId], predictions: Mapping[DetectionId, TrackId], timestamps: Mapping[DetectionId, typing.Any], + ground_truth_taxa: Mapping[TrackId, typing.Any] | None = None, ) -> TrackingEvaluation: """Score predicted tracks against confirmed tracks. @@ -175,7 +191,8 @@ def evaluate_tracks( ``timestamps`` gives each ground-truth detection a sortable capture time. Only detections in ``ground_truth`` are scored. A predicted track is cut down to those detections, so its links are between consecutive scored detections; a scored detection missing from - ``predictions`` counts as a predicted track of its own. + ``predictions`` counts as a predicted track of its own. ``ground_truth_taxa`` optionally + gives each confirmed track's determination, to count merges across species. """ scored = list(ground_truth) missing_times = [d for d in scored if timestamps.get(d) is None] @@ -239,6 +256,13 @@ def evaluate_tracks( ) ) + multi = [score for score in gt_scores if score.length > 1] + cross_species = None + if ground_truth_taxa is not None: + cross_species = sum( + 1 for spans in gt_by_pred.values() if len({ground_truth_taxa.get(gt_id) for gt_id in spans} - {None}) > 1 + ) + return TrackingEvaluation( ground_truth_tracks=len(gt_tracks), detections=len(scored), @@ -262,6 +286,10 @@ def evaluate_tracks( exactly_recovered=sum(1 for s in gt_scores if s.exactly_recovered), mean_completeness=(sum(s.completeness for s in gt_scores) / len(gt_scores)) if gt_scores else None, mean_purity=(sum(s.purity for s in pred_scores) / len(pred_scores)) if pred_scores else None, + multi_detection_tracks=len(multi), + multi_detection_exactly_recovered=sum(1 for s in multi if s.exactly_recovered), + multi_detection_mean_completeness=(sum(s.completeness for s in multi) / len(multi)) if multi else None, + cross_species_merges=cross_species, ground_truth_track_scores=gt_scores, predicted_track_scores=pred_scores, ) @@ -297,6 +325,148 @@ def tracks_from_links( return track_of +def summarise_session( + predictions: Mapping[DetectionId, TrackId], labels: Mapping[DetectionId, tuple[typing.Any, float]] +) -> dict[str, typing.Any]: + """What tracking does to a whole session, confirmed or not: occurrences and species to review. + + ``predictions`` maps every detection in the session to its predicted track; ``labels`` + gives a detection's top label as ``(taxon, score)``. Before tracking every detection is + its own occurrence; after, each track is one, determined by its highest-scoring label + (the way an occurrence takes its best prediction). Unlabelled detections add no species. + The median track length is over tracks of two or more detections, since most detections + in a busy session stay alone and would pin the median at one. + """ + members: dict[TrackId, list[DetectionId]] = collections.defaultdict(list) + for detection_id, track_id in predictions.items(): + members[track_id].append(detection_id) + lengths = sorted(len(m) for m in members.values()) + multi_lengths = [n for n in lengths if n > 1] + + def determination(detection_ids: list[DetectionId]) -> typing.Any: + scored = [(labels[d][1], d) for d in detection_ids if d in labels] + if not scored: + return None + _, best = min(scored, key=lambda item: (-item[0], _sort_key(item[1]))) + return labels[best][0] + + after = {determination(m) for m in members.values()} - {None} + before = {labels[d][0] for d in predictions if d in labels} - {None} + return { + "detections": len(predictions), + "occurrences_before": len(predictions), + "occurrences_after": len(members), + "multi_detection_tracks": len(multi_lengths), + "track_length_median": statistics.median(multi_lengths) if multi_lengths else None, + "track_length_max": lengths[-1] if lengths else None, + "unique_determinations_before": len(before), + "unique_determinations_after": len(after), + } + + +def sweep_row( + index: int, + settings: Mapping[str, typing.Any], + scope: typing.Any, + evaluation: TrackingEvaluation, + session: Mapping[str, typing.Any] | None, + links_proposed: int, +) -> dict[str, typing.Any]: + """One line of a sweep table: one setting scored over one session, or over all of them.""" + session = session or {} + return { + "run": index, + "settings": dict(settings), + "scope": scope, + "links_proposed": links_proposed, + "link_precision": evaluation.link_precision, + "link_recall": evaluation.link_recall, + "link_f1": evaluation.link_f1, + "pairwise_precision": evaluation.pairwise_precision, + "pairwise_recall": evaluation.pairwise_recall, + "multi_detection_exactly_recovered": evaluation.multi_detection_exactly_recovered, + "multi_detection_tracks": evaluation.multi_detection_tracks, + "multi_detection_mean_completeness": evaluation.multi_detection_mean_completeness, + "ground_truth_singletons": evaluation.ground_truth_singletons, + "fragmented_tracks": evaluation.fragmented_tracks, + "cross_individual_merges": evaluation.merges, + "cross_species_merges": evaluation.cross_species_merges, + "links_with_conflicting_labels": session.get("links_with_conflicting_labels"), + "multi_detection_tracks_predicted": session.get("multi_detection_tracks"), + "track_length_median": session.get("track_length_median"), + "track_length_max": session.get("track_length_max"), + "occurrences_before": session.get("occurrences_before"), + "occurrences_after": session.get("occurrences_after"), + "unique_determinations_before": session.get("unique_determinations_before"), + "unique_determinations_after": session.get("unique_determinations_after"), + } + + +def format_sweep_markdown(rows: list[Mapping[str, typing.Any]]) -> str: + """Sweep rows as Markdown: one table per scope (each session, then overall), one line per setting. + + Only the settings that vary between runs are shown, to keep the tables narrow. + """ + + def fmt(value: typing.Any) -> str: + # Scores are cut, not rounded, to three places, so only a perfect score reads 1.000. + if value is None: + return "n/a" + if isinstance(value, float): + return f"{math.floor(value * 1000) / 1000:.3f}" + return str(value) + + def setting(value: typing.Any) -> str: + return "n/a" if value is None else str(value) + + all_settings = [row["settings"] for row in rows] + keys = sorted({key for settings in all_settings for key in settings}) + varying = [key for key in keys if len({json.dumps(s.get(key), default=str) for s in all_settings}) > 1] + columns = [ + ("links", lambda r: fmt(r["links_proposed"])), + ("link P", lambda r: fmt(r["link_precision"])), + ("link R", lambda r: fmt(r["link_recall"])), + ("link F1", lambda r: fmt(r["link_f1"])), + ("pair P", lambda r: fmt(r["pairwise_precision"])), + ("pair R", lambda r: fmt(r["pairwise_recall"])), + ("exact multi", lambda r: f"{r['multi_detection_exactly_recovered']}/{r['multi_detection_tracks']}"), + ("multi compl.", lambda r: fmt(r["multi_detection_mean_completeness"])), + ("GT singletons", lambda r: fmt(r["ground_truth_singletons"])), + ("fragmented", lambda r: fmt(r["fragmented_tracks"])), + ("x-indiv merges", lambda r: fmt(r["cross_individual_merges"])), + ("x-species merges", lambda r: fmt(r["cross_species_merges"])), + ("label-conflict links", lambda r: fmt(r.get("links_with_conflicting_labels"))), + ("tracks >1 det", lambda r: fmt(r["multi_detection_tracks_predicted"])), + ("track len med/max", lambda r: f"{fmt(r['track_length_median'])}/{fmt(r['track_length_max'])}"), + ("occurrences", lambda r: f"{fmt(r['occurrences_before'])} -> {fmt(r['occurrences_after'])}"), + ( + "species", + lambda r: f"{fmt(r['unique_determinations_before'])} -> {fmt(r['unique_determinations_after'])}", + ), + ] + scopes: list[typing.Any] = [] + for row in rows: + if row["scope"] not in scopes: + scopes.append(row["scope"]) + scopes.sort(key=lambda scope: (scope == "overall", _sort_key(scope))) + + lines: list[str] = [] + for scope in scopes: + lines.append(f"## {'Overall' if scope == 'overall' else f'Session {scope}'}") + lines.append("") + header = ["run", *varying, *(name for name, _ in columns)] + lines.append("| " + " | ".join(header) + " |") + lines.append("|" + "---|" * len(header)) + for row in rows: + if row["scope"] != scope: + continue + cells = [str(row["run"]), *(setting(row["settings"].get(key)) for key in varying)] + cells.extend(render(row) for _, render in columns) + lines.append("| " + " | ".join(cells) + " |") + lines.append("") + return "\n".join(lines) + + # CSV adapter: the tracks export writes one row per detection, grouped by occurrence. diff --git a/ami/ml/post_processing/tracking_task.py b/ami/ml/post_processing/tracking_task.py index 403f3c5ae..52a1829b7 100644 --- a/ami/ml/post_processing/tracking_task.py +++ b/ami/ml/post_processing/tracking_task.py @@ -74,13 +74,82 @@ class TrackingConfig(pydantic.BaseModel): # or the project's default among several (see resolve_feature_algorithm). feature_extraction_algorithm_id: int | None = None + # Weight of each cost term. At 1.0 each the cost is the plain sum described above. + appearance_weight: float = pydantic.Field(1.0, ge=0) + iou_weight: float = pydantic.Field(1.0, ge=0) + size_weight: float = pydantic.Field(1.0, ge=0) + distance_weight: float = pydantic.Field(1.0, ge=0) + + # Species gate: two detections whose top labels both score at least species_gate_min_score + # and name unrelated taxa (neither is an ancestor of the other) are either never linked + # ("forbid") or have species_gate_penalty added to their cost ("penalty"). + species_gate: typing.Literal["off", "penalty", "forbid"] = "off" + species_gate_min_score: float = pydantic.Field(0.5, ge=0, le=1) + species_gate_penalty: float = pydantic.Field(1.0, ge=0) + # Which classifier's labels the gate compares. Scores from different models are not on + # one scale, so left unset the gate uses the only classifier that labelled the session + # and skips a session labelled by several. + species_label_algorithm_id: int | None = None + + # Activity scaling: the more detections a pair of captures holds, the more the distance + # term weighs, so a crowded sheet tolerates less movement. "log" multiplies it by + # log(1 + n) / log(1 + activity_reference_count) when that exceeds 1; "steps" uses the + # multiplier of the highest [count, multiplier] step in activity_steps that n reaches. + activity_scaling: typing.Literal["off", "log", "steps"] = "off" + activity_reference_count: int = pydantic.Field(5, ge=1) + activity_steps: list[tuple[int, float]] = [] + + # Stationary-first pass: pairs whose centre moved at most stationary_max_shift (share of + # the image diagonal), overlap by at least stationary_min_iou and cost less than + # stationary_cost_threshold are linked before any other pair, so a moving insect cannot + # take the place of one sitting still. With stationary_allow_missing_features, such a + # pair is linked on geometry alone even when require_features would skip it. + stationary_first: bool = False + stationary_max_shift: float = pydantic.Field(0.01, ge=0) + stationary_min_iou: float = pydantic.Field(0.7, ge=0, le=1) + stationary_cost_threshold: float = pydantic.Field(0.2, ge=0) + stationary_allow_missing_features: bool = False + + # Appearance calibration: cosine similarity at or above the ceiling costs 0, at or below the + # floor costs 1, linearly between. Each feature extractor has its own similarity range, so the + # defaults (0 and 1, the plain 1 - similarity) suit none in particular. + appearance_similarity_floor: float = pydantic.Field(0.0, ge=0, le=1) + appearance_similarity_ceiling: float = pydantic.Field(1.0, ge=0, le=1) + # Appearance gate: never link two detections whose embeddings are less similar than this. + # Pairs missing an embedding are not gated. + appearance_min_similarity: float | None = pydantic.Field(None, ge=0, le=1) + # Move rule: when two embeddings are at least this similar, the overlap term becomes at most + # the centre shift in box sizes divided by motion_max_shift, so an insect that moved clear of + # its old box can still link. Off (None) keeps the overlap term as it is. + motion_min_similarity: float | None = pydantic.Field(None, ge=0, le=1) + motion_max_shift: float = pydantic.Field(3.0, gt=0) + + @pydantic.validator("activity_steps") + def _steps_ascend(cls, steps: list[tuple[int, float]]) -> list[tuple[int, float]]: + counts = [count for count, _ in steps] + if counts != sorted(set(counts)): + raise ValueError("activity_steps counts must be strictly increasing") + if any(count < 0 or multiplier <= 0 for count, multiplier in steps): + raise ValueError("activity_steps need counts >= 0 and multipliers > 0") + return steps + @pydantic.root_validator(skip_on_failure=True) def _exactly_one_scope(cls, values: dict) -> dict: scopes = [values.get("source_image_collection_id"), values.get("event_ids") or None] if sum(s is not None for s in scopes) != 1: raise ValueError("Provide exactly one of source_image_collection_id or event_ids") + if values.get("activity_scaling") == "steps" and not values.get("activity_steps"): + raise ValueError("activity_scaling 'steps' needs activity_steps") + if values.get("appearance_similarity_floor", 0.0) >= values.get("appearance_similarity_ceiling", 1.0): + raise ValueError("appearance_similarity_floor must be below appearance_similarity_ceiling") return values + def link_options(self) -> "LinkOptions": + """The settings that decide which pairs link, apart from the threshold and feature requirement.""" + values = {field.name: getattr(self, field.name) for field in dataclasses.fields(LinkOptions)} + values["activity_steps"] = tuple(tuple(step) for step in self.activity_steps) + return LinkOptions(**values) + class Config: extra = "forbid" @@ -139,6 +208,266 @@ def total_cost(f1, f2, bb1, bb2, diag) -> float: return (1 - cosine_similarity(f1, f2)) + geometry +@dataclasses.dataclass(frozen=True) +class LinkOptions: + """The optional link rules of ``TrackingConfig``; the defaults leave linking as the plain cost sum.""" + + appearance_weight: float = 1.0 + iou_weight: float = 1.0 + size_weight: float = 1.0 + distance_weight: float = 1.0 + species_gate: str = "off" + species_gate_min_score: float = 0.5 + species_gate_penalty: float = 1.0 + activity_scaling: str = "off" + activity_reference_count: int = 5 + activity_steps: tuple[tuple[int, float], ...] = () + stationary_first: bool = False + stationary_max_shift: float = 0.01 + stationary_min_iou: float = 0.7 + stationary_cost_threshold: float = 0.2 + stationary_allow_missing_features: bool = False + appearance_similarity_floor: float = 0.0 + appearance_similarity_ceiling: float = 1.0 + appearance_min_similarity: float | None = None + motion_min_similarity: float | None = None + motion_max_shift: float = 3.0 + + +DEFAULT_LINK_OPTIONS = LinkOptions() + + +@dataclasses.dataclass(frozen=True) +class PairTerms: + """The raw cost terms of one pair of detections. ``appearance`` is 1 - cosine similarity, + or None when either detection has no embedding. ``shift`` is the centre shift in box sizes + (the geometric mean of the two boxes' sides), used only by the move rule.""" + + appearance: float | None + iou: float + size_ratio: float + distance: float + shift: float | None = None + + +@dataclasses.dataclass(frozen=True) +class TopLabel: + """A detection's highest-scoring terminal label and the ids of that taxon's ancestors.""" + + taxon_id: int + score: float + ancestor_ids: frozenset[int] = frozenset() + + +def shift_in_box_sizes(bb1, bb2) -> float: + area1 = (bb1[2] - bb1[0] + 1) * (bb1[3] - bb1[1] + 1) + area2 = (bb2[2] - bb2[0] + 1) * (bb2[3] - bb2[1] + 1) + # Every pair is scored with this, move rule or not, so a malformed box must not raise. + # An infinite shift leaves the overlap term as it is. + if area1 <= 0 or area2 <= 0: + return math.inf + shift = math.dist(((bb1[0] + bb1[2]) / 2, (bb1[1] + bb1[3]) / 2), ((bb2[0] + bb2[2]) / 2, (bb2[1] + bb2[3]) / 2)) + return shift / math.sqrt(math.sqrt(area1 * area2)) + + +def pair_terms(f1, f2, bb1, bb2, diag: float) -> PairTerms: + appearance = None if f1 is None or f2 is None else 1 - cosine_similarity(f1, f2) + return PairTerms( + appearance, iou(bb1, bb2), box_ratio(bb1, bb2), distance_ratio(bb1, bb2, diag), shift_in_box_sizes(bb1, bb2) + ) + + +def appearance_term(appearance: float, options: LinkOptions) -> float: + """The appearance cost after calibration: 0 at or above the similarity ceiling, 1 at or below the floor.""" + floor, ceiling = options.appearance_similarity_floor, options.appearance_similarity_ceiling + if floor == 0.0 and ceiling == 1.0: + return appearance # Returned untouched so the default cost stays bit-identical. + similarity = 1 - appearance + return min(1.0, max(0.0, (ceiling - similarity) / (ceiling - floor))) + + +def overlap_term(terms: PairTerms, options: LinkOptions) -> float: + """1 - IoU, or with the move rule and look-alike embeddings, at most the shift over motion_max_shift.""" + term = 1 - terms.iou + if ( + options.motion_min_similarity is not None + and terms.appearance is not None + and terms.shift is not None + and 1 - terms.appearance >= options.motion_min_similarity + ): + term = min(term, terms.shift / options.motion_max_shift) + return term + + +def weighted_cost(terms: PairTerms, options: LinkOptions = DEFAULT_LINK_OPTIONS, distance_multiplier=1.0) -> float: + """The matching cost from its terms. At the default options it equals ``total_cost`` + exactly: the terms are summed in the same order and multiplying by 1.0 changes nothing.""" + geometry = ( + options.iou_weight * overlap_term(terms, options) + + options.size_weight * (1 - terms.size_ratio) + + options.distance_weight * distance_multiplier * terms.distance + ) + if terms.appearance is None: + return geometry + return options.appearance_weight * appearance_term(terms.appearance, options) + geometry + + +def activity_multiplier(detection_count: int, options: LinkOptions) -> float: + """How much more the distance term weighs for a pair of captures holding this many detections.""" + if options.activity_scaling == "log": + return max(1.0, math.log1p(detection_count) / math.log1p(options.activity_reference_count)) + if options.activity_scaling == "steps": + multiplier = 1.0 + for count, step_multiplier in options.activity_steps: + if detection_count >= count: + multiplier = step_multiplier + return multiplier + return 1.0 + + +def labels_conflict(a: TopLabel | None, b: TopLabel | None, min_score: float) -> bool: + """Do two confident labels name unrelated taxa? A genus and one of its species do not conflict.""" + if a is None or b is None or a.score < min_score or b.score < min_score: + return False + if a.taxon_id == b.taxon_id: + return False + return a.taxon_id not in b.ancestor_ids and b.taxon_id not in a.ancestor_ids + + +def is_stationary(terms: PairTerms, options: LinkOptions) -> bool: + return terms.distance <= options.stationary_max_shift and terms.iou >= options.stationary_min_iou + + +def choose_links( + pairs: Iterable[tuple[typing.Any, typing.Any, PairTerms]], + cost_threshold: float, + require_features: bool = True, + options: LinkOptions = DEFAULT_LINK_OPTIONS, + detection_count: int = 0, + labels: typing.Mapping[typing.Any, TopLabel] | None = None, + key=lambda detection: detection, +) -> list[tuple[typing.Any, typing.Any, float]]: + """Pick the links between two adjacent captures from the scored pairs, one link per detection a side. + + With ``stationary_first``, pairs that sit still are taken first (lowest cost first), then + every other candidate below ``cost_threshold`` is. ``key`` gives the id that breaks ties + between equal costs, so the order is the same across runs. + """ + labels = labels or {} + multiplier = activity_multiplier(detection_count, options) + stationary: list[tuple[typing.Any, typing.Any, float]] = [] + candidates: list[tuple[typing.Any, typing.Any, float]] = [] + for det, nxt, terms in pairs: + missing = terms.appearance is None + if ( + options.appearance_min_similarity is not None + and not missing + and 1 - terms.appearance < options.appearance_min_similarity + ): + continue + cost = weighted_cost(terms, options, multiplier) + if options.species_gate != "off" and labels_conflict( + labels.get(key(det)), labels.get(key(nxt)), options.species_gate_min_score + ): + if options.species_gate == "forbid": + continue + cost += options.species_gate_penalty + if ( + options.stationary_first + and cost < options.stationary_cost_threshold + and (not missing or not require_features or options.stationary_allow_missing_features) + and is_stationary(terms, options) + ): + stationary.append((det, nxt, cost)) + elif (not missing or not require_features) and cost < cost_threshold: + candidates.append((det, nxt, cost)) + + claimed_current: set = set() + claimed_next: set = set() + links: list[tuple[typing.Any, typing.Any, float]] = [] + for group in (stationary, candidates): + # Secondary keys keep tied costs deterministic across runs. + group.sort(key=lambda x: (x[2], key(x[0]), key(x[1]))) + for det, nxt, cost in group: + if key(det) in claimed_current or key(nxt) in claimed_next: + continue + claimed_current.add(key(det)) + claimed_next.add(key(nxt)) + links.append((det, nxt, cost)) + return links + + +def _own_terminal_labels(detection_ids: Iterable[int]): + # Rows copied from another classification (``applied_to`` set, as tracking and class + # masking leave) are left out, so a label is what a classifier said about this crop. + return Classification.objects.filter( + detection_id__in=list(detection_ids), + terminal=True, + applied_to__isnull=True, + taxon_id__isnull=False, + score__isnull=False, + ) + + +class AmbiguousSpeciesLabels(ValueError): + """A session is labelled by several classifiers and none was chosen for species comparisons.""" + + +def resolve_label_algorithm(detection_ids: Iterable[int], configured_id: int | None) -> int | None: + """The one classifier whose labels a session's species comparisons use. + + A configured id is used as given. Otherwise it is the only classifier with terminal + labels on these detections, or None when there are none. Several classifiers raise + ``AmbiguousSpeciesLabels``: their scores are not on one scale, so mixing them is noise. + """ + if configured_id is not None: + return configured_id + candidates = sorted( + _own_terminal_labels(detection_ids).order_by().values_list("algorithm_id", flat=True).distinct() + ) + if len(candidates) > 1: + raise AmbiguousSpeciesLabels( + f"Detections are labelled by {len(candidates)} classifiers ({candidates}); " + "set species_label_algorithm_id to pick one." + ) + return candidates[0] if candidates else None + + +def top_labels(detection_ids: Iterable[int], algorithm_id: int | None) -> dict[int, TopLabel]: + """Each detection's highest-scoring terminal label from one classifier, in two queries. + + ``algorithm_id`` None returns no labels, so the caller must resolve the classifier first + (``resolve_label_algorithm``) rather than mix labels from several. + """ + from ami.main.models import Taxon + + if algorithm_id is None: + return {} + best: dict[int, tuple[int, float]] = {} + rows = ( + _own_terminal_labels(detection_ids) + .filter(algorithm_id=algorithm_id) + .order_by("detection_id", "-score", "-pk") + .values_list("detection_id", "taxon_id", "score") + ) + for detection_id, taxon_id, score in rows: + best.setdefault(detection_id, (taxon_id, score)) + ancestors: dict[int, frozenset[int]] = {} + for taxon_id, parents in Taxon.objects.filter(pk__in={t for t, _ in best.values()}).values_list( + "pk", "parents_json" + ): + ids = set() + for parent in parents or []: + parent_id = parent.get("id") if isinstance(parent, dict) else getattr(parent, "id", None) + if parent_id is not None: + ids.add(int(parent_id)) + ancestors[taxon_id] = frozenset(ids) + return { + detection_id: TopLabel(taxon_id, score, ancestors.get(taxon_id, frozenset())) + for detection_id, (taxon_id, score) in best.items() + } + + def get_unique_feature_algorithm_for_event(event: Event) -> tuple[Algorithm | None, list[Algorithm]]: """ Return ``(unique_algorithm, all_candidates)``. @@ -483,40 +812,37 @@ def select_links( diag: float, cost_threshold: float, require_features: bool = True, + options: LinkOptions = DEFAULT_LINK_OPTIONS, + labels: typing.Mapping[int, TopLabel] | None = None, ) -> list[tuple[Detection, Detection, float]]: """The links tracking makes between two adjacent captures, lowest cost first, saving nothing. A pair is a candidate when its matching cost is below ``cost_threshold``; with - ``require_features``, a detection with no embedding in ``vectors`` never is. Candidates - are taken lowest cost first, and each detection is linked at most once on either side. - Tracking runs and the session view's link preview both call this, so they cannot differ. + ``require_features``, a detection with no embedding in ``vectors`` never is (unless the + stationary pass allows it). Each detection is linked at most once on either side; see + ``choose_links`` for the order. Tracking runs and the session view's link preview both + call this, so they cannot differ. """ - candidates: list[tuple[Detection, Detection, float]] = [] + skip_missing = require_features and not (options.stationary_first and options.stationary_allow_missing_features) + pairs = [] for det in current_detections: det_vec = vectors.get(det.pk) - if det_vec is None and require_features: + if det_vec is None and skip_missing: continue for nxt in next_detections: nxt_vec = vectors.get(nxt.pk) - if nxt_vec is None and require_features: + if nxt_vec is None and skip_missing: continue - cost = total_cost(det_vec, nxt_vec, det.bbox, nxt.bbox, diag) - if cost < cost_threshold: - candidates.append((det, nxt, cost)) - - # Secondary keys (det.pk, nxt.pk) keep tied costs deterministic across runs. - candidates.sort(key=lambda x: (x[2], x[0].pk, x[1].pk)) - - claimed_current: set[int] = set() - claimed_next: set[int] = set() - links: list[tuple[Detection, Detection, float]] = [] - for det, nxt, cost in candidates: - if det.pk in claimed_current or nxt.pk in claimed_next: - continue - claimed_current.add(det.pk) - claimed_next.add(nxt.pk) - links.append((det, nxt, cost)) - return links + pairs.append((det, nxt, pair_terms(det_vec, nxt_vec, det.bbox, nxt.bbox, diag))) + return choose_links( + pairs, + cost_threshold, + require_features, + options, + detection_count=max(len(current_detections), len(next_detections)), + labels=labels, + key=lambda detection: detection.pk, + ) def select_transition_links( @@ -527,11 +853,19 @@ def select_transition_links( cost_threshold: float, algorithm: Algorithm | None, require_features: bool = True, + options: LinkOptions = DEFAULT_LINK_OPTIONS, + labels: typing.Mapping[int, TopLabel] | None = None, + vectors: dict[int, typing.Any] | None = None, ) -> list[tuple[Detection, Detection, float]]: - """The links tracking makes between two adjacent captures, reading embeddings but saving nothing.""" - vectors: dict[int, typing.Any] = {} - if algorithm is not None: - vectors = vectors_for_detections([det.pk for det in [*current_detections, *next_detections]], algorithm.pk) + """The links tracking makes between two adjacent captures, reading embeddings but saving nothing. + + ``vectors`` replaces the embeddings read from ``algorithm``, for scoring embeddings supplied + by the caller. + """ + if vectors is None: + vectors = {} + if algorithm is not None: + vectors = vectors_for_detections([det.pk for det in [*current_detections, *next_detections]], algorithm.pk) return select_links( current_detections, next_detections, @@ -539,6 +873,8 @@ def select_transition_links( image_diagonal(image_width, image_height), cost_threshold, require_features, + options, + labels, ) @@ -596,6 +932,7 @@ def iter_transition_links( algorithm: Algorithm | None, config: TrackingConfig, logger: logging.Logger, + vectors: dict[int, typing.Any] | None = None, ) -> Iterator[list[tuple[Detection, Detection, float]] | None]: """Yield the proposed links for each pair of consecutive captures, in order, saving nothing. @@ -603,6 +940,15 @@ def iter_transition_links( generator is lazy, so a caller that saves each transition's links before asking for the next one sees detections as they stand after its own writes. """ + options = config.link_options() + labels = None + if options.species_gate != "off": + # One read for the session, so the gate costs no query per pair of captures. + detection_ids = list( + Detection.objects.valid().filter(source_image__in=source_images).values_list("pk", flat=True) + ) + label_algorithm_id = resolve_label_algorithm(detection_ids, config.species_label_algorithm_id) + labels = top_labels(detection_ids, label_algorithm_id) transitions = len(source_images) - 1 for i in range(transitions): cur = source_images[i] @@ -622,6 +968,9 @@ def iter_transition_links( cost_threshold=config.cost_threshold, algorithm=algorithm, require_features=config.require_features, + options=options, + labels=labels, + vectors=vectors, ) @@ -630,20 +979,88 @@ def propose_event_links( algorithm: Algorithm | None, config: TrackingConfig, logger: logging.Logger, + vectors: dict[int, typing.Any] | None = None, ) -> list[tuple[int, int, float]]: """The ``(detection_id, next_detection_id, cost)`` links a tracking run would make in one event, treating every detection as unlinked and writing nothing. Tracking runs choose links the same way (``iter_transition_links``), so an evaluation - built on this scores what the task would do on the same detections. + built on this scores what the task would do on the same detections. ``vectors`` replaces + the stored embeddings, as in ``select_transition_links``. """ source_images = list(event.captures.order_by("timestamp")) links: list[tuple[int, int, float]] = [] - for transition in iter_transition_links(source_images, algorithm, config, logger): + for transition in iter_transition_links(source_images, algorithm, config, logger, vectors): links.extend((det.pk, nxt.pk, cost) for det, nxt, cost in transition or []) return links +@dataclasses.dataclass +class TransitionPairs: + """Every pair of detections across two adjacent captures, with its cost terms, for re-scoring + under many settings without reading the database again.""" + + pairs: list[tuple[int, int, PairTerms]] + detection_count: int + + +def event_transition_pairs( + event: Event, algorithm: Algorithm | None, vectors: dict[int, typing.Any] | None = None +) -> tuple[list[TransitionPairs], list[int]]: + """The scored pairs of every transition in one event, and every valid detection id in it. + + ``links_from_transition_pairs`` over the result gives the same links as + ``propose_event_links`` for any setting: the same terms feed ``choose_links``. + """ + source_images = list(event.captures.order_by("timestamp")) + detections = list( + Detection.objects.valid().filter(source_image__in=source_images).only("pk", "bbox", "source_image_id") + ) + by_capture: dict[int, list[Detection]] = collections.defaultdict(list) + for detection in sorted(detections, key=lambda d: d.pk): + by_capture[detection.source_image_id].append(detection) + if vectors is None: + vectors = vectors_for_detections([d.pk for d in detections], algorithm.pk) if algorithm is not None else {} + # Convert once per detection rather than once per pair; the values are unchanged. + arrays = {pk: np.asarray(vector) for pk, vector in vectors.items()} + + transitions: list[TransitionPairs] = [] + for cur, nxt in zip(source_images, source_images[1:]): + if not cur.width or not cur.height: + continue + diag = image_diagonal(cur.width, cur.height) + current, following = by_capture.get(cur.pk, []), by_capture.get(nxt.pk, []) + pairs = [ + (a.pk, b.pk, pair_terms(arrays.get(a.pk), arrays.get(b.pk), a.bbox, b.bbox, diag)) + for a in current + for b in following + ] + transitions.append(TransitionPairs(pairs, max(len(current), len(following)))) + return transitions, [d.pk for d in detections] + + +def links_from_transition_pairs( + transitions: Iterable[TransitionPairs], + config: TrackingConfig, + labels: typing.Mapping[int, TopLabel] | None = None, +) -> list[tuple[int, int, float]]: + """The links a run with ``config`` makes over pre-scored transitions (see ``event_transition_pairs``).""" + options = config.link_options() + links: list[tuple[int, int, float]] = [] + for transition in transitions: + links.extend( + choose_links( + transition.pairs, + config.cost_threshold, + config.require_features, + options, + detection_count=transition.detection_count, + labels=labels if options.species_gate != "off" else None, + ) + ) + return links + + def assign_occurrences_by_tracking_images( event: Event, logger: logging.Logger, @@ -808,20 +1225,27 @@ def _stage_progress(p: float, _idx=idx, _total=total) -> None: overall = ((_idx - 1) + p) / _total self.update_progress(overall) - counters = assign_occurrences_by_tracking_images( - event=event, - logger=self.logger, - algorithm=algorithm, - config=self.config, - record_as=self.algorithm, - progress_cb=_stage_progress, - history=TrackingHistory( - settings=self.config.dict(exclude={"source_image_collection_id", "event_ids"}), - feature_algorithm_id=algorithm.pk if algorithm is not None else None, - job=self.job, - algorithm=self.algorithm, - ), - ) + try: + counters = assign_occurrences_by_tracking_images( + event=event, + logger=self.logger, + algorithm=algorithm, + config=self.config, + record_as=self.algorithm, + progress_cb=_stage_progress, + history=TrackingHistory( + settings=self.config.dict(exclude={"source_image_collection_id", "event_ids"}), + feature_algorithm_id=algorithm.pk if algorithm is not None else None, + job=self.job, + algorithm=self.algorithm, + ), + ) + except AmbiguousSpeciesLabels as error: + # Raised before the first link is saved, so the session is left as it was. + self.logger.warning(f"Skipping event {event.pk}: {error}") + totals["events_skipped"] += 1 + skip_reasons["its species gate cannot tell which classifier's labels to compare"] += 1 + continue totals["events_tracked"] += 1 tracked_event_ids.append(event.pk) totals["links_created"] += counters.get("links_created", 0) diff --git a/docs/claude/INDEX.md b/docs/claude/INDEX.md index 2171d6d53..6460a8efa 100644 --- a/docs/claude/INDEX.md +++ b/docs/claude/INDEX.md @@ -45,6 +45,12 @@ archived. | `planning/2026-05-28-captures-processed-filter-design.md` | Design: captures "Processed / Not processed" filter | | `planning/2026-05-28-captures-processed-filter-plan.md` | Implementation plan (checkbox tasks) for the captures processed filter | +## Reports (point-in-time findings) + +| File | Description | +|---|---| +| `reports/2026-10-02-tracking-parameter-tuning.md` | Tracking parameter tuning on a partner's evaluation project (three cameras: busy micromoth, medium macromoth, quiet nights) and three by-eye audits of the proposed tracks. Recommends setting D3 (threshold 0.9, BioCLIP appearance, gate 0.47, log activity scaling, species penalty 0.5 at 0.2); measured saving 22.7% / 45.9% / 93.2% with 0 joins in 175 sampled long tracks. Covers the structural threshold-1.0 fact, why confirmed-track metrics hide joins, BioCLIP vs classifier-backbone similarity scales (AUC equal, scale not), what did not work, staff-only vs member fields, production readiness and PR landing order. Keywords: tracking, tuning, cost_threshold, BioCLIP, embeddings, species gate, activity scaling, visual audit, joins, merges | + ## Archive / session snapshots | File | Description | diff --git a/docs/claude/reference/occurrence-tracking.md b/docs/claude/reference/occurrence-tracking.md index cf5220976..94a90cd01 100644 --- a/docs/claude/reference/occurrence-tracking.md +++ b/docs/claude/reference/occurrence-tracking.md @@ -68,6 +68,22 @@ now-empty siblings. | `skip_if_human_identifications` | True | Protect reviewed sessions. | | `feature_extraction_algorithm_id` | None | Disambiguate when several classifiers ran. | | `require_completely_processed_session` | False | Off deliberately; see below. | +| `appearance_weight`, `iou_weight`, `size_weight`, `distance_weight` | 1.0 | Weight of each cost term. At 1.0 the cost is the plain sum above, bit for bit. | +| `species_gate` (`off`/`penalty`/`forbid`), `species_gate_min_score`, `species_gate_penalty`, `species_label_algorithm_id` | off, 0.5, 1.0, None | Forbid or penalise a link whose two top labels are both confident and name unrelated taxa (ancestor/descendant is not a conflict). Labels: highest-score terminal classification per detection from ONE classifier (`species_label_algorithm_id`, else the only one in the session; several ⇒ run skips the session, `evaluate_tracking` errors), rows with `applied_to` excluded; read once per session (`resolve_label_algorithm` + `top_labels`). | +| `activity_scaling` (`off`/`log`/`steps`), `activity_reference_count`, `activity_steps` | off, 5, [] | Multiply the distance term by `log(1+n)/log(1+ref)` (at least 1) or by a step table, where n is the larger detection count of the two captures. Crowded sheets tolerate less movement. | +| `stationary_first`, `stationary_max_shift`, `stationary_min_iou`, `stationary_cost_threshold`, `stationary_allow_missing_features` | False, 0.01, 0.7, 0.2, False | Link pairs that barely moved before any other pair; optionally even without embeddings. | +| `appearance_similarity_floor`, `appearance_similarity_ceiling` | 0, 1 | Map cosine similarity onto the appearance cost: 0 at or above the ceiling, 1 at or below the floor. The defaults are the plain `1 - similarity`. Set per feature extractor (see below). | +| `appearance_min_similarity` | None | Never link a pair whose two embeddings are less similar than this; pairs without an embedding are not gated. | +| `motion_min_similarity`, `motion_max_shift` | None, 3.0 | For pairs at least this similar, the overlap term becomes `min(1 - IoU, shift / motion_max_shift)`, the shift measured in box sizes, so a moved insect can link below a threshold of 1. | + +Everything after `require_completely_processed_session` is staff-only (not in +`MEMBER_POST_PROCESSING_TASKS` in `registry.py`). The matcher is `choose_links()`; the pure +helpers are `pair_terms`, `weighted_cost`, `activity_multiplier`, `labels_conflict`, +`is_stationary`, `appearance_term`, `overlap_term`, `shift_in_box_sizes`. Structural fact worth knowing when tuning: `(1 - IoU)` is 1 for boxes that do +not overlap, so at `cost_threshold <= 1.0` with `iou_weight=1` only overlapping boxes can link, +whatever the other terms say. Linking a moth that moved further needs a threshold above 1 or a +lower `iou_weight`, and that is where the activity scaling earns its keep (see +`tracking-evaluation.md`). ## The trap that matters most diff --git a/docs/claude/reference/tracking-evaluation.md b/docs/claude/reference/tracking-evaluation.md index e8967f80d..56d19042b 100644 --- a/docs/claude/reference/tracking-evaluation.md +++ b/docs/claude/reference/tracking-evaluation.md @@ -61,6 +61,55 @@ Output: per-session scores and an overall score pooled over all scored sessions. keys `project_id`, `config`, `events[]` (`event_id`, `feature_extraction_algorithm_id`, `note`, `links_proposed`, `evaluation`), `skipped_events[]`, `overall`. +### Sweeps + +```bash +python manage.py evaluate_tracking --project --sweep grid.json --output-dir out/ [--per-track-csv out/tracks.csv] \ + [--vectors-file vectors.npz] [--config '{"species_gate": "forbid"}'] +``` + +- `--sweep` takes inline JSON or a file: a list of settings, or `{"base": {...}, "grid": {name: [values]}, + "configs": [...]}`. The grid is a cartesian product on top of `base`; a grid value that is an object is + merged whole, so one axis can set a mode and its parameters together. Every setting is validated by + `TrackingConfig`; scope fields are refused (the command picks the sessions). +- Pairs and their cost terms are read and computed once per session and extractor + (`event_transition_pairs`) and re-scored per setting (`links_from_transition_pairs`, the same + `choose_links` a run uses; a test pins that both give the same links). 168 settings over three + one-hour sessions (about 12k detections) took 47 s locally. +- Writes `sweep.json` (every run, with per-session and overall evaluations, and flat `rows`) and + `sweep.md` (one table per session, then overall; only the settings that vary are shown; scores + are cut, not rounded, to three places so only a perfect score prints 1.000). +- `--vectors-file`: an `.npz` with `detection_ids` and `vectors`, compared instead of stored + embeddings (for extractors whose vectors are not stored as classifications). +- Added scores: `multi_detection_*` (confirmed tracks of 2+ detections; single-detection tracks + are recovered by doing nothing and never count as exact there), `cross_species_merges` + (predicted tracks joining confirmed tracks with different determinations), and per session + `summarise_session`: occurrences and distinct determinations before (one per detection) and + after tracking, predicted tracks of 2+ detections and their median/max length, and + `links_with_conflicting_labels` (links whose two confident labels, score >= 0.5, name + unrelated taxa: a session-wide proxy for wrong links, since confirmed tracks only reveal a + merge of two confirmed tracks). The proxy has a noise floor: labels flicker on the same insect. +- For a hard guarantee against writes on a copied database, also run with + `PGOPTIONS="-c default_transaction_read_only=on"` in the container environment. + +### Calibrating the appearance term for a feature extractor + +Each extractor has its own cosine similarity range, so the appearance settings are chosen per extractor from +confirmed tracks. On a partner's evaluation project (three one-hour sessions, 40 multi-detection tracks), measured: + +| | classifier backbone (2048-d) | BioCLIP (1024-d) | +|---|---|---| +| true consecutive pairs, similarity p1 / p25 / p50 | 0.958 / 0.988 / 0.992 | 0.47 / 0.89 / 0.95 | +| different insects in adjacent captures within 3 box sizes, p50 / p95 | 0.957 / 0.979 | 0.39 / 0.78 | +| AUC, true pairs vs those near negatives | 0.980 | 0.979 | +| true pairs that moved clear of their box, p50 | 0.981 | 0.74 | + +Both rank pairs about equally well; the backbone squeezes them into 0.87–1, so its plain `1 - similarity` +term barely changes the cost. A rule of thumb used there (not optimised): ceiling = p25 of true pairs, +floor = p50 of the near negatives, gate (`appearance_min_similarity`) = p1 of true pairs. Only 29 of 1,212 +true links were moves; the move rule recovered a few of them at threshold 1.0 without merges but added +about one doubtful link per correct move, so it stays an experiment. + ### 2. Outside Antenna, from exported CSVs ```bash @@ -109,8 +158,6 @@ print("\n".join(result.summary_lines())) ## Next steps (not built) -- Parameter sweeps: run the command over a grid of `--cost-threshold` values, with and without - features, and table link F1 against threshold. A small wrapper script is enough to start. - Per-species breakdown: group the per-track scores by the confirmed occurrence's determination. Untested idea; needs the determination added to the command's output. - A dry-run tracking job (#1416) that stores proposed links for review in the UI would reuse @@ -125,6 +172,9 @@ print("\n".join(result.summary_lines())) `evaluate_csv_files`, `main`; no Django imports. - `ami/ml/post_processing/tracking_task.py`: `iter_transition_links` (read-only, shared by the task and the evaluator), `propose_event_links`, `select_transition_links`, `save_links`. -- `ami/main/management/commands/evaluate_tracking.py`: the command. +- `ami/main/management/commands/evaluate_tracking.py`: the command, `expand_sweep`, `load_vectors_file`. +- `tracking_evaluation.py`: `summarise_session`, `sweep_row`, `format_sweep_markdown` (no Django). +- Tests for the optional rules and sweeps: `ami/ml/post_processing/tests/test_tracking_cost_terms.py` + (includes the regression test pinning default links to a frozen copy of the old matcher). - Tests: `ami/ml/post_processing/tests/test_tracking_evaluation.py` (metrics on hand-built cases, CSV adapter, command on a synthetic session, and proposed links equal to the links a run saves). diff --git a/docs/claude/reports/2026-10-02-tracking-parameter-tuning.md b/docs/claude/reports/2026-10-02-tracking-parameter-tuning.md new file mode 100644 index 000000000..c6624ddb4 --- /dev/null +++ b/docs/claude/reports/2026-10-02-tracking-parameter-tuning.md @@ -0,0 +1,253 @@ +# Tracking parameter tuning and a by-eye audit of proposed tracks (2026-10-02) + +This report summarises how the occurrence-tracking parameters were tuned on a partner's evaluation +project, and what a by-eye review of the resulting tracks found. It covers three cameras: a busy night +dominated by micromoths ("the busy micromoth night"), a medium night with larger moths ("the medium +macromoth night") and a quiet night with a few large moths ("the quiet night"). The partner confirmed tracks +in a one-hour window of each night; those windows are the benchmark. + +Related code: #1442 (cost terms and the sweep harness), #1439 (detection embeddings, feature-only jobs, +occurrence history), #1444 (resetting a session's tracking). Reference docs: `reference/occurrence-tracking.md`, +`reference/tracking-evaluation.md`, `reference/detection-embeddings.md`. + +**Tags.** [M] measured: a real tracking job, or the tracker's own code run read-only and scored against the +confirmed tracks, or a count made by eye. [S] simulated: the tracker's linking code run read-only without +ground truth; wherever a simulated setting was also run for real, the two matched detection for detection. +[E] estimated: a sample rate multiplied out, or a figure from reading code. Intervals are 95% Wilson +intervals. "Best case" counts tracks the reviewer marked `unsure` as correct, "worst case" counts them as joins. + +## 1. Summary + +**Recommended setting: D3.** Link consecutive detections on box overlap plus BioCLIP 2.5 appearance, with a +hard similarity floor, stricter matching in crowded captures and a penalty when the species classifier +confidently names two unrelated taxa (exact values in section 7). + +Full nights, real runs [M]: + +| night | detections = occurrences before | occurrences after D3 | fewer to review | unique determinations | +|---|---|---|---|---| +| busy micromoth night | 36,578 | 28,270 | **22.7%** | 873 → 822 | +| medium macromoth night | 13,094 | 7,084 | **45.9%** | 601 → 519 | +| quiet night | 4,983 | 340 | **93.2%** | 56 → 47 | + +Joins of two different insects, by eye, two seeded samples combined [M]: long tracks (4+ captures) **0 / 75** +on the busy night and **0 / 75** on the medium night (upper bound 4.9% each), **0 / 25** on the quiet night +(every long track); short tracks (2–3 captures) **1 / 40** (0.4–12.9%), 0 / 39 and 0 / 16. + +For comparison [M]: the old default (threshold 0.2) joined no insects in 120 sampled tracks but saves only +9–28% on busy and medium nights; threshold 1.0 on geometry saves 29% / 50% / 93% but joined insects in 6.3% +(worst case 15%) of long busy-night tracks; threshold 1.5 joined insects in 90% of long tracks on the busy +micromoth hour. + +**Limits.** One reviewer, working from crops and capture views, cannot see two insects of the same species +swapping on one spot, so same-species joins are undercounted everywhere. Three cameras and three nights; the +benchmark hours are parts of the same nights. + +**Decisions for the partner:** D3 versus D1 (slightly more saving, one definite join and several unsure tracks); +whether resting moths split by the species penalty are acceptable until a fix lands; who sets the staff-only +parameters; whether a specialist looks at four links that cannot be settled from the images. + +## 2. Data + +- **Benchmark hours** [M]: three one-hour windows of 180 captures at about 20 s intervals; 8,259 / 782 / 3,127 + detections (busy micromoth / quiet / medium macromoth). 49 partner-confirmed tracks, 40 of 2+ detections and + 9 singletons; 1,212 ground-truth links, 29 of them moves (boxes do not overlap). +- **Full nights** [M]: about 1,080 captures each; 36,578 / 13,094 / 4,983 detections. Only the benchmark hour + had been processed originally; the rest was detected, classified and embedded for this study. +- **BioCLIP 2.5 vectors for every detection** (1,024-d), produced by feature-only ML jobs that send existing + detections to the processing service. +- **Limits of the ground truth.** Confirmed tracks are clear, well-separated moths, so scores against them say + little about crowded captures. With classifier-backbone vectors required, only 83.4% of ground-truth links + have a vector at both ends, which caps recall at 0.834. + +## 3. Method + +**Cost.** For two detections in consecutive captures: +`(1 − cosine similarity) + (1 − IoU) + (1 − area ratio) + centre shift / diagonal`. Pairs below +`cost_threshold` are claimed greedily, cheapest first. Options added in #1442 (off by default, staff-only): +term weights, appearance calibration, an appearance gate, activity scaling of the distance term, a species +gate, stationary-first and a move rule. + +**Structural fact.** `1 − IoU` is 1 for boxes that do not overlap and every other term is at least 0, so at a +threshold of 1.0 or less only overlapping boxes can link. Joins between confirmed tracks appear only above 1.0, +and moving moths are never linked at the recommended settings. + +**Sweeps** (read-only, rolled back) [M]: about 2,500 settings over three rounds (classifier-backbone vectors, then +BioCLIP vs backbone with calibration, gates and a move rule), 47 s to 7.5 min per grid. + +**Why confirmed-track metrics hide joins.** Precision and merges are scored only inside confirmed tracks. The +first sweep reported **0 merges** for threshold 1.5 with activity scaling; by eye, **90%** of its long tracks on +the busy micromoth hour joined different insects [M]. From then on every candidate was judged by eye. + +**By-eye audit.** Three audits (464, 1,130 and 316 sampled tracks). Seeded samples of tracks not wholly inside a +confirmed track: 40 of 4+ captures and 20 of 2–3 per setting and night. The last two audits were blind to the +setting. Every frame shown up to 100 frames; longer tracks show every link overlapping below IoU 0.85 and every +10th frame. Ambiguous tracks got enlarged crops, the full capture with every detection drawn, a four-capture +view around a link, and database checks for a detection left behind at the old position. Verdicts: same, merge, +unsure, background. Re-judged sequences kept their verdict in 128 of 129 and 76 of 77 cases. + +**Real runs against the simulation.** Five settings were run as real tracking jobs on the three full nights, +resetting between settings. In all 15 setting-night cells the real multi-detection occurrences equalled the +simulated tracks detection for detection [M], so audits of simulated and real tracks sample one population. + +## 4. Settings + +| setting | threshold | vectors | extra | +|---|---|---|---| +| A (default) | 0.2 | required | none | +| B | 1.0 | not required (classifier backbone where present) | none | +| C | 1.5 | not required (backbone) | activity scaling, log | +| D1 | 1.0 | BioCLIP | gate 0.47, activity scaling log (reference 5) | +| D2 | 1.0 | BioCLIP | D1 + species penalty 0.5 at score 0.2 | +| **D3** | **0.9** | BioCLIP | D2's options | + +## 5. Results + +**Benchmark hours against confirmed tracks** (12,168 detections) [M]: + +| setting | link P | link R | exact multi-detection tracks (of 40) | occurrences after | +|---|---|---|---|---| +| A | 1.000 | 0.624 | 7 | 9,774 | +| B | 1.000 | 0.960 | 22 | 7,415 | +| C | 1.000 | 0.983 | 28 | 3,226 | +| D1 | 1.000 | 0.939 | 22 | 7,834 | +| D2 | 1.000 | 0.931 | 22 | 7,935 | +| **D3** | 1.000 | **0.917** | **20** | **8,064** | + +**Full nights, occurrences after (saving)**: + +| setting | busy micromoth night | medium macromoth night | quiet night | source | +|---|---|---|---|---| +| A (BioCLIP) | 33,365 (8.8%) | 10,232 (21.9%) | 818 (83.6%) | [M] | +| B (backbone) | 26,110 (28.6%) | 6,502 (50.3%) | 327 (93.4%) | [S] | +| C (backbone) | 10,172 (72.2%) | 3,822 (70.8%) | 275 (94.5%) | [S] | +| D1 | 27,573 (24.6%) | 6,847 (47.7%) | 329 (93.4%) | [M] | +| D2 | 27,861 (23.8%) | 6,924 (47.1%) | 330 (93.4%) | [M] | +| **D3** | **28,270 (22.7%)** | **7,084 (45.9%)** | **340 (93.2%)** | [M] | + +D3 keeps 79% / 91% / 100% of B's saving. Most occurrences left on busy nights are single detections; tracking +shortens the list mainly by collapsing moths that sit still. + +**Joins by eye, long tracks (4+ captures), best / worst case** [M]: + +| setting | busy night | medium night | quiet night | hour windows | +|---|---|---|---|---| +| A | not audited | not audited | not audited | 0 / 120 | +| B | 5/80 = 6.3% / 12/80 = 15.0% | 1/40 / 2–3 of 40 | 0 / 19 | 4/122 / 6/122 | +| C | not audited | not audited | not audited | 94/123 = 76% | +| D1 | 0/80 / 3/80 = 3.8% | 1/76 = 1.3% | 0 / 19 | 0/81 / 2/81 | +| D2 | 0/40 / 0/40 | 0/40 / 0/40 | 0 / 20 | 1/81 / 2/81 | +| **D3** | **0/75 (0–4.9%)** | **0/75 (0–4.9%)** | **0 / 25** | 0/81 / 1/81 | + +Short tracks (2–3 captures), D3: 1/40 on the busy night, 0/39 medium, 0/16 quiet [M]. Scaled to the busy night's +2,779 short tracks, that one join is about 70 joined short tracks, with a very wide interval [E]. + +Every definite join found is between visibly different insects, and almost all follow one pattern: one insect +leaves, a different one lands on the same spot within one capture interval, the boxes overlap, and the track +continues. D3 still makes two known wrong links (IoU 0.79 and 0.73): one "leaves and lands", one neighbour swap +where the first insect is still detected just beside the box. + +**Run time** [M]: real tracking jobs 53 s to 7 m 41 s per night (D3 on the busy night 7 m 41 s); resetting three +nights 9–59 s. + +## 6. Vectors: BioCLIP 2.5 against the classifier backbone [M] + +| | classifier backbone | BioCLIP 2.5 | +|---|---|---| +| coverage (benchmark detections) | 74% | 100% | +| negative components | none (ReLU; 72% exactly zero) | about 50% | +| random pair similarity, median | 0.97 | 0.51 | +| same insect, median | 0.992 | 0.95 | +| different insect within 3 box sizes, median | 0.957 | 0.39 | +| AUC, same vs nearby different | 0.980 | 0.979 | + +Both rank pairs equally well; only BioCLIP has a usable scale. The backbone is trained with a classification loss +behind a ReLU, so every crop shares one large direction and its appearance cost moves by about 0.05 between true +and false pairs, next to geometry terms of up to 3. BioCLIP is trained contrastively and spreads crops over the +sphere. The joins found by eye sit where geometry already costs about 0.7; a BioCLIP similarity of 0.41–0.67 pushes +them over the threshold. Switching setting B to BioCLIP vectors blocks 9 of 10 known join links, and activity +scaling blocks the tenth (fitted on those same links). + +Calibration (a floor and ceiling on similarity) rescues the backbone but makes look-alike joins cheaper with +BioCLIP. The 0.47 gate is the 1st percentile of true BioCLIP pairs and costs about 0.006 recall. Centring the +backbone vectors (subtract the mean, renormalise) spreads random-pair similarity to −0.17–0.26; its effect on +tracking is untested. BioCLIP at threshold 0.2 looked worse than the backbone (link R 0.547 vs 0.624) only because +its wider scale taxes true pairs more at the same threshold. + +## 7. What did not work [M] + +- Threshold 0.2: recall 0.624, 7/40 tracks exact. +- Thresholds above 1.0 (setting C): the track hops to the nearest insect; 76% of long tracks joined. +- The "different-label link" proxy as a join detector: missed all 4 joins at B, caught 19 of 346 wrong links at C. +- Calibration with BioCLIP; gates above 0.6; species forbid at score 0.2 (loses 127 of 1,205 confirmed links); + thresholds 1.1–1.2. +- The move rule (3–8 of 29 moving links, about one doubtful link per correct move) and stationary-first (no change + without required features). +- A similarity floor below 0.80 for partial overlaps: blocks both remaining D3 joins but cuts 645 links on the + busy and medium nights, and 30 of 30 sampled were the same insect. +- A static-object rule: the longest track (930 captures) is a real resting moth, and the only static non-insect + objects found are 2-frame sheet folds. + +## 8. Recommendation + +```json +{ + "cost_threshold": 0.9, + "require_features": false, + "feature_extraction_algorithm_id": "", + "appearance_min_similarity": 0.47, + "activity_scaling": "log", + "activity_reference_count": 5, + "species_gate": "penalty", + "species_gate_penalty": 0.5, + "species_gate_min_score": 0.2, + "species_label_algorithm_id": "" +} +``` + +Other fields at their defaults. Algorithm ids differ per deployment and must be looked up. + +**Who can change what.** Project members who can run tracking may set the sessions, `cost_threshold`, +`require_features`, the feature extractor and `require_completely_processed_session`. Everything else in the block +above is staff-only. A member-started run therefore gets the defaults for those fields, not D3; running D3 needs a +superuser, stored per-project settings (not built), or opening some fields to members. Lowering the threshold is +safe for a member to try; raising it above 1.0 is what produced setting C. + +**Follow-ups to discuss:** skip the species penalty when boxes overlap at IoU ≥ 0.9 (no known wrong link overlaps +above 0.79, and resting moths whose label flips would stay whole); a neighbour-swap tie-break in the assignment +(as a hard block it cuts 59 links over three nights, 1 of 20 sampled wrong). + +## 9. Risks and open questions + +- Same-species swaps on one spot are invisible in crops; every join rate undercounts them. +- The benchmark hours are parts of the full nights; three cameras in total. +- Samples are small: one join moves a night's rate by 2.5% (long) or 5% (short). +- Four links remain unsure after a second look (dark capture, frame edge, colour change in place). +- The species penalty split a resting moth of 930 captures into three occurrences at label flips between related + species. How often this happens was not counted. +- Moving moths are not linked at thresholds ≤ 1.0. +- Resetting a session left tracking history from undone runs on occurrences; a fix is pending on #1444. +- The merge picker scores with default settings, not the settings the run used. + +## 10. Production readiness + +| item | state | +|---|---| +| pgvector on the production database | not checked; required by the embeddings table | +| BioCLIP on a processing service | draft PR in the processing-service repo; needs a GPU worker | +| Embedding existing detections | feature-only ML jobs (#1439); measured 32 detections/s on one RTX 3090 inside the service, about 3 s per capture end to end, dominated by image download [M]; roughly one hour per 1,000-capture night per worker [E] | +| Result backend | sync ML jobs can end FAILURE after saves succeed (#1443), and job creation can 500 on an idle broker (#1437); run ML jobs one at a time until both land | +| Staff-only fields through the API | refused for every user on the PR heads; a superuser exemption is pending | +| Algorithm ids | look up per deployment | + +**Landing order:** #1435 (CI) → main into #1272 → #1432 → #1439 → #1442 → #1444; #1441 after #1432; #1437, #1438 +and #1443 to main independently. Then deploy behind the per-project tracking flag. + +## 11. Next steps + +1. Agree the decisions in section 1 with the partner. +2. Land the pending fixes on their PR branches and the stack in the order above. +3. Check pgvector in production; deploy BioCLIP; embed existing detections. +4. Run D3 on the partner's nights; the partner reviews in the UI. +5. If wanted, build the species-gate overlap exemption and re-audit a sample. +6. Test D3 on a camera or night it was not tuned on.