diff --git a/ami/base/model_references.py b/ami/base/model_references.py new file mode 100644 index 000000000..38993454e --- /dev/null +++ b/ami/base/model_references.py @@ -0,0 +1,104 @@ +"""Record ids stored in JSON, resolved to ``{type, id, name}`` so the UI can show and link them. + +A model opts in by declaring ``reference_type`` (and ``reference_name_field``, the field or method +shown as its name; None shows ``#``). A pydantic schema marks a field that holds such an id, or a +list of them, with ``model_reference()``. ``resolve_model_references`` reads the names with one query +per type, and the UI maps each type to a page in ``ui/src/utils/model-references.ts``. +""" + +import collections +import dataclasses +import functools +import typing + +import pydantic +from django.apps import apps + + +@dataclasses.dataclass(frozen=True) +class ModelRef: + """A record a stored value names. ``name`` is None when the record no longer exists.""" + + type: str + id: int + name: str | None + + +def model_reference(ref_type: str, default: typing.Any = None, **field_options: typing.Any) -> typing.Any: + """Declare a pydantic field that holds the id of a record of ``ref_type``, or a list of such ids. + + ``field_options`` go to ``pydantic.Field``. + """ + return pydantic.Field(default, reference=ref_type, **field_options) + + +def field_references(schema: type[pydantic.BaseModel]) -> dict[str, str]: + """The schema's fields declared with ``model_reference()``, mapped to the reference type.""" + return { + name: field.field_info.extra["reference"] + for name, field in schema.__fields__.items() + if "reference" in field.field_info.extra + } + + +def field_titles(schema: type[pydantic.BaseModel]) -> dict[str, str | None]: + """Every field of the schema, in declaration order, mapped to its ``title`` or None.""" + return {name: field.field_info.title for name, field in schema.__fields__.items()} + + +def is_record_id(value: typing.Any) -> bool: + """Whether a stored value can be a primary key: an int that is not a bool.""" + return isinstance(value, int) and not isinstance(value, bool) + + +def record_ids(value: typing.Any) -> list[int]: + """The record ids a stored value holds: one id, a list of ids, or none for anything else.""" + if is_record_id(value): + return [value] + if isinstance(value, list) and all(is_record_id(item) for item in value): + return value + return [] + + +@functools.cache +def reference_types() -> dict[str, tuple[type, str | None]]: + """Each declared reference type mapped to its model and name field.""" + types: dict[str, tuple[type, str | None]] = {} + for model in apps.get_models(): + ref_type = getattr(model, "reference_type", None) + if ref_type is None: + continue + if ref_type in types: + raise ValueError(f"Reference type {ref_type!r} is declared by {types[ref_type][0]} and {model}.") + types[ref_type] = (model, getattr(model, "reference_name_field", None)) + return types + + +def unmapped_reference_types(schemas: typing.Iterable[type[pydantic.BaseModel]]) -> set[str]: + """Reference types the schemas declare that no model declares; resolving them would fail.""" + declared = {ref_type for schema in schemas for ref_type in field_references(schema).values()} + return declared - set(reference_types()) + + +def resolve_model_references(wanted: typing.Iterable[tuple[str, int]]) -> dict[tuple[str, int], ModelRef]: + """Each ``(type, id)`` as a ``ModelRef``, reading names with one query per type. + + An unknown type raises KeyError: a schema declared a reference type no model declares. + """ + ids_by_type: dict[str, set[int]] = collections.defaultdict(set) + for ref_type, ref_id in wanted: + ids_by_type[ref_type].add(ref_id) + resolved: dict[tuple[str, int], ModelRef] = {} + for ref_type, ids in ids_by_type.items(): + model, name_field = reference_types()[ref_type] + rows = model.objects.filter(pk__in=ids) + if name_field is None: + names = {pk: f"#{pk}" for pk in rows.values_list("pk", flat=True)} + elif callable(getattr(model, name_field)): + # A name computed in Python, such as a session's date: load the rows and call it. + names = {row.pk: str(getattr(row, name_field)()) for row in rows} + else: + names = dict(rows.values_list("pk", name_field)) + for ref_id in ids: + resolved[(ref_type, ref_id)] = ModelRef(ref_type, ref_id, names.get(ref_id)) + return resolved diff --git a/ami/base/serializers.py b/ami/base/serializers.py index e39ede52b..bf0598c0e 100644 --- a/ami/base/serializers.py +++ b/ami/base/serializers.py @@ -180,3 +180,11 @@ def clean(self) -> dict[str, typing.Any]: if self.is_valid(raise_exception=True): return self.validated_data raise api_exceptions.ValidationError("Invalid filter parameters") + + +class ModelRefSerializer(serializers.Serializer): + """A record a stored value names (see ``ami.base.model_references``). ``name`` is null when it no longer exists.""" + + type = serializers.CharField(help_text="What kind of record: taxa_list, capture_set, algorithm, ...") + id = serializers.IntegerField() + name = serializers.CharField(allow_null=True) diff --git a/ami/jobs/job_config.py b/ami/jobs/job_config.py new file mode 100644 index 000000000..107076ea7 --- /dev/null +++ b/ami/jobs/job_config.py @@ -0,0 +1,78 @@ +"""A job's config as labelled rows, with the records it names resolved, for showing it to people. + +A post-processing job stores its config in ``params["config"]``, validated by its task's +``config_schema``. The schema gives each field its title, its order and, through ``model_reference()``, +the type of record an id names. +""" + +import dataclasses +import typing + +from rest_framework import serializers + +from ami.base.model_references import ModelRef, field_references, field_titles, record_ids, resolve_model_references +from ami.base.serializers import ModelRefSerializer +from ami.jobs.models import Job +from ami.ml.post_processing.registry import get_postprocessing_task + + +@dataclasses.dataclass +class JobConfigField: + """One field of a job's config: its title from the task's config schema, and the records it names, if any.""" + + key: str + label: str + value: typing.Any + refs: list[ModelRef] = dataclasses.field(default_factory=list) + + +class JobConfigFieldSerializer(serializers.Serializer): + """One field of a job's config. ``label`` is the title the task's config schema gives it, else the key.""" + + key = serializers.CharField() + label = serializers.CharField() + value = serializers.JSONField(allow_null=True) + refs = ModelRefSerializer(many=True, help_text="The records the field names: one per id it holds, in order.") + + +def job_config(job: Job | None) -> dict | None: + """The config a post-processing job ran with, or None for any other job or malformed params.""" + params = job.params if job is not None else None + config = params.get("config") if isinstance(params, dict) else None + return config if isinstance(config, dict) else None + + +def job_config_fields(jobs: typing.Iterable[Job]) -> dict[int, list[JobConfigField]]: + """Each job's config fields by job id, resolving every record id they name with one query per type.""" + specs = {job.pk: _field_specs(job) for job in jobs} + wanted = [ + (ref_type, ref_id) + for fields in specs.values() + for _, _, value, ref_type in fields + if ref_type + for ref_id in record_ids(value) + ] + resolved = resolve_model_references(wanted) if wanted else {} + return { + job_id: [ + JobConfigField(key, label, value, [resolved[(ref_type, i)] for i in record_ids(value)] if ref_type else []) + for key, label, value, ref_type in fields + ] + for job_id, fields in specs.items() + } + + +def _field_specs(job: Job) -> list[tuple[str, str, typing.Any, str | None]]: + """``(key, label, value, reference type)`` per field, labelled, typed and ordered by the task's config schema. + + The stored config's key order is not kept (Postgres orders a JSON object's keys), so fields follow the + schema and any key it does not declare comes after. A job whose task is not registered shows its + config by key, naming no records. + """ + config = job_config(job) or {} + task_key = job.params.get("task") if config else None + task = get_postprocessing_task(task_key) if isinstance(task_key, str) else None + titles = field_titles(task.config_schema) if task else {} + references = field_references(task.config_schema) if task else {} + keys = [key for key in titles if key in config] + [key for key in config if key not in titles] + return [(key, titles.get(key) or key, config[key], references.get(key)) for key in keys] diff --git a/ami/jobs/models.py b/ami/jobs/models.py index ff65f31f2..cc422a7ed 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -1,4 +1,5 @@ import datetime +import json import logging import random import time @@ -913,6 +914,12 @@ def run(cls, job: "Job"): raise ValueError(f"Unknown post-processing task '{task_key}'") task = task_cls(job=job, **config) + # Store the config as the task's schema validated it, defaults included, so the job + # records exactly what this run used. The occurrence history shows it. + validated_config = json.loads(task.config.json()) + if validated_config != config: + job.params = {**params, "config": validated_config} + job.save(update_fields=["params", "updated_at"]) task.run() job.progress.update_stage(cls.key, status=JobState.SUCCESS, progress=1) job.finished_at = datetime.datetime.now() diff --git a/ami/jobs/tests/test_job_config.py b/ami/jobs/tests/test_job_config.py new file mode 100644 index 000000000..2b3ca0f4f --- /dev/null +++ b/ami/jobs/tests/test_job_config.py @@ -0,0 +1,83 @@ +from unittest import mock + +import pydantic +from django.test import TestCase + +from ami.base.model_references import ModelRef, model_reference, unmapped_reference_types +from ami.jobs.job_config import JobConfigField, job_config_fields +from ami.jobs.models import Job +from ami.main.models import TaxaList +from ami.ml.models import Algorithm +from ami.ml.post_processing.registry import POSTPROCESSING_TASKS +from ami.tests.fixtures.main import setup_test_project + + +class JobConfigFieldsTestCase(TestCase): + """A job's config as labelled rows, in its task's schema order, with the records it names resolved.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.project, _ = setup_test_project(reuse=False) + + def _fields(self, params) -> list[JobConfigField]: + job = Job.objects.create(project=self.project, name="Run", job_type_key="post_processing", params=params) + return job_config_fields([job])[job.pk] + + def test_labels_order_and_references_come_from_the_task_config_schema(self): + taxa_list = TaxaList.objects.create(name="Kept species") + classifier = Algorithm.objects.create(name="Classifier", key="config-test-classifier") + config = { + "reweight": True, + "algorithm_id": classifier.pk, + "taxa_list_id": taxa_list.pk, + "occurrence_id": True, + "source_image_collection_id": None, + } + self.assertEqual( + self._fields({"task": "class_masking", "config": config}), + [ + JobConfigField("source_image_collection_id", "Capture set", None), + # A value that is not an id names no record, whatever the schema declares. + JobConfigField("occurrence_id", "Occurrence", True), + JobConfigField( + "taxa_list_id", "Species list", taxa_list.pk, [ModelRef("taxa_list", taxa_list.pk, "Kept species")] + ), + JobConfigField( + "algorithm_id", "Classifier", classifier.pk, [ModelRef("algorithm", classifier.pk, "Classifier")] + ), + JobConfigField("reweight", "Re-weighted scores", True), + ], + ) + + def test_a_list_of_ids_names_one_record_per_id_in_order(self): + kept, other = TaxaList.objects.create(name="Kept"), TaxaList.objects.create(name="Other") + + class ListConfig(pydantic.BaseModel): + taxa_list_ids: list[int] = model_reference("taxa_list", [], title="Species lists") + + task = mock.Mock(config_schema=ListConfig) + with mock.patch("ami.jobs.job_config.get_postprocessing_task", return_value=task): + fields = self._fields({"task": "lists", "config": {"taxa_list_ids": [other.pk, kept.pk, 999999]}}) + self.assertEqual( + fields[0].refs, + [ + ModelRef("taxa_list", other.pk, "Other"), + ModelRef("taxa_list", kept.pk, "Kept"), + ModelRef("taxa_list", 999999, None), + ], + ) + + def test_a_job_without_a_registered_task_shows_its_config_by_key_with_no_references(self): + self.assertEqual( + self._fields({"config": {"taxa_list_id": 3, "size_threshold": 0.01}}), + [ + JobConfigField("taxa_list_id", "taxa_list_id", 3), + JobConfigField("size_threshold", "size_threshold", 0.01), + ], + ) + + def test_a_job_whose_params_are_not_an_object_has_no_config(self): + self.assertEqual(self._fields(["not", "an", "object"]), []) + + def test_every_reference_a_task_declares_can_be_resolved(self): + self.assertEqual(unmapped_reference_types(task.config_schema for task in POSTPROCESSING_TASKS.values()), set()) diff --git a/ami/jobs/tests/test_jobs.py b/ami/jobs/tests/test_jobs.py index fbed09b8f..c8c80a85e 100644 --- a/ami/jobs/tests/test_jobs.py +++ b/ami/jobs/tests/test_jobs.py @@ -1799,10 +1799,11 @@ def test_query_count_does_not_grow_with_the_number_of_jobs(self): """No per-job work: listing eight jobs runs the same queries as listing three, for a member whose permissions would otherwise be resolved on every row. Cachalot is off so every query counts.""" - from cachalot.api import cachalot_disabled from django.db import connection from django.test.utils import CaptureQueriesContext + from ami.tests.fixtures.queries import no_query_cache + self.client.force_authenticate(self.user) # The owner is a member of their project. def list_choices() -> tuple[int, int]: @@ -1811,17 +1812,12 @@ def list_choices() -> tuple[int, int]: self.assertEqual(response.status_code, status.HTTP_200_OK) return len(response.json()["results"]), len(queries.captured_queries) - disabled = cachalot_disabled() - disabled.__enter__() - try: + with no_query_cache(): three_rows, three_rows_queries = list_choices() Job.objects.bulk_create( Job(project=self.project, name=f"More {index}", job_type_key=MLJob.key) for index in range(5) ) eight_rows, eight_rows_queries = list_choices() - finally: - # cachalot_disabled() does not restore itself when the block raises. - disabled.__exit__(None, None, None) self.assertEqual((three_rows, eight_rows), (3, 8)) self.assertEqual(eight_rows_queries, three_rows_queries) diff --git a/ami/main/admin.py b/ami/main/admin.py index 55c84701f..47b383a00 100644 --- a/ami/main/admin.py +++ b/ami/main/admin.py @@ -605,6 +605,8 @@ class ClassificationAdmin(IdSearchAdminMixin, admin.ModelAdmin[Classification]): # every taxon / detection / classification — the latter makes the change page # unusable on a large database. autocomplete_fields = ("detection", "taxon", "algorithm", "category_map", "applied_to") + # Algorithm results have no admin to search, so the link is an id input rather than a select. + raw_id_fields = ("algorithm_result",) # A digit term jumps to that classification by id (IdSearchAdminMixin); text searches taxon name. search_fields = ("taxon__name",) # Order by -id (indexed PK) rather than the model's -created_at (no index). diff --git a/ami/main/api/__init__.py b/ami/main/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ami/main/api/occurrence_history/__init__.py b/ami/main/api/occurrence_history/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ami/main/api/occurrence_history/serializers.py b/ami/main/api/occurrence_history/serializers.py new file mode 100644 index 000000000..61a8e03af --- /dev/null +++ b/ami/main/api/occurrence_history/serializers.py @@ -0,0 +1,132 @@ +"""Serializers for an occurrence's history: identifications, predictions and algorithm results as one list.""" + +from drf_spectacular.utils import PolymorphicProxySerializer, extend_schema_field +from rest_framework import serializers + +from ami.jobs.job_config import JobConfigFieldSerializer +from ami.main.api.serializers import TaxonParentSerializer + + +class HistoryUserSerializer(serializers.Serializer): + """A person in an occurrence's history: name and picture only, never an email address.""" + + id = serializers.IntegerField() + name = serializers.CharField() + image = serializers.ImageField(allow_null=True) + + +class HistoryAlgorithmSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + key = serializers.CharField() + + +class HistoryTaxonSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + rank = serializers.CharField() + parents = TaxonParentSerializer(many=True, read_only=True, source="parents_json") + + +class HistoryJobSerializer(serializers.Serializer): + """A job in the history. ``config`` is filled per entry by ``HistoryEntryBaseSerializer``.""" + + id = serializers.IntegerField() + name = serializers.CharField() + config = JobConfigFieldSerializer( + many=True, read_only=True, help_text="The job's config, in the order its task's config schema declares it." + ) + + +class ReplacedClassificationSerializer(serializers.Serializer): + id = serializers.IntegerField() + taxon = HistoryTaxonSerializer(allow_null=True) + score = serializers.FloatField(allow_null=True) + + +class CreatedClassificationSerializer(serializers.Serializer): + """A classification a run created, and the one it replaced (null when none, or when it was deleted).""" + + id = serializers.IntegerField(source="classification.id") + taxon = HistoryTaxonSerializer(source="classification.taxon", allow_null=True) + score = serializers.FloatField(source="classification.score", allow_null=True) + terminal = serializers.BooleanField(source="classification.terminal") + detection_id = serializers.IntegerField(source="classification.detection_id") + replaced = ReplacedClassificationSerializer(allow_null=True) + + +class HistoryEntryBaseSerializer(serializers.Serializer): + """The fields every entry of an occurrence's history has.""" + + id = serializers.IntegerField(help_text="Primary key of the row in the table ``type`` names.") + timestamp = serializers.DateTimeField() + user = HistoryUserSerializer(allow_null=True) + algorithm = HistoryAlgorithmSerializer(allow_null=True) + job = serializers.SerializerMethodField() + taxon = HistoryTaxonSerializer(allow_null=True, help_text="The identified or predicted taxon.") + score = serializers.FloatField(allow_null=True, help_text="A prediction's score; null for other entries.") + + @extend_schema_field(HistoryJobSerializer(allow_null=True)) + def get_job(self, entry) -> dict | None: + if entry.job is None: + return None + return { + **HistoryJobSerializer(entry.job).data, + "config": JobConfigFieldSerializer(entry.job_config, many=True).data, + } + + +class IdentificationDetailsSerializer(serializers.Serializer): + comment = serializers.CharField(allow_blank=True) + withdrawn = serializers.BooleanField() + + +class PredictionDetailsSerializer(serializers.Serializer): + terminal = serializers.BooleanField() + + +class IdentificationEntrySerializer(HistoryEntryBaseSerializer): + type = serializers.ChoiceField(choices=["identification"]) + details = IdentificationDetailsSerializer() + + +class PredictionEntrySerializer(HistoryEntryBaseSerializer): + type = serializers.ChoiceField(choices=["prediction"]) + details = PredictionDetailsSerializer() + + +class AlgorithmResultEntrySerializer(HistoryEntryBaseSerializer): + """What a post-processing run decided about the occurrence, with the classifications it created.""" + + type = serializers.ChoiceField(choices=["algorithm_result"]) + kind = serializers.CharField() + value = serializers.FloatField( + allow_null=True, help_text="The kind's headline figure, for sorting and filtering; not a confidence." + ) + data = serializers.JSONField(help_text="The kind's figures, validated against its data model.") + determination_before = HistoryTaxonSerializer(allow_null=True) + determination_after = HistoryTaxonSerializer(allow_null=True) + classifications = CreatedClassificationSerializer( + many=True, help_text="The classifications the run created, best score first." + ) + + +HISTORY_ENTRY_SERIALIZERS = { + "identification": IdentificationEntrySerializer, + "prediction": PredictionEntrySerializer, + "algorithm_result": AlgorithmResultEntrySerializer, +} + +# The history endpoint's response: a plain oneOf with a literal ``type`` field. A result's ``data`` is +# published as JSON; the server validates it against its kind's model, and the UI types it by kind. +# Publishing one typed component per kind waits for a client generated from the schema (#1482). +OCCURRENCE_HISTORY_ENTRY_SCHEMA = PolymorphicProxySerializer( + component_name="OccurrenceHistoryEntry", + serializers=list(HISTORY_ENTRY_SERIALIZERS.values()), + resource_type_field_name=None, + many=True, +) + + +def serialize_history(entries, context) -> list[dict]: + return [HISTORY_ENTRY_SERIALIZERS[entry.type](entry, context=context).data for entry in entries] diff --git a/ami/main/api/occurrence_history/tests.py b/ami/main/api/occurrence_history/tests.py new file mode 100644 index 000000000..114b8d869 --- /dev/null +++ b/ami/main/api/occurrence_history/tests.py @@ -0,0 +1,448 @@ +"""Algorithm results and the occurrence history endpoint that reads them.""" + +import datetime +import re + +from django.db import connection +from django.test import TestCase +from django.test.utils import CaptureQueriesContext +from rest_framework.test import APITestCase + +from ami.jobs.models import Job +from ami.main.api.occurrence_history.timeline import occurrence_timeline +from ami.main.models import ( + Classification, + Detection, + Identification, + Occurrence, + SourceImage, + SourceImageCollection, + TaxaList, + Taxon, +) +from ami.ml.models import Algorithm, AlgorithmResult +from ami.ml.post_processing.class_masking import ClassMaskingResultData +from ami.ml.post_processing.small_size_filter import SizeFilterResultData +from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project +from ami.tests.fixtures.queries import no_query_cache +from ami.users.models import User +from ami.users.roles import BasicMember, ProjectManager + +# Measured: two savepoints, the project, the occurrence, the results with their algorithm and job, +# their taxa, the classifications they created, the classifications those replaced, identifications, +# predictions with their jobs, and one query per type of record the entries name (here capture sets). +HISTORY_QUERIES = 11 + +SIZE_FILTER = SizeFilterResultData.kind +CLASS_MASKING = ClassMaskingResultData.kind + + +class OccurrenceHistorySchemaTestCase(TestCase): + """The OpenAPI schema publishes the history as a oneOf of identification, prediction and result entries.""" + + def test_the_history_entries_are_a_oneof_tagged_by_type(self): + from drf_spectacular.generators import SchemaGenerator + + components = SchemaGenerator(api_version="api").get_schema(request=None, public=True)["components"]["schemas"] + entry_names = {ref["$ref"].rsplit("/", 1)[-1] for ref in components["OccurrenceHistoryEntry"]["oneOf"]} + self.assertEqual(entry_names, {"IdentificationEntry", "PredictionEntry", "AlgorithmResultEntry"}) + # Named enums, not drf-spectacular's hashed fallback names. + self.assertIn("AlgorithmResultEntryTypeEnum", components) + + +class OccurrenceFixtureTestCase(APITestCase): + """One occurrence of four classified detections, with a project manager and a basic member.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.project, cls.deployment = setup_test_project(reuse=False) + create_taxa(project=cls.project) + create_captures(deployment=cls.deployment, num_nights=1, images_per_night=4, interval_minutes=1) + captures = list(SourceImage.objects.filter(deployment=cls.deployment).order_by("timestamp")) + cls.taxon = Taxon.objects.filter(projects=cls.project).order_by("pk").first() + + cls.manager = User.objects.create_user(email="history-manager@insectai.org") # type: ignore + cls.reader = User.objects.create_user(email="history-reader@insectai.org") # type: ignore + ProjectManager.assign_user(cls.manager, cls.project) + BasicMember.assign_user(cls.reader, cls.project) + + cls.occurrence = Occurrence.objects.create( + event=captures[0].event, deployment=cls.deployment, project=cls.project + ) + cls.detections = [] + for capture in captures[:4]: + detection = Detection.objects.create( + source_image=capture, timestamp=capture.timestamp, bbox=[10, 10, 40, 40], occurrence=cls.occurrence + ) + detection.classifications.create(taxon=cls.taxon, score=0.9, timestamp=capture.timestamp) + cls.detections.append(detection) + cls.occurrence.save() + + +class OccurrenceHistoryEndpointTestCase(OccurrenceFixtureTestCase): + """GET /occurrences/{id}/history/ merges results, identifications and predictions, newest first.""" + + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + cls.size_filter = Algorithm.objects.create(name="Small size filter", key="size-filter-history-test") + cls.job = Job.objects.create( + project=cls.project, + name="Size filter run", + job_type_key="post_processing", + params={"task": "small_size_filter", "config": {"size_threshold": 0.01}}, + ) + cls.other_taxon = Taxon.objects.filter(projects=cls.project).exclude(pk=cls.taxon.pk).first() + cls.superuser = User.objects.create_superuser(email="history-super@insectai.org") # type: ignore + cls.outsider = User.objects.create_user(email="history-outsider@insectai.org") # type: ignore + + def url(self, occurrence: Occurrence | None = None) -> str: + return f"/api/v2/occurrences/{(occurrence or self.occurrence).pk}/history/?project_id={self.project.pk}" + + def get(self, user: User | None = None): + self.client.force_authenticate(user=user or self.reader) + response = self.client.get(self.url()) + self.assertEqual(response.status_code, 200, response.data) + return response.data + + def _add_history(self, start: datetime.datetime, rounds: int = 1) -> None: + """Per round: a size filter result with the classification it created, and an identification.""" + for i in range(rounds): + at = start + datetime.timedelta(hours=4 * i) + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.size_filter, + job=self.job, + kind=SIZE_FILTER, + value=0.001, + data={ + "relative_size": 0.001, + "determination_before_id": self.other_taxon.pk, + "determination_after_id": self.taxon.pk, + }, + timestamp=at, + ) + Classification.objects.create( + detection=self.detections[0], + taxon=self.taxon, + score=0.1, + algorithm=self.size_filter, + timestamp=at, + job=self.job, + algorithm_result=result, + ) + identification = Identification.objects.create( + occurrence=self.occurrence, user=self.reader, taxon=self.other_taxon, comment=f"round {i}" + ) + Identification.objects.filter(pk=identification.pk).update(created_at=at + datetime.timedelta(hours=1)) + + def test_entries_are_merged_newest_first_and_a_result_carries_the_classifications_it_created(self): + # Predictions are stamped with created_at (now), so the history rows sit in the past. + self._add_history(datetime.datetime.now() - datetime.timedelta(days=2)) + data = self.get() + + types = [entry["type"] for entry in data] + self.assertEqual(types[-2:], ["identification", "algorithm_result"]) + # The fixture's detections share one tied top prediction, shown once; the result's + # classification is shown inside the result, not as a prediction of its own. + self.assertEqual(types[:-2], ["prediction"]) + self.assertIsNone(data[0]["algorithm"]) + timestamps = [entry["timestamp"] for entry in data] + self.assertEqual(timestamps, sorted(timestamps, reverse=True)) + + identification, result = data[-2:] + self.assertEqual(set(identification["user"]), {"id", "name", "image"}) + self.assertEqual(identification["user"]["id"], self.reader.pk) + self.assertEqual(identification["taxon"]["id"], self.other_taxon.pk) + self.assertEqual(identification["details"]["comment"], "round 0") + self.assertEqual(result["kind"], "size_filter") + self.assertEqual(result["algorithm"]["key"], self.size_filter.key) + self.assertEqual( + result["job"], + { + "id": self.job.pk, + "name": self.job.name, + # The label is the title the size filter's config schema gives the setting. + "config": [{"key": "size_threshold", "label": "Size threshold", "value": 0.01, "refs": []}], + }, + ) + self.assertIsNone(result["taxon"]) + self.assertEqual(result["determination_after"]["id"], self.taxon.pk) + self.assertEqual(result["determination_before"]["id"], self.other_taxon.pk) + self.assertIsNone(result["score"]) + self.assertEqual(result["value"], 0.001) + self.assertEqual(result["data"]["relative_size"], 0.001) + created = Classification.objects.get(algorithm=self.size_filter) + self.assertEqual( + result["classifications"], + [ + { + "id": created.pk, + "taxon": {"id": self.taxon.pk, "name": self.taxon.name, "rank": self.taxon.rank, "parents": []}, + "score": 0.1, + "terminal": True, + "detection_id": self.detections[0].pk, + "replaced": None, + } + ], + ) + self.assertNotIn("email", str(data)) + + def test_a_masked_prediction_shows_inside_its_result_and_the_original_stays_as_a_prediction(self): + """Class masking demotes the original classification and links its replacement to the result.""" + classifier = Algorithm.objects.create(name="Classifier", key="classifier-history-test") + masker = Algorithm.objects.create(name="Masked classifier", key="masker-history-test") + taxa_list = TaxaList.objects.create(name="Kept species") + job = Job.objects.create( + project=self.project, + name="Masking run", + job_type_key="post_processing", + params={"task": "class_masking", "config": {"taxa_list_id": taxa_list.pk, "algorithm_id": classifier.pk}}, + ) + original = Classification.objects.get(detection=self.detections[1]) + Classification.objects.filter(pk=original.pk).update(terminal=False, algorithm=classifier) + # A second classifier on the same detection that the run did not re-score. + bystander_algorithm = Algorithm.objects.create(name="Second classifier", key="bystander-history-test") + bystander = Classification.objects.create( + detection=self.detections[1], + taxon=self.taxon, + score=0.99, + terminal=True, + algorithm=bystander_algorithm, + timestamp=datetime.datetime.now(), + ) + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=masker, + job=job, + kind=CLASS_MASKING, + value=0.4, + data={ + "excluded_probability": 0.4, + "new_winner_original_rank": 2, + "determination_before_id": self.taxon.pk, + "determination_after_id": self.other_taxon.pk, + }, + ) + masked = Classification.objects.create( + detection=self.detections[1], + taxon=self.other_taxon, + score=0.95, + algorithm=masker, + timestamp=datetime.datetime.now(), + applied_to=original, + job=job, + algorithm_result=result, + ) + data = self.get() + + result_entry = next(entry for entry in data if entry["type"] == "algorithm_result") + self.assertEqual([c["id"] for c in result_entry["classifications"]], [masked.pk]) + # The masked classification shows inside the result, not as a prediction; the original + # keeps its card, demoted to non-terminal. + predictions = { + entry["algorithm"]["key"] if entry["algorithm"] else None: entry + for entry in data + if entry["type"] == "prediction" + } + self.assertEqual(set(predictions), {None, classifier.key, bystander_algorithm.key}) + self.assertEqual(predictions[bystander_algorithm.key]["id"], bystander.pk) + demoted = predictions[classifier.key] + self.assertEqual(demoted["id"], original.pk) + self.assertFalse(demoted["details"]["terminal"]) + config = {field["key"]: field for field in result_entry["job"]["config"]} + self.assertEqual( + config["taxa_list_id"], + { + "key": "taxa_list_id", + "label": "Species list", + "value": taxa_list.pk, + "refs": [{"type": "taxa_list", "id": taxa_list.pk, "name": "Kept species"}], + }, + ) + self.assertEqual(config["algorithm_id"]["label"], "Classifier") + self.assertEqual( + config["algorithm_id"]["refs"], [{"type": "algorithm", "id": classifier.pk, "name": classifier.name}] + ) + # The top prediction before masking is the classification the masked one replaced. + self.assertEqual(result_entry["classifications"][0]["replaced"]["id"], original.pk) + self.assertEqual(result_entry["classifications"][0]["replaced"]["taxon"]["id"], self.taxon.pk) + self.assertEqual(result_entry["value"], 0.4) + + def test_each_algorithm_shows_one_prediction_preferring_terminal_then_latest(self): + """Tied top scores would otherwise show the same prediction once per detection.""" + classifier = Algorithm.objects.create(name="Tied classifier", key="tied-classifier-history-test") + other = Algorithm.objects.create(name="Second tied classifier", key="second-tied-classifier-history-test") + now = datetime.datetime.now() + + def classify(detection, algorithm, terminal, created_at): + classification = Classification.objects.create( + detection=detection, taxon=self.taxon, score=0.8, algorithm=algorithm, terminal=terminal, timestamp=now + ) + Classification.objects.filter(pk=classification.pk).update(created_at=created_at) + return classification + + classify(self.detections[0], classifier, False, now) + terminal = classify(self.detections[1], classifier, True, now - datetime.timedelta(hours=1)) + classify(self.detections[2], other, True, now - datetime.timedelta(hours=1)) + latest = classify(self.detections[3], other, True, now) + + predictions = [entry for entry in self.get() if entry["type"] == "prediction"] + by_algorithm = { + entry["algorithm"]["key"] if entry["algorithm"] else None: entry["id"] for entry in predictions + } + self.assertEqual(len(predictions), 3) + self.assertEqual(by_algorithm[classifier.key], terminal.pk) + self.assertEqual(by_algorithm[other.key], latest.pk) + self.assertIn(None, by_algorithm) + + def test_query_count_does_not_grow_with_the_entries_and_no_score_arrays_are_loaded(self): + self.client.force_authenticate(user=self.reader) + # A setting that names a record, and created classifications that replaced one, so the + # reference and replaced-classification queries are counted too. + scope = SourceImageCollection.objects.create(project=self.project, name="Scope") + self.job.params = { + "task": "small_size_filter", + "config": {"size_threshold": 0.01, "source_image_collection_id": scope.pk}, + } + self.job.save() + original = Classification.objects.get(detection=self.detections[0]) + now = datetime.datetime.now() + for rounds, total in ((1, 1), (2, 3)): + self._add_history(now - datetime.timedelta(days=3 * total), rounds=rounds) + Classification.objects.filter(algorithm=self.size_filter).update(applied_to=original) + with self.subTest(rounds=total), no_query_cache(): + with CaptureQueriesContext(connection) as queries, self.assertNumQueries(HISTORY_QUERIES): + response = self.client.get(self.url()) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2 * total + 1) + self.assertFalse( + [q["sql"] for q in queries.captured_queries if '"main_classification"."scores"' in q["sql"]] + ) + # A job's progress and logs can be large JSON documents; the history shows neither. + self.assertFalse( + [ + q["sql"] + for q in queries.captured_queries + if re.search(r'"jobs_job"\."(progress|logs)"', q["sql"]) + ] + ) + + def test_visible_to_whoever_can_open_the_occurrence(self): + """Member, non-member, anonymous and superuser, on a public and on a draft project.""" + expected = { + False: {"member": 200, "non-member": 200, "anonymous": 200, "superuser": 200}, + True: {"member": 200, "non-member": 404, "anonymous": 404, "superuser": 200}, + } + users = {"member": self.reader, "non-member": self.outsider, "anonymous": None, "superuser": self.superuser} + for draft, codes in expected.items(): + self.project.draft = draft + self.project.save() + for label, user in users.items(): + with self.subTest(draft=draft, user=label): + self.client.force_authenticate(user=user) + detail = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") + history = self.client.get(self.url()) + self.assertEqual(history.status_code, codes[label]) + self.assertEqual(history.status_code, detail.status_code) + + def test_an_occurrence_the_default_filters_hide_still_has_a_history(self): + """The session view lists occurrences with the default filters off, so their history must open too.""" + self.project.default_filters_score_threshold = 0.95 + self.project.save() + self.client.force_authenticate(user=self.reader) + detail = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") + self.assertEqual(detail.status_code, 404) + self.assertEqual(self.client.get(self.url()).status_code, 200) + + def test_the_list_filters_do_not_hide_the_history(self): + """A list filter in the query string names no occurrence, so it must not turn the history into a 404.""" + self.client.force_authenticate(user=self.reader) + response = self.client.get(f"{self.url()}&event=0&taxon=0&search=nothing-matches") + self.assertEqual(response.status_code, 200) + + def test_a_result_lists_what_each_created_classification_replaced(self): + classifier = Algorithm.objects.create(name="Classifier", key="replaced-test-classifier") + masker = Algorithm.objects.create(name="Masked", key="replaced-test-masker") + original = Classification.objects.get(detection=self.detections[1]) + Classification.objects.filter(pk=original.pk).update(terminal=False, algorithm=classifier) + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=masker, + kind=CLASS_MASKING, + value=0.4, + data={"excluded_probability": 0.4}, + ) + masked = Classification.objects.create( + detection=self.detections[1], + taxon=self.other_taxon, + score=0.9, + algorithm=masker, + timestamp=datetime.datetime.now(), + applied_to=original, + algorithm_result=result, + ) + # The original was deleted (applied_to is SET_NULL until #1472), so nothing is shown as replaced. + orphan = Classification.objects.create( + detection=self.detections[2], + taxon=self.other_taxon, + score=0.8, + algorithm=masker, + timestamp=datetime.datetime.now(), + applied_to=None, + algorithm_result=result, + ) + entry = next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") + by_id = {c.classification.pk: c for c in entry.classifications} + self.assertEqual(by_id[masked.pk].replaced.pk, original.pk) + self.assertEqual(by_id[masked.pk].replaced.taxon_id, original.taxon_id) + self.assertIsNone(by_id[orphan.pk].replaced) + + def test_classifications_link_to_a_result_only_through_algorithm_result(self): + """A classification sharing the result's job but not linked to it stays an ordinary prediction.""" + AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.size_filter, + job=self.job, + kind=SIZE_FILTER, + data={"relative_size": 0.001}, + ) + unlinked = Classification.objects.create( + detection=self.detections[0], + taxon=self.taxon, + score=0.99, + algorithm=self.size_filter, + timestamp=datetime.datetime.now(), + job=self.job, + ) + entries = occurrence_timeline(self.occurrence) + result_entry = next(e for e in entries if e.type == "algorithm_result") + self.assertEqual(result_entry.classifications, []) + self.assertIn(unlinked.pk, [e.id for e in entries if e.type == "prediction"]) + + def test_predictions_carry_the_job_that_wrote_them(self): + Classification.objects.filter(detection__occurrence=self.occurrence).update(job=self.job) + predictions = [e for e in occurrence_timeline(self.occurrence) if e.type == "prediction"] + self.assertTrue(predictions) + self.assertTrue(all(e.job == self.job for e in predictions)) + + def test_a_result_without_a_job_has_no_config(self): + AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.size_filter, + job=None, + kind=SIZE_FILTER, + data={"relative_size": 0.001}, + ) + entry = next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") + self.assertIsNone(entry.job) + self.assertEqual(entry.job_config, []) + + def test_a_result_of_a_kind_no_longer_registered_still_shows_in_the_history(self): + """A row written by a branch with an extra kind, or before a kind was renamed, must not break the history.""" + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, algorithm=self.size_filter, kind=SIZE_FILTER, data={"relative_size": 0.001} + ) + AlgorithmResult.objects.filter(pk=result.pk).update(kind="rank_rollup") + entry = next(e for e in self.get() if e["type"] == "algorithm_result") + self.assertEqual(entry["kind"], "rank_rollup") diff --git a/ami/main/api/occurrence_history/timeline.py b/ami/main/api/occurrence_history/timeline.py new file mode 100644 index 000000000..072c1bd62 --- /dev/null +++ b/ami/main/api/occurrence_history/timeline.py @@ -0,0 +1,197 @@ +"""An occurrence's history: what algorithms and people did to it, newest first. + +``occurrence_timeline`` merges an occurrence's algorithm results, identifications and predictions +into one list of ``OccurrenceTimelineEntry`` for the history endpoint. What a post-processing run +decided about an occurrence is an ``AlgorithmResult``; identifications and predictions keep their +own tables. +""" + +from __future__ import annotations + +import dataclasses +import datetime +import typing + +from ami.jobs.job_config import JobConfigField, job_config_fields +from ami.main.models import Classification, Identification, Occurrence, Taxon, User +from ami.ml.models import Algorithm, AlgorithmResult + +if typing.TYPE_CHECKING: + from ami.jobs.models import Job + +# A job's progress and logs can be large JSON documents, and the history shows neither. +JOB_FIELDS_NOT_SHOWN = ("job__progress", "job__logs") + + +@dataclasses.dataclass +class CreatedClassification: + """A classification a run created, with the one it replaced (through applied_to), if that still exists.""" + + classification: Classification + replaced: Classification | None + + +@dataclasses.dataclass +class OccurrenceTimelineEntry: + """One entry of an occurrence's merged history. Sections below say which fields each type fills.""" + + # --- Every entry + type: str # "algorithm_result" | "identification" | "prediction" + id: int + timestamp: datetime.datetime + user: User | None = None # identification + algorithm: Algorithm | None = None # result, prediction + job: Job | None = None # result, prediction + job_config: list[JobConfigField] = dataclasses.field(default_factory=list) # empty without a job + taxon: Taxon | None = None # identification, prediction + score: float | None = None # prediction + # --- Algorithm result + kind: str | None = None + value: float | None = None # the kind's headline figure, not a confidence + data: dict = dataclasses.field(default_factory=dict) # validated per kind + determination_before: Taxon | None = None + determination_after: Taxon | None = None + classifications: list[CreatedClassification] = dataclasses.field(default_factory=list) + # --- Identification and prediction + details: dict = dataclasses.field(default_factory=dict) + + +def occurrence_timeline(occurrence: Occurrence) -> list[OccurrenceTimelineEntry]: + """Results, identifications and predictions of one occurrence, merged newest first. + + A result comes with the classifications its run created, so the history shows the run and + what it changed as one entry; those classifications are not listed again as predictions. + Every other algorithm contributes one prediction, its best. Ids the entries show are resolved + to references with one query per type. The query count does not grow + with the entries. + """ + results = list( + AlgorithmResult.objects.filter(occurrence=occurrence) + .select_related("algorithm", "job") + .defer(*JOB_FIELDS_NOT_SHOWN) + .order_by("-timestamp", "-pk") + ) + taxa = _by_id( + Taxon, + {result.data.get(key) for result in results for key in ("determination_before_id", "determination_after_id")}, + ) + created_by_result = _classifications_created_by(results, occurrence) + created_ids = {c.classification.pk for created in created_by_result.values() for c in created} + entries = [ + OccurrenceTimelineEntry( + type="algorithm_result", + id=result.pk, + timestamp=result.timestamp, + algorithm=result.algorithm, + job=result.job, + value=result.value, + kind=result.kind, + data=result.data, + determination_before=taxa.get(result.data.get("determination_before_id")), + determination_after=taxa.get(result.data.get("determination_after_id")), + classifications=created_by_result.get(result.pk, []), + ) + for result in results + ] + + identifications = Identification.objects.filter(occurrence=occurrence).select_related("user", "taxon") + entries.extend( + OccurrenceTimelineEntry( + type="identification", + id=identification.pk, + timestamp=identification.created_at, + user=identification.user, + taxon=identification.taxon, + details={ + "comment": identification.comment or "", + "withdrawn": identification.withdrawn, + }, + ) + for identification in identifications + ) + + entries.extend( + OccurrenceTimelineEntry( + type="prediction", + id=prediction.pk, + timestamp=prediction.created_at, + algorithm=prediction.algorithm, + job=prediction.job, + taxon=prediction.taxon, + score=prediction.score, + details={"terminal": prediction.terminal}, + ) + for prediction in _one_prediction_per_algorithm(occurrence) + if prediction.pk not in created_ids + ) + + config_fields = job_config_fields({entry.job.pk: entry.job for entry in entries if entry.job}.values()) + for entry in entries: + if entry.job is not None: + entry.job_config = config_fields[entry.job.pk] + entries.sort(key=lambda entry: (entry.timestamp, entry.id), reverse=True) + return entries + + +def _by_id(model, ids: set) -> dict: + ids.discard(None) + return {row.pk: row for row in model.objects.filter(pk__in=ids)} if ids else {} + + +def _classifications_created_by( + results: list[AlgorithmResult], occurrence: Occurrence +) -> dict[int, list[CreatedClassification]]: + """The classifications each result's run created on the occurrence, keyed by result id. + + A classification names its result through ``algorithm_result``. Each comes with the + classification it replaced, read in one more query. + """ + if not results: + return {} + created = list( + Classification.objects.filter( + detection__occurrence=occurrence, algorithm_result_id__in=[result.pk for result in results] + ) + .select_related("taxon", "algorithm") + # The score arrays can hold tens of thousands of values per row and are not shown here. + .defer("scores", "logits") + .order_by("-score", "-pk") + ) + # A separate query, not select_related("applied_to"): .defer() does not reach a self-join, + # so the replaced rows' score arrays would load. + replaced_ids = {c.applied_to_id for c in created if c.applied_to_id is not None} + replaced = ( + { + c.pk: c + for c in Classification.objects.filter(pk__in=replaced_ids) + .select_related("taxon") + .only("pk", "score", "taxon", "taxon__name", "taxon__rank", "taxon__parents_json") + } + if replaced_ids + else {} + ) + by_result: dict[int, list[CreatedClassification]] = {} + for classification in created: + by_result.setdefault(classification.algorithm_result_id, []).append( + CreatedClassification(classification, replaced.get(classification.applied_to_id)) + ) + return by_result + + +def _one_prediction_per_algorithm(occurrence: Occurrence) -> list[Classification]: + """The best prediction of each algorithm: highest score, then terminal, then latest. + + ``Occurrence.predictions()`` keeps every classification tied for an algorithm's top + score, which would show the same prediction once per detection. + """ + + def rank(prediction: Classification) -> tuple: + score = prediction.score if prediction.score is not None else float("-inf") + return (score, prediction.terminal, prediction.created_at, prediction.pk) + + best: dict[int | None, Classification] = {} + for prediction in occurrence.predictions().select_related("job").defer("scores", "logits", *JOB_FIELDS_NOT_SHOWN): + current = best.get(prediction.algorithm_id) + if current is None or rank(prediction) > rank(current): + best[prediction.algorithm_id] = prediction + return list(best.values()) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 4f8b19224..46aed982e 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -32,6 +32,8 @@ from ami.base.permissions import IsActiveStaffOrReadOnly, IsProjectMemberOrReadOnly, ObjectPermission from ami.base.serializers import FilterParamsSerializer, SingleParamSerializer from ami.base.views import ProjectMixin +from ami.main.api.occurrence_history.serializers import OCCURRENCE_HISTORY_ENTRY_SCHEMA, serialize_history +from ami.main.api.occurrence_history.timeline import occurrence_timeline from ami.main.api.schemas import limit_doc_param, project_id_doc_param from ami.main.api.serializers import TagSerializer from ami.main.models_future.identifications import create_identifications_batch, resolve_occurrences @@ -1522,11 +1524,17 @@ def get_serializer_class(self): else: return OccurrenceSerializer + # Actions that read one occurrence's history. They build their own queries and never + # serialize the occurrence, and the default filters, which shape lists, do not hide it. + HISTORY_ACTIONS = ("history",) + def get_queryset(self) -> QuerySet["Occurrence"]: project = self.get_active_project() qs = super().get_queryset().valid() # type: ignore if project: qs = qs.filter(project=project) + if self.action in self.HISTORY_ACTIONS: + return qs qs = qs.select_related( "determination", "deployment", @@ -1575,6 +1583,25 @@ def get_queryset(self) -> QuerySet["Occurrence"]: def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) + @extend_schema( + operation_id="occurrences_history_retrieve", + parameters=[project_id_doc_param], + filters=False, + responses=OCCURRENCE_HISTORY_ENTRY_SCHEMA, + ) + @action(detail=True, methods=["get"], name="history", pagination_class=None, filter_backends=[]) + def history(self, request: Request, pk=None) -> Response: + """Everything that happened to this occurrence, newest first. + + Merges post-processing results with the classifications they created, identifications + and predictions into one list. Available to anyone who can open the occurrence, including + for an occurrence that the project's default filters hide from lists and the detail view. + The list's filters do not apply: the history is for one occurrence, found by its id. + """ + occurrence = self.get_object() + entries = occurrence_timeline(occurrence) + return Response(serialize_history(entries, {"request": request})) + @extend_schema(parameters=[project_id_doc_param], responses=AlgorithmSerializer(many=True)) @action(detail=False, methods=["get"], name="algorithms") def algorithms(self, request: Request) -> Response: diff --git a/ami/main/migrations/0098_classification_algorithm_result.py b/ami/main/migrations/0098_classification_algorithm_result.py new file mode 100644 index 000000000..59238b19f --- /dev/null +++ b/ami/main/migrations/0098_classification_algorithm_result.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.10 on 2026-10-05 19:29 +"""The link from a classification to the algorithm result it was created for. Its index is built +concurrently in 0099, so this ALTER TABLE holds its lock only for adding a nullable column.""" + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("ml", "0029_algorithm_result"), + ("main", "0097_detection_and_classification_job_indexes"), + ] + + operations = [ + migrations.AddField( + model_name="classification", + name="algorithm_result", + field=models.ForeignKey( + blank=True, + db_index=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="classifications", + to="ml.algorithmresult", + ), + ), + ] diff --git a/ami/main/migrations/0099_classification_algorithm_result_index.py b/ami/main/migrations/0099_classification_algorithm_result_index.py new file mode 100644 index 000000000..83d081317 --- /dev/null +++ b/ami/main/migrations/0099_classification_algorithm_result_index.py @@ -0,0 +1,36 @@ +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + """Index the classification's link to its algorithm result, which deleting a result looks up. + + Partial, because pipeline classifications never have a result, so the index stays small. + Built CONCURRENTLY on this large table, which needs a non-atomic migration; see 0093 for + why the statement timeout is cleared and restored around the build. + """ + + atomic = False + + dependencies = [ + ("main", "0098_classification_algorithm_result"), + ] + + operations = [ + migrations.RunSQL( + sql="SET statement_timeout = 0;", + reverse_sql=migrations.RunSQL.noop, + ), + AddIndexConcurrently( + model_name="classification", + index=models.Index( + condition=models.Q(algorithm_result__isnull=False), + fields=["algorithm_result"], + name="cls_algorithm_result_idx", + ), + ), + migrations.RunSQL( + sql="RESET statement_timeout;", + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index d064e353e..6f91ea374 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -1303,6 +1303,9 @@ class EventManager(models.Manager.from_queryset(EventQuerySet)): class Event(BaseModel): """A monitoring session""" + reference_type = "session" + reference_name_field = "name" + objects: EventManager = EventManager() group_by = models.CharField( max_length=255, @@ -3072,7 +3075,12 @@ class ClassificationManager(models.Manager.from_queryset(ClassificationQuerySet) @final class Classification(BaseModel): - """The output of a classifier""" + """A taxon classification of one detection: the taxon a classifier or a post-processing run assigned it. + + ``job`` records the run that wrote it, when there was one. ``algorithm_result`` is the + post-processing result the row was created for, so the occurrence history can show a run + with the classifications it produced; classifications from a pipeline leave it empty. + """ project_accessor = "detection__source_image__project" detection = models.ForeignKey( @@ -3114,6 +3122,15 @@ class Classification(BaseModel): related_name="classifications", db_index=False, ) + # Indexed by the partial index in Meta, not by the foreign key's own index. + algorithm_result = models.ForeignKey( + "ml.AlgorithmResult", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="classifications", + db_index=False, + ) applied_to = models.ForeignKey( "self", on_delete=models.SET_NULL, @@ -3139,6 +3156,13 @@ class Meta: models.Index( fields=["job", "detection"], name="cls_job_detection_idx", condition=models.Q(job__isnull=False) ), + # Deleting a result looks up its classifications. Partial, because pipeline + # classifications never have a result. + models.Index( + fields=["algorithm_result"], + name="cls_algorithm_result_idx", + condition=models.Q(algorithm_result__isnull=False), + ), ] def __str__(self) -> str: @@ -3526,15 +3550,17 @@ def not_processed_by_algorithm(self, algorithm_ids) -> "OccurrenceQuerySet": def created_or_updated_by_job(self, job_id: int) -> "OccurrenceQuerySet": """Occurrences created or updated by the given job. - An occurrence matches when the job created one of its detections, or a classification - on one of them, so several jobs can match the same occurrence. + An occurrence matches when the job created one of its detections, a classification + on one of them, or an algorithm result for it, so several jobs can match the same occurrence. - Identifications are not matched: people make them, not jobs. Two EXISTS subqueries - return each occurrence once, where a join would return one row per matching result. + Identifications are not matched: people make them, not jobs. EXISTS subqueries return + each occurrence once, where a join would return one row per matching result. """ + AlgorithmResult = apps.get_model("ml", "AlgorithmResult") return self.filter( Exists(Detection.objects.filter(occurrence=OuterRef("pk"), job_id=job_id)) | Exists(Classification.objects.filter(detection__occurrence=OuterRef("pk"), job_id=job_id)) + | Exists(AlgorithmResult.objects.filter(occurrence=OuterRef("pk"), job_id=job_id)) ) def with_timestamps(self): @@ -3765,6 +3791,9 @@ def get_queryset(self): class Occurrence(BaseModel): """An occurrence of a taxon, a sequence of one or more detections""" + reference_type = "occurrence" + reference_name_field = None + # @TODO change Determination to a nested field with a Taxon, User, Identification, etc like the serializer # this could be a OneToOneField to a Determination model or a JSONField validated by a Pydantic model determination = models.ForeignKey("Taxon", on_delete=models.SET_NULL, null=True, related_name="occurrences") @@ -4928,6 +4957,9 @@ class TaxaListManager(models.Manager.from_queryset(TaxaListQuerySet)): class TaxaList(BaseModel): """A checklist of taxa""" + reference_type = "taxa_list" + reference_name_field = "name" + name = models.CharField(max_length=255) description = models.TextField(blank=True) @@ -5112,6 +5144,9 @@ class SourceImageCollection(BaseModel): """ + reference_type = "capture_set" + reference_name_field = "name" + name = models.CharField(max_length=255) description = models.TextField(blank=True) # dataset_type = models.CharField( diff --git a/ami/main/tests.py b/ami/main/tests.py index 6c4a200de..c07564439 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -42,9 +42,11 @@ TaxonRank, group_images_into_events, ) +from ami.ml.models.algorithm_result import AlgorithmResult from ami.ml.models.pipeline import Pipeline from ami.ml.models.processing_service import ProcessingService from ami.ml.models.project_pipeline_config import ProjectPipelineConfig +from ami.ml.post_processing.small_size_filter import SizeFilterResultData from ami.tests.fixtures.main import ( create_captures, create_captures_from_files, @@ -7409,8 +7411,8 @@ def test_exclude_is_the_complement_of_include(self): class TestOccurrenceJobFilter(APITestCase): """ - Covers the ``?job=`` occurrence filter: occurrences with a detection or a - classification written by the given job. Each occurrence must appear once, however + Covers the ``?job=`` occurrence filter: occurrences with a detection, a classification + or an algorithm result written by the given job. Each occurrence must appear once, however many of its rows the job wrote, and a malformed id must be a 400, not a 500. """ @@ -7446,6 +7448,15 @@ def setUp(self): # Only the other job, and no job at all, must not match. self.occ_other = self._make_occurrence([(self.other_job, [self.other_job])]) self.occ_none = self._make_occurrence([(None, [None])]) + # The job wrote only an algorithm result, as a post-processing run that changes no prediction does. + self.occ_result = self._make_occurrence([(None, [None])]) + AlgorithmResult.objects.record( + occurrence=self.occ_result, + algorithm=self.algorithm, + job=self.job, + kind=SizeFilterResultData.kind, + data={"relative_size": 0.001}, + ) def _make_occurrence(self, detections) -> Occurrence: """``detections`` is a list of (detection job, [classification jobs]).""" @@ -7481,8 +7492,10 @@ def test_returns_each_occurrence_the_job_wrote_once(self): self.assertEqual(response.status_code, 200) ids = [row["id"] for row in response.json()["results"]] self.assertEqual(len(ids), len(set(ids))) - self.assertEqual(set(ids), {self.occ_detected.pk, self.occ_classified.pk, self.occ_multi.pk}) - self.assertEqual(response.json()["count"], 3) + self.assertEqual( + set(ids), {self.occ_detected.pk, self.occ_classified.pk, self.occ_multi.pk, self.occ_result.pk} + ) + self.assertEqual(response.json()["count"], 4) def test_other_job_matches_only_its_own_occurrences(self): response = self._list(self.other_job.pk) @@ -7499,23 +7512,18 @@ def test_filter_stays_inside_the_occurrence_query(self): """The filter adds no queries of its own: one statement for the rows and one for the count, with no ids read into Python first and no query per occurrence. Cachalot is off so every query counts.""" - from cachalot.api import cachalot_disabled + from ami.tests.fixtures.queries import no_query_cache def filtered(): return Occurrence.objects.filter(project=self.project).created_or_updated_by_job(self.job.pk) - disabled = cachalot_disabled() - disabled.__enter__() - try: + with no_query_cache(): with self.assertNumQueries(1): ids = {occurrence.pk for occurrence in filtered()} with self.assertNumQueries(1): count = filtered().count() - finally: - # cachalot_disabled() does not restore itself when the block raises. - disabled.__exit__(None, None, None) - self.assertEqual(ids, {self.occ_detected.pk, self.occ_classified.pk, self.occ_multi.pk}) - self.assertEqual(count, 3) + self.assertEqual(ids, {self.occ_detected.pk, self.occ_classified.pk, self.occ_multi.pk, self.occ_result.pk}) + self.assertEqual(count, 4) class TestCleanupNullOnlyOccurrencesCommand(TestCase): diff --git a/ami/ml/admin.py b/ami/ml/admin.py index 008b20e84..e07d9ad5f 100644 --- a/ami/ml/admin.py +++ b/ami/ml/admin.py @@ -3,6 +3,7 @@ from ami.main.admin import AdminBase, ProjectPipelineConfigInline from .models.algorithm import Algorithm, AlgorithmCategoryMap +from .models.algorithm_result import AlgorithmResult from .models.pipeline import Pipeline from .models.processing_service import ProcessingService @@ -32,6 +33,25 @@ class AlgorithmAdmin(AdminBase): ] +@admin.register(AlgorithmResult) +class AlgorithmResultAdmin(AdminBase): + """Read-only: results are written by their runs through the writer, which validates them.""" + + # Ids rather than the occurrence's name, which would join its deployment and determination per row. + list_display = ["id", "kind", "value", "occurrence_id", "algorithm", "job", "project", "timestamp"] + list_filter = ["kind", "algorithm", "project"] + list_select_related = ["algorithm", "job", "project"] + search_fields = ["=occurrence__id", "=job__id"] + ordering = ["-timestamp", "-pk"] + raw_id_fields = ["occurrence", "job"] + + def has_add_permission(self, request) -> bool: + return False + + def has_change_permission(self, request, obj=None) -> bool: + return False + + @admin.register(Pipeline) class PipelineAdmin(AdminBase): inlines = [ProjectPipelineConfigInline] diff --git a/ami/ml/migrations/0029_algorithm_result.py b/ami/ml/migrations/0029_algorithm_result.py new file mode 100644 index 000000000..3613f9cec --- /dev/null +++ b/ami/ml/migrations/0029_algorithm_result.py @@ -0,0 +1,75 @@ +# Generated by Django 4.2.10 on 2026-10-05 19:29 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + dependencies = [ + ("jobs", "0023_alter_job_job_type_key"), + ("main", "0097_detection_and_classification_job_indexes"), + ("ml", "0028_normalize_empty_endpoint_url_to_null"), + ] + + operations = [ + migrations.CreateModel( + name="AlgorithmResult", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("kind", models.CharField(max_length=32)), + ("value", models.FloatField(blank=True, null=True)), + ("data", models.JSONField(blank=True, default=dict)), + ("timestamp", models.DateTimeField(default=django.utils.timezone.now)), + ( + "algorithm", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="algorithm_results", + to="ml.algorithm", + ), + ), + ( + "job", + models.ForeignKey( + blank=True, + db_index=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="algorithm_results", + to="jobs.job", + ), + ), + ( + "occurrence", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="algorithm_results", + to="main.occurrence", + ), + ), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="algorithm_results", + to="main.project", + ), + ), + ], + options={ + "indexes": [ + models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), + models.Index( + condition=models.Q(("value__isnull", False)), + fields=["project", "kind", "value"], + name="algorithm_result_value", + ), + models.Index(fields=["job", "occurrence"], name="algorithm_result_job_occ"), + ], + }, + ), + ] diff --git a/ami/ml/models/__init__.py b/ami/ml/models/__init__.py index 5000c7f53..07eefda82 100644 --- a/ami/ml/models/__init__.py +++ b/ami/ml/models/__init__.py @@ -1,4 +1,5 @@ from ami.ml.models.algorithm import Algorithm, AlgorithmCategoryMap +from ami.ml.models.algorithm_result import AlgorithmResult from ami.ml.models.pipeline import Pipeline from ami.ml.models.processing_service import ProcessingService from ami.ml.models.project_pipeline_config import ProjectPipelineConfig @@ -6,6 +7,7 @@ __all__ = [ "Algorithm", "AlgorithmCategoryMap", + "AlgorithmResult", "Pipeline", "ProcessingService", "ProjectPipelineConfig", diff --git a/ami/ml/models/algorithm.py b/ami/ml/models/algorithm.py index 605f90861..e7c272946 100644 --- a/ami/ml/models/algorithm.py +++ b/ami/ml/models/algorithm.py @@ -228,6 +228,9 @@ def as_choice(self): class Algorithm(BaseModel): """A machine learning algorithm""" + reference_type = "algorithm" + reference_name_field = "name" + name = models.CharField(max_length=255) key = models.SlugField(max_length=255, unique=True) task_type = models.CharField( diff --git a/ami/ml/models/algorithm_result.py b/ami/ml/models/algorithm_result.py new file mode 100644 index 000000000..8153c69b5 --- /dev/null +++ b/ami/ml/models/algorithm_result.py @@ -0,0 +1,127 @@ +import logging +import typing + +from django.db import models +from django.db.models import Q +from django.utils import timezone + +from ami.base.models import BaseModel, BaseQuerySet +from ami.ml.results.schemas import result_value, validate_result_data + +logger = logging.getLogger(__name__) + + +class AlgorithmResultQuerySet(BaseQuerySet): + def bulk_create(self, objs, *args, **kwargs): + objs = list(objs) + for obj in objs: + obj.fill_from_data() + return super().bulk_create(objs, *args, **kwargs) + + def bulk_update(self, objs, fields, *args, **kwargs): + objs = list(objs) + if "data" in fields: + for obj in objs: + obj.fill_from_data() + fields = [*fields, "value"] if "value" not in fields else fields + return super().bulk_update(objs, fields, *args, **kwargs) + + def record(self, **fields) -> "AlgorithmResult": + """Insert one result.""" + results = self.record_many([self.model(**fields)]) + if not results: + raise ValueError("The occurrence has no project.") + return results[0] + + def record_many(self, results: typing.Iterable["AlgorithmResult"]) -> list["AlgorithmResult"]: + """Insert results, each with its project copied from its occurrence. + + Results are only ever added: a later run's result sits beside the earlier ones. A result + whose occurrence has no project is skipped with a warning rather than failing a run that + has already changed data. The returned list holds the results that were written, in order. + """ + kept: list[AlgorithmResult] = [] + for result in results: + if result.project_id is None: + result.project_id = result.occurrence.project_id + if result.project_id is None: + # Occurrences without a project are a data problem tracked on #1188; skipping keeps the run going. + logger.warning( + f"Not recording the {result.kind} result for Occurrence #{result.occurrence_id}: " + "it has no project." + ) + continue + kept.append(result) + return self.bulk_create(kept) if kept else [] + + +@typing.final +class AlgorithmResult(BaseModel): + """What one run of an algorithm decided about one occurrence, with the figures only that run knew. + + Occurrences are the only target today. Detections and captures are expected to become targets + of this same table; see ami/ml/results/README.md. + + The determination never reads this table. A run that changes an occurrence's taxon does so + through the ``Classification`` rows it creates, and those rows point back here through + ``Classification.algorithm_result`` so the history can show a run with what it changed. + ``data`` holds the run's own figures, validated against the model for ``kind`` + (ami/ml/results/schemas.py), and ``value`` repeats the one figure lists filter and sort on. The + ``extra`` object inside ``data`` is stored and returned by the API only; nothing reads it for + logic, and a value a feature needs becomes a typed field. Every run adds its own result, + so running a method twice leaves two results on the occurrence, one per job. Write through + ``AlgorithmResult.objects.record`` or ``record_many``. Tracking and rank roll-ups are the + next kinds expected. See #1431. + """ + + # Copied from the occurrence when the result is written, so per-project + # queries and permission checks need no join. + project = models.ForeignKey("main.Project", on_delete=models.CASCADE, related_name="algorithm_results") + # Indexed together with the timestamp, below. + occurrence = models.ForeignKey( + "main.Occurrence", on_delete=models.CASCADE, related_name="algorithm_results", db_index=False + ) + algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="algorithm_results") + # The run that wrote the result. Deleting the job keeps the result, with no run to link to. + # Indexed together with the occurrence, below. + job = models.ForeignKey( + "jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="algorithm_results", db_index=False + ) + # No ``choices``: the registry of data models in ami/ml/results/schemas.py is the list of kinds, + # and every write path rejects a kind without one, so a new kind needs no migration. + kind = models.CharField(max_length=32) + # The kind's headline figure, for filtering and sorting, copied from the data field its model + # names in ``value_field`` on every write. + value = models.FloatField(null=True, blank=True) + data = models.JSONField(default=dict, blank=True) + timestamp = models.DateTimeField(default=timezone.now) + + objects = AlgorithmResultQuerySet.as_manager() + + class Meta: + indexes = [ + models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), + models.Index( + fields=["project", "kind", "value"], + condition=Q(value__isnull=False), + name="algorithm_result_value", + ), + # Leads with the job, for the ?job= occurrence filter and for deleting a job. + models.Index(fields=["job", "occurrence"], name="algorithm_result_job_occ"), + ] + + def __str__(self) -> str: + return f"#{self.pk} {self.kind} for Occurrence #{self.occurrence_id} from Algorithm #{self.algorithm_id}" + + def fill_from_data(self) -> None: + """Validate ``data`` against the kind's model and set ``value`` to the figure the kind names.""" + self.data = validate_result_data(self.kind, self.data) + self.value = result_value(self.kind, self.data) + + def save(self, *args, **kwargs): + self.fill_from_data() + if self.project_id is None: + self.project_id = self.occurrence.project_id + if self.project_id is None: + raise ValueError("The occurrence has no project.") + super().save(*args, **kwargs) diff --git a/ami/ml/post_processing/base.py b/ami/ml/post_processing/base.py index 8f197f192..d64a4837f 100644 --- a/ami/ml/post_processing/base.py +++ b/ami/ml/post_processing/base.py @@ -8,6 +8,7 @@ from ami.ml.models import Algorithm from ami.ml.models.algorithm import AlgorithmTaskType +from ami.ml.results.schemas import AlgorithmResultData if typing.TYPE_CHECKING: from ami.jobs.models import Job @@ -27,6 +28,9 @@ class BasePostProcessingTask(abc.ABC): key: str name: str config_schema: type[pydantic.BaseModel] + # The result data models (ami/ml/results/schemas.py) of the algorithm results the task writes; + # empty for a task that writes none. Its config is declared by ``config_schema``. + result_models: tuple[type[AlgorithmResultData], ...] = () def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index 18b22c26d..ab4443dcb 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -8,9 +8,12 @@ from django.db.models import QuerySet from django.utils import timezone +from ami.base.model_references import model_reference from ami.main.models import Classification, Occurrence, SourceImageCollection, TaxaList from ami.ml.models.algorithm import Algorithm, AlgorithmTaskType from ami.ml.post_processing.base import BasePostProcessingTask +from ami.ml.results.schemas import DeterminationSnapshot +from ami.ml.results.writer import AlgorithmResultWriter if typing.TYPE_CHECKING: from ami.jobs.models import Job @@ -23,16 +26,16 @@ class ClassMaskingConfig(pydantic.BaseModel): # capture set is the bulk path; a single occurrence is the spot/dev path (fast # feedback while tuning a taxa list). This mirrors SmallSizeFilterConfig's # discriminated-scope shape — the shared pattern for per-occurrence triggers. - source_image_collection_id: int | None = None - occurrence_id: int | None = None + source_image_collection_id: int | None = model_reference("capture_set", None, title="Capture set") + occurrence_id: int | None = model_reference("occurrence", None, title="Occurrence") # The taxa list to keep: classes whose taxon is not in this list are masked out. - taxa_list_id: int + taxa_list_id: int = model_reference("taxa_list", ..., title="Species list") # The source classifier whose terminal classifications are re-scored. - algorithm_id: int + algorithm_id: int = model_reference("algorithm", ..., title="Classifier") # When True (default), renormalise the kept classes' scores to sum to 1 after # masking. When False, the kept classes retain their original absolute scores and # the excluded classes are zeroed; the chosen species is identical either way. - reweight: bool = True + reweight: bool = pydantic.Field(True, title="Re-weighted scores") @pydantic.root_validator(skip_on_failure=True) def _exactly_one_scope(cls, values: dict) -> dict: @@ -45,6 +48,18 @@ class Config: extra = "forbid" +class ClassMaskingResultData(DeterminationSnapshot): + """Figures from the occurrence's winning detection: the one whose masked classification scores highest.""" + + kind: typing.ClassVar[str] = "class_masking" + value_field: typing.ClassVar[str | None] = "excluded_probability" + + # One minus the kept mass of the unmasked softmax: what the list removed, not an out-of-distribution score. + excluded_probability: float + # Where the class that wins after masking ranked before it; 1 means it was already the top. + new_winner_original_rank: int | None = None + + def make_classifications_filtered_by_taxa_list( classifications: QuerySet[Classification], taxa_list: TaxaList, @@ -81,6 +96,12 @@ def make_classifications_filtered_by_taxa_list( New classifications record ``job`` as the run that wrote them. + Every occurrence with a re-scored classification gets one algorithm result, written in + the same transaction as the batch that changes it, and its new classifications point at + that result (see ``AlgorithmResultWriter``). The result records the probability the list excluded + for the occurrence's winning detection; the original top prediction is the classification + the winning one replaced (``applied_to``). + Returns final counters (checked / masked / occurrences updated) for stage metrics. """ taxa_in_list = set(taxa_list.taxa.all()) @@ -123,6 +144,12 @@ def make_classifications_filtered_by_taxa_list( timestamp = timezone.now() masked_count = 0 + results = AlgorithmResultWriter( + kind=ClassMaskingResultData.kind, + algorithm=new_algorithm, + job=job, + timestamp=timestamp, + ) # Sizing the scope and expanding the category map both take time on a large # classifier, so report before touching a row. @@ -198,6 +225,16 @@ def make_classifications_filtered_by_taxa_list( detection = classification.detection if detection is not None and detection.occurrence is not None: occurrences_to_update.add(detection.occurrence) + # The occurrence's result carries the figures of its winning detection: + # the one whose masked classification scores highest (see AlgorithmResultWriter). + results.note( + detection.occurrence, + { + "excluded_probability": float(1.0 - kept_sum), + "new_winner_original_rank": int((full_softmax > full_softmax[top_index]).sum()) + 1, + }, + rank=score, + ) # Flush every batch_size items and at the final item. The flush fires even # when nothing was accumulated so the job health-check sees a heartbeat during @@ -206,6 +243,7 @@ def make_classifications_filtered_by_taxa_list( with transaction.atomic(): if classifications_to_demote: Classification.objects.bulk_update(classifications_to_demote, ["terminal", "updated_at"]) + results.start_batch(occurrences_to_update, classifications_to_add) if classifications_to_add: Classification.objects.bulk_create(classifications_to_add) # Count an occurrence only when saving its new terminal classification @@ -217,6 +255,7 @@ def make_classifications_filtered_by_taxa_list( occurrence.save(update_determination=True) if occurrence.pk is not None and occurrence.determination_id != prev: changed_occurrence_ids.add(occurrence.pk) + results.finish_batch(occurrences_to_update) classifications_to_demote.clear() classifications_to_add.clear() @@ -246,6 +285,7 @@ class ClassMaskingTask(BasePostProcessingTask): key = "class_masking" name = "Class masking" config_schema = ClassMaskingConfig + result_models = (ClassMaskingResultData,) def _get_or_create_masking_algorithm( self, source_algorithm: Algorithm, taxa_list: TaxaList, *, reweight: bool diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index f0b8289eb..283185cff 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -1,9 +1,15 @@ +import typing + import pydantic -from django.db.models import QuerySet +from django.db import transaction +from django.db.models import Exists, OuterRef, QuerySet from django.utils import timezone +from ami.base.model_references import model_reference from ami.main.models import Classification, Detection, Occurrence, SourceImageCollection, Taxon, TaxonRank from ami.ml.post_processing.base import BasePostProcessingTask +from ami.ml.results.schemas import DeterminationSnapshot +from ami.ml.results.writer import AlgorithmResultWriter from ami.ml.schemas import BoundingBox @@ -12,9 +18,9 @@ class SmallSizeFilterConfig(pydantic.BaseModel): # set is the bulk path; a single occurrence is the spot/dev path (fast feedback # while tuning a filter). This discriminated-scope shape is the pattern other # post-processing tasks copy when they gain per-occurrence / per-event triggers. - source_image_collection_id: int | None = None - occurrence_id: int | None = None - size_threshold: float = 0.0008 + source_image_collection_id: int | None = model_reference("capture_set", None, title="Capture set") + occurrence_id: int | None = model_reference("occurrence", None, title="Occurrence") + size_threshold: float = pydantic.Field(0.0008, title="Size threshold") @pydantic.validator("size_threshold") def _threshold_in_unit_interval(cls, v: float) -> float: @@ -33,10 +39,21 @@ class Config: extra = "forbid" +class SizeFilterResultData(DeterminationSnapshot): + """Figures from the occurrence's smallest filtered detection.""" + + kind: typing.ClassVar[str] = "size_filter" + value_field: typing.ClassVar[str | None] = "relative_size" + + # The detection's box area as a fraction of its image. + relative_size: float + + class SmallSizeFilterTask(BasePostProcessingTask): key = "small_size_filter" name = "Small size filter" config_schema = SmallSizeFilterConfig + result_models = (SizeFilterResultData,) def _scoped_detections(self, config: SmallSizeFilterConfig) -> tuple[QuerySet[Detection], str]: """Resolve the detections to examine from whichever scope the config carries. @@ -78,6 +95,11 @@ def run(self) -> None: self.logger.info(f"=== Starting {self.name} ===") detections, scope_desc = self._scoped_detections(config) + # A detection this filter already flagged keeps its flag. Flagging it again would only add an + # identical classification, and a retried job would repeat its own work. + detections = detections.exclude( + Exists(Classification.objects.filter(detection=OuterRef("pk"), algorithm=self.algorithm)) + ) total = detections.count() self.logger.info(f"Found {total} detections in {scope_desc}") @@ -90,8 +112,61 @@ def run(self) -> None: updated_occurrence_ids: set[int] = set() modified_occurrences = 0 checked = 0 + # One algorithm result per flagged occurrence, written with the batch that flags it. + results = AlgorithmResultWriter( + kind=SizeFilterResultData.kind, + algorithm=self.algorithm, + job=self.job, + ) + + def flush(i: int) -> None: + """Write the rows gathered so far, in one transaction, and report progress as of detection ``i``.""" + nonlocal checked, modified_detections, modified_occurrences + checked = i + modified_detections += len(detections_to_update) + + with transaction.atomic(): + results.start_batch(occcurrences_to_update, classifications_to_create) + self.logger.info(f"Creating {len(classifications_to_create)} new classifications") + Classification.objects.bulk_create(classifications_to_create) + classifications_to_create.clear() + + self.logger.info(f"Marking {len(detections_to_update)} detections as {not_identifiable_taxon.name}") + for det in detections_to_update: + det.updated_at = timezone.now() + Detection.objects.bulk_update(detections_to_update, ["updated_at"]) + detections_to_update.clear() + + self.logger.info(f"Updating {len(occcurrences_to_update)} occurrences") + for occ in occcurrences_to_update: + # Count an occurrence only when flagging its detection actually + # changes the determination. Re-saving recomputes it in place, so + # an occurrence pinned to a human identification keeps its taxon + # and must not inflate the metric. + prev_determination_id = occ.determination_id + occ.save(update_determination=True) + if occ.pk is not None and occ.determination_id != prev_determination_id: + updated_occurrence_ids.add(occ.pk) + results.finish_batch(occcurrences_to_update) + modified_occurrences = len(updated_occurrence_ids) + occcurrences_to_update.clear() + + progress = i / total if total > 0 else 1.0 + self.update_progress(progress) + self.report_stage_metrics( + { + "detections_checked": checked, + "detections_flagged": modified_detections, + "occurrences_updated": modified_occurrences, + } + ) + i = 0 for i, det in enumerate(detections.iterator(), start=1): + # Write each full batch of 100 before the next begins. The flush sits ahead of the + # skip checks below so a skipped detection never swallows the batch before it. + if i > 1 and (i - 1) % 100 == 0: + flush(i - 1) bbox = det.get_bbox() if not bbox: self.logger.debug(f"Detection {det.pk}: no bbox, skipping") @@ -132,45 +207,10 @@ def run(self) -> None: detections_to_update.add(det) if det.occurrence is not None: occcurrences_to_update.add(det.occurrence) + # The occurrence's result carries the size of its smallest flagged detection. + results.note(det.occurrence, {"relative_size": rel_area}, rank=-rel_area) self.logger.debug(f"Marking detection {det.pk} as {not_identifiable_taxon.name}") - # Update progress every 100 detections - if i % 100 == 0 or i == total: - checked = i - modified_detections += len(detections_to_update) - - # with transaction.atomic(): - self.logger.info(f"Creating {len(classifications_to_create)} new classifications") - Classification.objects.bulk_create(classifications_to_create) - classifications_to_create.clear() - - self.logger.info(f"Marking {len(detections_to_update)} detections as {not_identifiable_taxon.name}") - for det in detections_to_update: - det.updated_at = timezone.now() - Detection.objects.bulk_update(detections_to_update, ["updated_at"]) - detections_to_update.clear() - - self.logger.info(f"Updating {len(occcurrences_to_update)} occurrences") - for occ in occcurrences_to_update: - # Count an occurrence only when flagging its detection actually - # changes the determination. Re-saving recomputes it in place, so - # an occurrence pinned to a human identification keeps its taxon - # and must not inflate the metric. - prev_determination_id = occ.determination_id - occ.save(update_determination=True) - if occ.pk is not None and occ.determination_id != prev_determination_id: - updated_occurrence_ids.add(occ.pk) - modified_occurrences = len(updated_occurrence_ids) - occcurrences_to_update.clear() - - progress = i / total if total > 0 else 1.0 - self.update_progress(progress) - self.report_stage_metrics( - { - "detections_checked": checked, - "detections_flagged": modified_detections, - "occurrences_updated": modified_occurrences, - } - ) + flush(i) self.logger.info(f"=== Completed {self.name}: {modified_detections} of {total} detections modified ===") diff --git a/ami/ml/post_processing/tests/test_class_masking.py b/ami/ml/post_processing/tests/test_class_masking.py index 541a70195..bd82dc1aa 100644 --- a/ami/ml/post_processing/tests/test_class_masking.py +++ b/ami/ml/post_processing/tests/test_class_masking.py @@ -26,7 +26,7 @@ TaxonRank, group_images_into_events, ) -from ami.ml.models import Algorithm, AlgorithmCategoryMap +from ami.ml.models import Algorithm, AlgorithmCategoryMap, AlgorithmResult from ami.ml.models.algorithm import AlgorithmTaskType from ami.ml.post_processing.class_masking import ClassMaskingTask, make_classifications_filtered_by_taxa_list from ami.tests.fixtures.main import create_taxa, setup_test_project @@ -383,6 +383,101 @@ def test_task_run_occurrence_scope(self): self.assertIsNotNone(new_clf) self.assertEqual(new_clf.taxon, self.species_taxa[0]) + # ----- algorithm results ----------------------------------------------- + + def _masking_fixture(self, name: str = "History list") -> tuple[TaxaList, Detection, Occurrence]: + """One occurrence whose top class (index 2) the taxa list excludes, so masking re-scores it.""" + logits = [2.0, 1.0, 5.0] + taxa_list = TaxaList.objects.create(name=name) + taxa_list.taxa.set(self.species_taxa[:2]) + det, occ = self._detection_with_occurrence() + self._create_classification_with_logits(det, self.species_taxa[2], _softmax(logits), logits) + occ.save() + return taxa_list, det, occ + + def test_a_rescored_occurrence_gets_one_result_linked_to_the_classification_it_created(self): + taxa_list, det, occ = self._masking_fixture() + + # The second run finds nothing left to mask, so the first run's result stays the only one. + for _ in range(2): + ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() + + result = AlgorithmResult.objects.get(occurrence=occ, kind="class_masking") + self.assertIn(result.kind, {model.kind for model in ClassMaskingTask.result_models}) + self.assertEqual(result.project_id, self.project.pk) + self.assertTrue(result.algorithm.key.startswith(f"{self.algorithm.key}_filtered_by_taxa_list_{taxa_list.pk}")) + # Index 2 held e^5 / (e^2 + e^1 + e^5) of the probability and was excluded; index 0 was second. + scores = _softmax([2.0, 1.0, 5.0]) + self.assertAlmostEqual(result.value, scores[2]) + self.assertAlmostEqual(result.data.pop("excluded_probability"), scores[2]) + self.assertEqual( + result.data, + { + "new_winner_original_rank": 2, + "determination_before_id": self.species_taxa[2].pk, + "determination_after_id": self.species_taxa[0].pk, + "extra": {}, + }, + ) + masked = result.classifications.get() + self.assertEqual((masked.taxon, masked.terminal), (self.species_taxa[0], True)) + # The top prediction before masking is the classification the masked one replaced. + self.assertEqual(masked.applied_to.taxon, self.species_taxa[2]) + self.assertAlmostEqual(masked.applied_to.score, scores[2]) + self.assertEqual(masked.applied_to.taxon, self.species_taxa[2]) + + def test_an_occurrence_masking_does_not_change_gets_no_result(self): + logits = [2.0, 1.0, 5.0] + taxa_list = TaxaList.objects.create(name="Keeps everything") + taxa_list.taxa.set(self.species_taxa) + det, occ = self._detection_with_occurrence() + self._create_classification_with_logits(det, self.species_taxa[2], _softmax(logits), logits) + + ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() + + self.assertFalse(AlgorithmResult.objects.filter(occurrence=occ).exists()) + + def test_an_occurrence_spanning_batches_gets_one_result_covering_every_batch(self): + taxa_list, det1, occ = self._masking_fixture() + det2 = Detection.objects.create(source_image=self.collection.images.first(), bbox=[0, 0, 100, 100]) + occ.detections.add(det2) + logits = [2.0, 1.0, 5.0] + self._create_classification_with_logits(det2, self.species_taxa[2], _softmax(logits), logits) + new_algorithm = Algorithm.objects.create( + name="masked_span", + key="masked_span_test", + task_type=AlgorithmTaskType.CLASSIFICATION.value, + category_map=self.algorithm.category_map, + ) + + make_classifications_filtered_by_taxa_list( + classifications=Classification.objects.filter(detection__in=[det1, det2]), + taxa_list=taxa_list, + algorithm=self.algorithm, + new_algorithm=new_algorithm, + batch_size=1, + ) + + result = AlgorithmResult.objects.get(occurrence=occ) + self.assertEqual(result.data["determination_after_id"], self.species_taxa[0].pk) + self.assertEqual( + sorted(result.classifications.values_list("detection_id", flat=True)), sorted([det1.pk, det2.pk]) + ) + + def test_an_occurrence_without_a_project_is_still_rescored_but_gets_no_result(self): + """A missing project must not fail a run after it has changed data; the result is skipped with a warning.""" + taxa_list, det, occ = self._masking_fixture() + Occurrence.objects.filter(pk=occ.pk).update(project=None) + + with self.assertLogs("ami.ml.models.algorithm_result", level="WARNING") as logs: + ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() + + self.assertIn(f"Occurrence #{occ.pk}", logs.output[0]) + occ.refresh_from_db() + self.assertEqual(occ.determination, self.species_taxa[0]) + self.assertFalse(AlgorithmResult.objects.filter(occurrence=occ).exists()) + self.assertIsNone(Classification.objects.get(applied_to__detection=det).algorithm_result) + # ----- batched commit + heartbeat ------------------------------------- def test_batched_commit_flushes_multiple_times(self): diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py new file mode 100644 index 000000000..5f0127e5f --- /dev/null +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -0,0 +1,186 @@ +"""What the small size filter writes: a "Not identifiable" classification per flagged detection, in batches.""" + +import logging +from unittest import mock + +from django.test import TestCase + +from ami.jobs.models import Job, PostProcessingJob +from ami.main.models import Classification, Detection, Occurrence, SourceImage, Taxon +from ami.ml.models import AlgorithmResult +from ami.ml.post_processing.small_size_filter import SizeFilterResultData, SmallSizeFilterTask +from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project + +logger = logging.getLogger(__name__) + + +class SizeFilterTestCase(TestCase): + @classmethod + def setUpTestData(cls) -> None: + cls.project, cls.deployment = setup_test_project(reuse=False) + create_taxa(project=cls.project) + create_captures(deployment=cls.deployment, num_nights=1, images_per_night=3, interval_minutes=1) + cls.captures = list(SourceImage.objects.filter(deployment=cls.deployment).order_by("timestamp")) + SourceImage.objects.filter(pk__in=[c.pk for c in cls.captures]).update(width=1000, height=1000) + cls.event = cls.captures[0].event + cls.taxon = Taxon.objects.filter(projects=cls.project).order_by("pk").first() + + def _singleton(self, capture: SourceImage, bbox: list[int]) -> Occurrence: + """One detection in an occurrence of its own, classified as the fixture taxon.""" + occurrence = Occurrence.objects.create(event=self.event, deployment=self.deployment, project=self.project) + detection = Detection.objects.create( + source_image=capture, bbox=bbox, timestamp=capture.timestamp, occurrence=occurrence + ) + detection.classifications.create(taxon=self.taxon, score=0.9, timestamp=capture.timestamp) + occurrence.save() + return occurrence + + def run_filter(self, occurrence: Occurrence, **kwargs) -> None: + SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=0.01, **kwargs).run() + + +class SizeFilterBatchTestCase(SizeFilterTestCase): + def test_a_skipped_last_detection_still_saves_the_batch_before_it(self): + """A detection the scan skips (no box) must not swallow the flagged ones gathered before it.""" + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + # Inserted last, so the scan reaches it last; it has no box and is skipped. + Detection.objects.create( + source_image=self.captures[1], bbox=None, timestamp=self.captures[1].timestamp, occurrence=occurrence + ) + + self.run_filter(occurrence) + + occurrence.refresh_from_db() + self.assertEqual(occurrence.determination.name, "Not identifiable") + + def test_a_detection_above_the_threshold_is_left_alone(self): + occurrence = self._singleton(self.captures[0], [0, 0, 500, 500]) + self.run_filter(occurrence) + occurrence.refresh_from_db() + self.assertEqual(occurrence.determination, self.taxon) + self.assertEqual(occurrence.detections.get().classifications.count(), 1) + + +class SizeFilterJobTestCase(SizeFilterTestCase): + def _job(self, occurrence: Occurrence, **config) -> Job: + return Job.objects.create( + project=self.project, + name="Size filter", + job_type_key="post_processing", + params={"task": "small_size_filter", "config": {"occurrence_id": occurrence.pk, **config}}, + ) + + def test_the_job_stores_the_settings_the_run_used_with_their_defaults(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + job = self._job(occurrence) + + PostProcessingJob.run(job) + + job.refresh_from_db() + self.assertEqual( + job.params["config"], + {"occurrence_id": occurrence.pk, "source_image_collection_id": None, "size_threshold": 0.0008}, + ) + self.assertEqual(job.status, "SUCCESS") + + def test_a_run_records_its_job_on_each_classification_it_writes(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + job = self._job(occurrence, size_threshold=0.01) + + PostProcessingJob.run(job) + + classification = occurrence.detections.get().classifications.get(taxon__name="Not identifiable") + self.assertEqual(classification.job_id, job.pk) + self.assertEqual(job.classifications.get().pk, classification.pk) + self.assertEqual(job.algorithm_results.get().pk, classification.algorithm_result_id) + + def test_a_retried_job_reuses_its_own_result_instead_of_adding_another(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + job = self._job(occurrence, size_threshold=0.01) + + PostProcessingJob.run(job) + first = job.algorithm_results.get() + PostProcessingJob.run(job) + + self.assertEqual(list(job.algorithm_results.values_list("pk", flat=True)), [first.pk]) + self.assertEqual(occurrence.algorithm_results.count(), 1) + self.assertEqual(job.classifications.count(), 1) + + +class SizeFilterResultsTestCase(SizeFilterTestCase): + """The size filter leaves one algorithm result per occurrence it flags, written with the batch that flags it.""" + + def records(self): + return AlgorithmResult.objects.filter(kind=SizeFilterResultData.kind).order_by("pk") + + def test_a_flagged_occurrence_gets_one_result_linked_to_the_classifications_it_created(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + Detection.objects.create( + source_image=self.captures[1], + bbox=[0, 0, 12, 12], + timestamp=self.captures[1].timestamp, + occurrence=occurrence, + ) + + self.run_filter(occurrence) + + result = self.records().get() + detection_ids = sorted(occurrence.detections.values_list("pk", flat=True)) + self.assertEqual((result.occurrence_id, result.project_id), (occurrence.pk, self.project.pk)) + self.assertEqual(result.algorithm.key, "small_size_filter") + self.assertIn(result.kind, {model.kind for model in SmallSizeFilterTask.result_models}) + # The smallest flagged detection, 10 by 10 pixels on a 1000 by 1000 image, represents the occurrence. + self.assertAlmostEqual(result.value, 0.0001) + self.assertAlmostEqual(result.data.pop("relative_size"), 0.0001) + self.assertEqual( + result.data, + { + "determination_before_id": self.taxon.pk, + "determination_after_id": Taxon.objects.get(name="Not identifiable").pk, + "extra": {}, + }, + ) + self.assertEqual(sorted(result.classifications.values_list("detection_id", flat=True)), detection_ids) + + def test_a_second_run_with_a_larger_threshold_adds_a_result_for_what_it_newly_flags(self): + """Each run that flags something shows up in the history; a detection already flagged is left alone.""" + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + medium = Detection.objects.create( + source_image=self.captures[1], + bbox=[0, 0, 150, 150], + timestamp=self.captures[1].timestamp, + occurrence=occurrence, + ) + medium.classifications.create(taxon=self.taxon, score=0.9, timestamp=self.captures[1].timestamp) + for threshold in (0.01, 0.05): + SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=threshold).run() + + self.assertEqual(self.records().filter(occurrence=occurrence).count(), 2) + flags = Classification.objects.filter(detection__occurrence=occurrence, taxon__name="Not identifiable") + # Each detection carries exactly one flag: the second run did not flag the small one again. + self.assertEqual( + sorted(flags.values_list("detection_id", flat=True)), + sorted(occurrence.detections.values_list("pk", flat=True)), + ) + + def test_running_again_with_the_same_threshold_flags_nothing_twice(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + for _ in range(2): + self.run_filter(occurrence) + self.assertEqual(self.records().filter(occurrence=occurrence).count(), 1) + self.assertEqual(occurrence.detections.get().classifications.filter(taxon__name="Not identifiable").count(), 1) + + def test_an_occurrence_with_nothing_flagged_gets_no_result(self): + occurrence = self._singleton(self.captures[0], [0, 0, 500, 500]) + self.run_filter(occurrence) + self.assertFalse(self.records().exists()) + + def test_a_batch_that_fails_leaves_neither_its_classifications_nor_its_result(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + + with mock.patch.object(Occurrence, "save", side_effect=RuntimeError("determination update failed")): + with self.assertRaises(RuntimeError): + self.run_filter(occurrence) + + self.assertFalse(self.records().exists()) + self.assertFalse(occurrence.detections.get().classifications.filter(taxon__name="Not identifiable").exists()) diff --git a/ami/ml/results/README.md b/ami/ml/results/README.md new file mode 100644 index 000000000..1a66dc0d9 --- /dev/null +++ b/ami/ml/results/README.md @@ -0,0 +1,62 @@ +# Algorithm results + +An algorithm result (`ami/ml/models/algorithm_result.py`) records what one run of a method decided +about one occurrence: the figures only that run knew, a headline `value`, and the occurrence's +determination before and after, with the algorithm and job that produced it. Post-processing tasks +write them today. The determination never reads them; it still comes only from identifications and +classifications. The occurrence history (`ami/main/api/occurrence_history/`, +`GET /occurrences/{id}/history/`) shows each result with the classifications its run created. + +Processing services cannot write results yet: their contract (`ami/ml/schemas.py`) has no place for +them and must not name occurrences, which Antenna creates. Classifications and feature vectors keep their +own tables, with the same algorithm and job provenance. + +## Adding a kind + +1. **Data model** in the task's module, beside its config schema (`ClassMaskingResultData` in + `class_masking.py`), subclassing `AlgorithmResultData` from `schemas.py`. Set `kind` and + `value_field` (the figure lists filter and sort on; every write copies it into `value`). + Subclass `DeterminationSnapshot` if the run can change the determination. Record only what the + run alone knows: its config stays on the job, the new taxon on the classifications it creates, and + the prediction it replaced on their `applied_to`. Anything else the method returns goes in + `extra`, which nothing reads for logic. +2. **Task**: declare the model in the task's `result_models` (which registers the kind), give every `config_schema` field a + `title`, and declare fields that hold a record id with `model_reference("capture_set", ...)` + (`ami/base/model_references.py`); `ami/jobs/job_config.py` labels and links them. A new reference type + is declared on its model (`reference_type`, `reference_name_field`) and routed in + `ui/src/utils/model-references.ts`. +3. **Writing**: use `AlgorithmResultWriter` (`writer.py`) as `class_masking.py` and + `small_size_filter.py` do. Per batch, inside one transaction: `note()` each changed detection's + figures, `start_batch()` before inserting classifications (it points them at their result), then + save the occurrences and call `finish_batch()`. +4. **UI**: add the data interface and the kind to `ui/src/data-services/models/occurrence-history.ts` + and a stats case to `ui/src/pages/occurrence-details/history/algorithm-result.tsx`. + The timeline skips kinds it does not know, so the server side can ship first. +5. **Tests**: one result per touched occurrence with the expected `data` and `value`; created + classifications carry `algorithm_result`, `job` and, where they replaced one, `applied_to`; the + history shows the result. `python manage.py test ami.ml.results ami.main.api.occurrence_history.tests`. + +No migration, serializer or OpenAPI change is needed: `kind` is a plain `CharField`, every write path +validates `data` against the registry, and the API publishes `data` as JSON. + +## Rules + +- **Results are only added.** Each run adds one result per occurrence it touches; running a method + twice leaves two. A retried job reuses its own results. A list that needs an algorithm's latest + value takes the latest by timestamp. +- **A run gets its own `Algorithm` only when it changes what can be output.** Class masking makes one + per source classifier and species list, because the list changes which species can be named. The + rest of the config stays on the job: the size filter uses one algorithm whatever its threshold. +- **Demote only what the run replaced**, set `job` on every row it creates, and never call + `.distinct()` on a classification queryset that includes the `scores` or `logits` arrays (#1376). +- **Occurrences without a project** get no result and a warning; the run carries on (#1188). +- **A result belongs to an occurrence, for now.** Class masking and the size filter work per + detection, so a result keeps the figures of one detection per occurrence: the winning one for + masking, the smallest flagged one for the size filter. Detections and captures are expected to + become targets of this table, and results to come from pipeline stages and processing services + too. Keep that open: occurrence-only figures go in a subclass such as `DeterminationSnapshot`, + never in `AlgorithmResultData`, and a result's target is a column, never a key in `data`. The plan + is in the PR that introduced this table (#1461), under "Looking ahead". +- **The determination before and after is a snapshot** of the occurrence when the run wrote the + result. Later runs change the occurrence and merging occurrences moves the result, so to evaluate + a stage, read the classifications it created rather than these two figures. diff --git a/ami/ml/results/__init__.py b/ami/ml/results/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ami/ml/results/schemas.py b/ami/ml/results/schemas.py new file mode 100644 index 000000000..0bc051bc5 --- /dev/null +++ b/ami/ml/results/schemas.py @@ -0,0 +1,75 @@ +"""The base for what each kind of algorithm result records, and the registry of kinds. + +A kind's model holds only what its run alone knows; the run's config lives on the job, and the new +taxon on the classifications the run created. ``extra`` takes whatever else a method returns and is +never read for logic. It imports nothing from Django, so models, writers and serializers can all +import it. A kind's model lives with the task that writes it and is listed in the task's +``result_models``; see README.md in this package. +""" + +import functools +import json +from typing import Any, ClassVar + +import pydantic + + +class AlgorithmResultData(pydantic.BaseModel): + # The kind this model validates; each concrete model sets it, and the registry is keyed by it. + kind: ClassVar[str] + # The data field the result's ``value`` repeats, for filtering and sorting; None for no value. + value_field: ClassVar[str | None] = None + + # Stored and returned by the API only, never read for logic; a value a feature needs becomes a typed field. + extra: dict[str, Any] = {} + + @pydantic.validator("extra") + def _extra_is_json(cls, value: dict[str, Any]) -> dict[str, Any]: + return json.loads(json.dumps(value)) + + class Config: + extra = "forbid" + + +class DeterminationSnapshot(AlgorithmResultData): + # The occurrence's determination before and after the run, as taxon ids; equal when it did not move. + # A snapshot of the occurrence at run time: later runs change it and merging occurrences moves the + # result, so to evaluate a stage, read the classifications it created, not these. + determination_before_id: int | None = None + determination_after_id: int | None = None + + +@functools.cache +def result_data_models() -> dict[str, type[AlgorithmResultData]]: + """Every result kind, from the ``result_models`` the post-processing tasks declare, keyed by kind.""" + from ami.ml.post_processing.registry import POSTPROCESSING_TASKS + + models: dict[str, type[AlgorithmResultData]] = {} + for task in POSTPROCESSING_TASKS.values(): + for model in task.result_models: + if models.setdefault(model.kind, model) is not model: + raise ValueError(f"Result kind {model.kind!r} is declared by two different data models.") + return models + + +def _schema_for(kind: str) -> type[AlgorithmResultData]: + schema = result_data_models().get(kind) + if schema is None: + raise ValueError(f"No result data model is registered for {kind!r}.") + return schema + + +def validate_result_data(kind: str, data: dict | AlgorithmResultData | None) -> dict: + """``data`` as JSON-ready values, or a ValueError when it does not fit the kind's model.""" + schema = _schema_for(kind) + if isinstance(data, AlgorithmResultData) and not isinstance(data, schema): + raise ValueError(f"{type(data).__name__} is not the result data model for {kind!r}.") + values = data.dict() if isinstance(data, AlgorithmResultData) else (data or {}) + # Round-trip through JSON so datetimes are stored as ISO strings. + return json.loads(schema.parse_obj(values).json()) + + +def result_value(kind: str, data: dict) -> float | None: + """The figure the kind's ``value_field`` names in validated ``data``, which a result stores as ``value``.""" + field = _schema_for(kind).value_field + return data.get(field) if field else None diff --git a/ami/ml/results/tests.py b/ami/ml/results/tests.py new file mode 100644 index 000000000..3368bffb3 --- /dev/null +++ b/ami/ml/results/tests.py @@ -0,0 +1,140 @@ +from django.test import TestCase +from django.urls import reverse + +from ami.main.models import Occurrence +from ami.ml.models import Algorithm, AlgorithmResult +from ami.ml.post_processing.small_size_filter import SizeFilterResultData +from ami.tests.fixtures.main import setup_test_project +from ami.users.models import User + +SIZE_FILTER = SizeFilterResultData.kind + + +class AlgorithmResultTestCase(TestCase): + """A result's data fits its kind, its value follows its data, and its project is its occurrence's.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.project, cls.deployment = setup_test_project(reuse=False) + cls.occurrence = Occurrence.objects.create(project=cls.project, deployment=cls.deployment) + cls.algorithm = Algorithm.objects.create(name="Size filter", key="size-filter-result-test") + + def _size_filter(self, data: dict | None, **fields) -> AlgorithmResult: + fields.setdefault("occurrence", self.occurrence) + return AlgorithmResult(algorithm=self.algorithm, kind=SIZE_FILTER, data=data, **fields) + + def test_data_that_does_not_fit_its_kind_is_refused_on_every_write_path(self): + for label, data in ( + ("missing field", {}), + ("wrong type", {"relative_size": "small"}), + ("unknown field", {"relative_size": 0.001, "note": "x"}), + ("extra that is not JSON", {"relative_size": 0.001, "extra": {"when": object()}}), + ): + with self.subTest(label): + with self.assertRaises(ValueError): + self._size_filter(data).save() + with self.assertRaises(ValueError): + AlgorithmResult.objects.record_many([self._size_filter(data)]) + with self.assertRaises(ValueError): + AlgorithmResult.objects.record( + occurrence=self.occurrence, algorithm=self.algorithm, kind="unregistered", data={} + ) + self.assertFalse(AlgorithmResult.objects.exists()) + + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.algorithm, + kind=SIZE_FILTER, + value=0.001, + data={"relative_size": 0.001, "extra": {"service": "size-filter 1.2"}}, + ) + self.assertEqual(result.value, 0.001) + self.assertEqual(result.data["extra"], {"service": "size-filter 1.2"}) + result.data["relative_size"] = "small" + with self.assertRaises(ValueError): + AlgorithmResult.objects.bulk_update([result], ["data"]) + + def test_each_run_adds_a_result_beside_the_earlier_ones(self): + first = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.algorithm, + kind=SIZE_FILTER, + data={"relative_size": 0.01}, + ) + other_occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) + second, untouched = AlgorithmResult.objects.record_many( + [ + self._size_filter({"relative_size": 0.02}), + self._size_filter({"relative_size": 0.02}, occurrence=other_occurrence), + ] + ) + self.assertEqual(set(self.occurrence.algorithm_results.values_list("pk", flat=True)), {first.pk, second.pk}) + self.assertEqual(list(other_occurrence.algorithm_results.values_list("pk", flat=True)), [untouched.pk]) + + def test_project_comes_from_the_occurrence_and_a_result_without_one_is_skipped(self): + result = self._size_filter({"relative_size": 0.01}) + result.save() + self.assertEqual(result.project_id, self.project.pk) + + orphan = Occurrence.objects.create(project=None, deployment=self.deployment) + with self.assertLogs("ami.ml.models.algorithm_result", level="WARNING"): + written = AlgorithmResult.objects.record_many( + [ + self._size_filter({"relative_size": 0.02}), + self._size_filter({"relative_size": 0.03}, occurrence=orphan), + ] + ) + # The occurrence with a project still gets its result; the other is skipped, not fatal. + self.assertEqual([r.occurrence_id for r in written], [self.occurrence.pk]) + with self.assertRaises(ValueError): + self._size_filter({"relative_size": 0.03}, occurrence=orphan).save() + + def test_value_is_the_figure_the_kind_names_on_every_write_path(self): + saved = self._size_filter({"relative_size": 0.004}, value=0.9) + saved.save() + [created] = AlgorithmResult.objects.bulk_create( + [self._size_filter({"relative_size": 0.005}, value=None, project=self.project)] + ) + created.data = {"relative_size": 0.006} + AlgorithmResult.objects.bulk_update([created], ["data"]) + created.refresh_from_db() + self.assertEqual((saved.value, created.value), (0.004, 0.006)) + + +class AlgorithmResultAdminTestCase(TestCase): + """Staff can list, filter and open results in the admin, but not add or edit them.""" + + @classmethod + def setUpTestData(cls) -> None: + project, deployment = setup_test_project(reuse=False) + algorithm = Algorithm.objects.create(name="Size filter", key="size-filter-admin-test") + cls.results = [ + AlgorithmResult.objects.record( + occurrence=Occurrence.objects.create(project=project, deployment=deployment), + algorithm=algorithm, + kind=SIZE_FILTER, + data={"relative_size": size}, + ) + for size in (0.001, 0.002) + ] + cls.superuser = User.objects.create_superuser(email="results-admin@insectai.org", password="x") + + def setUp(self) -> None: + self.client.force_login(self.superuser) + + def test_results_are_listed_filtered_and_shown_read_only(self): + changelist = reverse("admin:ml_algorithmresult_changelist") + response = self.client.get(changelist, {"kind": SIZE_FILTER}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context["cl"].result_count, 2) + + result = self.results[0] + response = self.client.get(reverse("admin:ml_algorithmresult_change", args=[result.pk])) + self.assertEqual(response.status_code, 200) + self.assertFalse(response.context["has_change_permission"]) + self.assertEqual(self.client.get(reverse("admin:ml_algorithmresult_add")).status_code, 403) + + response = self.client.post(reverse("admin:ml_algorithmresult_change", args=[result.pk]), {"kind": "x"}) + self.assertEqual(response.status_code, 403) + result.refresh_from_db() + self.assertEqual(result.kind, SIZE_FILTER) diff --git a/ami/ml/results/writer.py b/ami/ml/results/writer.py new file mode 100644 index 000000000..4a05b0eca --- /dev/null +++ b/ami/ml/results/writer.py @@ -0,0 +1,98 @@ +"""One algorithm result per occurrence a post-processing run touches, written inside each batch. + +A run that re-scores or flags detections changes classifications and determinations in batches, +each in its own transaction. ``AlgorithmResultWriter`` writes the result rows in those same transactions, +so a batch either lands with its results or not at all. Every occurrence the run touches gets +exactly one result from the run: it is created by the first batch that touches the occurrence, +before the classifications that batch inserts so they can point at it, and later batches update it +with the figures of the detection that represents the occurrence and the determination after their +saves. A retried job reuses the results it already wrote rather than adding a second one each. +""" + +from __future__ import annotations + +import datetime +import typing + +from ami.ml.models import AlgorithmResult + +if typing.TYPE_CHECKING: + from ami.jobs.models import Job + from ami.main.models import Classification, Occurrence + from ami.ml.models import Algorithm + + +class AlgorithmResultWriter: + def __init__( + self, + *, + kind: str, + algorithm: Algorithm, + job: Job | None, + timestamp: datetime.datetime | None = None, + ): + self.kind = kind + self.algorithm = algorithm + self.job = job + self.timestamp = timestamp + self.results: dict[int, AlgorithmResult] = {} + # Occurrences with no project; tried once, then left alone. + self.skipped: set[int] = set() + # Per occurrence, the figures of the detection that represents it so far, with their rank. + self.figures: dict[int, tuple[float, dict]] = {} + + def note(self, occurrence: Occurrence, figures: dict, rank: float) -> None: + """Offer one changed detection's figures for the occurrence; the highest ``rank`` wins the result.""" + current = self.figures.get(occurrence.pk) + if current is None or rank > current[0]: + self.figures[occurrence.pk] = (rank, figures) + + def start_batch(self, occurrences: typing.Iterable[Occurrence], classifications: typing.Iterable[Classification]): + """Create the results for occurrences the run has not seen and point the batch's new classifications at theirs. + + Call inside the batch transaction, before the classifications are inserted and before the + occurrences are saved, so a result's ``determination_before_id`` is what the run found. + """ + new = [o for o in occurrences if o.pk not in self.results and o.pk not in self.skipped] + for result in self._written_by_this_job(new): + self.results[result.occurrence_id] = result + new = [o for o in new if o.pk not in self.results] + written = AlgorithmResult.objects.record_many( + AlgorithmResult( + occurrence=occurrence, + algorithm=self.algorithm, + job=self.job, + kind=self.kind, + data={**self.figures[occurrence.pk][1], "determination_before_id": occurrence.determination_id}, + **({"timestamp": self.timestamp} if self.timestamp else {}), + ) + for occurrence in new + ) + for result in written: + self.results[result.occurrence_id] = result + self.skipped.update(o.pk for o in new if o.pk not in self.results) + for classification in classifications: + classification.algorithm_result = self.results.get(classification.detection.occurrence_id) + + def _written_by_this_job(self, occurrences: list[Occurrence]) -> list[AlgorithmResult]: + """Results an earlier attempt of the same job wrote for these occurrences, latest last.""" + if self.job is None or not occurrences: + return [] + return list( + AlgorithmResult.objects.filter( + job=self.job, algorithm=self.algorithm, kind=self.kind, occurrence__in=[o.pk for o in occurrences] + ).order_by("timestamp", "pk") + ) + + def finish_batch(self, occurrences: typing.Iterable[Occurrence]) -> None: + """Record each occurrence's determination after the batch's saves and its best figures so far.""" + touched = [] + for occurrence in occurrences: + result = self.results.get(occurrence.pk) + if result is None: + continue + figures = self.figures[occurrence.pk][1] + result.data = {**result.data, **figures, "determination_after_id": occurrence.determination_id} + touched.append(result) + if touched: + AlgorithmResult.objects.bulk_update(touched, ["data", "value"]) diff --git a/ami/tests/fixtures/queries.py b/ami/tests/fixtures/queries.py new file mode 100644 index 000000000..5101e4fa4 --- /dev/null +++ b/ami/tests/fixtures/queries.py @@ -0,0 +1,17 @@ +import contextlib + +from cachalot.api import cachalot_disabled + + +@contextlib.contextmanager +def no_query_cache(): + """``cachalot_disabled()`` that is restored even when the block raises, so one failing test stays one. + + Use it around query-count assertions, which the query cache would otherwise hide. + """ + context = cachalot_disabled() + context.__enter__() + try: + yield + finally: + context.__exit__(None, None, None) diff --git a/ami/tests/test_model_references.py b/ami/tests/test_model_references.py new file mode 100644 index 000000000..5fbb12c18 --- /dev/null +++ b/ami/tests/test_model_references.py @@ -0,0 +1,46 @@ +import datetime + +import pydantic +from django.test import TestCase + +from ami.base.model_references import ( + ModelRef, + model_reference, + reference_types, + resolve_model_references, + unmapped_reference_types, +) +from ami.main.models import Event, TaxaList +from ami.ml.models import Algorithm +from ami.tests.fixtures.queries import no_query_cache + + +class ReferenceTestCase(TestCase): + """Record ids stored in JSON become {type, id, name}; a deleted row keeps its id with no name.""" + + def test_resolves_each_type_in_one_query_and_marks_missing_rows(self): + taxa_list = TaxaList.objects.create(name="Kept species") + algorithm = Algorithm.objects.create(name="Classifier", key="ref-test-classifier") + wanted = [("taxa_list", taxa_list.pk), ("algorithm", algorithm.pk), ("taxa_list", 999999)] + with no_query_cache(), self.assertNumQueries(2): + refs = resolve_model_references(wanted) + self.assertEqual(refs[("taxa_list", taxa_list.pk)], ModelRef("taxa_list", taxa_list.pk, "Kept species")) + self.assertEqual(refs[("algorithm", algorithm.pk)].name, "Classifier") + self.assertEqual(refs[("taxa_list", 999999)], ModelRef("taxa_list", 999999, None)) + + def test_a_name_computed_by_a_method_is_called_on_each_row(self): + session = Event.objects.create(group_by="2026-07-01", start=datetime.datetime(2026, 7, 1, 21, 0)) + refs = resolve_model_references([("session", session.pk)]) + self.assertEqual(refs[("session", session.pk)], ModelRef("session", session.pk, "Wednesday, Jul 1 2026")) + + def test_types_come_from_the_models_that_declare_them(self): + self.assertTrue({"algorithm", "capture_set", "occurrence", "session", "taxa_list"} <= set(reference_types())) + self.assertIs(reference_types()["taxa_list"][0], TaxaList) + + def test_a_schema_naming_a_type_no_model_declares_is_reported(self): + """Resolving such a field would fail at request time, so a test over real schemas keeps this empty.""" + + class Probe(pydantic.BaseModel): + merged_into_id: int | None = model_reference("not_a_type") + + self.assertEqual(unmapped_reference_types([Probe]), {"not_a_type"}) diff --git a/config/settings/base.py b/config/settings/base.py index 48580fed6..5fc1652d2 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -554,6 +554,13 @@ def _celery_result_backend_url(redis_url): "VERSION": "1.0.0", # "SERVE_PERMISSIONS": ["rest_framework.permissions.IsAdminUser"], "SERVE_PERMISSIONS": ["rest_framework.permissions.AllowAny"], + # Names for the single-value enums that tag each occurrence history entry (see + # OCCURRENCE_HISTORY_ENTRY_SCHEMA); without them drf-spectacular falls back to hashed names. + "ENUM_NAME_OVERRIDES": { + "AlgorithmResultEntryTypeEnum": ["algorithm_result"], + "IdentificationEntryTypeEnum": ["identification"], + "PredictionEntryTypeEnum": ["prediction"], + }, } # Your stuff... # ------------------------------------------------------------------------------ diff --git a/docs/claude/INDEX.md b/docs/claude/INDEX.md index 61f0980d8..feb8f9308 100644 --- a/docs/claude/INDEX.md +++ b/docs/claude/INDEX.md @@ -11,6 +11,7 @@ archived. |---|---| | `reference/canonical-patterns.md` | Existing helpers/patterns to reuse before writing new ones, with file:line refs (SingleParamSerializer, ProjectMixin, permissions, schemas, fixtures). Keywords: reuse, helpers, conventions, DRF | | `reference/query-patterns.md` | DB model relationship table, composite indexes, prefetch/select_related patterns, full custom QuerySet method catalog, query anti-patterns. Keywords: N+1, indexes, ORM, performance | +| `../../ami/ml/results/README.md` | (Developer guide, lives next to the code.) Algorithm results overview, how to add a result kind (data model with `kind` and `value_field`, task `result_models`, config fields with `title` and `model_reference()` from `ami/base/model_references.py`, `AlgorithmResultWriter` batch pattern, UI card, tests), and the rules (append-only, own Algorithm only when the output space changes, project-less occurrences). Job config rows: `ami/jobs/job_config.py`. Keywords: AlgorithmResult, kind, registry, post-processing, history, OpenAPI, DetectionEmbedding | | `reference/api-stats-pattern.md` | How to add aggregate/leaderboard/chart endpoints (`//stats//`): GenericViewSet + @action, pure querysets in models_future. Keywords: stats, charts, aggregation | | `reference/monitoring-async-jobs.md` | Monitoring/debugging async_api (NATS JetStream) jobs: ORM, REST, consumer state, Redis counters, worker logs. Keywords: NATS, async, jobs, monitoring | | `reference/worktree-testing.md` | Testing git-worktree changes against the main Docker stack: bind-mount route vs duplicate-stack route, caveats, cleanup. Keywords: worktree, docker compose, override | diff --git a/ui/src/components/model-ref-value/model-ref-value.tsx b/ui/src/components/model-ref-value/model-ref-value.tsx new file mode 100644 index 000000000..5ccb4cbbb --- /dev/null +++ b/ui/src/components/model-ref-value/model-ref-value.tsx @@ -0,0 +1,45 @@ +import { Fragment } from 'react' +import { Link } from 'react-router-dom' +import { + ServerModelRef, + getModelRefLabel, + linkFor, +} from 'utils/model-references' + +/** A record the API names: a link to its page, or plain text when it was deleted or has no page. */ +export const ModelRefValue = ({ + projectId, + reference, +}: { + projectId: string + reference: ServerModelRef +}) => { + const to = linkFor(reference, projectId) + const label = getModelRefLabel(reference) + + return to ? ( + + {label} + + ) : ( + <>{label} + ) +} + +/** Several records the API names, comma separated, each a link when it has a page. */ +export const ModelRefValues = ({ + projectId, + references, +}: { + projectId: string + references: ServerModelRef[] +}) => ( + <> + {references.map((reference, index) => ( + + {index > 0 && ', '} + + + ))} + +) diff --git a/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts b/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts new file mode 100644 index 000000000..9e3a2cdc7 --- /dev/null +++ b/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts @@ -0,0 +1,33 @@ +import { API_ROUTES, API_URL } from 'data-services/constants' +import { ServerOccurrenceHistoryEntry } from 'data-services/models/occurrence-history' +import { useAuthorizedQuery } from '../auth/useAuthorizedQuery' + +// Under the occurrences prefix, so every mutation that invalidates occurrences refreshes it. +export const getOccurrenceHistoryQueryKey = (occurrenceId: string) => [ + API_ROUTES.OCCURRENCES, + occurrenceId, + 'history', +] + +export const useOccurrenceHistory = ({ + occurrenceId, + projectId, +}: { + occurrenceId: string + projectId?: string +}): { + entries?: ServerOccurrenceHistoryEntry[] + isLoading: boolean + error?: unknown +} => { + const params = new URLSearchParams(projectId ? { project_id: projectId } : {}) + const { data, isLoading, error } = useAuthorizedQuery< + ServerOccurrenceHistoryEntry[] + >({ + enabled: !!occurrenceId, + queryKey: getOccurrenceHistoryQueryKey(occurrenceId), + url: `${API_URL}/${API_ROUTES.OCCURRENCES}/${occurrenceId}/history/?${params}`, + }) + + return { entries: data, isLoading, error } +} diff --git a/ui/src/data-services/models/occurrence-details.ts b/ui/src/data-services/models/occurrence-details.ts index 39bcbfb39..53bfeb935 100644 --- a/ui/src/data-services/models/occurrence-details.ts +++ b/ui/src/data-services/models/occurrence-details.ts @@ -1,5 +1,5 @@ import { getFormatedTimeString } from 'utils/date/getFormatedTimeString/getFormatedTimeString' -import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { UserPermission } from 'utils/user/types' import { Algorithm } from './algorithm' import { Occurrence, ServerOccurrence } from './occurrence' @@ -67,12 +67,10 @@ export class OccurrenceDetails extends Occurrence { user: i.user ? { id: `${i.user.id}`, - name: i.user.name?.length - ? i.user.name - : translate(STRING.ANONYMOUS_USER), + name: getUserLabel(i.user), image: i.user.image, } - : { name: translate(STRING.ANONYMOUS_USER) }, + : { name: getUserLabel(null) }, comment: i.comment, userPermissions: i.user_permissions, createdAt: i.created_at, diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts new file mode 100644 index 000000000..2188f2c24 --- /dev/null +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -0,0 +1,378 @@ +import { getOccurrenceHistoryQueryKey } from 'data-services/hooks/occurrences/useOccurrenceHistory' +import { API_ROUTES } from 'data-services/constants' +import { Algorithm } from './algorithm' +import { HumanIdentification, MachinePrediction } from './occurrence-details' +import { + getFallbackTimelineItems, + getJobConfigFields, + getResultPrediction, + getTimelineItems, + ServerOccurrenceHistoryEntry, +} from './occurrence-history' +import { Taxon } from './taxa' +import { UserPermission } from 'utils/user/types' + +const NOCTUA = { id: 3, name: 'Noctua pronuba', rank: 'SPECIES' } +const XESTIA = { id: 4, name: 'Xestia c-nigrum', rank: 'SPECIES' } + +const base = { + algorithm: null, + job: null, + score: null, + taxon: null, + timestamp: '2026-04-29T22:00:00', + user: null, +} + +const JOB = { + id: 30, + name: 'Masking run', + config: [ + { + key: 'taxa_list_id', + label: 'Species list', + value: 2, + refs: [{ type: 'taxa_list', id: 2, name: 'Kept' }], + }, + ], +} + +const identificationEntry = (id: number): ServerOccurrenceHistoryEntry => ({ + ...base, + id, + details: { + comment: 'Looks right', + withdrawn: false, + }, + taxon: NOCTUA, + type: 'identification', + user: { id: 9, image: null, name: '' }, +}) + +const predictionEntry = ( + id: number, + taxon: typeof NOCTUA | null = XESTIA, + job: typeof JOB | null = null +): ServerOccurrenceHistoryEntry => ({ + ...base, + algorithm: { id: 7, key: 'classifier', name: 'Classifier' }, + id, + job, + details: { + terminal: true, + }, + score: 0.8, + taxon, + type: 'prediction', +}) + +const classMasking: ServerOccurrenceHistoryEntry = { + ...base, + algorithm: { id: 12, key: 'mask', name: 'Masked classifier' }, + classifications: [ + { + detection_id: 1, + id: 20, + replaced: { id: 6, score: 0.83, taxon: XESTIA }, + score: 0.7, + taxon: NOCTUA, + terminal: true, + }, + { + detection_id: 2, + id: 21, + replaced: null, + score: 0.4, + taxon: XESTIA, + terminal: true, + }, + ], + data: { + determination_after_id: 3, + determination_before_id: 4, + excluded_probability: 0.38, + extra: {}, + new_winner_original_rank: 2, + }, + determination_after: NOCTUA, + determination_before: XESTIA, + id: 5, + job: JOB, + kind: 'class_masking', + score: null, + type: 'algorithm_result', + value: 0.38, +} + +const ownIdentification: HumanIdentification = { + comment: '', + createdAt: '2026-04-29T22:00:00', + id: '1', + taxon: new Taxon({ ...NOCTUA, id: `${NOCTUA.id}`, cover_image_url: null }), + user: { name: 'Someone' }, + userPermissions: [UserPermission.Delete], +} + +const ownPrediction = ( + id: string, + algorithmId: number, + taxon = NOCTUA, + createdAt = '2026-04-29T21:00:00' +): MachinePrediction => ({ + algorithm: new Algorithm({ id: algorithmId }), + createdAt, + id, + score: 0.9, + taxon: new Taxon({ ...taxon, id: `${taxon.id}`, cover_image_url: null }), + terminal: true, + userPermissions: [UserPermission.Update], +}) + +describe('getTimelineItems', () => { + test('maps each entry type to its card, keeping the server order', () => { + const items = getTimelineItems({ + entries: [classMasking, identificationEntry(2), predictionEntry(6)], + identifications: [], + predictions: [], + }) + + expect(items.map((item) => item.type)).toEqual([ + 'algorithm_result', + 'identification', + 'prediction', + ]) + }) + + test("reuses the occurrence's own records, which carry the viewer's permissions", () => { + const prediction = ownPrediction('6', 7) + const items = getTimelineItems({ + entries: [identificationEntry(1), predictionEntry(6)], + identifications: [ownIdentification], + predictions: [prediction], + }) + + expect(items[0]).toMatchObject({ identification: ownIdentification }) + expect(items[1]).toMatchObject({ prediction }) + }) + + test('builds a read-only card when the occurrence lacks the record', () => { + const [item] = getTimelineItems({ + determinationTaxonId: '3', + entries: [identificationEntry(2)], + identifications: [], + predictions: [], + }) + + expect(item.type === 'identification' && item.identification).toMatchObject( + { + applied: true, + comment: 'Looks right', + user: { id: '9', name: 'Anonymous user' }, + userPermissions: [], + } + ) + }) + + test('carries the job that wrote a prediction as a reference', () => { + const items = getTimelineItems({ + entries: [predictionEntry(6, XESTIA, JOB), predictionEntry(8)], + identifications: [], + predictions: [], + }) + + expect(items[0]).toMatchObject({ + job: { type: 'job', id: 30, name: 'Masking run' }, + }) + expect(items[1]).not.toHaveProperty('job', expect.anything()) + }) + + test('leaves out results of a kind it has no card for, and entries it cannot build a card from', () => { + // A kind the server added before the UI has a card for it, e.g. tracking. + const unknown = { + ...classMasking, + kind: 'tracking', + } as unknown as ServerOccurrenceHistoryEntry + + expect( + getTimelineItems({ + entries: [ + unknown, + predictionEntry(6, null), + { ...predictionEntry(7), algorithm: null }, + { ...identificationEntry(2), taxon: null }, + ], + identifications: [], + predictions: [], + }) + ).toEqual([]) + }) +}) + +describe('getFallbackTimelineItems', () => { + test('merges identifications and predictions newest first', () => { + const items = getFallbackTimelineItems({ + identifications: [ownIdentification], + predictions: [ + ownPrediction('6', 7, NOCTUA, '2026-04-29T23:00:00'), + ownPrediction('7', 12, NOCTUA, '2026-04-29T20:00:00'), + ], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-6', + 'identification-1', + 'prediction-7', + ]) + }) + + test('breaks timestamp ties on id, newest first, as the server does', () => { + const items = getFallbackTimelineItems({ + identifications: [ownIdentification], + predictions: [ownPrediction('6', 7, NOCTUA, ownIdentification.createdAt)], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-6', + 'identification-1', + ]) + }) +}) + +describe('fallback predictions', () => { + const tied = (id: string, terminal: boolean, createdAt: string) => ({ + ...ownPrediction(id, 7, NOCTUA, createdAt), + terminal, + }) + + test('keep one per algorithm when detections tie for its top score', () => { + const items = getFallbackTimelineItems({ + identifications: [], + predictions: [ + tied('6', true, '2026-04-29T21:00:00'), + tied('7', true, '2026-04-29T22:00:00'), + ownPrediction('8', 12), + ], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-7', + 'prediction-8', + ]) + }) + + test('prefer a terminal prediction over a later intermediate one', () => { + const items = getFallbackTimelineItems({ + identifications: [], + predictions: [ + tied('6', true, '2026-04-29T21:00:00'), + tied('7', false, '2026-04-29T22:00:00'), + ], + }) + + expect(items.map((item) => item.id)).toEqual(['prediction-6']) + }) +}) + +describe('getResultPrediction', () => { + test('is the best classification the run created, marked applied when it is the determination', () => { + expect(getResultPrediction(classMasking as never, [], '3')).toMatchObject({ + applied: true, + id: '20', + score: 0.7, + taxon: { id: '3', name: 'Noctua pronuba' }, + }) + expect(getResultPrediction(classMasking as never, [], '4')).toMatchObject({ + applied: false, + }) + }) + + test("is the occurrence's own record of the classification when it has one", () => { + const own = { id: '20', score: 0.7 } as never + + expect(getResultPrediction(classMasking as never, [own], '3')).toBe(own) + }) + + test('keeps the taxon ranks the history sends', () => { + const withParents = { + ...classMasking, + classifications: classMasking.classifications.map((c) => ({ + ...c, + taxon: c.taxon && { + ...c.taxon, + parents: [{ id: 1, name: 'Noctuidae', rank: 'FAMILY' }], + }, + })), + } + + expect( + getResultPrediction(withParents as never, [], '3')?.taxon.ranks + ).toMatchObject([{ id: '1', name: 'Noctuidae', rank: 'FAMILY' }]) + }) + + test('is undefined for a run that created no classification', () => { + expect( + getResultPrediction( + { ...classMasking, classifications: [] } as never, + [], + '3' + ) + ).toBeUndefined() + }) +}) + +describe('getOccurrenceHistoryQueryKey', () => { + test('sits under the occurrences key that mutations invalidate', () => { + expect(getOccurrenceHistoryQueryKey('100')[0]).toBe(API_ROUTES.OCCURRENCES) + }) +}) + +describe('getJobConfigFields', () => { + test('lists the config a job ran with, with its labels and records, leaving out unset fields', () => { + const deletedList: { type: string; id: number; name: string | null } = { + type: 'taxa_list', + id: 2, + name: null, + } + const field = ( + key: string, + value: unknown, + refs: (typeof deletedList)[] = [] + ) => ({ + key, + label: `Label of ${key}`, + value, + refs, + }) + expect( + getJobConfigFields({ + id: 1, + name: 'Size filter', + config: [ + field('occurrence_id', 4, [ + { + type: 'occurrence', + id: 4, + name: '#4', + }, + ]), + field('reweight', true), + field('size_threshold', 0.01), + field('source_image_collection_id', null), + // A deleted list keeps its reference with no name, so the card shows its id as text. + field('taxa_list_id', 2, [deletedList]), + ], + }) + ).toEqual([ + field('occurrence_id', 4, [{ type: 'occurrence', id: 4, name: '#4' }]), + field('reweight', true), + field('taxa_list_id', 2, [deletedList]), + ]) + }) + + test('is empty for a job without config', () => { + expect(getJobConfigFields({ id: 1, name: 'Pipeline', config: [] })).toEqual( + [] + ) + expect(getJobConfigFields(null)).toEqual([]) + }) +}) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts new file mode 100644 index 000000000..b46832ca0 --- /dev/null +++ b/ui/src/data-services/models/occurrence-history.ts @@ -0,0 +1,440 @@ +import { STRING, translate } from 'utils/language' +import { ServerModelRef } from 'utils/model-references' +import { getUserLabel } from 'utils/user/getUserLabel' +import { Algorithm } from './algorithm' +import { HumanIdentification, MachinePrediction } from './occurrence-details' +import { Taxon } from './taxa' + +/* + * The server types below mirror the OccurrenceHistoryEntry components in the OpenAPI schema; + * replace them with generated types when the UI adopts OpenAPI generation. + */ + +export interface ServerHistoryTaxon { + id: number + name: string + /** The taxon's ancestors, highest rank first. */ + parents?: { id: number; name: string; rank: string }[] + rank: string +} + +export interface ServerHistoryUser { + id: number + image: string | null + name: string +} + +export interface ServerHistoryAlgorithm { + id: number + key: string + name: string +} + +/** One field of a job's config. `label` is the title the task's config schema gives it, else the key. */ +export interface ServerJobConfigField { + key: string + label: string + /** The records the field names: one per id it holds, in order. */ + refs: ServerModelRef[] + value: unknown +} + +export interface ServerHistoryJob { + id: number + name: string + config: ServerJobConfigField[] +} + +/** The classification a run's classification replaced, when there was one and it still exists. */ +export interface ServerReplacedClassification { + id: number + score: number | null + taxon: ServerHistoryTaxon | null +} + +/** A classification a post-processing run created, listed inside the run's result. */ +export interface ServerCreatedClassification { + detection_id: number + id: number + replaced: ServerReplacedClassification | null + score: number | null + /** Null when the taxon was deleted. */ + taxon: ServerHistoryTaxon | null + terminal: boolean +} + +interface ServerDeterminationSnapshot { + determination_after_id: number | null + determination_before_id: number | null + /** Whatever else the method returned; stored as JSON and not interpreted. */ + extra: Record +} + +export interface ClassMaskingResultData extends ServerDeterminationSnapshot { + /** The share of the source classifier's probability outside the species list. */ + excluded_probability: number + /** Where the class that wins after masking ranked before it; 1 means it was already the top. */ + new_winner_original_rank: number | null +} + +export interface SizeFilterResultData extends ServerDeterminationSnapshot { + /** The filtered detection's box area as a fraction of its image. */ + relative_size: number +} + +export interface ServerIdentificationDetails { + comment: string + withdrawn: boolean +} + +export interface ServerPredictionDetails { + terminal: boolean +} + +/** The fields every entry has. */ +interface ServerHistoryEntryBase { + algorithm: ServerHistoryAlgorithm | null + id: number + job: ServerHistoryJob | null + /** A prediction's score; null for other entries. */ + score: number | null + /** The identified or predicted taxon; null for a result. */ + taxon: ServerHistoryTaxon | null + timestamp: string + user: ServerHistoryUser | null +} + +interface ServerResultEntry + extends ServerHistoryEntryBase { + /** The classifications the run created, best score first. */ + classifications: ServerCreatedClassification[] + data: Data + determination_after: ServerHistoryTaxon | null + determination_before: ServerHistoryTaxon | null + kind: Kind + type: 'algorithm_result' + /** The kind's headline figure, for sorting and filtering; not a confidence. */ + value: number | null +} + +export type ClassMaskingResultEntry = ServerResultEntry< + 'class_masking', + ClassMaskingResultData +> +export type SizeFilterResultEntry = ServerResultEntry< + 'size_filter', + SizeFilterResultData +> +export type AlgorithmResultEntry = + | ClassMaskingResultEntry + | SizeFilterResultEntry + +export interface IdentificationEntry extends ServerHistoryEntryBase { + details: ServerIdentificationDetails + type: 'identification' +} + +export interface PredictionEntry extends ServerHistoryEntryBase { + details: ServerPredictionDetails + type: 'prediction' +} + +export type ServerOccurrenceHistoryEntry = + | AlgorithmResultEntry + | IdentificationEntry + | PredictionEntry + +export type TimelineItem = + | { + type: 'identification' + id: string + identification: HumanIdentification + } + | { + type: 'prediction' + id: string + prediction: MachinePrediction + /** The job that wrote the prediction, when the history names one. */ + job?: ServerModelRef + } + | { type: 'algorithm_result'; id: string; entry: AlgorithmResultEntry } + +/** The result kinds this UI has a card for, with their names; results of any other kind are skipped. */ +const RESULT_KIND_LABELS: Record = { + class_masking: STRING.HISTORY_CLASS_MASKING, + size_filter: STRING.HISTORY_SIZE_FILTER, +} + +/** What to call a result's kind, e.g. "Class masking". */ +export const getResultKindLabel = (entry: AlgorithmResultEntry) => + translate(RESULT_KIND_LABELS[entry.kind]) + +export const convertHistoryTaxon = (taxon: ServerHistoryTaxon) => + new Taxon({ + ...taxon, + id: `${taxon.id}`, + cover_image_url: null, + parents: taxon.parents?.map((parent) => ({ + ...parent, + id: `${parent.id}`, + cover_image_url: null, + })), + }) + +const toIdentification = ( + entry: IdentificationEntry & { taxon: ServerHistoryTaxon }, + determinationTaxonId?: string +): HumanIdentification => { + const taxon = convertHistoryTaxon(entry.taxon) + + return { + applied: taxon.id === determinationTaxonId, + comment: entry.details.comment, + createdAt: entry.timestamp, + id: `${entry.id}`, + overridden: entry.details.withdrawn, + taxon, + user: entry.user + ? { + id: `${entry.user.id}`, + image: entry.user.image ?? undefined, + name: getUserLabel(entry.user), + } + : { name: getUserLabel(entry.user) }, + userPermissions: [], + } +} + +/** A prediction built from the history, for when the occurrence has no record of its own for it. */ +const toPrediction = ( + { + algorithm, + id, + score, + taxon: historyTaxon, + terminal, + timestamp, + }: { + algorithm: ServerHistoryAlgorithm + id: number + score: number | null + taxon: ServerHistoryTaxon + terminal: boolean + timestamp: string + }, + determinationTaxonId?: string +): MachinePrediction => { + const taxon = convertHistoryTaxon(historyTaxon) + + return { + algorithm: new Algorithm(algorithm), + applied: taxon.id === determinationTaxonId, + createdAt: timestamp, + id: `${id}`, + overridden: taxon.id !== determinationTaxonId, + score: score ?? 0, + taxon, + terminal, + userPermissions: [], + } +} + +const isAlgorithmResult = ( + entry: ServerOccurrenceHistoryEntry +): entry is AlgorithmResultEntry => + entry.type === 'algorithm_result' && entry.kind in RESULT_KIND_LABELS + +/** + * The history as cards to render, newest first. Identifications and predictions reuse the + * occurrence's own records when it has them, since only those carry the viewer's permissions. + */ +export const getTimelineItems = ({ + determinationTaxonId, + entries, + identifications, + predictions, +}: { + determinationTaxonId?: string + entries: ServerOccurrenceHistoryEntry[] + identifications: HumanIdentification[] + predictions: MachinePrediction[] +}): TimelineItem[] => { + return entries.flatMap((entry): TimelineItem[] => { + const id = `${entry.type}-${entry.id}` + + switch (entry.type) { + case 'identification': { + const { taxon } = entry + const identification = + identifications.find((i) => i.id === `${entry.id}`) ?? + (taxon + ? toIdentification({ ...entry, taxon }, determinationTaxonId) + : undefined) + + return identification + ? [{ type: 'identification', id, identification }] + : [] + } + case 'prediction': { + const { algorithm, taxon } = entry + const prediction = + predictions.find((p) => p.id === `${entry.id}`) ?? + (algorithm && taxon + ? toPrediction( + { + ...entry, + algorithm, + taxon, + terminal: entry.details.terminal, + }, + determinationTaxonId + ) + : undefined) + const job: ServerModelRef | undefined = entry.job + ? { type: 'job', id: entry.job.id, name: entry.job.name } + : undefined + + return prediction ? [{ type: 'prediction', id, prediction, job }] : [] + } + case 'algorithm_result': + return isAlgorithmResult(entry) + ? [{ type: 'algorithm_result', id, entry }] + : [] + default: + return [] + } + }) +} + +/** + * The best prediction of each algorithm: highest score, then terminal, then latest, then id. + * The occurrence lists every detection tied for an algorithm's top score, and the history keeps one. + */ +const onePredictionPerAlgorithm = (predictions: MachinePrediction[]) => { + const rank = (p: MachinePrediction) => [ + p.score ?? -Infinity, + p.terminal ? 1 : 0, + new Date(p.createdAt).getTime() || 0, + Number(p.id) || 0, + ] + const isBetter = (a: MachinePrediction, b: MachinePrediction) => { + const [rankA, rankB] = [rank(a), rank(b)] + const index = rankA.findIndex((value, i) => value !== rankB[i]) + + return index !== -1 && rankA[index] > rankB[index] + } + const best = new Map() + predictions.forEach((prediction) => { + const key = `${prediction.algorithm?.id}` + const current = best.get(key) + if (!current || isBetter(prediction, current)) { + best.set(key, prediction) + } + }) + + return [...best.values()] +} + +/** + * Identifications and predictions merged newest first, for while the history loads or when it is + * unavailable. Predictions and tie breaks follow the server, so cards keep their place once it arrives. + */ +export const getFallbackTimelineItems = ({ + identifications, + predictions, +}: { + identifications: HumanIdentification[] + predictions: MachinePrediction[] +}): TimelineItem[] => + [ + ...identifications.map( + (identification): TimelineItem => ({ + type: 'identification', + id: `identification-${identification.id}`, + identification, + }) + ), + ...onePredictionPerAlgorithm(predictions).map( + (prediction): TimelineItem => ({ + type: 'prediction', + id: `prediction-${prediction.id}`, + prediction, + }) + ), + ] + .map((item) => ({ + item, + time: new Date(getCreatedAt(item)).getTime(), + id: Number(getSourceId(item)), + })) + .sort((a, b) => b.time - a.time || b.id - a.id) + .map(({ item }) => item) + +const getSourceId = (item: TimelineItem) => { + switch (item.type) { + case 'identification': + return item.identification.id + case 'prediction': + return item.prediction.id + default: + return item.entry.id + } +} + +const getCreatedAt = (item: TimelineItem) => { + switch (item.type) { + case 'identification': + return item.identification.createdAt + case 'prediction': + return item.prediction.createdAt + default: + return item.entry.timestamp + } +} + +/** + * The prediction a result stands for: the best classification its run created, if any. The + * occurrence's own record is used when it has one, since only that carries the viewer's permissions. + */ +export const getResultPrediction = ( + entry: AlgorithmResultEntry, + predictions: MachinePrediction[], + determinationTaxonId?: string +): MachinePrediction | undefined => { + const best = entry.classifications.find( + (classification) => classification.taxon !== null + ) + if (!best?.taxon || !entry.algorithm) { + return undefined + } + return ( + predictions.find((p) => p.id === `${best.id}`) ?? + toPrediction( + { + ...best, + algorithm: entry.algorithm, + taxon: best.taxon, + timestamp: entry.timestamp, + }, + determinationTaxonId + ) + ) +} + +/** One field of a job's config, when the job set it. */ +export const getJobConfigField = (job: ServerHistoryJob | null, key: string) => + job?.config.find((field) => field.key === key) + +/** Config fields the result card already shows in a row of their own. */ +const CONFIG_SHOWN_ELSEWHERE = ['size_threshold'] + +/** A job's config fields to show as rows, leaving out unset ones and those shown elsewhere. */ +export const getJobConfigFields = ( + job: ServerHistoryJob | null +): ServerJobConfigField[] => + (job?.config ?? []).filter( + ({ key, value }) => + value !== null && + value !== undefined && + !CONFIG_SHOWN_ELSEWHERE.includes(key) + ) diff --git a/ui/src/data-services/models/occurrence.ts b/ui/src/data-services/models/occurrence.ts index 468f670f1..db4fe3b4b 100644 --- a/ui/src/data-services/models/occurrence.ts +++ b/ui/src/data-services/models/occurrence.ts @@ -1,6 +1,6 @@ import { getFormatedDateString } from 'utils/date/getFormatedDateString/getFormatedDateString' import { getFormatedTimeString } from 'utils/date/getFormatedTimeString/getFormatedTimeString' -import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' import { UserPermission } from 'utils/user/types' import { Taxon } from './taxa' @@ -104,11 +104,9 @@ export class Occurrence { return verifiedBy ? { id: `${verifiedBy.id}`, - name: verifiedBy.name?.length - ? verifiedBy.name - : translate(STRING.ANONYMOUS_USER), + name: getUserLabel(verifiedBy), } - : { name: translate(STRING.ANONYMOUS_USER) } + : { name: getUserLabel(null) } } get durationLabel(): string | undefined { diff --git a/ui/src/pages/occurrence-details/history/algorithm-result.tsx b/ui/src/pages/occurrence-details/history/algorithm-result.tsx new file mode 100644 index 000000000..912e6d0c9 --- /dev/null +++ b/ui/src/pages/occurrence-details/history/algorithm-result.tsx @@ -0,0 +1,226 @@ +import { + ModelRefValue, + ModelRefValues, +} from 'components/model-ref-value/model-ref-value' +import { + AlgorithmResultEntry, + getJobConfigField, + getJobConfigFields, + getResultKindLabel, + getResultPrediction, + ServerHistoryTaxon, +} from 'data-services/models/occurrence-history' +import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' +import { FilterIcon, LucideIcon, RulerIcon } from 'lucide-react' +import { BasicTooltip, IdentificationCard } from 'nova-ui-kit' +import { useParams } from 'react-router-dom' +import { getModelRefLabel } from 'utils/model-references' +import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' +import { STRING, translate } from 'utils/language' +import { UserInfo } from 'utils/user/types' +import { HistoryStat, HistoryStats, HistoryTime } from './history-stats' +import { MachinePrediction } from './machine-prediction' + +const KIND_ICONS: Record = { + class_masking: FilterIcon, + size_filter: RulerIcon, +} + +const formatConfigValue = (value: unknown) => { + if (typeof value === 'boolean') { + return translate(value ? STRING.YES : STRING.NO) + } + if (Array.isArray(value)) { + return value.join(', ') + } + + return typeof value === 'object' ? JSON.stringify(value) : `${value}` +} + +/** The card's subtitle: for class masking the classifier and species list, otherwise the algorithm's name. */ +const getSubTitle = (entry: AlgorithmResultEntry) => { + if (entry.kind === 'class_masking') { + const refFor = (key: string) => getJobConfigField(entry.job, key)?.refs[0] + const classifier = refFor('algorithm_id') + + return translate(STRING.HISTORY_MASKING_SUBTITLE, { + algorithm: classifier + ? getModelRefLabel(classifier) + : entry.algorithm?.name ?? translate(STRING.VALUE_NOT_AVAILABLE), + list: getModelRefLabel(refFor('taxa_list_id')), + }) + } + + return entry.algorithm?.name +} + +/** A fraction as a percentage with at most one decimal, e.g. 0.0315 reads "3.2%". */ +const formatPercent = (fraction: number) => + `${Math.round(fraction * 1000) / 10}%` + +/** The determination row, left out when there was no determination before or after. */ +const getDeterminationStats = ( + before: ServerHistoryTaxon | null, + after: ServerHistoryTaxon | null +): HistoryStat[] => { + if (!before && !after) { + return [] + } + + const notAvailable = translate(STRING.VALUE_NOT_AVAILABLE) + const value = + before?.id === after?.id + ? translate(STRING.HISTORY_DETERMINATION_UNCHANGED, { + name: after?.name ?? notAvailable, + }) + : translate(STRING.HISTORY_DETERMINATION_CHANGED, { + after: after?.name ?? notAvailable, + before: before?.name ?? notAvailable, + }) + + return [{ label: translate(STRING.HISTORY_DETERMINATION), value }] +} + +export const AlgorithmResult = ({ + currentUser, + entry, + occurrence, +}: { + currentUser?: UserInfo + entry: AlgorithmResultEntry + occurrence: Occurrence +}) => { + const { projectId } = useParams() + const Icon = KIND_ICONS[entry.kind] + const prediction = getResultPrediction( + entry, + occurrence.machinePredictions, + occurrence.determinationTaxon?.id + ) + + const stats: HistoryStat[] = getDeterminationStats( + entry.determination_before, + entry.determination_after + ) + switch (entry.kind) { + case 'class_masking': { + stats.push({ + label: translate(STRING.HISTORY_EXCLUDED_PROBABILITY), + value: formatPercent(entry.data.excluded_probability), + }) + // The top prediction before masking: what the run's best classification replaced. + const replaced = entry.classifications.find( + (c) => c.taxon !== null + )?.replaced + if (replaced?.taxon) { + stats.push({ + label: translate(STRING.HISTORY_ORIGINAL_PREDICTION), + value: + replaced.score !== null + ? `${replaced.taxon.name} (${replaced.score.toFixed(2)})` + : replaced.taxon.name, + }) + } + break + } + case 'size_filter': { + const threshold = getJobConfigField(entry.job, 'size_threshold')?.value + stats.push({ + label: translate(STRING.HISTORY_DETECTION_SIZE), + value: + typeof threshold === 'number' + ? translate(STRING.HISTORY_DETECTION_SIZE_WITH_THRESHOLD, { + size: formatPercent(entry.data.relative_size), + threshold: formatPercent(threshold), + }) + : translate(STRING.HISTORY_DETECTION_SIZE_VALUE, { + size: formatPercent(entry.data.relative_size), + }), + }) + break + } + } + stats.push({ + label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), + value: new Set(entry.classifications.map((c) => c.detection_id)).size, + }) + getJobConfigFields(entry.job).forEach(({ label, value, refs }) => { + stats.push({ + label, + value: refs.length ? ( + + ) : ( + formatConfigValue(value) + ), + }) + }) + if (entry.algorithm) { + stats.push({ + label: translate(STRING.FIELD_LABEL_ALGORITHM), + value: ( + + ), + }) + } + if (entry.job) { + stats.push({ + label: translate(STRING.FIELD_LABEL_JOB), + value: ( + + ), + }) + } + + const header = { + avatar: , + avatarTooltip: translate(STRING.HISTORY_ALGORITHM_RESULT), + subTitle: getSubTitle(entry), + title: getResultKindLabel(entry), + } + + // The run's best new classification, with the classifier's other top predictions and the run's details. + if (prediction) { + return ( + + + + ) + } + + return ( +
+ + + + {header.avatar} + + + } + subTitle={header.subTitle} + title={header.title} + > + + +
+ ) +} diff --git a/ui/src/pages/occurrence-details/history/history-stats.tsx b/ui/src/pages/occurrence-details/history/history-stats.tsx new file mode 100644 index 000000000..4a0dff556 --- /dev/null +++ b/ui/src/pages/occurrence-details/history/history-stats.tsx @@ -0,0 +1,27 @@ +import { Fragment, ReactNode } from 'react' + +export interface HistoryStat { + label: string + value: ReactNode +} + +// Labels get at most 40% of the width and wrap first, so values keep the room they need. +export const HistoryStats = ({ stats }: { stats: HistoryStat[] }) => + stats.length ? ( +
+ {stats.map(({ label, value }) => ( + + {label} + + {value} + + + ))} +
+ ) : null + +export const HistoryTime = ({ label }: { label: string }) => ( + + {label} + +) diff --git a/ui/src/pages/occurrence-details/identification-card/human-identification.tsx b/ui/src/pages/occurrence-details/history/human-identification.tsx similarity index 100% rename from ui/src/pages/occurrence-details/identification-card/human-identification.tsx rename to ui/src/pages/occurrence-details/history/human-identification.tsx diff --git a/ui/src/pages/occurrence-details/identification-card/machine-avatar.svg b/ui/src/pages/occurrence-details/history/machine-avatar.svg similarity index 100% rename from ui/src/pages/occurrence-details/identification-card/machine-avatar.svg rename to ui/src/pages/occurrence-details/history/machine-avatar.svg diff --git a/ui/src/pages/occurrence-details/history/machine-prediction.tsx b/ui/src/pages/occurrence-details/history/machine-prediction.tsx new file mode 100644 index 000000000..dec018282 --- /dev/null +++ b/ui/src/pages/occurrence-details/history/machine-prediction.tsx @@ -0,0 +1,296 @@ +import { ErrorState } from 'components/error-state/error-state' +import { TaxonDetails } from 'components/taxon-details/taxon-details' +import { useClassificationDetails } from 'data-services/hooks/identifications/useClassificationDetails' +import { + MachinePrediction as Identification, + OccurrenceDetails as Occurrence, +} from 'data-services/models/occurrence-details' +import { Taxon } from 'data-services/models/taxa' +import { Loader2 } from 'lucide-react' +import { + BasicTooltip, + Collapsible, + IdentificationCard, + IdentificationDetails, + IdentificationScore, +} from 'nova-ui-kit' +import { ReactNode, useState } from 'react' +import { Link, useNavigate, useParams } from 'react-router-dom' +import { APP_ROUTES } from 'utils/constants' +import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' +import { getAppRoute } from 'utils/getAppRoute' +import { STRING, translate } from 'utils/language' +import { ServerModelRef } from 'utils/model-references' +import { UserInfo, UserPermission } from 'utils/user/types' +import { Agree } from '../agree/agree' +import { ModelRefValue } from 'components/model-ref-value/model-ref-value' +import { HistoryStats } from './history-stats' +import machineAvatar from './machine-avatar.svg' + +export const MachinePrediction = ({ + avatar, + avatarTooltip, + children, + currentUser, + identification, + job, + occurrence, + subTitle, + timestamp, + title, +}: { + /** Header overrides for a card that shows a run's prediction rather than a classifier's. */ + avatar?: ReactNode + avatarTooltip?: string + /** Details shown below the other predictions when the card is expanded. */ + children?: ReactNode + currentUser?: UserInfo + identification: Identification + /** The job that wrote the prediction, shown as a row when the history names one. */ + job?: ServerModelRef + occurrence: Occurrence + /** Replaces the terminal/intermediate label. */ + subTitle?: string + timestamp?: string + title?: string +}) => { + const [open, setOpen] = useState(false) + const navigate = useNavigate() + const { projectId } = useParams() + const formattedTime = getFormatedDateTimeString({ + date: new Date(timestamp ?? identification.createdAt), + }) + + return ( +
+ + {formattedTime} + + + + {avatar ?? } + + + } + collapsible + collapsibleTriggerTooltip={getExpandTooltip(open)} + onOpenChange={setOpen} + open={open} + subTitle={ + subTitle ?? + (identification.terminal + ? translate(STRING.TERMINAL_CLASSIFICATION) + : translate(STRING.INTERMEDIATE_CLASSIFICATION)) + } + title={ + title ?? + identification.algorithm?.name ?? + translate(STRING.MACHINE_SUGGESTION) + } + onTitleClick={ + identification.algorithm + ? () => + navigate( + APP_ROUTES.ALGORITHM_DETAILS({ + projectId: projectId as string, + algorithmId: identification.algorithm?.id, + }) + ) + : undefined + } + > + + + + + {job ? ( + + ), + }, + ]} + /> + ) : null} + {children} + + + +
+ ) +} + +const PredictionRow = ({ + applied, + children, + score, + taxon, +}: { + applied?: boolean + children?: ReactNode + score: number + taxon: Taxon +}) => { + const { projectId } = useParams() + + return ( + +
+
+ +
+ +
+ + + + +
+ {children} +
+ + ) +} + +const AgreeablePredictionRow = ({ + applied, + currentUser, + occurrence, + predictionId, + score, + taxon, +}: { + applied?: boolean + currentUser?: UserInfo + occurrence: Occurrence + predictionId: string + score: number + taxon: Taxon +}) => ( + + {occurrence.userPermissions.includes(UserPermission.Update) && ( + + )} + +) + +/** The runner-up taxa of a classification, loaded when its card is expanded. */ +const MorePredictions = ({ + currentUser, + occurrence, + open, + predictionId, + showEmpty = true, + shownTaxonId, +}: { + currentUser?: UserInfo + occurrence: Occurrence + open: boolean + predictionId: string + showEmpty?: boolean + /** The taxon the card already shows, left out of the list. */ + shownTaxonId: string +}) => { + const { classification, error, isLoading } = useClassificationDetails( + predictionId, + open + ) + const topN = classification?.topN.filter( + ({ taxon }) => taxon.id !== shownTaxonId + ) + + if (isLoading) { + return ( +
+ +
+ ) + } + + if (error) { + return ( +
+ +
+ ) + } + + if (topN && topN.length === 0) { + return showEmpty ? ( +
+ + {translate(STRING.NO_MORE_PREDICTIONS)} + +
+ ) : null + } + + return ( + <> + {topN?.map(({ score, taxon }) => ( + + ))} + + ) +} + +const getExpandTooltip = (open: boolean) => + translate( + open + ? STRING.FEWER_PREDICTIONS_AND_DETAILS + : STRING.MORE_PREDICTIONS_AND_DETAILS + ) diff --git a/ui/src/pages/occurrence-details/history/occurrence-timeline.tsx b/ui/src/pages/occurrence-details/history/occurrence-timeline.tsx new file mode 100644 index 000000000..549b9dc2d --- /dev/null +++ b/ui/src/pages/occurrence-details/history/occurrence-timeline.tsx @@ -0,0 +1,97 @@ +import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' +import { + getFallbackTimelineItems, + getTimelineItems, + ServerOccurrenceHistoryEntry, +} from 'data-services/models/occurrence-history' +import { Loader2Icon } from 'lucide-react' +import { useMemo } from 'react' +import { STRING, translate } from 'utils/language' +import { UserInfo } from 'utils/user/types' +import { AlgorithmResult } from './algorithm-result' +import { HumanIdentification } from './human-identification' +import { MachinePrediction } from './machine-prediction' + +export const OccurrenceTimeline = ({ + currentUser, + entries, + error, + isLoading, + occurrence, +}: { + currentUser?: UserInfo + entries?: ServerOccurrenceHistoryEntry[] + error?: unknown + isLoading: boolean + occurrence: Occurrence +}) => { + const items = useMemo( + () => + entries + ? getTimelineItems({ + determinationTaxonId: occurrence.determinationTaxon?.id, + entries, + identifications: occurrence.humanIdentifications, + predictions: occurrence.machinePredictions, + }) + : getFallbackTimelineItems({ + identifications: occurrence.humanIdentifications, + predictions: occurrence.machinePredictions, + }), + [entries, occurrence] + ) + + return ( + <> + {/* The fallback cards stand in while the history loads, so the spinner only fills an empty list. */} + {isLoading && !items.length ? ( +
+ +
+ ) : null} + {error ? ( +

+ {translate(STRING.HISTORY_LOAD_ERROR)} +

+ ) : null} + {!isLoading && !error && !items.length ? ( +

+ {translate(STRING.HISTORY_EMPTY)} +

+ ) : null} + {items.map((item) => { + switch (item.type) { + case 'identification': + return ( + + ) + case 'prediction': + return ( + + ) + case 'algorithm_result': + return ( + + ) + } + })} + + ) +} diff --git a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx deleted file mode 100644 index 5150c078a..000000000 --- a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx +++ /dev/null @@ -1,226 +0,0 @@ -import { ErrorState } from 'components/error-state/error-state' -import { TaxonDetails } from 'components/taxon-details/taxon-details' -import { useClassificationDetails } from 'data-services/hooks/identifications/useClassificationDetails' -import { - MachinePrediction as Identification, - OccurrenceDetails as Occurrence, -} from 'data-services/models/occurrence-details' -import { Taxon } from 'data-services/models/taxa' -import { Loader2 } from 'lucide-react' -import { - BasicTooltip, - Collapsible, - IdentificationCard, - IdentificationDetails, - IdentificationScore, -} from 'nova-ui-kit' -import { ReactNode, useState } from 'react' -import { Link, useNavigate, useParams } from 'react-router-dom' -import { APP_ROUTES } from 'utils/constants' -import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' -import { getAppRoute } from 'utils/getAppRoute' -import { STRING, translate } from 'utils/language' -import { UserInfo, UserPermission } from 'utils/user/types' -import { Agree } from '../agree/agree' -import machineAvatar from './machine-avatar.svg' - -export const MachinePrediction = ({ - currentUser, - identification, - occurrence, -}: { - currentUser?: UserInfo - identification: Identification - occurrence: Occurrence -}) => { - const [open, setOpen] = useState(false) - const { classification, error, isLoading } = useClassificationDetails( - identification.id, - open - ) - const navigate = useNavigate() - const { projectId } = useParams() - const topN = classification?.topN.filter( - ({ taxon }) => taxon.id !== identification.taxon.id - ) - const formattedTime = getFormatedDateTimeString({ - date: new Date(identification.createdAt), - }) - const showAgree = occurrence.userPermissions.includes(UserPermission.Update) - - return ( -
- - {formattedTime} - - } - collapsible - collapsibleTriggerTooltip={ - open ? 'Hide top predictions' : 'Show top predictions' - } - onOpenChange={setOpen} - open={open} - subTitle={ - identification.terminal - ? translate(STRING.TERMINAL_CLASSIFICATION) - : translate(STRING.INTERMEDIATE_CLASSIFICATION) - } - title={ - identification.algorithm?.name ?? translate(STRING.MACHINE_SUGGESTION) - } - onTitleClick={ - identification.algorithm - ? () => - navigate( - APP_ROUTES.ALGORITHM_DETAILS({ - projectId: projectId as string, - algorithmId: identification.algorithm?.id, - }) - ) - : undefined - } - > - - {showAgree && ( - - )} - - - - - {topN?.map(({ score, taxon }) => { - const applied = taxon.id === occurrence.determinationTaxon.id - - return ( - - {showAgree && ( - - )} - - ) - })} - - - -
- ) -} - -const MachinePredictionDetails = ({ - applied, - children, - score, - taxon, -}: { - applied?: boolean - children: ReactNode - score: number - taxon: Taxon -}) => { - const { projectId } = useParams() - - return ( - -
-
- -
- -
- - - - -
- {children} -
- - ) -} - -const FetchDetails = ({ - empty, - error, - isLoading, -}: { - empty?: boolean - error: unknown - isLoading: boolean -}) => { - if (isLoading) { - return ( -
- -
- ) - } - - if (error) { - return ( -
- -
- ) - } - - if (empty) { - return ( -
- No more predictions to show -
- ) - } - - return null -} diff --git a/ui/src/pages/occurrence-details/occurrence-details.tsx b/ui/src/pages/occurrence-details/occurrence-details.tsx index 5bc502d89..2421fad93 100644 --- a/ui/src/pages/occurrence-details/occurrence-details.tsx +++ b/ui/src/pages/occurrence-details/occurrence-details.tsx @@ -3,6 +3,7 @@ import { BlueprintItem, } from 'components/blueprint-collection/blueprint-collection' import { TaxonDetails } from 'components/taxon-details/taxon-details' +import { useOccurrenceHistory } from 'data-services/hooks/occurrences/useOccurrenceHistory' import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' import { SearchIcon } from 'lucide-react' import { @@ -26,8 +27,7 @@ import { useUser } from 'utils/user/userContext' import { useUserInfo } from 'utils/user/userInfoContext' import { Agree } from './agree/agree' import { IdQuickActions } from './id-quick-actions/id-quick-actions' -import { HumanIdentification } from './identification-card/human-identification' -import { MachinePrediction } from './identification-card/machine-prediction' +import { OccurrenceTimeline } from './history/occurrence-timeline' import styles from './occurrence-details.module.scss' import { StatusLabel } from './status-label/status-label' import { SuggestId } from './suggest-id/suggest-id' @@ -57,6 +57,10 @@ export const OccurrenceDetails = ({ const navigate = useNavigate() const location = useLocation() const [suggestIdOpen, setSuggestIdOpen] = useState(false) + const history = useOccurrenceHistory({ + occurrenceId: occurrence.id, + projectId, + }) const canUpdate = occurrence.userPermissions.includes(UserPermission.Update) const blueprintItems = useMemo( @@ -256,24 +260,13 @@ export const OccurrenceDetails = ({ )} - {occurrence.humanIdentifications.map((i) => ( - - ))} - - {occurrence.machinePredictions.map((p) => ( - - ))} + diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 8fdd8d3b7..c1a3d3442 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -121,6 +121,8 @@ export enum STRING { FIELD_LABEL_ID, FIELD_LABEL_IMAGE, FIELD_LABEL_JOB_STATUS, + ENTITY_ID, + FIELD_LABEL_ALGORITHM, FIELD_LABEL_JOB, FIELD_LABEL_JOBS, FIELD_LABEL_KEY, @@ -325,6 +327,21 @@ export enum STRING { EXTERNAL_RESOURCES, FORGOT_PASSWORD_DETAILS, FORGOT_PASSWORD, + HISTORY_ALGORITHM_RESULT, + HISTORY_CLASS_MASKING, + HISTORY_DETECTION_SIZE, + HISTORY_DETECTION_SIZE_VALUE, + HISTORY_DETECTION_SIZE_WITH_THRESHOLD, + HISTORY_DETECTIONS_AFFECTED, + HISTORY_DETERMINATION, + HISTORY_DETERMINATION_CHANGED, + HISTORY_DETERMINATION_UNCHANGED, + HISTORY_EMPTY, + HISTORY_EXCLUDED_PROBABILITY, + HISTORY_MASKING_SUBTITLE, + HISTORY_ORIGINAL_PREDICTION, + HISTORY_LOAD_ERROR, + HISTORY_SIZE_FILTER, ID_APPLIED, INFO, INTERMEDIATE_CLASSIFICATION, @@ -333,9 +350,14 @@ export enum STRING { LEAVE_TEAM, LOADING_DATA, LOADING_HIGHER_RESOLUTION, + MACHINE_PREDICTION, MACHINE_PREDICTION_SCORE, MACHINE_SUGGESTION, MANAGE_ACCESS_FOR, + MORE_PREDICTIONS_AND_DETAILS, + NO_MORE_PREDICTIONS, + PREDICTIONS_NOT_LOADED, + FEWER_PREDICTIONS_AND_DETAILS, MOST_IDENTIFICATIONS, MOST_OBSERVED_TAXA, NEW_ID, @@ -383,6 +405,8 @@ export enum STRING { VIEW_IN_SESSION, VIEW_OCCURRENCES, VIEW_SETTINGS, + NO, + YES, YOU, } @@ -488,6 +512,8 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.FIELD_LABEL_ID]: 'ID', [STRING.FIELD_LABEL_IMAGE]: 'Cover image', [STRING.FIELD_LABEL_JOB_STATUS]: 'Job status', + [STRING.ENTITY_ID]: '#{{id}}', + [STRING.FIELD_LABEL_ALGORITHM]: 'Algorithm', [STRING.FIELD_LABEL_JOB]: 'Job', [STRING.FIELD_LABEL_JOBS]: 'Jobs', [STRING.FIELD_LABEL_KEY]: 'Key', @@ -764,6 +790,23 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.EXTERNAL_RESOURCES]: 'External resources', [STRING.FORGOT_PASSWORD_DETAILS]: `No worries, we'll send you reset instructions.`, [STRING.FORGOT_PASSWORD]: 'Forgot password?', + [STRING.HISTORY_ALGORITHM_RESULT]: 'Algorithm result', + [STRING.HISTORY_CLASS_MASKING]: 'Class masking', + [STRING.HISTORY_DETECTION_SIZE]: 'Detection size', + [STRING.HISTORY_DETECTION_SIZE_VALUE]: '{{size}} of the image', + [STRING.HISTORY_DETECTION_SIZE_WITH_THRESHOLD]: + '{{size}} of the image (threshold {{threshold}})', + [STRING.HISTORY_DETECTIONS_AFFECTED]: 'Detections changed', + [STRING.HISTORY_DETERMINATION]: 'Determination', + [STRING.HISTORY_DETERMINATION_CHANGED]: '{{before}} → {{after}}', + [STRING.HISTORY_DETERMINATION_UNCHANGED]: '{{name}} (unchanged)', + [STRING.HISTORY_EMPTY]: 'Nothing has happened to this occurrence yet.', + [STRING.HISTORY_EXCLUDED_PROBABILITY]: 'Probability outside the list', + [STRING.HISTORY_MASKING_SUBTITLE]: '{{algorithm}}, filtered by {{list}}', + [STRING.HISTORY_ORIGINAL_PREDICTION]: 'Original top prediction', + [STRING.HISTORY_LOAD_ERROR]: + 'Could not load the full history. Showing identifications and predictions only.', + [STRING.HISTORY_SIZE_FILTER]: 'Size filter', [STRING.ID_APPLIED]: 'ID applied', [STRING.INFO]: 'Info', [STRING.INTERMEDIATE_CLASSIFICATION]: 'Intermediate classification', @@ -772,9 +815,14 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.LEAVE_TEAM]: 'Leave team', [STRING.LOADING_DATA]: 'Loading data', [STRING.LOADING_HIGHER_RESOLUTION]: 'Loading higher resolution', + [STRING.MACHINE_PREDICTION]: 'Machine prediction', [STRING.MACHINE_PREDICTION_SCORE]: 'Machine prediction score\n{{score}}', [STRING.MACHINE_SUGGESTION]: 'Machine suggestion', [STRING.MANAGE_ACCESS_FOR]: 'Manage access for {{user}}.', + [STRING.MORE_PREDICTIONS_AND_DETAILS]: 'More predictions and details', + [STRING.NO_MORE_PREDICTIONS]: 'No more predictions to show', + [STRING.PREDICTIONS_NOT_LOADED]: 'Could not load top predictions', + [STRING.FEWER_PREDICTIONS_AND_DETAILS]: 'Hide predictions and details', [STRING.MOST_IDENTIFICATIONS]: 'Most identifications', [STRING.MOST_OBSERVED_TAXA]: 'Most observed taxa', [STRING.NEW_ID]: 'New ID', @@ -821,6 +869,8 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.VIEW_IN_SESSION]: 'View in session', [STRING.VIEW_OCCURRENCES]: 'View occurrences', [STRING.VIEW_SETTINGS]: 'View settings', + [STRING.NO]: 'No', + [STRING.YES]: 'Yes', [STRING.YOU]: 'You', } diff --git a/ui/src/utils/model-references.test.ts b/ui/src/utils/model-references.test.ts new file mode 100644 index 000000000..8f996a5d9 --- /dev/null +++ b/ui/src/utils/model-references.test.ts @@ -0,0 +1,39 @@ +import { linkFor } from './model-references' + +// utils/constants reads import.meta, which ts-jest cannot compile; these mirror its route shapes. +jest.mock('utils/constants', () => ({ + APP_ROUTES: { + ALGORITHM_DETAILS: (p: { projectId: string; algorithmId: string }) => + `/projects/${p.projectId}/algorithms/${p.algorithmId}`, + CAPTURES: (p: { projectId: string }) => `/projects/${p.projectId}/captures`, + JOB_DETAILS: (p: { projectId: string; jobId: string }) => + `/projects/${p.projectId}/jobs/${p.jobId}`, + OCCURRENCE_DETAILS: (p: { projectId: string; occurrenceId: string }) => + `/projects/${p.projectId}/occurrences/${p.occurrenceId}`, + TAXA_LIST_DETAILS: (p: { projectId: string; taxaListId: string }) => + `/projects/${p.projectId}/taxa-lists/${p.taxaListId}`, + TAXON_DETAILS: (p: { projectId: string; taxonId: string }) => + `/projects/${p.projectId}/taxa/${p.taxonId}`, + }, +})) + +describe('linkFor', () => { + it('links a known type to its page', () => { + expect(linkFor({ type: 'job', id: 7, name: 'Run' }, '1')).toBe( + '/projects/1/jobs/7' + ) + expect(linkFor({ type: 'taxa_list', id: 3, name: 'Kept' }, '1')).toBe( + '/projects/1/taxa-lists/3' + ) + expect(linkFor({ type: 'capture_set', id: 5, name: 'Set' }, '1')).toBe( + '/projects/1/captures?collections=5' + ) + }) + + it('does not link a deleted row or an unknown type', () => { + expect(linkFor({ type: 'job', id: 7, name: null }, '1')).toBeUndefined() + expect( + linkFor({ type: 'something_new', id: 1, name: 'x' }, '1') + ).toBeUndefined() + }) +}) diff --git a/ui/src/utils/model-references.ts b/ui/src/utils/model-references.ts new file mode 100644 index 000000000..610686a01 --- /dev/null +++ b/ui/src/utils/model-references.ts @@ -0,0 +1,43 @@ +import { APP_ROUTES } from 'utils/constants' +import { getAppRoute } from 'utils/getAppRoute' +import { STRING, translate } from 'utils/language' + +/** Another record the API mentions. `name` is null when the record no longer exists. */ +export interface ServerModelRef { + type: string + id: number + name: string | null +} + +/** Where each type of record is shown in the app. A type missing here is shown as text. */ +const ROUTES: Record string> = { + algorithm: (projectId, id) => + APP_ROUTES.ALGORITHM_DETAILS({ projectId, algorithmId: id }), + capture_set: (projectId, id) => + getAppRoute({ + to: APP_ROUTES.CAPTURES({ projectId }), + filters: { collections: id }, + }), + job: (projectId, id) => APP_ROUTES.JOB_DETAILS({ projectId, jobId: id }), + occurrence: (projectId, id) => + APP_ROUTES.OCCURRENCE_DETAILS({ projectId, occurrenceId: id }), + session: (projectId, id) => + APP_ROUTES.SESSION_DETAILS({ projectId, sessionId: id }), + taxa_list: (projectId, id) => + APP_ROUTES.TAXA_LIST_DETAILS({ projectId, taxaListId: id }), +} + +/** Where a reference links to, or undefined when the record is gone or has no page. */ +export const linkFor = (ref: ServerModelRef, projectId: string) => + ref.name === null ? undefined : ROUTES[ref.type]?.(projectId, `${ref.id}`) + +/** A reference's name, its id when the record was deleted, or "not available" when there is none. */ +export const getModelRefLabel = (reference?: ServerModelRef) => { + if (!reference) { + return translate(STRING.VALUE_NOT_AVAILABLE) + } + + return ( + reference.name ?? translate(STRING.ENTITY_ID, { id: `${reference.id}` }) + ) +} diff --git a/ui/src/utils/user/getUserLabel.test.ts b/ui/src/utils/user/getUserLabel.test.ts new file mode 100644 index 000000000..1dbbb5624 --- /dev/null +++ b/ui/src/utils/user/getUserLabel.test.ts @@ -0,0 +1,17 @@ +import { getUserLabel } from './getUserLabel' + +describe('getUserLabel', () => { + test('uses the display name of a user who has one', () => { + expect(getUserLabel({ name: 'Ada' })).toBe('Ada') + }) + + test('reads "Anonymous user" for a user without a display name, as the identification cards do', () => { + expect(getUserLabel({ name: '' })).toBe('Anonymous user') + expect(getUserLabel({ name: null })).toBe('Anonymous user') + }) + + test('reads "Anonymous user" for an action with no user', () => { + expect(getUserLabel(undefined)).toBe('Anonymous user') + expect(getUserLabel(null)).toBe('Anonymous user') + }) +}) diff --git a/ui/src/utils/user/getUserLabel.ts b/ui/src/utils/user/getUserLabel.ts new file mode 100644 index 000000000..f82ecaf95 --- /dev/null +++ b/ui/src/utils/user/getUserLabel.ts @@ -0,0 +1,8 @@ +import { STRING, translate } from 'utils/language' + +/** + * The name to show for the author of an action: their display name, else "Anonymous user". The + * same rule the identification cards use, so one person reads the same on every card. + */ +export const getUserLabel = (user?: { name?: string | null } | null) => + user?.name?.length ? user.name : translate(STRING.ANONYMOUS_USER)