From 0af9c0e89cb177d16d99ba404e55528d4e4e224f Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 8 Oct 2026 14:19:15 -0400 Subject: [PATCH 1/9] feat(occurrences): keep a fixed list of occurrences as a set A project needs the same occurrences again later: comparing two classifiers only means something if both saw the same rows, and a review pass wants the list it started from. A filter answers differently as data arrives, so the membership is stored rather than described. An OccurrenceSet holds its occurrences and the projects it belongs to. A set with no project is global and is offered everywhere, following how TaxaList already treats a list with no project. Membership is decided when the set is created and nothing adds to or removes from one afterwards, because anything recorded against a set was measured on exactly those occurrences. Creating, renaming and deleting are gated on new project permissions, held by the roles that already curate a project's data. A global set has no single project to check against, so it cannot be edited through the API at all. The occurrence list takes an occurrence_set filter, and the sets are offered as choices the same way capture sets are. --- ami/base/permissions.py | 29 ++ ami/main/api/serializers.py | 54 ++++ ami/main/api/views.py | 80 +++++- ami/main/migrations/0098_occurrence_set.py | 102 +++++++ .../0099_grant_occurrence_set_permissions.py | 98 +++++++ ami/main/models.py | 76 ++++++ ami/main/test_occurrence_sets.py | 254 ++++++++++++++++++ ami/users/roles.py | 5 + config/api_router.py | 1 + 9 files changed, 698 insertions(+), 1 deletion(-) create mode 100644 ami/main/migrations/0098_occurrence_set.py create mode 100644 ami/main/migrations/0099_grant_occurrence_set_permissions.py create mode 100644 ami/main/test_occurrence_sets.py diff --git a/ami/base/permissions.py b/ami/base/permissions.py index 328aa8384..af2636c94 100644 --- a/ami/base/permissions.py +++ b/ami/base/permissions.py @@ -198,6 +198,35 @@ def has_permission(self, request, view): return super().has_permission(request, view) +class OccurrenceSetPermission(ObjectPermission): + """ + Permission for the occurrence-set route, where list and create have no object yet. + + Listing follows the project: anyone who may see the project may see which sets it has, + since only a set's name and size are exposed. Creating one is gated on the project's + ``create_occurrenceset`` permission. Everything else falls through to the object check, + which refuses a global set because it has no project to check against. + """ + + def has_permission(self, request, view): + from ami.main.models import Project + + if view.action == "list": + # The project id is required for this action, so reaching here means it was + # given and the caller may see it. + return view.get_active_project() is not None + + if view.action == "create": + # Read from the payload rather than the query string: the set names the project + # it will belong to, and that is the project whose permission must allow it. + project = Project.objects.filter(pk=request.data.get("project_id")).first() + if not project: + return False + return request.user.has_perm(Project.Permissions.CREATE_OCCURRENCE_SET, project) + + return super().has_permission(request, view) + + class UserMembershipPermission(ObjectPermission): """ Custom permission for UserProjectMembershipViewSet. diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 1ca588980..5611106ff 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -26,6 +26,7 @@ Event, Identification, Occurrence, + OccurrenceSet, Page, Project, ProjectSettingsMixin, @@ -1418,6 +1419,59 @@ def to_representation(self, instance): return {key: value for key, value in data.items() if value is not None} +class OccurrenceSetSerializer(DefaultSerializer): + """A fixed list of occurrences. ``occurrence_ids`` is accepted only when it is created.""" + + project_id = serializers.PrimaryKeyRelatedField(queryset=Project.objects.all(), write_only=True) + occurrence_ids = serializers.PrimaryKeyRelatedField( + queryset=Occurrence.objects.all(), many=True, write_only=True, allow_empty=False + ) + occurrences_count = serializers.SerializerMethodField() + + class Meta: + model = OccurrenceSet + fields = [ + "id", + "details", + "name", + "description", + "project_id", + "occurrence_ids", + "occurrences_count", + "created_at", + "updated_at", + ] + + def get_occurrences_count(self, obj) -> int: + return getattr(obj, "annotated_occurrences_count", None) or obj.occurrences.count() + + def validate(self, attrs): + # Scores recorded against a set were measured on these occurrences, so moving them + # would change what those numbers mean. Another set is the way to change the list. + if self.instance is not None and "occurrence_ids" in attrs: + raise serializers.ValidationError( + {"occurrence_ids": "A set's occurrences cannot be changed. Create another set instead."} + ) + + project = attrs.get("project_id") + occurrences = attrs.get("occurrence_ids") + if project and occurrences: + outside = sorted(o.pk for o in occurrences if o.project_id != project.pk) + if outside: + raise serializers.ValidationError( + {"occurrence_ids": f"Not occurrences in this project: {outside[:10]}"} + ) + return attrs + + def create(self, validated_data): + project = validated_data.pop("project_id") + occurrences = validated_data.pop("occurrence_ids") + occurrence_set = OccurrenceSet.objects.create(**validated_data) + occurrence_set.projects.add(project) + occurrence_set.occurrences.set(occurrences) + return occurrence_set + + class SourceImageCollectionSerializer(DefaultSerializer): source_images = serializers.SerializerMethodField() kwargs = SourceImageCollectionCommonKwargsSerializer(required=False, partial=True) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 99709eb29..40abdef53 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -29,7 +29,12 @@ from ami.base.metadata import ResponseSchemaMetadata from ami.base.models import BaseQuerySet from ami.base.pagination import LimitOffsetPaginationWithPermissions -from ami.base.permissions import IsActiveStaffOrReadOnly, IsProjectMemberOrReadOnly, ObjectPermission +from ami.base.permissions import ( + IsActiveStaffOrReadOnly, + IsProjectMemberOrReadOnly, + ObjectPermission, + OccurrenceSetPermission, +) from ami.base.serializers import FilterParamsSerializer, SingleParamSerializer from ami.base.views import ProjectMixin from ami.main.api.schemas import limit_doc_param, project_id_doc_param @@ -49,6 +54,7 @@ Event, Identification, Occurrence, + OccurrenceSet, Page, Project, ProjectQuerySet, @@ -84,6 +90,7 @@ ModelAgreementSerializer, OccurrenceListSerializer, OccurrenceSerializer, + OccurrenceSetSerializer, PageListSerializer, PageSerializer, ProjectListSerializer, @@ -953,6 +960,74 @@ class ChoicesPagination(LimitOffsetPaginationWithPermissions): max_limit = 100 +class OccurrenceSetViewSet(DefaultViewSet, ProjectMixin): + """ + Fixed lists of occurrences, so the same data can be used again later. + + Membership is decided when a set is created and cannot be changed afterwards: there is + no endpoint to add or remove an occurrence. Anything that compares results over time + depends on the list standing still, and a set that grows quietly makes every number + recorded against it mean something different. + + A set belonging to no project is global and is offered to every project, but it cannot + be edited here, because it has no single project whose permissions would govern it. + """ + + queryset = OccurrenceSet.objects.all() + serializer_class = OccurrenceSetSerializer + permission_classes = [OccurrenceSetPermission] + # No put: a full replace would include the occurrences. + http_method_names = ["get", "post", "patch", "delete", "head", "options"] + ordering_fields = ["name", "created_at", "updated_at"] + search_fields = ["name"] + ordering = ["name"] + # Listing without a project would answer with every set on the platform, so it is + # required there. A detail route names one set, which carries its own project, and the + # create payload names the project it belongs to. + require_project = False + require_project_for_list = True + + def create(self, request, *args, **kwargs): + """ + Create without the base class's unsaved-instance permission check. + + ``DefaultViewSet.create`` builds ``Model(**validated_data)`` to check object + permissions before saving. A set's project and its occurrences arrive as payload + fields rather than columns, so that instance cannot be built. The same check runs + earlier instead, in ``OccurrenceSetPermission``, against the project the payload + names. + """ + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + + @extend_schema(parameters=[project_id_doc_param], responses=OccurrenceSetSerializer(many=True)) + @action(detail=False, methods=["get"], name="choices") + def choices(self, request: Request) -> Response: + """ + Choices for the occurrence-set filter and pickers. + + Follows SourceImageCollectionViewSet.choices: most recently updated first and + enough of them that a dropdown never has to page. + """ + self.ordering_fields = ["id", "created_at", "updated_at", "name"] + queryset = self.filter_queryset(self.get_queryset()) + paginator = ChoicesPagination() + page = paginator.paginate_queryset(queryset, request, view=self) + serializer = self.get_serializer(page, many=True) + return paginator.get_paginated_response(serializer.data) + + def get_queryset(self) -> QuerySet["OccurrenceSet"]: + qs = super().get_queryset().annotate(annotated_occurrences_count=models.Count("occurrences")) + project = self.get_active_project() + if project is not None: + # A set with no project is global, so it is offered everywhere. + return qs.for_project(project) + return qs.visible_for_user(self.request.user) + + class SourceImageCollectionViewSet(DefaultViewSet, ProjectMixin): """ Endpoint for viewing capture sets or samples of captures. @@ -1495,6 +1570,9 @@ class OccurrenceFilterSet(FilterSet): """ detections__source_image = RelatedIdFilter() + # Named for what it means to a reader rather than for the reverse accessor, which is + # called evaluation_sets because scoring was the first thing to use one. + occurrence_set = RelatedIdFilter(field_name="evaluation_sets") class Meta: model = Occurrence diff --git a/ami/main/migrations/0098_occurrence_set.py b/ami/main/migrations/0098_occurrence_set.py new file mode 100644 index 000000000..26153b846 --- /dev/null +++ b/ami/main/migrations/0098_occurrence_set.py @@ -0,0 +1,102 @@ +# Generated by Django 4.2.10 on 2026-10-08 13:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0097_detection_and_classification_job_indexes"), + ] + + operations = [ + migrations.AlterModelOptions( + name="project", + options={ + "ordering": ["-priority", "created_at"], + "permissions": [ + ("create_identification", "Can create identifications"), + ("update_identification", "Can update identifications"), + ("delete_identification", "Can delete identifications"), + ("create_job", "Can create a job"), + ("update_job", "Can update a job"), + ("run_ml_job", "Can run/retry/cancel ML jobs"), + ("run_populate_captures_collection_job", "Can run/retry/cancel Populate Collection jobs"), + ("run_data_storage_sync_job", "Can run/retry/cancel Data Storage Sync jobs"), + ("run_regroup_events_job", "Can run/retry/cancel Regroup Events jobs"), + ("run_data_export_job", "Can run/retry/cancel Data Export jobs"), + ("run_single_image_ml_job", "Can process a single capture"), + ("run_post_processing_job", "Can run/retry/cancel Post-Processing jobs"), + ("delete_job", "Can delete a job"), + ("create_deployment", "Can create a deployment"), + ("delete_deployment", "Can delete a deployment"), + ("update_deployment", "Can update a deployment"), + ("sync_deployment", "Can sync images to a deployment"), + ("regroup_sessions_deployment", "Can regroup deployment captures into sessions"), + ("create_occurrenceset", "Can create an occurrence set"), + ("update_occurrenceset", "Can rename or describe an occurrence set"), + ("delete_occurrenceset", "Can delete an occurrence set"), + ("create_sourceimagecollection", "Can create a collection"), + ("update_sourceimagecollection", "Can update a collection"), + ("delete_sourceimagecollection", "Can delete a collection"), + ("populate_sourceimagecollection", "Can populate a collection"), + ("create_sourceimage", "Can create a source image"), + ("update_sourceimage", "Can update a source image"), + ("delete_sourceimage", "Can delete a source image"), + ("star_sourceimage", "Can star a source image"), + ("create_sourceimageupload", "Can create a source image upload"), + ("update_sourceimageupload", "Can update a source image upload"), + ("delete_sourceimageupload", "Can delete a source image upload"), + ("create_s3storagesource", "Can create storage"), + ("delete_s3storagesource", "Can delete storage"), + ("update_s3storagesource", "Can update storage"), + ("test_s3storagesource", "Can test storage connection"), + ("create_site", "Can create a site"), + ("delete_site", "Can delete a site"), + ("update_site", "Can update a site"), + ("create_device", "Can create a device"), + ("delete_device", "Can delete a device"), + ("update_device", "Can update a device"), + ("view_userprojectmembership", "Can view project members"), + ("create_userprojectmembership", "Can add a user to the project"), + ("update_userprojectmembership", "Can update a user's project membership and role in the project"), + ("delete_userprojectmembership", "Can remove a user from the project"), + ("create_dataexport", "Can create a data export"), + ("update_dataexport", "Can update a data export"), + ("delete_dataexport", "Can delete a data export"), + ("create_projectpipelineconfig", "Can register pipelines for the project"), + ("update_projectpipelineconfig", "Can update pipeline configurations"), + ("delete_projectpipelineconfig", "Can remove pipelines from the project"), + ("create_taxalist", "Can create a taxa list"), + ("update_taxalist", "Can update a taxa list"), + ("delete_taxalist", "Can delete a taxa list"), + ("view_private_data", "Can view private data"), + ], + }, + ), + migrations.CreateModel( + name="OccurrenceSet", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("name", models.CharField(max_length=255)), + ("description", models.TextField(blank=True)), + ( + "occurrences", + models.ManyToManyField(blank=True, related_name="evaluation_sets", to="main.occurrence"), + ), + ( + "projects", + models.ManyToManyField( + blank=True, + help_text="Projects this set belongs to. A set with none is available everywhere.", + related_name="occurrence_sets", + to="main.project", + ), + ), + ], + options={ + "ordering": ["name"], + }, + ), + ] diff --git a/ami/main/migrations/0099_grant_occurrence_set_permissions.py b/ami/main/migrations/0099_grant_occurrence_set_permissions.py new file mode 100644 index 000000000..f69e2fcda --- /dev/null +++ b/ami/main/migrations/0099_grant_occurrence_set_permissions.py @@ -0,0 +1,98 @@ +""" +Grant the new occurrence-set permissions to the roles that already curate a project's data. + +``create_roles_for_project`` gives new projects their role permissions, and a +``post_migrate`` signal re-syncs every project on migrate, so this backfill is belt and +braces. It is kept so the grant is explicit rather than a side effect of a signal, and so +the reverse is written down. + +Guardian reads **object-level** rows: ``get_perms(user, project)`` and +``user.has_perm("create_occurrenceset", project)`` both look at +``GroupObjectPermission``. Adding the permission to ``group.permissions`` alone would not +show up there, so each project gets a row per role group. + +``ProjectManager`` inherits ``MLDataManager``'s permissions, so both groups are covered. +""" + +from django.db import migrations +from django.db.models import Q + +PERMISSIONS = ("create_occurrenceset", "update_occurrenceset", "delete_occurrenceset") +ROLE_GROUP_SUFFIXES = ("_MLDataManager", "_ProjectManager") + + +def _permissions(apps): + Permission = apps.get_model("auth", "Permission") + ContentType = apps.get_model("contenttypes", "ContentType") + try: + project_ct = ContentType.objects.get(app_label="main", model="project") + except ContentType.DoesNotExist: + return [], None + found = list(Permission.objects.filter(codename__in=PERMISSIONS, content_type=project_ct)) + return found, project_ct + + +def _project_pk_from_group(group): + # Group names are "{project_pk}_{project_name}_{RoleName}"; the pk is the immutable + # leading segment, since a project name may itself contain underscores. + try: + return int(group.name.split("_", 1)[0]) + except (ValueError, IndexError): + return None + + +def _role_groups(apps): + Group = apps.get_model("auth", "Group") + query = Q() + for suffix in ROLE_GROUP_SUFFIXES: + query |= Q(name__endswith=suffix) + return Group.objects.filter(query) + + +def grant(apps, schema_editor): + GroupObjectPermission = apps.get_model("guardian", "GroupObjectPermission") + permissions, project_ct = _permissions(apps) + if not permissions: + return + + for group in _role_groups(apps): + project_pk = _project_pk_from_group(group) + if project_pk is None: + continue + for permission in permissions: + group.permissions.add(permission) + GroupObjectPermission.objects.get_or_create( + permission=permission, + content_type=project_ct, + object_pk=str(project_pk), + group=group, + ) + + +def revoke(apps, schema_editor): + GroupObjectPermission = apps.get_model("guardian", "GroupObjectPermission") + permissions, project_ct = _permissions(apps) + if not permissions: + return + + for group in _role_groups(apps): + project_pk = _project_pk_from_group(group) + if project_pk is None: + continue + for permission in permissions: + group.permissions.remove(permission) + GroupObjectPermission.objects.filter( + permission=permission, + content_type=project_ct, + object_pk=str(project_pk), + group=group, + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0098_occurrence_set"), + ("guardian", "0002_generic_permissions_index"), + ] + + operations = [migrations.RunPython(grant, revoke)] diff --git a/ami/main/models.py b/ami/main/models.py index a4c0c82f1..03959c548 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -461,6 +461,11 @@ class Permissions: SYNC_DEPLOYMENT = "sync_deployment" REGROUP_SESSIONS_DEPLOYMENT = "regroup_sessions_deployment" + # Occurrence set permissions + CREATE_OCCURRENCE_SET = "create_occurrenceset" + UPDATE_OCCURRENCE_SET = "update_occurrenceset" + DELETE_OCCURRENCE_SET = "delete_occurrenceset" + # Collection permissions CREATE_COLLECTION = "create_sourceimagecollection" UPDATE_COLLECTION = "update_sourceimagecollection" @@ -543,6 +548,9 @@ class Meta: ("sync_deployment", "Can sync images to a deployment"), ("regroup_sessions_deployment", "Can regroup deployment captures into sessions"), # Collection permissions + ("create_occurrenceset", "Can create an occurrence set"), + ("update_occurrenceset", "Can rename or describe an occurrence set"), + ("delete_occurrenceset", "Can delete an occurrence set"), ("create_sourceimagecollection", "Can create a collection"), ("update_sourceimagecollection", "Can update a collection"), ("delete_sourceimagecollection", "Can delete a collection"), @@ -4912,6 +4920,74 @@ def html(self) -> str: ] +class OccurrenceSetQuerySet(BaseQuerySet): + def for_project(self, project) -> models.QuerySet: + """Sets this project can use: its own, plus any that belong to no project.""" + return self.filter(models.Q(projects=project) | models.Q(projects__isnull=True)).distinct() + + def visible_for_user(self, user) -> models.QuerySet: + """ + Global sets stay visible; project sets follow their project. + + The inherited filter keeps a row only if it reaches a non-draft project, which a + set belonging to no project never does. Without this a global set is hidden from + everyone but a superuser, which is the opposite of what global means. Only a set's + name and size are exposed, never the occurrences inside it. + """ + visible = super().visible_for_user(user) + return self.filter(models.Q(pk__in=visible.values("pk")) | models.Q(projects__isnull=True)).distinct() + + +@final +class OccurrenceSet(BaseModel): + """ + A fixed list of occurrences, kept so the same ones can be used again later. + + Membership is stored rather than described by a filter, because a filter answers + differently as data arrives. Anything that compares results over time needs the list to + stand still: scoring two classifiers is only meaningful if both saw the same + occurrences. + + A set with no projects is global, which is how one set is shared across the platform; + that follows how TaxaList already treats a list with no project. + """ + + name = models.CharField(max_length=255) + description = models.TextField(blank=True) + occurrences = models.ManyToManyField("main.Occurrence", related_name="evaluation_sets", blank=True) + projects = models.ManyToManyField( + "main.Project", + related_name="occurrence_sets", + blank=True, + help_text="Projects this set belongs to. A set with none is available everywhere.", + ) + + objects = OccurrenceSetQuerySet.as_manager() + + class Meta: + ordering = ["name"] + + def __str__(self) -> str: + return f"{self.name} ({self.occurrences.count()} occurrences)" + + @property + def is_global(self) -> bool: + return not self.projects.exists() + + def get_project(self): + """ + The project whose permissions govern this set. + + BaseModel returns None for a many-to-many project relation, and + ``check_permission`` refuses every action without a project, so a global set cannot + be renamed or deleted through the API at all. That is the behaviour we want: a set + shared across projects changing under one of them would quietly change what every + other project's stored scores were measured on. Global sets stay a deliberate, + out-of-band act. + """ + return self.projects.first() + + class SourceImageCollectionQuerySet(BaseQuerySet): def with_source_images_count(self): return self.annotate( diff --git a/ami/main/test_occurrence_sets.py b/ami/main/test_occurrence_sets.py new file mode 100644 index 000000000..ad84c7c58 --- /dev/null +++ b/ami/main/test_occurrence_sets.py @@ -0,0 +1,254 @@ +""" +Occurrence sets: a fixed list of occurrences, decided once. + +The point of the model is that the list does not move. Anything comparing results over +time — scoring two classifiers, re-running an export — is only meaningful if both runs saw +the same occurrences. So these tests are mostly about what the API refuses. +""" + +import datetime +import uuid + +from rest_framework import status +from rest_framework.test import APITestCase + +from ami.main.models import ( + Classification, + Deployment, + Detection, + Event, + Occurrence, + OccurrenceSet, + Project, + S3StorageSource, + SourceImage, + Taxon, + TaxonRank, +) +from ami.users.models import User +from ami.users.roles import BasicMember, MLDataManager + + +class OccurrenceSetFixture(APITestCase): + """ + Occurrences are created directly rather than through the capture fixtures. + + These tests are about what the endpoint accepts and refuses, and need nothing from a + detection. Going through create_captures/create_occurrences pulls in event grouping and + a storage read for image dimensions, neither of which works in a test, and the failures + land at random on whichever test ran first. + """ + + def setUp(self) -> None: + self.owner = User.objects.create_user(email="owner@example.com", password="testpass123") + self.curator = User.objects.create_user(email="curator@example.com", password="testpass123") + self.member = User.objects.create_user(email="member@example.com", password="testpass123") + + self.project = Project.objects.create(name="Set Project", owner=self.owner) + MLDataManager.assign_user(self.curator, self.project) + BasicMember.assign_user(self.member, self.project) + + self.occurrences = [self.make_occurrence(self.project) for _ in range(3)] + self.url = "/api/v2/occurrences/sets/" + + @staticmethod + def make_occurrence(project: Project) -> Occurrence: + """ + One occurrence the list endpoint will actually return. + + Built here rather than through the capture fixtures, which time their images from + the current clock and group them into events by reading image dimensions from + storage. Neither works in a test, and the failures land at random on whichever test + ran first. The list only shows occurrences with a real detection and a + determination, so both are set. + """ + storage = S3StorageSource.objects.create(name=f"source {uuid.uuid4().hex[:6]}", project=project, bucket="test") + deployment = Deployment.objects.create( + name=f"station {uuid.uuid4().hex[:6]}", project=project, data_source=storage + ) + night = datetime.datetime(2026, 6, 1, 22, 0, 0, tzinfo=datetime.timezone.utc) + event = Event.objects.create(deployment=deployment, project=project, start=night) + image = SourceImage.objects.create( + deployment=deployment, + project=project, + event=event, + timestamp=night, + path=f"test/{uuid.uuid4().hex[:8]}.jpg", + ) + taxon = Taxon.objects.create(name=f"Testus {uuid.uuid4().hex[:6]}", rank=TaxonRank.SPECIES.name) + taxon.projects.add(project) + detection = Detection.objects.create(source_image=image, timestamp=night, bbox=[0.1, 0.1, 0.2, 0.2]) + Classification.objects.create(detection=detection, taxon=taxon, score=0.9, timestamp=night) + occurrence = detection.associate_new_occurrence() + occurrence.refresh_from_db() + return occurrence + + def _create(self, user, **overrides): + payload = { + "name": "Blind set", + "description": "", + "project_id": self.project.pk, + "occurrence_ids": [o.pk for o in self.occurrences], + } + payload.update(overrides) + self.client.force_authenticate(user=user) + return self.client.post(self.url, payload, format="json") + + +class TestCreatingASet(OccurrenceSetFixture): + def test_a_curator_can_create_one(self): + response = self._create(self.curator) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.data) + created = OccurrenceSet.objects.get(pk=response.data["id"]) + self.assertEqual(created.occurrences.count(), 3) + self.assertEqual(list(created.projects.all()), [self.project]) + + def test_a_member_without_the_permission_cannot(self): + response = self._create(self.member) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertFalse(OccurrenceSet.objects.exists()) + + def test_an_anonymous_caller_cannot(self): + response = self._create(None) + + self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) + self.assertFalse(OccurrenceSet.objects.exists()) + + def test_an_occurrence_from_another_project_is_refused(self): + """Otherwise a set quietly reaches into data its project cannot see.""" + other = Project.objects.create(name="Other Project", owner=self.owner) + outsider = self.make_occurrence(other) + + response = self._create(self.curator, occurrence_ids=[self.occurrences[0].pk, outsider.pk]) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("occurrence_ids", response.data) + self.assertFalse(OccurrenceSet.objects.exists()) + + def test_an_empty_set_is_refused(self): + response = self._create(self.curator, occurrence_ids=[]) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + +class TestMembershipDoesNotMove(OccurrenceSetFixture): + def setUp(self) -> None: + super().setUp() + self.set = OccurrenceSet.objects.create(name="Blind set") + self.set.projects.add(self.project) + self.set.occurrences.add(*self.occurrences) + self.detail_url = f"{self.url}{self.set.pk}/" + self.client.force_authenticate(user=self.curator) + + def test_the_occurrences_cannot_be_changed(self): + response = self.client.patch(self.detail_url, {"occurrence_ids": [self.occurrences[0].pk]}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(self.set.occurrences.count(), 3) + + def test_the_name_and_description_can_be_changed(self): + response = self.client.patch(self.detail_url, {"name": "Renamed", "description": "why"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_200_OK, response.data) + self.set.refresh_from_db() + self.assertEqual(self.set.name, "Renamed") + + def test_there_is_no_put(self): + """A full replace would carry the occurrences with it.""" + response = self.client.put(self.detail_url, {"name": "Replaced"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED) + + def test_a_curator_can_delete_one(self): + response = self.client.delete(self.detail_url) + + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertFalse(OccurrenceSet.objects.filter(pk=self.set.pk).exists()) + + +class TestFilteringOccurrencesBySet(OccurrenceSetFixture): + """The occurrence list can be narrowed to one set, which is how a set is reviewed.""" + + def setUp(self) -> None: + super().setUp() + self.set = OccurrenceSet.objects.create(name="Blind set") + self.set.projects.add(self.project) + self.set.occurrences.add(self.occurrences[0], self.occurrences[1]) + self.client.force_authenticate(user=self.curator) + + def _ids(self, **params): + # apply_defaults=false: the list hides occurrences without a determination or a + # score by default, and these fixtures are bare rows made to exercise the filter. + response = self.client.get( + "/api/v2/occurrences/", + {"project_id": self.project.pk, "apply_defaults": "false", **params}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK, response.data) + return {row["id"] for row in response.data["results"]} + + def test_only_the_sets_occurrences_come_back(self): + self.assertEqual( + self._ids(occurrence_set=self.set.pk), + {self.occurrences[0].pk, self.occurrences[1].pk}, + ) + + def test_without_the_filter_every_occurrence_comes_back(self): + self.assertEqual(self._ids(), {o.pk for o in self.occurrences}) + + +class TestGlobalSets(OccurrenceSetFixture): + def setUp(self) -> None: + super().setUp() + self.global_set = OccurrenceSet.objects.create(name="Platform-wide set") + self.client.force_authenticate(user=self.curator) + + def _names(self, response): + return [row["name"] for row in response.data["results"]] + + def test_a_global_set_is_offered_to_every_project(self): + response = self.client.get(self.url, {"project_id": self.project.pk}) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn(self.global_set.name, self._names(response)) + + def test_a_global_set_cannot_be_edited_through_the_api(self): + """It has no single project whose permissions would govern it.""" + response = self.client.patch(f"{self.url}{self.global_set.pk}/", {"name": "Mine now"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.global_set.refresh_from_db() + self.assertEqual(self.global_set.name, "Platform-wide set") + + +class TestListing(OccurrenceSetFixture): + def setUp(self) -> None: + super().setUp() + self.set = OccurrenceSet.objects.create(name="Blind set") + self.set.projects.add(self.project) + self.set.occurrences.add(*self.occurrences) + + def test_a_project_is_required(self): + """Without one the endpoint would list every set on the platform.""" + self.client.force_authenticate(user=self.curator) + response = self.client.get(self.url) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_the_size_is_reported_without_the_occurrences(self): + self.client.force_authenticate(user=self.curator) + response = self.client.get(self.url, {"project_id": self.project.pk}) + + row = next(r for r in response.data["results"] if r["name"] == "Blind set") + self.assertEqual(row["occurrences_count"], 3) + self.assertNotIn("occurrences", row) + + def test_another_project_does_not_see_it(self): + other = Project.objects.create(name="Other Project", owner=self.owner) + MLDataManager.assign_user(self.curator, other) + self.client.force_authenticate(user=self.curator) + + response = self.client.get(self.url, {"project_id": other.pk}) + + self.assertNotIn("Blind set", [row["name"] for row in response.data["results"]]) diff --git a/ami/users/roles.py b/ami/users/roles.py index 718146e41..ae68565d8 100644 --- a/ami/users/roles.py +++ b/ami/users/roles.py @@ -154,6 +154,11 @@ class MLDataManager(Role): Project.Permissions.UPDATE_COLLECTION, Project.Permissions.DELETE_COLLECTION, Project.Permissions.POPULATE_COLLECTION, + # Whoever curates the data a model is measured on is the same person who manages + # the data it learns from, so this sits with the collection permissions. + Project.Permissions.CREATE_OCCURRENCE_SET, + Project.Permissions.UPDATE_OCCURRENCE_SET, + Project.Permissions.DELETE_OCCURRENCE_SET, } diff --git a/config/api_router.py b/config/api_router.py index 56a01c591..ebf5588f7 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -41,6 +41,7 @@ router.register(r"detections", views.DetectionViewSet) # Register before r"occurrences" — see docs/claude/reference/api-stats-pattern.md (guarded by regression test). router.register(r"occurrences/stats", views.OccurrenceStatsViewSet, basename="occurrence-stats") +router.register(r"occurrences/sets", views.OccurrenceSetViewSet) router.register(r"occurrences", views.OccurrenceViewSet) router.register(r"taxa/lists", views.TaxaListViewSet) # NESTED: /taxa/lists/{taxalist_id}/taxa/ From db96cd232c8eeb1631f20c3a1d0d7dbd59446ae4 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 8 Oct 2026 14:19:23 -0400 Subject: [PATCH 2/9] feat(ui): filter occurrences by set Adds the occurrence set to the occurrence filter panel, picked from the set choices endpoint the same way a capture set is. A set is only useful if you can look at what is in it, and this is where someone reviewing one starts. The field is carried over from other views like the existing filters, so arriving with a set already chosen shows it in the panel where it can be cleared. --- ui/src/components/filtering/filter-control.tsx | 2 ++ .../filtering/filters/occurrence-set-filter.tsx | 17 +++++++++++++++++ ui/src/data-services/constants.ts | 2 ++ ui/src/pages/occurrences/occurrence-filters.ts | 1 + ui/src/utils/language.ts | 5 +++++ ui/src/utils/useFilters.ts | 7 +++++++ 6 files changed, 34 insertions(+) create mode 100644 ui/src/components/filtering/filters/occurrence-set-filter.tsx diff --git a/ui/src/components/filtering/filter-control.tsx b/ui/src/components/filtering/filter-control.tsx index 742620c79..538d8dfab 100644 --- a/ui/src/components/filtering/filter-control.tsx +++ b/ui/src/components/filtering/filter-control.tsx @@ -9,6 +9,7 @@ import { CaptureSetFilter } from './filters/capture-set-filter' import { DateFilter } from './filters/date-filter' import { DeviceFilter } from './filters/device-filter' import { ImageFilter } from './filters/image-filter' +import { OccurrenceSetFilter } from './filters/occurrence-set-filter' import { PipelineFilter } from './filters/pipeline-filter' import { SessionFilter } from './filters/session-filter' import { SiteFilter } from './filters/site-filter' @@ -44,6 +45,7 @@ const ComponentMap: { not_algorithm: NotAlgorithmFilter, not_tag_id: TagFilter, not_taxa_list_id: TaxaListFilter, + occurrence_set: OccurrenceSetFilter, pipeline: PipelineFilter, source_image_collection: CaptureSetFilter, source_image_single: ImageFilter, diff --git a/ui/src/components/filtering/filters/occurrence-set-filter.tsx b/ui/src/components/filtering/filters/occurrence-set-filter.tsx new file mode 100644 index 000000000..3e6b2c620 --- /dev/null +++ b/ui/src/components/filtering/filters/occurrence-set-filter.tsx @@ -0,0 +1,17 @@ +import { API_ROUTES } from 'data-services/constants' +import { EntityPicker } from 'nova-ui-kit' +import { FilterProps } from './types' + +export const OccurrenceSetFilter = ({ onAdd, onClear, value }: FilterProps) => ( + { + if (value) { + onAdd(value) + } else { + onClear() + } + }} + value={value} + /> +) diff --git a/ui/src/data-services/constants.ts b/ui/src/data-services/constants.ts index 59bae6ede..26891161f 100644 --- a/ui/src/data-services/constants.ts +++ b/ui/src/data-services/constants.ts @@ -3,6 +3,8 @@ export const API_URL = '/api/v2' export const API_ROUTES = { ALGORITHM: 'ml/algorithms', CAPTURE_SET_CHOICES: 'captures/collections/choices', + OCCURRENCE_SETS: 'occurrences/sets', + OCCURRENCE_SET_CHOICES: 'occurrences/sets/choices', CAPTURE_SETS: 'captures/collections', CAPTURES: 'captures', CLASSIFICATIONS: 'classifications', diff --git a/ui/src/pages/occurrences/occurrence-filters.ts b/ui/src/pages/occurrences/occurrence-filters.ts index e63d5093c..6a79c7741 100644 --- a/ui/src/pages/occurrences/occurrence-filters.ts +++ b/ui/src/pages/occurrences/occurrence-filters.ts @@ -11,6 +11,7 @@ export const FILTERS_TO_OCCURRENCES = [ 'verified', 'verified_by_me', 'collection', + 'occurrence_set', 'date_start', 'date_end', 'deployment', diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 38e20a5e4..2158dccfb 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -86,6 +86,7 @@ export enum STRING { FIELD_LABEL_AVG_TEMP, FIELD_LABEL_BEST_SCORE, FIELD_LABEL_CAPTURE_SET, + FIELD_LABEL_OCCURRENCE_SET, FIELD_LABEL_CAPTURE, FIELD_LABEL_CAPTURES_WITH_DETECTIONS, FIELD_LABEL_CAPTURES, @@ -277,6 +278,7 @@ export enum STRING { /* TOOLTIPS */ TOOLTIP_ALGORITHM, TOOLTIP_CAPTURE_SET, + TOOLTIP_OCCURRENCE_SET, TOOLTIP_JOB_FILTER, TOOLTIP_CAPTURE, TOOLTIP_DEPLOYMENT, @@ -450,6 +452,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.FIELD_LABEL_AVG_TEMP]: 'Avg temp', [STRING.FIELD_LABEL_BEST_SCORE]: 'Best score', [STRING.FIELD_LABEL_CAPTURE_SET]: 'Capture set', + [STRING.FIELD_LABEL_OCCURRENCE_SET]: 'Occurrence set', [STRING.FIELD_LABEL_CAPTURE]: 'Capture', [STRING.FIELD_LABEL_CAPTURES_WITH_DETECTIONS]: 'Captures with detections', [STRING.FIELD_LABEL_CAPTURES]: 'Captures', @@ -693,6 +696,8 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { /* TOOLTIPS */ [STRING.TOOLTIP_ALGORITHM]: 'An algorithm is used in a pipeline for processing captures.', + [STRING.TOOLTIP_OCCURRENCE_SET]: + 'A fixed list of occurrences, saved so the same ones can be looked at again. Its contents do not change after it is created.', [STRING.TOOLTIP_CAPTURE_SET]: 'A capture set is a group of captures. A capture set contains all or some captures in a project. Capture sets are useful when specifying what captures to process.', [STRING.TOOLTIP_JOB_FILTER]: diff --git a/ui/src/utils/useFilters.ts b/ui/src/utils/useFilters.ts index 48fa644ba..3437328f7 100644 --- a/ui/src/utils/useFilters.ts +++ b/ui/src/utils/useFilters.ts @@ -45,6 +45,13 @@ export const AVAILABLE_FILTERS = (projectId: string): FilterConfig[] => [ }, }, }, + { + label: translate(STRING.FIELD_LABEL_OCCURRENCE_SET), + field: 'occurrence_set', + tooltip: { + text: translate(STRING.TOOLTIP_OCCURRENCE_SET), + }, + }, { label: translate(STRING.FIELD_LABEL_CAPTURE_SET), field: 'collection', // This is for viewing occurrences by capture set. @TODO: Can we update this key to "capture_set_id" to streamline? From be8423d886118c9fdf950ccbd96d903c6ab06424 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 8 Oct 2026 14:19:29 -0400 Subject: [PATCH 3/9] feat(ui): save the selected occurrences as a set Someone filtering the occurrence list to the rows they care about had no way to keep that selection. Selecting occurrences now offers saving them as a set, next to the identification actions already there. The action does not change any occurrence, so it is not behind update rights on them; the endpoint gates it on the project's own permission instead. --- .../occurrence-sets/useCreateOccurrenceSet.ts | 41 +++++++++++ .../create-occurrence-set-popover.tsx | 40 +++++++++++ .../create-occurrence-set.tsx | 70 +++++++++++++++++++ .../pages/occurrences/occurrences-actions.tsx | 29 ++++---- ui/src/utils/language.ts | 5 ++ 5 files changed, 172 insertions(+), 13 deletions(-) create mode 100644 ui/src/data-services/hooks/occurrence-sets/useCreateOccurrenceSet.ts create mode 100644 ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx create mode 100644 ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set.tsx diff --git a/ui/src/data-services/hooks/occurrence-sets/useCreateOccurrenceSet.ts b/ui/src/data-services/hooks/occurrence-sets/useCreateOccurrenceSet.ts new file mode 100644 index 000000000..8d6b6154c --- /dev/null +++ b/ui/src/data-services/hooks/occurrence-sets/useCreateOccurrenceSet.ts @@ -0,0 +1,41 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query' +import axios from 'axios' +import { API_ROUTES, API_URL, SUCCESS_TIMEOUT } from 'data-services/constants' +import { getAuthHeader } from 'data-services/utils' +import { useUser } from 'utils/user/userContext' + +interface OccurrenceSetFieldValues { + name: string + description?: string + projectId: string + occurrenceIds: string[] +} + +const convertToServerFieldValues = (fieldValues: OccurrenceSetFieldValues) => ({ + name: fieldValues.name, + description: fieldValues.description ?? '', + project_id: fieldValues.projectId, + occurrence_ids: fieldValues.occurrenceIds.map((id) => Number(id)), +}) + +export const useCreateOccurrenceSet = (onSuccess?: (id: string) => void) => { + const { user } = useUser() + const queryClient = useQueryClient() + + const { mutateAsync, isLoading, isSuccess, reset, error } = useMutation({ + mutationFn: (fieldValues: OccurrenceSetFieldValues) => + axios.post<{ id: number }>( + `${API_URL}/${API_ROUTES.OCCURRENCE_SETS}/`, + convertToServerFieldValues(fieldValues), + { headers: getAuthHeader(user) } + ), + onSuccess: ({ data }) => { + queryClient.invalidateQueries([API_ROUTES.OCCURRENCE_SETS]) + queryClient.invalidateQueries([API_ROUTES.OCCURRENCE_SET_CHOICES]) + onSuccess?.(`${data.id}`) + setTimeout(reset, SUCCESS_TIMEOUT) + }, + }) + + return { createOccurrenceSet: mutateAsync, isLoading, isSuccess, error } +} diff --git a/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx new file mode 100644 index 000000000..0fcaa2256 --- /dev/null +++ b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx @@ -0,0 +1,40 @@ +import { BookmarkPlusIcon } from 'lucide-react' +import { BasicTooltip, Button, Popover } from 'nova-ui-kit' +import { useState } from 'react' +import { STRING, translate } from 'utils/language' +import { CreateOccurrenceSet } from './create-occurrence-set' + +interface CreateOccurrenceSetPopoverProps { + occurrenceIds: string[] +} + +export const CreateOccurrenceSetPopover = ({ + occurrenceIds, +}: CreateOccurrenceSetPopoverProps) => { + const [open, setOpen] = useState(false) + + return ( + + + + + + + + setOpen(false)} + /> + + + ) +} diff --git a/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set.tsx b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set.tsx new file mode 100644 index 000000000..f249c36f9 --- /dev/null +++ b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set.tsx @@ -0,0 +1,70 @@ +import { FormError } from 'components/form/layout/layout' +import { useCreateOccurrenceSet } from 'data-services/hooks/occurrence-sets/useCreateOccurrenceSet' +import { Loader2Icon } from 'lucide-react' +import { Button, Input } from 'nova-ui-kit' +import { useState } from 'react' +import { useParams } from 'react-router-dom' +import { STRING, translate } from 'utils/language' +import { parseServerError } from 'utils/parseServerError/parseServerError' + +interface CreateOccurrenceSetProps { + occurrenceIds: string[] + onCancel: () => void +} + +export const CreateOccurrenceSet = ({ + occurrenceIds, + onCancel, +}: CreateOccurrenceSetProps) => { + const { projectId } = useParams() + const [name, setName] = useState('') + const { createOccurrenceSet, isLoading, error } = + useCreateOccurrenceSet(onCancel) + const formError = error ? parseServerError(error)?.message : undefined + + return ( + <> + {formError && ( + + )} +
+
+ setName(e.target.value)} + value={name} + /> +
+ + {translate(STRING.MESSAGE_OCCURRENCE_SET_IS_FIXED, { + count: `${occurrenceIds.length}`, + })} + +
+ + +
+
+ + ) +} diff --git a/ui/src/pages/occurrences/occurrences-actions.tsx b/ui/src/pages/occurrences/occurrences-actions.tsx index c3bfae4d7..b35c875e7 100644 --- a/ui/src/pages/occurrences/occurrences-actions.tsx +++ b/ui/src/pages/occurrences/occurrences-actions.tsx @@ -3,6 +3,7 @@ import { useCreateIdentifications } from 'data-services/hooks/identifications/us import { Occurrence } from 'data-services/models/occurrence' import { AlertCircleIcon, CheckIcon, Loader2Icon } from 'lucide-react' import { BasicTooltip, Button } from 'nova-ui-kit' +import { CreateOccurrenceSetPopover } from 'pages/occurrences/create-occurrence-set/create-occurrence-set-popover' import { IdQuickActions } from 'pages/occurrence-details/id-quick-actions/id-quick-actions' import { SuggestIdPopover } from 'pages/occurrence-details/suggest-id/suggest-id-popover' import { useMemo } from 'react' @@ -27,22 +28,24 @@ export const OccurrencesActions = ({ ({ userPermissions }) => !userPermissions.includes(UserPermission.Update) ) - if (!canUpdate) { - return null - } + const occurrenceIds = occurrences.map((occurrence) => occurrence.id) return (
- - occurrence.id)} - /> - occurrence.id)} - occurrenceTaxa={occurrences.map( - (occurrence) => occurrence.determinationTaxon - )} - /> + {/* Saving a set changes no occurrence, so it is not gated on update rights. */} + {canUpdate ? ( + <> + + + occurrence.determinationTaxon + )} + /> + + ) : null} +
) } diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 2158dccfb..7e727651d 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -134,6 +134,7 @@ export enum STRING { FIELD_LABEL_LOGS, FIELD_LABEL_LONGITUDE, FIELD_LABEL_MOST_RECENT, + CREATE_OCCURRENCE_SET, FIELD_LABEL_NAME, FIELD_LABEL_NEW_PASSWORD, FIELD_LABEL_NUM_PIPELINES_REGISTERED, @@ -192,6 +193,7 @@ export enum STRING { MESSAGE_CHANGE_PASSWORD, MESSAGE_CONFIGURE_LOCATION, MESSAGE_COULD_NOT_SAVE, + MESSAGE_OCCURRENCE_SET_IS_FIXED, MESSAGE_DATA_SOURCE_NOT_CONFIGURED, MESSAGE_DEFAULT_FILTERS, MESSAGE_DEFAULT_PIPELINE, @@ -500,6 +502,9 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.FIELD_LABEL_LOGS]: 'Logs', [STRING.FIELD_LABEL_LONGITUDE]: 'Longitude', [STRING.FIELD_LABEL_MOST_RECENT]: 'Most recent', + [STRING.CREATE_OCCURRENCE_SET]: 'Create occurrence set', + [STRING.MESSAGE_OCCURRENCE_SET_IS_FIXED]: + 'Saves the {{count}} selected occurrence(s) as a set. Its contents do not change afterwards.', [STRING.FIELD_LABEL_NAME]: 'Name', [STRING.FIELD_LABEL_NEW_PASSWORD]: 'New password', [STRING.FIELD_LABEL_NUM_PIPELINES_REGISTERED]: 'Pipelines registered', From 4094265c04f658c80cfaba22a934d9e23b40ae80 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 8 Oct 2026 14:37:50 -0400 Subject: [PATCH 4/9] fix(ui): show the occurrence set filter in the filter panel Registering a filter in the shared list is not enough for it to appear: the occurrences page renders one FilterControl per field it offers, and the set was missing from that list, so the filter existed everywhere except on screen. It sits under More filters beside the capture set, and that section now opens on arrival when a set is already applied, as it does for the other filters there. --- ui/src/pages/occurrences/occurrences.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui/src/pages/occurrences/occurrences.tsx b/ui/src/pages/occurrences/occurrences.tsx index e0aed9b24..5eb4f2f21 100644 --- a/ui/src/pages/occurrences/occurrences.tsx +++ b/ui/src/pages/occurrences/occurrences.tsx @@ -110,6 +110,7 @@ export const Occurrences = () => { defaultOpen={someActive( [ 'collection', + 'occurrence_set', 'deployment', 'deployment__device', 'deployment__research_site', @@ -123,6 +124,7 @@ export const Occurrences = () => { + From c731d37a7d6cbba1de9f331b48a89ad9f425eea2 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 8 Oct 2026 14:44:49 -0400 Subject: [PATCH 5/9] fix(ui): put creating a set in the toolbar rather than the selection bar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The selection bar holds identification actions and is hidden from anyone without update rights on the occurrences, so saving a set — which changes none of them — was unavailable to a reader who could still create one. It also sat there as an unlabelled icon among three others. It now sits beside Export as a labelled button, and appears only while something is selected, since that is the only time it does anything. --- .../create-occurrence-set-popover.tsx | 23 +++++++-------- .../pages/occurrences/occurrences-actions.tsx | 29 +++++++++---------- ui/src/pages/occurrences/occurrences.tsx | 4 +++ ui/src/utils/language.ts | 2 ++ 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx index 0fcaa2256..59025acc0 100644 --- a/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx +++ b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx @@ -1,5 +1,5 @@ import { BookmarkPlusIcon } from 'lucide-react' -import { BasicTooltip, Button, Popover } from 'nova-ui-kit' +import { Button, Popover } from 'nova-ui-kit' import { useState } from 'react' import { STRING, translate } from 'utils/language' import { CreateOccurrenceSet } from './create-occurrence-set' @@ -15,17 +15,16 @@ export const CreateOccurrenceSetPopover = ({ return ( - - - - - + + + !userPermissions.includes(UserPermission.Update) ) - const occurrenceIds = occurrences.map((occurrence) => occurrence.id) + if (!canUpdate) { + return null + } return (
- {/* Saving a set changes no occurrence, so it is not gated on update rights. */} - {canUpdate ? ( - <> - - - occurrence.determinationTaxon - )} - /> - - ) : null} - + + occurrence.id)} + /> + occurrence.id)} + occurrenceTaxa={occurrences.map( + (occurrence) => occurrence.determinationTaxon + )} + />
) } diff --git a/ui/src/pages/occurrences/occurrences.tsx b/ui/src/pages/occurrences/occurrences.tsx index 5eb4f2f21..9279d3312 100644 --- a/ui/src/pages/occurrences/occurrences.tsx +++ b/ui/src/pages/occurrences/occurrences.tsx @@ -31,6 +31,7 @@ import { columns } from './occurrence-columns' import { OccurrenceDetailsDialog } from './occurrence-details-dialog' import { OccurrenceGallery } from './occurrence-gallery' import { OccurrenceStats } from './occurrence-stats' +import { CreateOccurrenceSetPopover } from './create-occurrence-set/create-occurrence-set-popover' import { OccurrencesActions } from './occurrences-actions' export const Occurrences = () => { @@ -165,6 +166,9 @@ export const Occurrences = () => { Export + {selectedItems.length > 0 && ( + + )} Date: Thu, 8 Oct 2026 16:21:37 -0400 Subject: [PATCH 6/9] feat(main): add a project permission for retraining a classifier Training is run per project and rewrites what every later identification is compared against, so it belongs behind its own permission rather than any existing run_*_job one. ML data managers get it with the other job permissions they already hold; the migration grants it to the groups that exist today so current managers do not lose the ability when the job appears. --- .../0100_train_classifier_job_permission.py | 77 +++++++++++++++ ...1_grant_train_classifier_job_permission.py | 94 +++++++++++++++++++ ami/main/models.py | 2 + ami/users/roles.py | 3 + 4 files changed, 176 insertions(+) create mode 100644 ami/main/migrations/0100_train_classifier_job_permission.py create mode 100644 ami/main/migrations/0101_grant_train_classifier_job_permission.py diff --git a/ami/main/migrations/0100_train_classifier_job_permission.py b/ami/main/migrations/0100_train_classifier_job_permission.py new file mode 100644 index 000000000..ccb070eb1 --- /dev/null +++ b/ami/main/migrations/0100_train_classifier_job_permission.py @@ -0,0 +1,77 @@ +# Generated by Django 4.2.10 on 2026-10-08 16:04 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0099_grant_occurrence_set_permissions"), + ] + + operations = [ + migrations.AlterModelOptions( + name="project", + options={ + "ordering": ["-priority", "created_at"], + "permissions": [ + ("create_identification", "Can create identifications"), + ("update_identification", "Can update identifications"), + ("delete_identification", "Can delete identifications"), + ("create_job", "Can create a job"), + ("update_job", "Can update a job"), + ("run_ml_job", "Can run/retry/cancel ML jobs"), + ("run_populate_captures_collection_job", "Can run/retry/cancel Populate Collection jobs"), + ("run_data_storage_sync_job", "Can run/retry/cancel Data Storage Sync jobs"), + ("run_regroup_events_job", "Can run/retry/cancel Regroup Events jobs"), + ("run_data_export_job", "Can run/retry/cancel Data Export jobs"), + ("run_single_image_ml_job", "Can process a single capture"), + ("run_post_processing_job", "Can run/retry/cancel Post-Processing jobs"), + ("run_train_classifier_job", "Can run/retry/cancel Train Classifier jobs"), + ("delete_job", "Can delete a job"), + ("create_deployment", "Can create a deployment"), + ("delete_deployment", "Can delete a deployment"), + ("update_deployment", "Can update a deployment"), + ("sync_deployment", "Can sync images to a deployment"), + ("regroup_sessions_deployment", "Can regroup deployment captures into sessions"), + ("create_occurrenceset", "Can create an occurrence set"), + ("update_occurrenceset", "Can rename or describe an occurrence set"), + ("delete_occurrenceset", "Can delete an occurrence set"), + ("create_sourceimagecollection", "Can create a collection"), + ("update_sourceimagecollection", "Can update a collection"), + ("delete_sourceimagecollection", "Can delete a collection"), + ("populate_sourceimagecollection", "Can populate a collection"), + ("create_sourceimage", "Can create a source image"), + ("update_sourceimage", "Can update a source image"), + ("delete_sourceimage", "Can delete a source image"), + ("star_sourceimage", "Can star a source image"), + ("create_sourceimageupload", "Can create a source image upload"), + ("update_sourceimageupload", "Can update a source image upload"), + ("delete_sourceimageupload", "Can delete a source image upload"), + ("create_s3storagesource", "Can create storage"), + ("delete_s3storagesource", "Can delete storage"), + ("update_s3storagesource", "Can update storage"), + ("test_s3storagesource", "Can test storage connection"), + ("create_site", "Can create a site"), + ("delete_site", "Can delete a site"), + ("update_site", "Can update a site"), + ("create_device", "Can create a device"), + ("delete_device", "Can delete a device"), + ("update_device", "Can update a device"), + ("view_userprojectmembership", "Can view project members"), + ("create_userprojectmembership", "Can add a user to the project"), + ("update_userprojectmembership", "Can update a user's project membership and role in the project"), + ("delete_userprojectmembership", "Can remove a user from the project"), + ("create_dataexport", "Can create a data export"), + ("update_dataexport", "Can update a data export"), + ("delete_dataexport", "Can delete a data export"), + ("create_projectpipelineconfig", "Can register pipelines for the project"), + ("update_projectpipelineconfig", "Can update pipeline configurations"), + ("delete_projectpipelineconfig", "Can remove pipelines from the project"), + ("create_taxalist", "Can create a taxa list"), + ("update_taxalist", "Can update a taxa list"), + ("delete_taxalist", "Can delete a taxa list"), + ("view_private_data", "Can view private data"), + ], + }, + ), + ] diff --git a/ami/main/migrations/0101_grant_train_classifier_job_permission.py b/ami/main/migrations/0101_grant_train_classifier_job_permission.py new file mode 100644 index 000000000..19c5cb8ec --- /dev/null +++ b/ami/main/migrations/0101_grant_train_classifier_job_permission.py @@ -0,0 +1,94 @@ +""" +Grant ``run_train_classifier_job`` to the role that already runs the other ML jobs. + +``create_roles_for_project`` gives new projects their role permissions, and a +``post_migrate`` signal re-syncs every project on migrate, so this backfill is belt and +braces. It is kept so the grant is explicit rather than a side effect of a signal. + +Guardian reads object-level rows, so each project gets a ``GroupObjectPermission``; adding +to ``group.permissions`` alone would not appear in ``get_perms(user, project)``. Mirrors +``0099_grant_occurrence_set_permissions``. +""" + +from django.db import migrations +from django.db.models import Q + +PERMISSION = "run_train_classifier_job" +ROLE_GROUP_SUFFIXES = ("_MLDataManager", "_ProjectManager") + + +def _permission(apps): + Permission = apps.get_model("auth", "Permission") + ContentType = apps.get_model("contenttypes", "ContentType") + try: + project_ct = ContentType.objects.get(app_label="main", model="project") + except ContentType.DoesNotExist: + return None, None + try: + return Permission.objects.get(codename=PERMISSION, content_type=project_ct), project_ct + except Permission.DoesNotExist: + return None, None + + +def _project_pk_from_group(group): + # Group names are "{project_pk}_{project_name}_{RoleName}"; the pk is the immutable + # leading segment, since a project name may itself contain underscores. + try: + return int(group.name.split("_", 1)[0]) + except (ValueError, IndexError): + return None + + +def _role_groups(apps): + Group = apps.get_model("auth", "Group") + query = Q() + for suffix in ROLE_GROUP_SUFFIXES: + query |= Q(name__endswith=suffix) + return Group.objects.filter(query) + + +def grant(apps, schema_editor): + GroupObjectPermission = apps.get_model("guardian", "GroupObjectPermission") + permission, project_ct = _permission(apps) + if permission is None: + return + + for group in _role_groups(apps): + project_pk = _project_pk_from_group(group) + if project_pk is None: + continue + group.permissions.add(permission) + GroupObjectPermission.objects.get_or_create( + permission=permission, + content_type=project_ct, + object_pk=str(project_pk), + group=group, + ) + + +def revoke(apps, schema_editor): + GroupObjectPermission = apps.get_model("guardian", "GroupObjectPermission") + permission, project_ct = _permission(apps) + if permission is None: + return + + for group in _role_groups(apps): + project_pk = _project_pk_from_group(group) + if project_pk is None: + continue + group.permissions.remove(permission) + GroupObjectPermission.objects.filter( + permission=permission, + content_type=project_ct, + object_pk=str(project_pk), + group=group, + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0100_train_classifier_job_permission"), + ("guardian", "0002_generic_permissions_index"), + ] + + operations = [migrations.RunPython(grant, revoke)] diff --git a/ami/main/models.py b/ami/main/models.py index 66f788ad4..91d1091f4 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -454,6 +454,7 @@ class Permissions: RUN_REGROUP_EVENTS_JOB = "run_regroup_events_job" RUN_DATA_EXPORT_JOB = "run_data_export_job" RUN_POST_PROCESSING_JOB = "run_post_processing_job" + RUN_TRAIN_CLASSIFIER_JOB = "run_train_classifier_job" DELETE_JOB = "delete_job" # Deployment permissions @@ -542,6 +543,7 @@ class Meta: ("run_data_export_job", "Can run/retry/cancel Data Export jobs"), ("run_single_image_ml_job", "Can process a single capture"), ("run_post_processing_job", "Can run/retry/cancel Post-Processing jobs"), + ("run_train_classifier_job", "Can run/retry/cancel Train Classifier jobs"), ("delete_job", "Can delete a job"), # Deployment permissions ("create_deployment", "Can create a deployment"), diff --git a/ami/users/roles.py b/ami/users/roles.py index ae68565d8..d05aa209a 100644 --- a/ami/users/roles.py +++ b/ami/users/roles.py @@ -146,6 +146,9 @@ class MLDataManager(Role): Project.Permissions.SYNC_DEPLOYMENT, Project.Permissions.RUN_REGROUP_EVENTS_JOB, Project.Permissions.RUN_DATA_EXPORT_JOB, + # Retraining a head is ML data work, so it sits with the other run_*_job + # permissions this role already holds. + Project.Permissions.RUN_TRAIN_CLASSIFIER_JOB, Project.Permissions.DELETE_OCCURRENCES, Project.Permissions.CREATE_PROJECT_PIPELINE_CONFIG, Project.Permissions.UPDATE_PROJECT_PIPELINE_CONFIG, From 3fef6c93f3e55c5ec7f1ad4548aba6b4af1510e8 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 8 Oct 2026 16:21:43 -0400 Subject: [PATCH 7/9] feat(main): let a project name the taxa list its classifier covers A head trained only on the species someone happened to verify cannot predict the rest, so a region's expected species have to come from somewhere other than the verified data. The project now names that list, and training reads it as the class list. It is optional: without one, the classes are whatever has been verified. --- .../0102_project_default_taxa_list.py | 25 +++++++++++++++++++ ami/main/models.py | 13 ++++++++++ 2 files changed, 38 insertions(+) create mode 100644 ami/main/migrations/0102_project_default_taxa_list.py diff --git a/ami/main/migrations/0102_project_default_taxa_list.py b/ami/main/migrations/0102_project_default_taxa_list.py new file mode 100644 index 000000000..5634621b9 --- /dev/null +++ b/ami/main/migrations/0102_project_default_taxa_list.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.10 on 2026-10-08 16:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0101_grant_train_classifier_job_permission"), + ] + + operations = [ + migrations.AddField( + model_name="project", + name="default_taxa_list", + field=models.ForeignKey( + blank=True, + help_text="The species this project expects to see. Used as the class list when retraining a classifier head, so the head covers the region rather than only the species someone happened to verify.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="default_for_projects", + to="main.taxalist", + ), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index 91d1091f4..616800277 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -429,6 +429,19 @@ def check_custom_permission(self, user, action: str) -> bool: # Fall back to default permission checking for other actions return super().check_custom_permission(user, action) + default_taxa_list = models.ForeignKey( + "TaxaList", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="default_for_projects", + help_text=( + "The species this project expects to see. Used as the class list when retraining " + "a classifier head, so the head covers the region rather than only the species " + "someone happened to verify." + ), + ) + class Permissions: """CRUD Permission names follow the convention: `create_`, `update_`, `delete_`, `view_`""" From d42e6b6ba997fcb4375dcedc3c75805e50b680e1 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 8 Oct 2026 16:23:06 -0400 Subject: [PATCH 8/9] feat(ml): build a classifier head training set from verified occurrences A head is only worth retraining on the crops a person has already named, so this collects verified detections together with the embeddings stored for them and writes one dataset file per run. Decisions worth knowing about: The train/test split is grouped by occurrence, not by detection. Crops from one occurrence are near-identical frames of the same insect, so splitting per detection puts the same animal on both sides and the held-out score flatters the head. It is also a hash of the occurrence id rather than a random draw, so the held-out set stays the same between runs and two heads can be compared. The vector width is read from the stored vectors rather than assumed, because the embedding column is unsized and two algorithms may differ. Species with too few verified crops are dropped, since one example cannot be both trained and evaluated on. The classes come from the project's taxa list where it has one, so the head covers the region and not only what has been verified, and the classes with no verified data are recorded in the file rather than silently left out. The metadata is declared as a schema instead of assembled as a dict, so the shape is checked once here rather than at every reader, and the processing service echoes it back for the job to store. The rows are also served over the API, paged at a limit set for their size: a 1024-dimension vector is roughly 20 KB of JSON, so the platform default of 10 is useless and no limit at all returns hundreds of megabytes. --- ami/base/pagination.py | 12 ++ ami/ml/schemas.py | 58 ++++++ ami/ml/serializers.py | 18 ++ ami/ml/test_training.py | 160 +++++++++++++++++ ami/ml/training/__init__.py | 41 +++++ ami/ml/training/dataset.py | 341 ++++++++++++++++++++++++++++++++++++ ami/ml/views.py | 152 +++++++++++++++- config/api_router.py | 1 + 8 files changed, 782 insertions(+), 1 deletion(-) create mode 100644 ami/ml/test_training.py create mode 100644 ami/ml/training/__init__.py create mode 100644 ami/ml/training/dataset.py diff --git a/ami/base/pagination.py b/ami/base/pagination.py index 9ebca7b21..ffcbda445 100644 --- a/ami/base/pagination.py +++ b/ami/base/pagination.py @@ -29,3 +29,15 @@ def _get_project(self): if hasattr(view, "get_active_project"): return view.get_active_project() return None + + +class TrainingDataPagination(LimitOffsetPagination): + """ + Paging for embedding rows, which are far bigger than a normal API row. + + A 1024-dimension vector serialises to roughly 20 KB of JSON, so the platform default of + 10 is uselessly small and an unbounded limit would return hundreds of megabytes. + """ + + default_limit = 200 + max_limit = 1000 diff --git a/ami/ml/schemas.py b/ami/ml/schemas.py index 2393dd62b..d1d00a575 100644 --- a/ami/ml/schemas.py +++ b/ami/ml/schemas.py @@ -503,3 +503,61 @@ class AlgorithmTrainingInfo(pydantic.BaseModel): class Config: extra = "allow" + +class NamedReference(pydantic.BaseModel): + """ + A row a training set came from. + + Carries the name as well as the id so a dataset still says where it came from after the + row it points at is deleted. + """ + + id: int + name: str + + +class TrainedAlgorithmReference(AlgorithmReference): + """The algorithm a training set was built for, at the version it was then.""" + + version: int | None = None + + +class TrainingDatasetSettings(pydantic.BaseModel): + """The settings a training set was built under, so a later run can be made to match.""" + + min_per_species: int + split_salt: str + test_fraction: float + split_grouped_by: str = pydantic.Field( + default="occurrence", + description=( + "What the train/test split is grouped by. Splitting per detection puts " + "near-identical crops on both sides." + ), + ) + + +class TrainingDatasetMetadata(pydantic.BaseModel): + """ + What a training set is, written inside the file and echoed back by the service. + + Declared rather than assembled as a dict so the shape is checked once, here, instead of + at each reader. + """ + + url: str + project: NamedReference + algorithm: TrainedAlgorithmReference + dimensions: int = pydantic.Field(description="Width of one feature vector, read from the stored vectors.") + dtype: str + classes: list[str] = pydantic.Field(description="The head's output classes, in order.") + counts: dict[str, int] = pydantic.Field(default_factory=dict, description="Verified crops per class.") + taxa_list: NamedReference | None = None + occurrence_set: NamedReference | None = None + classes_without_verified_data: list[str] = pydantic.Field(default_factory=list) + dropped_species: list[str] = pydantic.Field(default_factory=list) + rows: int + train: int + test: int + verified_detections_without_embedding: int = 0 + settings: TrainingDatasetSettings diff --git a/ami/ml/serializers.py b/ami/ml/serializers.py index 1f39b5b23..0ece73409 100644 --- a/ami/ml/serializers.py +++ b/ami/ml/serializers.py @@ -176,3 +176,21 @@ class PipelineRegistrationSerializer(serializers.Serializer): processing_service_name = serializers.CharField() pipelines = SchemaField(schema=list[PipelineConfigResponse], default=[]) + +class TrainingDataRowSerializer(serializers.Serializer): + """ + One verified crop: its human label and the embedding to train on. + + Not a ModelSerializer — the useful fields live on the related occurrence and on the + pgvector column, and the split is computed, so building the dict directly is clearer. + """ + + def to_representation(self, instance): + from ami.ml import training + + return training.row_as_dict( + instance, + salt=self.context.get("split_salt", training.DEFAULT_SPLIT_SALT), + test_fraction=self.context.get("test_fraction", training.DEFAULT_TEST_FRACTION), + include_features=self.context.get("include_features", True), + ) diff --git a/ami/ml/test_training.py b/ami/ml/test_training.py new file mode 100644 index 000000000..6554d5325 --- /dev/null +++ b/ami/ml/test_training.py @@ -0,0 +1,160 @@ +""" +Building a training set from an occurrence set. + +A run learns from a named set rather than from whatever happened to be verified that day, +so these tests are mostly about the set deciding what goes in. +""" + +import datetime +import uuid + +from django.test import TestCase + +from ami.main.models import ( + Classification, + Deployment, + Detection, + Event, + Identification, + Occurrence, + OccurrenceSet, + Project, + S3StorageSource, + SourceImage, + Taxon, + TaxonRank, +) +from ami.ml.models import Algorithm, AlgorithmCategoryMap +from ami.ml.models.embedding import DetectionEmbedding +from ami.ml.schemas import NamedReference +from ami.ml.training import NotEnoughVerifiedData, build_training_dataset, verified_training_rows +from ami.users.models import User + +VECTOR_LENGTH = 8 + + +class TrainingSetFixture(TestCase): + """ + Occurrences, identifications and vectors built directly. + + The capture fixtures time their images from datetime.now() and group events by reading + image dimensions from storage, neither of which is deterministic in a test. + """ + + def setUp(self) -> None: + self.user = User.objects.create_user(email="curator@example.com", password="testpass123") + self.project = Project.objects.create(name="Training Project", owner=self.user) + storage = S3StorageSource.objects.create(name="src", project=self.project, bucket="test") + self.deployment = Deployment.objects.create(name="station", project=self.project, data_source=storage) + night = datetime.datetime(2026, 6, 1, 22, 0, 0, tzinfo=datetime.timezone.utc) + self.event = Event.objects.create(deployment=self.deployment, project=self.project, start=night) + self.night = night + + self.algorithm = Algorithm.objects.create(name="Backbone", key="backbone", trainable=True) + self.taxa = [ + Taxon.objects.create(name=f"Trainus {name}", rank=TaxonRank.SPECIES.name) for name in ("alpha", "beta") + ] + for taxon in self.taxa: + taxon.projects.add(self.project) + self.algorithm.category_map = AlgorithmCategoryMap.objects.create( + labels=[t.name for t in self.taxa], + data=[{"index": i, "label": t.name} for i, t in enumerate(self.taxa)], + version="training-test", + ) + self.algorithm.save() + + def make_occurrence(self, taxon: Taxon, with_vector: bool = True) -> Occurrence: + """One verified occurrence, optionally carrying a feature vector.""" + image = SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + event=self.event, + timestamp=self.night, + path=f"test/{uuid.uuid4().hex[:8]}.jpg", + ) + detection = Detection.objects.create(source_image=image, timestamp=self.night, bbox=[0.1, 0.1, 0.2, 0.2]) + Classification.objects.create(detection=detection, taxon=taxon, score=0.9, timestamp=self.night) + occurrence = detection.associate_new_occurrence() + Identification.objects.create(occurrence=occurrence, taxon=taxon, user=self.user) + if with_vector: + DetectionEmbedding.objects.create( + detection=detection, + algorithm=self.algorithm, + project=self.project, + vector=[0.1] * VECTOR_LENGTH, + ) + occurrence.refresh_from_db() + return occurrence + + def make_set(self, name: str, occurrences: list[Occurrence]) -> OccurrenceSet: + occurrence_set = OccurrenceSet.objects.create(name=name) + occurrence_set.projects.add(self.project) + occurrence_set.occurrences.set(occurrences) + return occurrence_set + + +class TestTheSetDecidesWhatIsLearnedFrom(TrainingSetFixture): + def test_only_the_sets_occurrences_are_used(self): + inside = [self.make_occurrence(self.taxa[0]), self.make_occurrence(self.taxa[1])] + outside = self.make_occurrence(self.taxa[0]) + occurrence_set = self.make_set("Spring review", inside) + + rows = verified_training_rows(self.project, self.algorithm, occurrence_set) + used = {row.detection.occurrence_id for row in rows} + + self.assertEqual(used, {o.pk for o in inside}) + self.assertNotIn(outside.pk, used) + + def test_without_a_set_every_verified_occurrence_is_used(self): + """The helper still answers without one; it is the job that insists on a set.""" + occurrences = [self.make_occurrence(self.taxa[0]), self.make_occurrence(self.taxa[1])] + + rows = verified_training_rows(self.project, self.algorithm) + + self.assertEqual({row.detection.occurrence_id for row in rows}, {o.pk for o in occurrences}) + + def test_the_dataset_says_which_set_it_came_from(self): + occurrences = [self.make_occurrence(self.taxa[0]) for _ in range(4)] + occurrences += [self.make_occurrence(self.taxa[1]) for _ in range(4)] + occurrence_set = self.make_set("Spring review", occurrences) + + # Half held out: the split is a hash of the occurrence, so a handful of rows at the + # usual fifth can land entirely on one side, which the build refuses. + dataset = build_training_dataset( + project=self.project, + algorithm=self.algorithm, + min_per_species=1, + test_fraction=0.5, + occurrence_set=occurrence_set, + ) + + self.assertEqual( + dataset["metadata"].occurrence_set, + NamedReference(id=occurrence_set.pk, name="Spring review"), + ) + self.assertEqual(dataset["metadata"].rows, 8) + + def test_an_empty_set_has_nothing_to_train_on(self): + self.make_occurrence(self.taxa[0]) + empty = self.make_set("Nothing in here", []) + + with self.assertRaises(NotEnoughVerifiedData): + build_training_dataset( + project=self.project, algorithm=self.algorithm, min_per_species=1, occurrence_set=empty + ) + + def test_the_vector_width_comes_from_the_stored_vectors(self): + """Nothing assumes a width: a backbone of another size is read, not truncated.""" + occurrences = [self.make_occurrence(self.taxa[0]) for _ in range(6)] + occurrence_set = self.make_set("Spring review", occurrences) + + dataset = build_training_dataset( + project=self.project, + algorithm=self.algorithm, + min_per_species=1, + test_fraction=0.5, + occurrence_set=occurrence_set, + ) + + self.assertEqual(dataset["metadata"].dimensions, VECTOR_LENGTH) + diff --git a/ami/ml/training/__init__.py b/ami/ml/training/__init__.py new file mode 100644 index 000000000..6770457ad --- /dev/null +++ b/ami/ml/training/__init__.py @@ -0,0 +1,41 @@ +""" +Retraining a classifier head from the species people have verified. + +``dataset`` decides what the head learns from and writes it to storage. The names +re-exported here are the ones callers outside this package use; anything else is internal +to it. +""" + +from ami.ml.training.dataset import ( + DEFAULT_SPLIT_SALT, + DEFAULT_TEST_FRACTION, + SPLIT_TEST, + SPLIT_TRAIN, + SPLITS, + NotEnoughVerifiedData, + build_training_dataset, + count_missing_embeddings, + label_counts, + row_as_dict, + species_with_enough_examples, + split_for, + verified_occurrence_ids, + verified_training_rows, +) + +__all__ = [ + "DEFAULT_SPLIT_SALT", + "DEFAULT_TEST_FRACTION", + "SPLITS", + "SPLIT_TEST", + "SPLIT_TRAIN", + "NotEnoughVerifiedData", + "build_training_dataset", + "count_missing_embeddings", + "label_counts", + "row_as_dict", + "species_with_enough_examples", + "split_for", + "verified_occurrence_ids", + "verified_training_rows", +] diff --git a/ami/ml/training/dataset.py b/ami/ml/training/dataset.py new file mode 100644 index 000000000..d81a1e8dd --- /dev/null +++ b/ami/ml/training/dataset.py @@ -0,0 +1,341 @@ +""" +Build a classifier-head training set out of what people verified, and write it to storage. + +Two halves of one job: deciding what counts as training data — which occurrences a person +identified, which species have enough of them, which rows are held out — and turning that +into a single file. + +It is a file because a service is handed a URL, not the rows. A project with a few hundred +thousand verified labels runs to hundreds of megabytes, which is fragile to send in one +request and has to start over if the connection drops. +""" + +import hashlib +import logging +import pathlib +import tempfile +import typing +from typing import TYPE_CHECKING + +import numpy as np +from django.conf import settings +from django.core.files.storage import default_storage +from django.db.models import QuerySet +from django.utils.text import slugify + +from ami.main.models import DEFAULT_EMBEDDING_KEY, Identification, Project +from ami.ml.models.algorithm import Algorithm +from ami.ml.models.embedding import DetectionEmbedding +from ami.ml.schemas import NamedReference, TrainedAlgorithmReference, TrainingDatasetMetadata, TrainingDatasetSettings + +if TYPE_CHECKING: + from ami.jobs.models import Job + from ami.main.models import OccurrenceSet, TaxaList + +logger = logging.getLogger(__name__) + + +DEFAULT_SPLIT_SALT = "antenna-head-v1" +DEFAULT_TEST_FRACTION = 0.2 + +# The two sides of the split. Named so the API, the dataset builder and the export +# cannot drift apart on spelling. +SPLIT_TRAIN = "train" +SPLIT_TEST = "test" +SPLITS = (SPLIT_TRAIN, SPLIT_TEST) + + +def split_for( + occurrence_id: int, + salt: str = DEFAULT_SPLIT_SALT, + test_fraction: float = DEFAULT_TEST_FRACTION, +) -> str: + """ + Assign an occurrence to "train" or "test", the same way every time. + + Grouped by occurrence, not detection: an occurrence is one insect across several + frames, so a detection-level split puts near-identical crops on both sides and + overstates accuracy. Hash-based, not random, so the test set does not move when new + data arrives — an eval set that drifts cannot compare two heads. + """ + digest = hashlib.sha256(f"{salt}:{occurrence_id}".encode()).hexdigest() + return SPLIT_TEST if (int(digest[:8], 16) / 0xFFFFFFFF) < test_fraction else SPLIT_TRAIN + + +def verified_occurrence_ids(project: Project, occurrence_set: "OccurrenceSet | None" = None) -> QuerySet: + """ + Occurrences a person has identified and not taken back. + + Narrowed to an occurrence set when one is given. A set is fixed once created, so a run + that names one can be repeated and says for itself what it learned from; without one the + pool is whatever happened to be verified that day, which no later reader can reconstruct. + """ + identifications = Identification.objects.filter( + withdrawn=False, + occurrence__project=project, + occurrence__determination__isnull=False, + ) + if occurrence_set is not None: + identifications = identifications.filter(occurrence__evaluation_sets=occurrence_set) + return identifications.order_by().values_list("occurrence_id", flat=True).distinct() + + +def verified_training_rows( + project: Project, algorithm: Algorithm, occurrence_set: "OccurrenceSet | None" = None +) -> QuerySet[DetectionEmbedding]: + """ + Embeddings whose detection sits under a verified occurrence. + + Constrained to one algorithm on purpose: vectors from different backbones are in + different spaces and must never be mixed into one training set. + """ + return ( + DetectionEmbedding.objects.filter( + algorithm=algorithm, + key=DEFAULT_EMBEDDING_KEY, + detection__occurrence_id__in=verified_occurrence_ids(project, occurrence_set), + detection__occurrence__determination__isnull=False, + ) + .select_related("detection__occurrence__determination") + .order_by("pk") + ) + + +def count_missing_embeddings( + project: Project, algorithm: Algorithm, occurrence_set: "OccurrenceSet | None" = None +) -> int: + """Verified detections this algorithm has never embedded. They need a pipeline re-run.""" + from ami.main.models import Detection + + return ( + Detection.objects.filter(occurrence_id__in=verified_occurrence_ids(project)) + .exclude(embeddings__algorithm=algorithm) + .count() + ) + + +def label_counts( + project: Project, algorithm: Algorithm, occurrence_set: "OccurrenceSet | None" = None +) -> dict[str, int]: + """Verified crops per species, for the chosen algorithm.""" + from django.db.models import Count + + rows = ( + verified_training_rows(project, algorithm, occurrence_set) + .order_by() + .values("detection__occurrence__determination__name") + .annotate(n=Count("pk")) + ) + return {r["detection__occurrence__determination__name"]: r["n"] for r in rows} + + +def species_with_enough_examples(counts: dict[str, int], minimum: int) -> set[str]: + """A class with a single example cannot be both trained on and evaluated.""" + return {name for name, n in counts.items() if name and n >= minimum} + + +def row_as_dict( + embedding: DetectionEmbedding, + salt: str = DEFAULT_SPLIT_SALT, + test_fraction: float = DEFAULT_TEST_FRACTION, + include_features: bool = True, +) -> dict[str, typing.Any]: + """One training row, in the shape the training-data API returns.""" + occurrence = embedding.detection.occurrence + row = { + "detection_id": embedding.detection_id, + "occurrence_id": occurrence.pk if occurrence else None, + "label": occurrence.determination.name if occurrence and occurrence.determination else None, + "label_id": occurrence.determination_id if occurrence else None, + "split": split_for(occurrence.pk, salt, test_fraction) if occurrence else None, + } + if include_features: + row["features"] = embedding.vector.to_list() + return row + + +# Vectors are written as float16 because that is exactly what Postgres stores (halfvec), +# so nothing is lost. Measured on 10k rows: npz float16 is 21 MB against 227 MB of JSON, +# and 1.1s to build against 29s. +DATASET_DTYPE = np.float16 + +DATASET_DIRECTORY = "training" + +# Rows are pulled from the database in batches so a large project does not have to fit +# every vector in memory at once. +FETCH_BATCH_SIZE = 2000 + + +class NotEnoughVerifiedData(Exception): + """The project does not hold enough verified labels to train and evaluate on.""" + + +def build_training_dataset( + project: Project, + algorithm: Algorithm, + min_per_species: int = 2, + split_salt: str = DEFAULT_SPLIT_SALT, + test_fraction: float = DEFAULT_TEST_FRACTION, + taxa_list: "TaxaList | None" = None, + occurrence_set: "OccurrenceSet | None" = None, + job: "Job | None" = None, +) -> dict[str, typing.Any]: + """ + Collect verified labels and their embeddings, and save them as one npz file. + + Returns the storage path, the URL a service can fetch, and the metadata describing + what went in. Raises NotEnoughVerifiedData rather than writing a file nothing can be + trained on. + """ + counts = label_counts(project, algorithm, occurrence_set) + taxa_list = taxa_list or project.default_taxa_list + + if taxa_list: + # The taxa list decides what the head can predict; the verified crops only decide + # how well it predicts each one. Without this the head shrinks to whatever happened + # to be verified, which silently narrows the pipeline. + classes = sorted(taxa_list.taxa.exclude(name="").values_list("name", flat=True)) + if not classes: + raise NotEnoughVerifiedData(f"Taxa list '{taxa_list}' is empty, so there is nothing to train.") + keep = set(classes) + outside = sorted(name for name in counts if name and name not in keep) + else: + keep = species_with_enough_examples(counts, min_per_species) + if not keep: + raise NotEnoughVerifiedData( + f"No species in '{project.name}' has at least {min_per_species} verified crops with an " + f"embedding from {algorithm.key}. Verify more occurrences, or re-run the pipeline so the " + "verified detections get embeddings." + ) + classes = sorted(keep) + outside = sorted(set(counts) - keep) + + class_index = {name: i for i, name in enumerate(classes)} + + rows = verified_training_rows(project, algorithm, occurrence_set) + total = rows.count() + + # Asked rather than assumed: a vector's width belongs to the model that produced it, and + # the column is unsized so two algorithms may differ. + dimensions = DetectionEmbedding.objects.stored_length(algorithm.pk) + if dimensions is None: + raise NotEnoughVerifiedData( + f"No detection has a feature vector from '{algorithm.key}' yet, so there is nothing to train on." + ) + features = np.zeros((total, dimensions), dtype=DATASET_DTYPE) + labels = np.zeros(total, dtype=np.int64) + detection_ids = np.zeros(total, dtype=np.int64) + occurrence_ids = np.zeros(total, dtype=np.int64) + splits: list[str] = [] + + kept = 0 + for embedding in rows.iterator(chunk_size=FETCH_BATCH_SIZE): + occurrence = embedding.detection.occurrence + if not occurrence or not occurrence.determination: + continue + name = occurrence.determination.name + if name not in class_index: + continue + features[kept] = np.asarray(embedding.vector.to_list(), dtype=DATASET_DTYPE) + labels[kept] = class_index[name] + detection_ids[kept] = embedding.detection_id + occurrence_ids[kept] = occurrence.pk + splits.append(split_for(occurrence.pk, split_salt, test_fraction)) + kept += 1 + + if not kept: + raise NotEnoughVerifiedData("No verified detection has an embedding from this algorithm yet.") + + features = features[:kept] + labels = labels[:kept] + detection_ids = detection_ids[:kept] + occurrence_ids = occurrence_ids[:kept] + split_array = np.array(splits) + + n_train = int((split_array == "train").sum()) + n_test = int((split_array == "test").sum()) + if not n_train or not n_test: + raise NotEnoughVerifiedData( + f"The split left {n_train} training and {n_test} held-out rows. Both sides need rows " + "before a new head can be compared against the current one." + ) + + file_path = dataset_path(project, algorithm, job.pk if job else None) + # Declared, not assembled: the shape is checked here rather than at each reader, and + # the service echoes it back in its result. + metadata = TrainingDatasetMetadata( + url=f"{settings.MEDIA_URL}{file_path}", + project=NamedReference(id=project.pk, name=project.name), + algorithm=TrainedAlgorithmReference(key=algorithm.key, name=algorithm.name, version=algorithm.version), + dimensions=dimensions, + dtype=np.dtype(DATASET_DTYPE).name, + classes=classes, + # .get(): with a taxa list, a class can legitimately have no verified crops yet. + counts={name: counts.get(name, 0) for name in classes}, + taxa_list=NamedReference(id=taxa_list.pk, name=taxa_list.name) if taxa_list else None, + occurrence_set=(NamedReference(id=occurrence_set.pk, name=occurrence_set.name) if occurrence_set else None), + classes_without_verified_data=sorted(name for name in classes if not counts.get(name)), + dropped_species=outside, + rows=kept, + train=n_train, + test=n_test, + verified_detections_without_embedding=count_missing_embeddings(project, algorithm, occurrence_set), + settings=TrainingDatasetSettings( + min_per_species=min_per_species, + split_salt=split_salt, + test_fraction=test_fraction, + ), + ) + + file_path = _save( + file_path=file_path, + arrays={ + "features": features, + "labels": labels, + "detection_ids": detection_ids, + "occurrence_ids": occurrence_ids, + "split": split_array, + "classes": np.array(classes), + }, + metadata=metadata, + ) + + file_url = metadata.url + logger.info(f"Wrote training dataset with {kept} rows over {len(classes)} species to {file_path}") + return {"path": file_path, "url": file_url, "metadata": metadata} + + +def dataset_path(project: Project, algorithm: Algorithm, job_id: int | None) -> str: + """Where this project's training set for this algorithm is written.""" + stem = f"{slugify(project.name)}-{slugify(algorithm.key)}" + suffix = f"job-{job_id}" if job_id else "manual" + return f"{DATASET_DIRECTORY}/{stem}-{suffix}.npz" + + +def _save( + file_path: str, + arrays: dict[str, np.ndarray], + metadata: TrainingDatasetMetadata, +) -> str: + """ + Write the npz to storage and return its path. + + default_storage is the local filesystem in development and the project's S3 bucket in + production, so this follows wherever captures already live without a special case. + """ + with tempfile.TemporaryDirectory() as tmp: + local = pathlib.Path(tmp) / "dataset.npz" + # Uncompressed: embeddings are close to random, so compression buys about 10 per + # cent for real CPU. Metadata rides inside the archive so the file is self-describing. + np.savez(local, metadata=np.array(metadata.json()), **arrays) + if default_storage.exists(file_path): + # A re-run of the same job replaces its dataset instead of piling up copies. + default_storage.delete(file_path) + with open(local, "rb") as f: + saved_path = default_storage.save(file_path, f) + + if saved_path != file_path: + # The metadata inside the archive records the URL it was written to, so a rename + # by the storage backend would leave the file describing somewhere it is not. + logger.warning(f"Storage wrote the training set to {saved_path}, not {file_path}.") + return saved_path diff --git a/ami/ml/views.py b/ami/ml/views.py index 61a63ff7e..556997951 100644 --- a/ami/ml/views.py +++ b/ami/ml/views.py @@ -3,22 +3,28 @@ from django.db import transaction from django.db.models import Prefetch from django.db.models.query import QuerySet +from django.http import Http404 from django.utils.text import slugify from drf_spectacular.utils import extend_schema from rest_framework import exceptions as api_exceptions -from rest_framework import mixins, status, viewsets +from rest_framework import mixins, serializers, status, viewsets from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response +from ami.base.pagination import TrainingDataPagination from ami.base.permissions import ProjectPipelineConfigPermission +from ami.base.serializers import SingleParamSerializer from ami.base.views import ProjectMixin from ami.main.api.schemas import project_id_doc_param from ami.main.api.views import DefaultViewSet from ami.main.models import Project, SourceImage +from ami.ml import training from ami.ml.schemas import PipelineRegistrationResponse from .models.algorithm import Algorithm, AlgorithmCategoryMap +from .models.embedding import DetectionEmbedding from .models.pipeline import Pipeline from .models.processing_service import ProcessingService from .models.project_pipeline_config import ProjectPipelineConfig @@ -28,6 +34,7 @@ PipelineRegistrationSerializer, PipelineSerializer, ProcessingServiceSerializer, + TrainingDataRowSerializer, ) logger = logging.getLogger(__name__) @@ -298,3 +305,146 @@ def create(self, request, *args, **kwargs): processing_service.mark_seen(live=True) return Response(response.dict(), status=status.HTTP_201_CREATED) + + +class TrainingDataViewSet(ProjectMixin, mixins.ListModelMixin, viewsets.GenericViewSet): + """ + Verified detections and their embeddings, for retraining a classifier head. + + A head is trained on embeddings, not pixels, and the backbone that produced them is + frozen. So a trainer can pull this and fit a new head without touching the images. + + Requires `project_id` and `algorithm` (an algorithm key). Constraining to one + algorithm is not optional: vectors from different backbones are in different spaces. + + GET /api/v2/ml/training-data/?project_id=3&algorithm= + GET /api/v2/ml/training-data/summary/?project_id=3&algorithm= + """ + + queryset = DetectionEmbedding.objects.none() + serializer_class = TrainingDataRowSerializer + require_project = True + # Membership is enforced in get_queryset() via Project.objects.visible_for_user(): + # ObjectPermission maps a "list" action on a Project to check_custom_permission, which + # denies members, and IsAuthenticated alone would let any account read any project's + # verified labels. + permission_classes = [IsAuthenticated] + filter_backends: list = [] + pagination_class = TrainingDataPagination + + def _get_visible_project(self) -> Project: + """ + The requested project, if this user is allowed to see it. + + Verified labels are project data, so an account that cannot see the project must + not be able to read them. + """ + project = self.get_active_project() + if not project: + raise Http404("Project not found.") + visible = Project.objects.visible_for_user(self.request.user).filter(pk=project.pk).exists() + if not visible: + raise api_exceptions.PermissionDenied("You do not have access to this project.") + return project + + def _get_algorithm(self) -> Algorithm: + key = SingleParamSerializer[str].clean( + "algorithm", + serializers.CharField( + required=True, + help_text="Key of the algorithm whose embeddings to train on.", + ), + self.request.query_params, + ) + algorithm = Algorithm.objects.filter(key=key).first() + if not algorithm: + raise api_exceptions.NotFound(f"No algorithm with key '{key}'.") + return algorithm + + def _get_split_settings(self) -> tuple[str, float]: + salt = SingleParamSerializer[str].clean( + "split_salt", + serializers.CharField(required=False, default=training.DEFAULT_SPLIT_SALT), + self.request.query_params, + ) + fraction = SingleParamSerializer[float].clean( + "test_fraction", + serializers.FloatField( + required=False, + default=training.DEFAULT_TEST_FRACTION, + min_value=0, + max_value=0.99, + ), + self.request.query_params, + ) + return salt, fraction + + def get_queryset(self) -> QuerySet[DetectionEmbedding]: + project = self._get_visible_project() + qs = training.verified_training_rows(project, self._get_algorithm()) + + self._split_filter = SingleParamSerializer[str].clean( + "split", + serializers.ChoiceField(choices=list(training.SPLITS), required=False, allow_null=True, default=None), + self.request.query_params, + ) + return qs + + def get_serializer_context(self): + context = super().get_serializer_context() + salt, fraction = self._get_split_settings() + context["split_salt"] = salt + context["test_fraction"] = fraction + # Not url_boolean_param: it returns `value or default`, so a default of True can + # never be turned off. + context["include_features"] = SingleParamSerializer[bool].clean( + "include_features", + serializers.BooleanField(required=False, default=True), + self.request.query_params, + ) + return context + + @extend_schema(parameters=[project_id_doc_param]) + def list(self, request, *args, **kwargs): + response = super().list(request, *args, **kwargs) + split = getattr(self, "_split_filter", None) + if split: + # Filtering by split in Python rather than SQL: the assignment is a hash of the + # occurrence id, which Postgres cannot compute. Callers that need whole splits + # should page through everything and group client-side. + results = [row for row in response.data["results"] if row["split"] == split] + response.data["results"] = results + return response + + @extend_schema(parameters=[project_id_doc_param]) + @action(detail=False, methods=["get"]) + def summary(self, request, *args, **kwargs): + """Counts only. Cheap enough to poll before deciding whether a retrain is worth it.""" + project = self._get_visible_project() + algorithm = self._get_algorithm() + salt, fraction = self._get_split_settings() + + counts = training.label_counts(project, algorithm) + rows = training.verified_training_rows(project, algorithm) + splits = {name: 0 for name in training.SPLITS} + for occurrence_id in rows.values_list("detection__occurrence_id", flat=True): + splits[training.split_for(occurrence_id, salt, fraction)] += 1 + + return Response( + { + "project": {"id": project.pk, "name": project.name}, + "algorithm": {"key": algorithm.key, "name": algorithm.name, "version": algorithm.version}, + "dimensions": DetectionEmbedding.objects.stored_length(algorithm.pk), + "rows": sum(counts.values()), + "classes": len(counts), + "counts": dict(sorted(counts.items(), key=lambda kv: -kv[1])), + "train": splits[training.SPLIT_TRAIN], + "test": splits[training.SPLIT_TEST], + "verified_detections_without_embedding": training.count_missing_embeddings(project, algorithm), + "settings": { + "split_salt": salt, + "test_fraction": fraction, + "split_grouped_by": "occurrence", + }, + } + ) diff --git a/config/api_router.py b/config/api_router.py index ebf5588f7..790acbc53 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -54,6 +54,7 @@ router.register(r"taxa", views.TaxonViewSet) router.register(r"tags", views.TagViewSet) router.register(r"ml/algorithms", ml_views.AlgorithmViewSet) +router.register(r"ml/training-data", ml_views.TrainingDataViewSet, basename="training-data") router.register(r"ml/labels", ml_views.AlgorithmCategoryMapViewSet) router.register(r"ml/pipelines", ml_views.PipelineViewSet) router.register(r"ml/processing_services", ml_views.ProcessingServiceViewSet) From ae5da963e8f33509accc7d05b04bd04406178ab6 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 8 Oct 2026 16:31:37 -0400 Subject: [PATCH 9/9] feat(ml): summarise what a retrain would learn from, for the job form The training job form asks for a split ratio and an optional occurrence set. Both questions are unanswerable without knowing what is there, so the summary endpoint now answers for exactly the run being set up. It takes the occurrence set, or reports every verified occurrence when none is chosen, which is what a run without a set does. It defaults the settings to the algorithm's own training config rather than to module defaults, so the first numbers someone sees are the numbers a run would use, and it accepts the settings as parameters so the form can preview a changed ratio without starting a job. The same bounds a job is held to are applied here, since stats computed under settings no run could use are worse than no stats. It also reports the classes that survive min_per_species, and names the species dropped: a raw class count overstates what the head would come out knowing. While adding the set parameter: the endpoint was gated on project visibility, which a non-draft project grants to everyone, so any account could read every verified label in a project together with the vector for each crop. It is now gated on the same permission as the training job. --- ami/ml/test_training.py | 100 ++++++++++++++++++++++ ami/ml/views.py | 181 ++++++++++++++++++++++++++++------------ 2 files changed, 226 insertions(+), 55 deletions(-) diff --git a/ami/ml/test_training.py b/ami/ml/test_training.py index 6554d5325..896e7fc7a 100644 --- a/ami/ml/test_training.py +++ b/ami/ml/test_training.py @@ -9,6 +9,8 @@ import uuid from django.test import TestCase +from django.urls import reverse +from rest_framework.test import APIClient from ami.main.models import ( Classification, @@ -29,6 +31,7 @@ from ami.ml.schemas import NamedReference from ami.ml.training import NotEnoughVerifiedData, build_training_dataset, verified_training_rows from ami.users.models import User +from ami.users.roles import ProjectManager VECTOR_LENGTH = 8 @@ -158,3 +161,100 @@ def test_the_vector_width_comes_from_the_stored_vectors(self): self.assertEqual(dataset["metadata"].dimensions, VECTOR_LENGTH) + +class TestTheSummaryTheFormShows(TrainingSetFixture): + """ + The numbers the training job form puts in front of someone before they start a run. + + A form that asked for a split ratio without saying how many occurrences it would split + would be asking blind, so these cover what changes when the set or the ratio changes. + """ + + def setUp(self) -> None: + super().setUp() + ProjectManager.assign_user(self.user, self.project) + self.client = APIClient() + self.client.force_authenticate(user=self.user) + self.url = reverse("api:training-data-summary") + + def _summary(self, **params): + response = self.client.get( + self.url, + {"project_id": self.project.pk, "algorithm": self.algorithm.key, **params}, + ) + self.assertEqual(response.status_code, 200, response.data) + return response.data + + def test_without_a_set_it_counts_every_verified_occurrence(self): + for taxon in self.taxa: + self.make_occurrence(taxon) + + summary = self._summary() + + self.assertIsNone(summary["occurrence_set"]) + self.assertEqual(summary["occurrences"], 2) + self.assertEqual(summary["rows"], 2) + + def test_a_set_narrows_the_counts_and_is_named_back(self): + mine = self.make_set("Mine", [self.make_occurrence(self.taxa[0])]) + self.make_occurrence(self.taxa[1]) + + summary = self._summary(occurrence_set=mine.pk) + + self.assertEqual(summary["occurrence_set"], {"id": mine.pk, "name": "Mine"}) + self.assertEqual(summary["occurrences"], 1) + + def test_a_set_from_another_project_is_not_counted(self): + other = Project.objects.create(name="Someone else", owner=self.user) + theirs = OccurrenceSet.objects.create(name="Theirs") + theirs.projects.add(other) + + response = self.client.get( + self.url, + {"project_id": self.project.pk, "algorithm": self.algorithm.key, "occurrence_set": theirs.pk}, + ) + + self.assertEqual(response.status_code, 404) + + def test_the_split_ratio_moves_the_counts(self): + for taxon in self.taxa: + self.make_occurrence(taxon) + self.make_occurrence(taxon) + + everything_held_out = self._summary(test_fraction=0.99) + + self.assertEqual(everything_held_out["test"], everything_held_out["rows"]) + self.assertEqual(everything_held_out["train"], 0) + self.assertEqual(everything_held_out["settings"]["test_fraction"], 0.99) + + def test_a_ratio_no_run_could_use_is_refused(self): + response = self.client.get( + self.url, + {"project_id": self.project.pk, "algorithm": self.algorithm.key, "test_fraction": 1.5}, + ) + + self.assertEqual(response.status_code, 400) + + def test_species_without_enough_crops_are_reported_as_dropped(self): + self.make_occurrence(self.taxa[0]) + self.make_occurrence(self.taxa[1]) + self.make_occurrence(self.taxa[1]) + + summary = self._summary(min_per_species=2) + + self.assertEqual(summary["classes"], 2) + self.assertEqual(summary["trainable_classes"], 1) + self.assertEqual(summary["dropped_species"], [self.taxa[0].name]) + + def test_someone_who_cannot_retrain_cannot_read_the_training_set(self): + """ + A non-draft project is readable by anyone, so visibility alone would hand every + verified label and vector to any account. The permission is the gate. + """ + self.make_occurrence(self.taxa[0]) + outsider = User.objects.create_user(email="outsider@example.com", password="testpass123") + self.client.force_authenticate(user=outsider) + + response = self.client.get(self.url, {"project_id": self.project.pk, "algorithm": self.algorithm.key}) + + self.assertEqual(response.status_code, 403) diff --git a/ami/ml/views.py b/ami/ml/views.py index 556997951..516aa3444 100644 --- a/ami/ml/views.py +++ b/ami/ml/views.py @@ -1,5 +1,6 @@ import logging +import pydantic from django.db import transaction from django.db.models import Prefetch from django.db.models.query import QuerySet @@ -19,9 +20,9 @@ from ami.base.views import ProjectMixin from ami.main.api.schemas import project_id_doc_param from ami.main.api.views import DefaultViewSet -from ami.main.models import Project, SourceImage +from ami.main.models import OccurrenceSet, Project, SourceImage from ami.ml import training -from ami.ml.schemas import PipelineRegistrationResponse +from ami.ml.schemas import AlgorithmTrainingConfig, PipelineRegistrationResponse from .models.algorithm import Algorithm, AlgorithmCategoryMap from .models.embedding import DetectionEmbedding @@ -317,71 +318,119 @@ class TrainingDataViewSet(ProjectMixin, mixins.ListModelMixin, viewsets.GenericV Requires `project_id` and `algorithm` (an algorithm key). Constraining to one algorithm is not optional: vectors from different backbones are in different spaces. + `occurrence_set` narrows this to one saved set. Without it the rows are every verified + occurrence in the project, which is what a retrain does by default. + GET /api/v2/ml/training-data/?project_id=3&algorithm= - GET /api/v2/ml/training-data/summary/?project_id=3&algorithm= + GET /api/v2/ml/training-data/summary/?project_id=3&algorithm=&occurrence_set=7 """ queryset = DetectionEmbedding.objects.none() serializer_class = TrainingDataRowSerializer require_project = True - # Membership is enforced in get_queryset() via Project.objects.visible_for_user(): - # ObjectPermission maps a "list" action on a Project to check_custom_permission, which - # denies members, and IsAuthenticated alone would let any account read any project's - # verified labels. + # The project permission is checked per request, in _get_project_for_training(). + # ObjectPermission's has_object_permission only runs for detail actions, and both + # actions here are detail=False, so a viewset-level class would never see the project. permission_classes = [IsAuthenticated] filter_backends: list = [] pagination_class = TrainingDataPagination - def _get_visible_project(self) -> Project: + def _get_project_for_training(self) -> Project: """ - The requested project, if this user is allowed to see it. + The requested project, if this user may retrain its classifier. - Verified labels are project data, so an account that cannot see the project must - not be able to read them. + Project visibility is not enough of a gate here. A non-draft project is readable by + anyone, and what this returns is every verified label in it together with the vector + for each crop: the whole training set, not a page of occurrences. It is gated on the + same permission the training job is, so the people who can start a run are the ones + who can see what it would learn from. """ project = self.get_active_project() if not project: raise Http404("Project not found.") - visible = Project.objects.visible_for_user(self.request.user).filter(pk=project.pk).exists() - if not visible: - raise api_exceptions.PermissionDenied("You do not have access to this project.") + if not self.request.user.has_perm(Project.Permissions.RUN_TRAIN_CLASSIFIER_JOB, project): + raise api_exceptions.PermissionDenied("You do not have permission to retrain classifiers in this project.") return project def _get_algorithm(self) -> Algorithm: - key = SingleParamSerializer[str].clean( - "algorithm", - serializers.CharField( - required=True, - help_text="Key of the algorithm whose embeddings to train on.", - ), - self.request.query_params, - ) - algorithm = Algorithm.objects.filter(key=key).first() - if not algorithm: - raise api_exceptions.NotFound(f"No algorithm with key '{key}'.") - return algorithm - - def _get_split_settings(self) -> tuple[str, float]: - salt = SingleParamSerializer[str].clean( - "split_salt", - serializers.CharField(required=False, default=training.DEFAULT_SPLIT_SALT), + # Cached because the split settings fall back to this algorithm's own training + # config, so one request asks for it from more than one place. + if getattr(self, "_algorithm", None) is None: + key = SingleParamSerializer[str].clean( + "algorithm", + serializers.CharField( + required=True, + help_text="Key of the algorithm whose embeddings to train on.", + ), + self.request.query_params, + ) + algorithm = Algorithm.objects.filter(key=key).first() + if not algorithm: + raise api_exceptions.NotFound(f"No algorithm with key '{key}'.") + self._algorithm = algorithm + return self._algorithm + + def _get_occurrence_set(self, project: Project) -> OccurrenceSet | None: + """ + The set to count, or None for every verified occurrence in the project. + + Optional, because retraining on everything verified so far is the common case and + the form shows those numbers before anyone has chosen a set. + """ + occurrence_set_id = SingleParamSerializer[int].clean( + "occurrence_set", + serializers.IntegerField(required=False, allow_null=True, default=None), self.request.query_params, ) - fraction = SingleParamSerializer[float].clean( - "test_fraction", - serializers.FloatField( - required=False, - default=training.DEFAULT_TEST_FRACTION, - min_value=0, - max_value=0.99, + if not occurrence_set_id: + return None + # Scoped to the project: a set id from elsewhere must not report this project's + # verified labels, or any other project's. + occurrence_set = OccurrenceSet.objects.for_project(project).filter(pk=occurrence_set_id).first() + if not occurrence_set: + raise api_exceptions.NotFound(f"No occurrence set with id {occurrence_set_id} in this project.") + return occurrence_set + + def _get_split_settings(self) -> AlgorithmTrainingConfig: + """ + The settings these numbers are computed under. + + Defaulted to the algorithm's own training config rather than to the module defaults, + so a form that shows the stats before anything is edited shows what an actual run + would do. Anything the caller passes wins, which is how the form previews a changed + split ratio without starting a job. + """ + config = self._get_algorithm().training_config + params = self.request.query_params + overrides = { + "split_salt": SingleParamSerializer[str].clean( + "split_salt", + serializers.CharField(required=False, allow_null=True, default=None), + params, ), - self.request.query_params, - ) - return salt, fraction + "test_fraction": SingleParamSerializer[float].clean( + "test_fraction", + serializers.FloatField(required=False, allow_null=True, default=None), + params, + ), + "min_per_species": SingleParamSerializer[int].clean( + "min_per_species", + serializers.IntegerField(required=False, allow_null=True, default=None), + params, + ), + } + # A parameter that was not sent must not override the config with None. + overrides = {name: value for name, value in overrides.items() if value is not None} + try: + return AlgorithmTrainingConfig.for_run(config, overrides) + except pydantic.ValidationError as e: + # The same bounds a job is held to, refused here rather than reported as stats + # computed under settings no run could use. + raise api_exceptions.ValidationError({"detail": e.errors()}) def get_queryset(self) -> QuerySet[DetectionEmbedding]: - project = self._get_visible_project() - qs = training.verified_training_rows(project, self._get_algorithm()) + project = self._get_project_for_training() + qs = training.verified_training_rows(project, self._get_algorithm(), self._get_occurrence_set(project)) self._split_filter = SingleParamSerializer[str].clean( "split", @@ -392,9 +441,9 @@ def get_queryset(self) -> QuerySet[DetectionEmbedding]: def get_serializer_context(self): context = super().get_serializer_context() - salt, fraction = self._get_split_settings() - context["split_salt"] = salt - context["test_fraction"] = fraction + config = self._get_split_settings() + context["split_salt"] = config.split_salt + context["test_fraction"] = config.test_fraction # Not url_boolean_param: it returns `value or default`, so a default of True can # never be turned off. context["include_features"] = SingleParamSerializer[bool].clean( @@ -419,31 +468,53 @@ def list(self, request, *args, **kwargs): @extend_schema(parameters=[project_id_doc_param]) @action(detail=False, methods=["get"]) def summary(self, request, *args, **kwargs): - """Counts only. Cheap enough to poll before deciding whether a retrain is worth it.""" - project = self._get_visible_project() + """ + Counts only, for what a retrain would learn from right now. + + This is what the training job form shows before anyone starts a run: how much + verified data there is, how many species of it are usable, and how the split would + fall. Cheap enough to re-fetch whenever the chosen set or the split ratio changes. + """ + project = self._get_project_for_training() algorithm = self._get_algorithm() - salt, fraction = self._get_split_settings() + occurrence_set = self._get_occurrence_set(project) + config = self._get_split_settings() + + counts = training.label_counts(project, algorithm, occurrence_set) + kept = training.species_with_enough_examples(counts, config.min_per_species) + rows = training.verified_training_rows(project, algorithm, occurrence_set) - counts = training.label_counts(project, algorithm) - rows = training.verified_training_rows(project, algorithm) splits = {name: 0 for name in training.SPLITS} + occurrences = set() for occurrence_id in rows.values_list("detection__occurrence_id", flat=True): - splits[training.split_for(occurrence_id, salt, fraction)] += 1 + splits[training.split_for(occurrence_id, config.split_salt, config.test_fraction)] += 1 + occurrences.add(occurrence_id) return Response( { "project": {"id": project.pk, "name": project.name}, "algorithm": {"key": algorithm.key, "name": algorithm.name, "version": algorithm.version}, + # None means every verified occurrence in the project, which is what a run + # without a set does. + "occurrence_set": ({"id": occurrence_set.pk, "name": occurrence_set.name} if occurrence_set else None), "dimensions": DetectionEmbedding.objects.stored_length(algorithm.pk), "rows": sum(counts.values()), + "occurrences": len(occurrences), "classes": len(counts), + # What the head would actually learn, once species with too few crops are + # dropped. Showing only the raw class count overstates the run. + "trainable_classes": len(kept), + "dropped_species": sorted(name for name in counts if name and name not in kept), "counts": dict(sorted(counts.items(), key=lambda kv: -kv[1])), "train": splits[training.SPLIT_TRAIN], "test": splits[training.SPLIT_TEST], - "verified_detections_without_embedding": training.count_missing_embeddings(project, algorithm), + "verified_detections_without_embedding": training.count_missing_embeddings( + project, algorithm, occurrence_set + ), "settings": { - "split_salt": salt, - "test_fraction": fraction, + "min_per_species": config.min_per_species, + "split_salt": config.split_salt, + "test_fraction": config.test_fraction, "split_grouped_by": "occurrence", }, }