diff --git a/ami/base/permissions.py b/ami/base/permissions.py index 328aa8384..af2636c94 100644 --- a/ami/base/permissions.py +++ b/ami/base/permissions.py @@ -198,6 +198,35 @@ def has_permission(self, request, view): return super().has_permission(request, view) +class OccurrenceSetPermission(ObjectPermission): + """ + Permission for the occurrence-set route, where list and create have no object yet. + + Listing follows the project: anyone who may see the project may see which sets it has, + since only a set's name and size are exposed. Creating one is gated on the project's + ``create_occurrenceset`` permission. Everything else falls through to the object check, + which refuses a global set because it has no project to check against. + """ + + def has_permission(self, request, view): + from ami.main.models import Project + + if view.action == "list": + # The project id is required for this action, so reaching here means it was + # given and the caller may see it. + return view.get_active_project() is not None + + if view.action == "create": + # Read from the payload rather than the query string: the set names the project + # it will belong to, and that is the project whose permission must allow it. + project = Project.objects.filter(pk=request.data.get("project_id")).first() + if not project: + return False + return request.user.has_perm(Project.Permissions.CREATE_OCCURRENCE_SET, project) + + return super().has_permission(request, view) + + class UserMembershipPermission(ObjectPermission): """ Custom permission for UserProjectMembershipViewSet. diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 1ca588980..5611106ff 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -26,6 +26,7 @@ Event, Identification, Occurrence, + OccurrenceSet, Page, Project, ProjectSettingsMixin, @@ -1418,6 +1419,59 @@ def to_representation(self, instance): return {key: value for key, value in data.items() if value is not None} +class OccurrenceSetSerializer(DefaultSerializer): + """A fixed list of occurrences. ``occurrence_ids`` is accepted only when it is created.""" + + project_id = serializers.PrimaryKeyRelatedField(queryset=Project.objects.all(), write_only=True) + occurrence_ids = serializers.PrimaryKeyRelatedField( + queryset=Occurrence.objects.all(), many=True, write_only=True, allow_empty=False + ) + occurrences_count = serializers.SerializerMethodField() + + class Meta: + model = OccurrenceSet + fields = [ + "id", + "details", + "name", + "description", + "project_id", + "occurrence_ids", + "occurrences_count", + "created_at", + "updated_at", + ] + + def get_occurrences_count(self, obj) -> int: + return getattr(obj, "annotated_occurrences_count", None) or obj.occurrences.count() + + def validate(self, attrs): + # Scores recorded against a set were measured on these occurrences, so moving them + # would change what those numbers mean. Another set is the way to change the list. + if self.instance is not None and "occurrence_ids" in attrs: + raise serializers.ValidationError( + {"occurrence_ids": "A set's occurrences cannot be changed. Create another set instead."} + ) + + project = attrs.get("project_id") + occurrences = attrs.get("occurrence_ids") + if project and occurrences: + outside = sorted(o.pk for o in occurrences if o.project_id != project.pk) + if outside: + raise serializers.ValidationError( + {"occurrence_ids": f"Not occurrences in this project: {outside[:10]}"} + ) + return attrs + + def create(self, validated_data): + project = validated_data.pop("project_id") + occurrences = validated_data.pop("occurrence_ids") + occurrence_set = OccurrenceSet.objects.create(**validated_data) + occurrence_set.projects.add(project) + occurrence_set.occurrences.set(occurrences) + return occurrence_set + + class SourceImageCollectionSerializer(DefaultSerializer): source_images = serializers.SerializerMethodField() kwargs = SourceImageCollectionCommonKwargsSerializer(required=False, partial=True) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 99709eb29..40abdef53 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -29,7 +29,12 @@ from ami.base.metadata import ResponseSchemaMetadata from ami.base.models import BaseQuerySet from ami.base.pagination import LimitOffsetPaginationWithPermissions -from ami.base.permissions import IsActiveStaffOrReadOnly, IsProjectMemberOrReadOnly, ObjectPermission +from ami.base.permissions import ( + IsActiveStaffOrReadOnly, + IsProjectMemberOrReadOnly, + ObjectPermission, + OccurrenceSetPermission, +) from ami.base.serializers import FilterParamsSerializer, SingleParamSerializer from ami.base.views import ProjectMixin from ami.main.api.schemas import limit_doc_param, project_id_doc_param @@ -49,6 +54,7 @@ Event, Identification, Occurrence, + OccurrenceSet, Page, Project, ProjectQuerySet, @@ -84,6 +90,7 @@ ModelAgreementSerializer, OccurrenceListSerializer, OccurrenceSerializer, + OccurrenceSetSerializer, PageListSerializer, PageSerializer, ProjectListSerializer, @@ -953,6 +960,74 @@ class ChoicesPagination(LimitOffsetPaginationWithPermissions): max_limit = 100 +class OccurrenceSetViewSet(DefaultViewSet, ProjectMixin): + """ + Fixed lists of occurrences, so the same data can be used again later. + + Membership is decided when a set is created and cannot be changed afterwards: there is + no endpoint to add or remove an occurrence. Anything that compares results over time + depends on the list standing still, and a set that grows quietly makes every number + recorded against it mean something different. + + A set belonging to no project is global and is offered to every project, but it cannot + be edited here, because it has no single project whose permissions would govern it. + """ + + queryset = OccurrenceSet.objects.all() + serializer_class = OccurrenceSetSerializer + permission_classes = [OccurrenceSetPermission] + # No put: a full replace would include the occurrences. + http_method_names = ["get", "post", "patch", "delete", "head", "options"] + ordering_fields = ["name", "created_at", "updated_at"] + search_fields = ["name"] + ordering = ["name"] + # Listing without a project would answer with every set on the platform, so it is + # required there. A detail route names one set, which carries its own project, and the + # create payload names the project it belongs to. + require_project = False + require_project_for_list = True + + def create(self, request, *args, **kwargs): + """ + Create without the base class's unsaved-instance permission check. + + ``DefaultViewSet.create`` builds ``Model(**validated_data)`` to check object + permissions before saving. A set's project and its occurrences arrive as payload + fields rather than columns, so that instance cannot be built. The same check runs + earlier instead, in ``OccurrenceSetPermission``, against the project the payload + names. + """ + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + + @extend_schema(parameters=[project_id_doc_param], responses=OccurrenceSetSerializer(many=True)) + @action(detail=False, methods=["get"], name="choices") + def choices(self, request: Request) -> Response: + """ + Choices for the occurrence-set filter and pickers. + + Follows SourceImageCollectionViewSet.choices: most recently updated first and + enough of them that a dropdown never has to page. + """ + self.ordering_fields = ["id", "created_at", "updated_at", "name"] + queryset = self.filter_queryset(self.get_queryset()) + paginator = ChoicesPagination() + page = paginator.paginate_queryset(queryset, request, view=self) + serializer = self.get_serializer(page, many=True) + return paginator.get_paginated_response(serializer.data) + + def get_queryset(self) -> QuerySet["OccurrenceSet"]: + qs = super().get_queryset().annotate(annotated_occurrences_count=models.Count("occurrences")) + project = self.get_active_project() + if project is not None: + # A set with no project is global, so it is offered everywhere. + return qs.for_project(project) + return qs.visible_for_user(self.request.user) + + class SourceImageCollectionViewSet(DefaultViewSet, ProjectMixin): """ Endpoint for viewing capture sets or samples of captures. @@ -1495,6 +1570,9 @@ class OccurrenceFilterSet(FilterSet): """ detections__source_image = RelatedIdFilter() + # Named for what it means to a reader rather than for the reverse accessor, which is + # called evaluation_sets because scoring was the first thing to use one. + occurrence_set = RelatedIdFilter(field_name="evaluation_sets") class Meta: model = Occurrence diff --git a/ami/main/migrations/0098_occurrence_set.py b/ami/main/migrations/0098_occurrence_set.py new file mode 100644 index 000000000..26153b846 --- /dev/null +++ b/ami/main/migrations/0098_occurrence_set.py @@ -0,0 +1,102 @@ +# Generated by Django 4.2.10 on 2026-10-08 13:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0097_detection_and_classification_job_indexes"), + ] + + operations = [ + migrations.AlterModelOptions( + name="project", + options={ + "ordering": ["-priority", "created_at"], + "permissions": [ + ("create_identification", "Can create identifications"), + ("update_identification", "Can update identifications"), + ("delete_identification", "Can delete identifications"), + ("create_job", "Can create a job"), + ("update_job", "Can update a job"), + ("run_ml_job", "Can run/retry/cancel ML jobs"), + ("run_populate_captures_collection_job", "Can run/retry/cancel Populate Collection jobs"), + ("run_data_storage_sync_job", "Can run/retry/cancel Data Storage Sync jobs"), + ("run_regroup_events_job", "Can run/retry/cancel Regroup Events jobs"), + ("run_data_export_job", "Can run/retry/cancel Data Export jobs"), + ("run_single_image_ml_job", "Can process a single capture"), + ("run_post_processing_job", "Can run/retry/cancel Post-Processing jobs"), + ("delete_job", "Can delete a job"), + ("create_deployment", "Can create a deployment"), + ("delete_deployment", "Can delete a deployment"), + ("update_deployment", "Can update a deployment"), + ("sync_deployment", "Can sync images to a deployment"), + ("regroup_sessions_deployment", "Can regroup deployment captures into sessions"), + ("create_occurrenceset", "Can create an occurrence set"), + ("update_occurrenceset", "Can rename or describe an occurrence set"), + ("delete_occurrenceset", "Can delete an occurrence set"), + ("create_sourceimagecollection", "Can create a collection"), + ("update_sourceimagecollection", "Can update a collection"), + ("delete_sourceimagecollection", "Can delete a collection"), + ("populate_sourceimagecollection", "Can populate a collection"), + ("create_sourceimage", "Can create a source image"), + ("update_sourceimage", "Can update a source image"), + ("delete_sourceimage", "Can delete a source image"), + ("star_sourceimage", "Can star a source image"), + ("create_sourceimageupload", "Can create a source image upload"), + ("update_sourceimageupload", "Can update a source image upload"), + ("delete_sourceimageupload", "Can delete a source image upload"), + ("create_s3storagesource", "Can create storage"), + ("delete_s3storagesource", "Can delete storage"), + ("update_s3storagesource", "Can update storage"), + ("test_s3storagesource", "Can test storage connection"), + ("create_site", "Can create a site"), + ("delete_site", "Can delete a site"), + ("update_site", "Can update a site"), + ("create_device", "Can create a device"), + ("delete_device", "Can delete a device"), + ("update_device", "Can update a device"), + ("view_userprojectmembership", "Can view project members"), + ("create_userprojectmembership", "Can add a user to the project"), + ("update_userprojectmembership", "Can update a user's project membership and role in the project"), + ("delete_userprojectmembership", "Can remove a user from the project"), + ("create_dataexport", "Can create a data export"), + ("update_dataexport", "Can update a data export"), + ("delete_dataexport", "Can delete a data export"), + ("create_projectpipelineconfig", "Can register pipelines for the project"), + ("update_projectpipelineconfig", "Can update pipeline configurations"), + ("delete_projectpipelineconfig", "Can remove pipelines from the project"), + ("create_taxalist", "Can create a taxa list"), + ("update_taxalist", "Can update a taxa list"), + ("delete_taxalist", "Can delete a taxa list"), + ("view_private_data", "Can view private data"), + ], + }, + ), + migrations.CreateModel( + name="OccurrenceSet", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("name", models.CharField(max_length=255)), + ("description", models.TextField(blank=True)), + ( + "occurrences", + models.ManyToManyField(blank=True, related_name="evaluation_sets", to="main.occurrence"), + ), + ( + "projects", + models.ManyToManyField( + blank=True, + help_text="Projects this set belongs to. A set with none is available everywhere.", + related_name="occurrence_sets", + to="main.project", + ), + ), + ], + options={ + "ordering": ["name"], + }, + ), + ] diff --git a/ami/main/migrations/0099_grant_occurrence_set_permissions.py b/ami/main/migrations/0099_grant_occurrence_set_permissions.py new file mode 100644 index 000000000..f69e2fcda --- /dev/null +++ b/ami/main/migrations/0099_grant_occurrence_set_permissions.py @@ -0,0 +1,98 @@ +""" +Grant the new occurrence-set permissions to the roles that already curate a project's data. + +``create_roles_for_project`` gives new projects their role permissions, and a +``post_migrate`` signal re-syncs every project on migrate, so this backfill is belt and +braces. It is kept so the grant is explicit rather than a side effect of a signal, and so +the reverse is written down. + +Guardian reads **object-level** rows: ``get_perms(user, project)`` and +``user.has_perm("create_occurrenceset", project)`` both look at +``GroupObjectPermission``. Adding the permission to ``group.permissions`` alone would not +show up there, so each project gets a row per role group. + +``ProjectManager`` inherits ``MLDataManager``'s permissions, so both groups are covered. +""" + +from django.db import migrations +from django.db.models import Q + +PERMISSIONS = ("create_occurrenceset", "update_occurrenceset", "delete_occurrenceset") +ROLE_GROUP_SUFFIXES = ("_MLDataManager", "_ProjectManager") + + +def _permissions(apps): + Permission = apps.get_model("auth", "Permission") + ContentType = apps.get_model("contenttypes", "ContentType") + try: + project_ct = ContentType.objects.get(app_label="main", model="project") + except ContentType.DoesNotExist: + return [], None + found = list(Permission.objects.filter(codename__in=PERMISSIONS, content_type=project_ct)) + return found, project_ct + + +def _project_pk_from_group(group): + # Group names are "{project_pk}_{project_name}_{RoleName}"; the pk is the immutable + # leading segment, since a project name may itself contain underscores. + try: + return int(group.name.split("_", 1)[0]) + except (ValueError, IndexError): + return None + + +def _role_groups(apps): + Group = apps.get_model("auth", "Group") + query = Q() + for suffix in ROLE_GROUP_SUFFIXES: + query |= Q(name__endswith=suffix) + return Group.objects.filter(query) + + +def grant(apps, schema_editor): + GroupObjectPermission = apps.get_model("guardian", "GroupObjectPermission") + permissions, project_ct = _permissions(apps) + if not permissions: + return + + for group in _role_groups(apps): + project_pk = _project_pk_from_group(group) + if project_pk is None: + continue + for permission in permissions: + group.permissions.add(permission) + GroupObjectPermission.objects.get_or_create( + permission=permission, + content_type=project_ct, + object_pk=str(project_pk), + group=group, + ) + + +def revoke(apps, schema_editor): + GroupObjectPermission = apps.get_model("guardian", "GroupObjectPermission") + permissions, project_ct = _permissions(apps) + if not permissions: + return + + for group in _role_groups(apps): + project_pk = _project_pk_from_group(group) + if project_pk is None: + continue + for permission in permissions: + group.permissions.remove(permission) + GroupObjectPermission.objects.filter( + permission=permission, + content_type=project_ct, + object_pk=str(project_pk), + group=group, + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0098_occurrence_set"), + ("guardian", "0002_generic_permissions_index"), + ] + + operations = [migrations.RunPython(grant, revoke)] diff --git a/ami/main/models.py b/ami/main/models.py index a4c0c82f1..03959c548 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -461,6 +461,11 @@ class Permissions: SYNC_DEPLOYMENT = "sync_deployment" REGROUP_SESSIONS_DEPLOYMENT = "regroup_sessions_deployment" + # Occurrence set permissions + CREATE_OCCURRENCE_SET = "create_occurrenceset" + UPDATE_OCCURRENCE_SET = "update_occurrenceset" + DELETE_OCCURRENCE_SET = "delete_occurrenceset" + # Collection permissions CREATE_COLLECTION = "create_sourceimagecollection" UPDATE_COLLECTION = "update_sourceimagecollection" @@ -543,6 +548,9 @@ class Meta: ("sync_deployment", "Can sync images to a deployment"), ("regroup_sessions_deployment", "Can regroup deployment captures into sessions"), # Collection permissions + ("create_occurrenceset", "Can create an occurrence set"), + ("update_occurrenceset", "Can rename or describe an occurrence set"), + ("delete_occurrenceset", "Can delete an occurrence set"), ("create_sourceimagecollection", "Can create a collection"), ("update_sourceimagecollection", "Can update a collection"), ("delete_sourceimagecollection", "Can delete a collection"), @@ -4912,6 +4920,74 @@ def html(self) -> str: ] +class OccurrenceSetQuerySet(BaseQuerySet): + def for_project(self, project) -> models.QuerySet: + """Sets this project can use: its own, plus any that belong to no project.""" + return self.filter(models.Q(projects=project) | models.Q(projects__isnull=True)).distinct() + + def visible_for_user(self, user) -> models.QuerySet: + """ + Global sets stay visible; project sets follow their project. + + The inherited filter keeps a row only if it reaches a non-draft project, which a + set belonging to no project never does. Without this a global set is hidden from + everyone but a superuser, which is the opposite of what global means. Only a set's + name and size are exposed, never the occurrences inside it. + """ + visible = super().visible_for_user(user) + return self.filter(models.Q(pk__in=visible.values("pk")) | models.Q(projects__isnull=True)).distinct() + + +@final +class OccurrenceSet(BaseModel): + """ + A fixed list of occurrences, kept so the same ones can be used again later. + + Membership is stored rather than described by a filter, because a filter answers + differently as data arrives. Anything that compares results over time needs the list to + stand still: scoring two classifiers is only meaningful if both saw the same + occurrences. + + A set with no projects is global, which is how one set is shared across the platform; + that follows how TaxaList already treats a list with no project. + """ + + name = models.CharField(max_length=255) + description = models.TextField(blank=True) + occurrences = models.ManyToManyField("main.Occurrence", related_name="evaluation_sets", blank=True) + projects = models.ManyToManyField( + "main.Project", + related_name="occurrence_sets", + blank=True, + help_text="Projects this set belongs to. A set with none is available everywhere.", + ) + + objects = OccurrenceSetQuerySet.as_manager() + + class Meta: + ordering = ["name"] + + def __str__(self) -> str: + return f"{self.name} ({self.occurrences.count()} occurrences)" + + @property + def is_global(self) -> bool: + return not self.projects.exists() + + def get_project(self): + """ + The project whose permissions govern this set. + + BaseModel returns None for a many-to-many project relation, and + ``check_permission`` refuses every action without a project, so a global set cannot + be renamed or deleted through the API at all. That is the behaviour we want: a set + shared across projects changing under one of them would quietly change what every + other project's stored scores were measured on. Global sets stay a deliberate, + out-of-band act. + """ + return self.projects.first() + + class SourceImageCollectionQuerySet(BaseQuerySet): def with_source_images_count(self): return self.annotate( diff --git a/ami/main/test_occurrence_sets.py b/ami/main/test_occurrence_sets.py new file mode 100644 index 000000000..ad84c7c58 --- /dev/null +++ b/ami/main/test_occurrence_sets.py @@ -0,0 +1,254 @@ +""" +Occurrence sets: a fixed list of occurrences, decided once. + +The point of the model is that the list does not move. Anything comparing results over +time — scoring two classifiers, re-running an export — is only meaningful if both runs saw +the same occurrences. So these tests are mostly about what the API refuses. +""" + +import datetime +import uuid + +from rest_framework import status +from rest_framework.test import APITestCase + +from ami.main.models import ( + Classification, + Deployment, + Detection, + Event, + Occurrence, + OccurrenceSet, + Project, + S3StorageSource, + SourceImage, + Taxon, + TaxonRank, +) +from ami.users.models import User +from ami.users.roles import BasicMember, MLDataManager + + +class OccurrenceSetFixture(APITestCase): + """ + Occurrences are created directly rather than through the capture fixtures. + + These tests are about what the endpoint accepts and refuses, and need nothing from a + detection. Going through create_captures/create_occurrences pulls in event grouping and + a storage read for image dimensions, neither of which works in a test, and the failures + land at random on whichever test ran first. + """ + + def setUp(self) -> None: + self.owner = User.objects.create_user(email="owner@example.com", password="testpass123") + self.curator = User.objects.create_user(email="curator@example.com", password="testpass123") + self.member = User.objects.create_user(email="member@example.com", password="testpass123") + + self.project = Project.objects.create(name="Set Project", owner=self.owner) + MLDataManager.assign_user(self.curator, self.project) + BasicMember.assign_user(self.member, self.project) + + self.occurrences = [self.make_occurrence(self.project) for _ in range(3)] + self.url = "/api/v2/occurrences/sets/" + + @staticmethod + def make_occurrence(project: Project) -> Occurrence: + """ + One occurrence the list endpoint will actually return. + + Built here rather than through the capture fixtures, which time their images from + the current clock and group them into events by reading image dimensions from + storage. Neither works in a test, and the failures land at random on whichever test + ran first. The list only shows occurrences with a real detection and a + determination, so both are set. + """ + storage = S3StorageSource.objects.create(name=f"source {uuid.uuid4().hex[:6]}", project=project, bucket="test") + deployment = Deployment.objects.create( + name=f"station {uuid.uuid4().hex[:6]}", project=project, data_source=storage + ) + night = datetime.datetime(2026, 6, 1, 22, 0, 0, tzinfo=datetime.timezone.utc) + event = Event.objects.create(deployment=deployment, project=project, start=night) + image = SourceImage.objects.create( + deployment=deployment, + project=project, + event=event, + timestamp=night, + path=f"test/{uuid.uuid4().hex[:8]}.jpg", + ) + taxon = Taxon.objects.create(name=f"Testus {uuid.uuid4().hex[:6]}", rank=TaxonRank.SPECIES.name) + taxon.projects.add(project) + detection = Detection.objects.create(source_image=image, timestamp=night, bbox=[0.1, 0.1, 0.2, 0.2]) + Classification.objects.create(detection=detection, taxon=taxon, score=0.9, timestamp=night) + occurrence = detection.associate_new_occurrence() + occurrence.refresh_from_db() + return occurrence + + def _create(self, user, **overrides): + payload = { + "name": "Blind set", + "description": "", + "project_id": self.project.pk, + "occurrence_ids": [o.pk for o in self.occurrences], + } + payload.update(overrides) + self.client.force_authenticate(user=user) + return self.client.post(self.url, payload, format="json") + + +class TestCreatingASet(OccurrenceSetFixture): + def test_a_curator_can_create_one(self): + response = self._create(self.curator) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.data) + created = OccurrenceSet.objects.get(pk=response.data["id"]) + self.assertEqual(created.occurrences.count(), 3) + self.assertEqual(list(created.projects.all()), [self.project]) + + def test_a_member_without_the_permission_cannot(self): + response = self._create(self.member) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertFalse(OccurrenceSet.objects.exists()) + + def test_an_anonymous_caller_cannot(self): + response = self._create(None) + + self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) + self.assertFalse(OccurrenceSet.objects.exists()) + + def test_an_occurrence_from_another_project_is_refused(self): + """Otherwise a set quietly reaches into data its project cannot see.""" + other = Project.objects.create(name="Other Project", owner=self.owner) + outsider = self.make_occurrence(other) + + response = self._create(self.curator, occurrence_ids=[self.occurrences[0].pk, outsider.pk]) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("occurrence_ids", response.data) + self.assertFalse(OccurrenceSet.objects.exists()) + + def test_an_empty_set_is_refused(self): + response = self._create(self.curator, occurrence_ids=[]) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + +class TestMembershipDoesNotMove(OccurrenceSetFixture): + def setUp(self) -> None: + super().setUp() + self.set = OccurrenceSet.objects.create(name="Blind set") + self.set.projects.add(self.project) + self.set.occurrences.add(*self.occurrences) + self.detail_url = f"{self.url}{self.set.pk}/" + self.client.force_authenticate(user=self.curator) + + def test_the_occurrences_cannot_be_changed(self): + response = self.client.patch(self.detail_url, {"occurrence_ids": [self.occurrences[0].pk]}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(self.set.occurrences.count(), 3) + + def test_the_name_and_description_can_be_changed(self): + response = self.client.patch(self.detail_url, {"name": "Renamed", "description": "why"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_200_OK, response.data) + self.set.refresh_from_db() + self.assertEqual(self.set.name, "Renamed") + + def test_there_is_no_put(self): + """A full replace would carry the occurrences with it.""" + response = self.client.put(self.detail_url, {"name": "Replaced"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED) + + def test_a_curator_can_delete_one(self): + response = self.client.delete(self.detail_url) + + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertFalse(OccurrenceSet.objects.filter(pk=self.set.pk).exists()) + + +class TestFilteringOccurrencesBySet(OccurrenceSetFixture): + """The occurrence list can be narrowed to one set, which is how a set is reviewed.""" + + def setUp(self) -> None: + super().setUp() + self.set = OccurrenceSet.objects.create(name="Blind set") + self.set.projects.add(self.project) + self.set.occurrences.add(self.occurrences[0], self.occurrences[1]) + self.client.force_authenticate(user=self.curator) + + def _ids(self, **params): + # apply_defaults=false: the list hides occurrences without a determination or a + # score by default, and these fixtures are bare rows made to exercise the filter. + response = self.client.get( + "/api/v2/occurrences/", + {"project_id": self.project.pk, "apply_defaults": "false", **params}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK, response.data) + return {row["id"] for row in response.data["results"]} + + def test_only_the_sets_occurrences_come_back(self): + self.assertEqual( + self._ids(occurrence_set=self.set.pk), + {self.occurrences[0].pk, self.occurrences[1].pk}, + ) + + def test_without_the_filter_every_occurrence_comes_back(self): + self.assertEqual(self._ids(), {o.pk for o in self.occurrences}) + + +class TestGlobalSets(OccurrenceSetFixture): + def setUp(self) -> None: + super().setUp() + self.global_set = OccurrenceSet.objects.create(name="Platform-wide set") + self.client.force_authenticate(user=self.curator) + + def _names(self, response): + return [row["name"] for row in response.data["results"]] + + def test_a_global_set_is_offered_to_every_project(self): + response = self.client.get(self.url, {"project_id": self.project.pk}) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn(self.global_set.name, self._names(response)) + + def test_a_global_set_cannot_be_edited_through_the_api(self): + """It has no single project whose permissions would govern it.""" + response = self.client.patch(f"{self.url}{self.global_set.pk}/", {"name": "Mine now"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.global_set.refresh_from_db() + self.assertEqual(self.global_set.name, "Platform-wide set") + + +class TestListing(OccurrenceSetFixture): + def setUp(self) -> None: + super().setUp() + self.set = OccurrenceSet.objects.create(name="Blind set") + self.set.projects.add(self.project) + self.set.occurrences.add(*self.occurrences) + + def test_a_project_is_required(self): + """Without one the endpoint would list every set on the platform.""" + self.client.force_authenticate(user=self.curator) + response = self.client.get(self.url) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_the_size_is_reported_without_the_occurrences(self): + self.client.force_authenticate(user=self.curator) + response = self.client.get(self.url, {"project_id": self.project.pk}) + + row = next(r for r in response.data["results"] if r["name"] == "Blind set") + self.assertEqual(row["occurrences_count"], 3) + self.assertNotIn("occurrences", row) + + def test_another_project_does_not_see_it(self): + other = Project.objects.create(name="Other Project", owner=self.owner) + MLDataManager.assign_user(self.curator, other) + self.client.force_authenticate(user=self.curator) + + response = self.client.get(self.url, {"project_id": other.pk}) + + self.assertNotIn("Blind set", [row["name"] for row in response.data["results"]]) diff --git a/ami/users/roles.py b/ami/users/roles.py index 718146e41..ae68565d8 100644 --- a/ami/users/roles.py +++ b/ami/users/roles.py @@ -154,6 +154,11 @@ class MLDataManager(Role): Project.Permissions.UPDATE_COLLECTION, Project.Permissions.DELETE_COLLECTION, Project.Permissions.POPULATE_COLLECTION, + # Whoever curates the data a model is measured on is the same person who manages + # the data it learns from, so this sits with the collection permissions. + Project.Permissions.CREATE_OCCURRENCE_SET, + Project.Permissions.UPDATE_OCCURRENCE_SET, + Project.Permissions.DELETE_OCCURRENCE_SET, } diff --git a/config/api_router.py b/config/api_router.py index 56a01c591..ebf5588f7 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -41,6 +41,7 @@ router.register(r"detections", views.DetectionViewSet) # Register before r"occurrences" — see docs/claude/reference/api-stats-pattern.md (guarded by regression test). router.register(r"occurrences/stats", views.OccurrenceStatsViewSet, basename="occurrence-stats") +router.register(r"occurrences/sets", views.OccurrenceSetViewSet) router.register(r"occurrences", views.OccurrenceViewSet) router.register(r"taxa/lists", views.TaxaListViewSet) # NESTED: /taxa/lists/{taxalist_id}/taxa/ diff --git a/ui/src/components/filtering/filter-control.tsx b/ui/src/components/filtering/filter-control.tsx index 742620c79..538d8dfab 100644 --- a/ui/src/components/filtering/filter-control.tsx +++ b/ui/src/components/filtering/filter-control.tsx @@ -9,6 +9,7 @@ import { CaptureSetFilter } from './filters/capture-set-filter' import { DateFilter } from './filters/date-filter' import { DeviceFilter } from './filters/device-filter' import { ImageFilter } from './filters/image-filter' +import { OccurrenceSetFilter } from './filters/occurrence-set-filter' import { PipelineFilter } from './filters/pipeline-filter' import { SessionFilter } from './filters/session-filter' import { SiteFilter } from './filters/site-filter' @@ -44,6 +45,7 @@ const ComponentMap: { not_algorithm: NotAlgorithmFilter, not_tag_id: TagFilter, not_taxa_list_id: TaxaListFilter, + occurrence_set: OccurrenceSetFilter, pipeline: PipelineFilter, source_image_collection: CaptureSetFilter, source_image_single: ImageFilter, diff --git a/ui/src/components/filtering/filters/occurrence-set-filter.tsx b/ui/src/components/filtering/filters/occurrence-set-filter.tsx new file mode 100644 index 000000000..3e6b2c620 --- /dev/null +++ b/ui/src/components/filtering/filters/occurrence-set-filter.tsx @@ -0,0 +1,17 @@ +import { API_ROUTES } from 'data-services/constants' +import { EntityPicker } from 'nova-ui-kit' +import { FilterProps } from './types' + +export const OccurrenceSetFilter = ({ onAdd, onClear, value }: FilterProps) => ( + { + if (value) { + onAdd(value) + } else { + onClear() + } + }} + value={value} + /> +) diff --git a/ui/src/data-services/constants.ts b/ui/src/data-services/constants.ts index 59bae6ede..26891161f 100644 --- a/ui/src/data-services/constants.ts +++ b/ui/src/data-services/constants.ts @@ -3,6 +3,8 @@ export const API_URL = '/api/v2' export const API_ROUTES = { ALGORITHM: 'ml/algorithms', CAPTURE_SET_CHOICES: 'captures/collections/choices', + OCCURRENCE_SETS: 'occurrences/sets', + OCCURRENCE_SET_CHOICES: 'occurrences/sets/choices', CAPTURE_SETS: 'captures/collections', CAPTURES: 'captures', CLASSIFICATIONS: 'classifications', diff --git a/ui/src/data-services/hooks/occurrence-sets/useCreateOccurrenceSet.ts b/ui/src/data-services/hooks/occurrence-sets/useCreateOccurrenceSet.ts new file mode 100644 index 000000000..8d6b6154c --- /dev/null +++ b/ui/src/data-services/hooks/occurrence-sets/useCreateOccurrenceSet.ts @@ -0,0 +1,41 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query' +import axios from 'axios' +import { API_ROUTES, API_URL, SUCCESS_TIMEOUT } from 'data-services/constants' +import { getAuthHeader } from 'data-services/utils' +import { useUser } from 'utils/user/userContext' + +interface OccurrenceSetFieldValues { + name: string + description?: string + projectId: string + occurrenceIds: string[] +} + +const convertToServerFieldValues = (fieldValues: OccurrenceSetFieldValues) => ({ + name: fieldValues.name, + description: fieldValues.description ?? '', + project_id: fieldValues.projectId, + occurrence_ids: fieldValues.occurrenceIds.map((id) => Number(id)), +}) + +export const useCreateOccurrenceSet = (onSuccess?: (id: string) => void) => { + const { user } = useUser() + const queryClient = useQueryClient() + + const { mutateAsync, isLoading, isSuccess, reset, error } = useMutation({ + mutationFn: (fieldValues: OccurrenceSetFieldValues) => + axios.post<{ id: number }>( + `${API_URL}/${API_ROUTES.OCCURRENCE_SETS}/`, + convertToServerFieldValues(fieldValues), + { headers: getAuthHeader(user) } + ), + onSuccess: ({ data }) => { + queryClient.invalidateQueries([API_ROUTES.OCCURRENCE_SETS]) + queryClient.invalidateQueries([API_ROUTES.OCCURRENCE_SET_CHOICES]) + onSuccess?.(`${data.id}`) + setTimeout(reset, SUCCESS_TIMEOUT) + }, + }) + + return { createOccurrenceSet: mutateAsync, isLoading, isSuccess, error } +} diff --git a/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx new file mode 100644 index 000000000..59025acc0 --- /dev/null +++ b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx @@ -0,0 +1,39 @@ +import { BookmarkPlusIcon } from 'lucide-react' +import { Button, Popover } from 'nova-ui-kit' +import { useState } from 'react' +import { STRING, translate } from 'utils/language' +import { CreateOccurrenceSet } from './create-occurrence-set' + +interface CreateOccurrenceSetPopoverProps { + occurrenceIds: string[] +} + +export const CreateOccurrenceSetPopover = ({ + occurrenceIds, +}: CreateOccurrenceSetPopoverProps) => { + const [open, setOpen] = useState(false) + + return ( + + + + + + setOpen(false)} + /> + + + ) +} diff --git a/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set.tsx b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set.tsx new file mode 100644 index 000000000..f249c36f9 --- /dev/null +++ b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set.tsx @@ -0,0 +1,70 @@ +import { FormError } from 'components/form/layout/layout' +import { useCreateOccurrenceSet } from 'data-services/hooks/occurrence-sets/useCreateOccurrenceSet' +import { Loader2Icon } from 'lucide-react' +import { Button, Input } from 'nova-ui-kit' +import { useState } from 'react' +import { useParams } from 'react-router-dom' +import { STRING, translate } from 'utils/language' +import { parseServerError } from 'utils/parseServerError/parseServerError' + +interface CreateOccurrenceSetProps { + occurrenceIds: string[] + onCancel: () => void +} + +export const CreateOccurrenceSet = ({ + occurrenceIds, + onCancel, +}: CreateOccurrenceSetProps) => { + const { projectId } = useParams() + const [name, setName] = useState('') + const { createOccurrenceSet, isLoading, error } = + useCreateOccurrenceSet(onCancel) + const formError = error ? parseServerError(error)?.message : undefined + + return ( + <> + {formError && ( + + )} +
+
+ setName(e.target.value)} + value={name} + /> +
+ + {translate(STRING.MESSAGE_OCCURRENCE_SET_IS_FIXED, { + count: `${occurrenceIds.length}`, + })} + +
+ + +
+
+ + ) +} diff --git a/ui/src/pages/occurrences/occurrence-filters.ts b/ui/src/pages/occurrences/occurrence-filters.ts index e63d5093c..6a79c7741 100644 --- a/ui/src/pages/occurrences/occurrence-filters.ts +++ b/ui/src/pages/occurrences/occurrence-filters.ts @@ -11,6 +11,7 @@ export const FILTERS_TO_OCCURRENCES = [ 'verified', 'verified_by_me', 'collection', + 'occurrence_set', 'date_start', 'date_end', 'deployment', diff --git a/ui/src/pages/occurrences/occurrences.tsx b/ui/src/pages/occurrences/occurrences.tsx index e0aed9b24..9279d3312 100644 --- a/ui/src/pages/occurrences/occurrences.tsx +++ b/ui/src/pages/occurrences/occurrences.tsx @@ -31,6 +31,7 @@ import { columns } from './occurrence-columns' import { OccurrenceDetailsDialog } from './occurrence-details-dialog' import { OccurrenceGallery } from './occurrence-gallery' import { OccurrenceStats } from './occurrence-stats' +import { CreateOccurrenceSetPopover } from './create-occurrence-set/create-occurrence-set-popover' import { OccurrencesActions } from './occurrences-actions' export const Occurrences = () => { @@ -110,6 +111,7 @@ export const Occurrences = () => { defaultOpen={someActive( [ 'collection', + 'occurrence_set', 'deployment', 'deployment__device', 'deployment__research_site', @@ -123,6 +125,7 @@ export const Occurrences = () => { + @@ -163,6 +166,9 @@ export const Occurrences = () => { Export + {selectedItems.length > 0 && ( + + )} [ }, }, }, + { + label: translate(STRING.FIELD_LABEL_OCCURRENCE_SET), + field: 'occurrence_set', + tooltip: { + text: translate(STRING.TOOLTIP_OCCURRENCE_SET), + }, + }, { label: translate(STRING.FIELD_LABEL_CAPTURE_SET), field: 'collection', // This is for viewing occurrences by capture set. @TODO: Can we update this key to "capture_set_id" to streamline?