Build a classifier training set from verified occurrences, and serve it

ami/base/pagination.py

@@ -29,3 +29,15 @@ def _get_project(self):
         if hasattr(view, "get_active_project"):
             return view.get_active_project()
         return None
+
+
+class TrainingDataPagination(LimitOffsetPagination):
+    """
+    Paging for embedding rows, which are far bigger than a normal API row.
+
+    A 1024-dimension vector serialises to roughly 20 KB of JSON, so the platform default of
+    10 is uselessly small and an unbounded limit would return hundreds of megabytes.
+    """
+
+    default_limit = 200
+    max_limit = 1000

ami/base/permissions.py

@@ -198,6 +198,35 @@ def has_permission(self, request, view):
         return super().has_permission(request, view)
 
 
+class OccurrenceSetPermission(ObjectPermission):
+    """
+    Permission for the occurrence-set route, where list and create have no object yet.
+
+    Listing follows the project: anyone who may see the project may see which sets it has,
+    since only a set's name and size are exposed. Creating one is gated on the project's
+    ``create_occurrenceset`` permission. Everything else falls through to the object check,
+    which refuses a global set because it has no project to check against.
+    """
+
+    def has_permission(self, request, view):
+        from ami.main.models import Project
+
+        if view.action == "list":
+            # The project id is required for this action, so reaching here means it was
+            # given and the caller may see it.
+            return view.get_active_project() is not None
+
+        if view.action == "create":
+            # Read from the payload rather than the query string: the set names the project
+            # it will belong to, and that is the project whose permission must allow it.
+            project = Project.objects.filter(pk=request.data.get("project_id")).first()
+            if not project:
+                return False
+            return request.user.has_perm(Project.Permissions.CREATE_OCCURRENCE_SET, project)
+
+        return super().has_permission(request, view)
+
+
 class UserMembershipPermission(ObjectPermission):
     """
     Custom permission for UserProjectMembershipViewSet.

ami/main/api/serializers.py

@@ -26,6 +26,7 @@
     Event,
     Identification,
     Occurrence,
+    OccurrenceSet,
     Page,
     Project,
     ProjectSettingsMixin,
@@ -1418,6 +1419,59 @@ def to_representation(self, instance):
         return {key: value for key, value in data.items() if value is not None}
 
 
+class OccurrenceSetSerializer(DefaultSerializer):
+    """A fixed list of occurrences. ``occurrence_ids`` is accepted only when it is created."""
+
+    project_id = serializers.PrimaryKeyRelatedField(queryset=Project.objects.all(), write_only=True)
+    occurrence_ids = serializers.PrimaryKeyRelatedField(
+        queryset=Occurrence.objects.all(), many=True, write_only=True, allow_empty=False
+    )
+    occurrences_count = serializers.SerializerMethodField()
+
+    class Meta:
+        model = OccurrenceSet
+        fields = [
+            "id",
+            "details",
+            "name",
+            "description",
+            "project_id",
+            "occurrence_ids",
+            "occurrences_count",
+            "created_at",
+            "updated_at",
+        ]
+
+    def get_occurrences_count(self, obj) -> int:
+        return getattr(obj, "annotated_occurrences_count", None) or obj.occurrences.count()
+
+    def validate(self, attrs):
+        # Scores recorded against a set were measured on these occurrences, so moving them
+        # would change what those numbers mean. Another set is the way to change the list.
+        if self.instance is not None and "occurrence_ids" in attrs:
+            raise serializers.ValidationError(
+                {"occurrence_ids": "A set's occurrences cannot be changed. Create another set instead."}
+            )
+
+        project = attrs.get("project_id")
+        occurrences = attrs.get("occurrence_ids")
+        if project and occurrences:
+            outside = sorted(o.pk for o in occurrences if o.project_id != project.pk)
+            if outside:
+                raise serializers.ValidationError(
+                    {"occurrence_ids": f"Not occurrences in this project: {outside[:10]}"}
+                )
+        return attrs
+
+    def create(self, validated_data):
+        project = validated_data.pop("project_id")
+        occurrences = validated_data.pop("occurrence_ids")
+        occurrence_set = OccurrenceSet.objects.create(**validated_data)
+        occurrence_set.projects.add(project)
+        occurrence_set.occurrences.set(occurrences)
+        return occurrence_set
+
+
 class SourceImageCollectionSerializer(DefaultSerializer):
     source_images = serializers.SerializerMethodField()
     kwargs = SourceImageCollectionCommonKwargsSerializer(required=False, partial=True)

ami/main/api/views.py

@@ -29,7 +29,12 @@
 from ami.base.metadata import ResponseSchemaMetadata
 from ami.base.models import BaseQuerySet
 from ami.base.pagination import LimitOffsetPaginationWithPermissions
-from ami.base.permissions import IsActiveStaffOrReadOnly, IsProjectMemberOrReadOnly, ObjectPermission
+from ami.base.permissions import (
+    IsActiveStaffOrReadOnly,
+    IsProjectMemberOrReadOnly,
+    ObjectPermission,
+    OccurrenceSetPermission,
+)
 from ami.base.serializers import FilterParamsSerializer, SingleParamSerializer
 from ami.base.views import ProjectMixin
 from ami.main.api.schemas import limit_doc_param, project_id_doc_param
@@ -50,6 +55,7 @@
     Event,
     Identification,
     Occurrence,
+    OccurrenceSet,
     Page,
     Project,
     ProjectQuerySet,
@@ -85,6 +91,7 @@
     ModelAgreementSerializer,
     OccurrenceListSerializer,
     OccurrenceSerializer,
+    OccurrenceSetSerializer,
     PageListSerializer,
     PageSerializer,
     ProjectListSerializer,
@@ -954,6 +961,74 @@ class ChoicesPagination(LimitOffsetPaginationWithPermissions):
     max_limit = 100
 
 
+class OccurrenceSetViewSet(DefaultViewSet, ProjectMixin):
+    """
+    Fixed lists of occurrences, so the same data can be used again later.
+
+    Membership is decided when a set is created and cannot be changed afterwards: there is
+    no endpoint to add or remove an occurrence. Anything that compares results over time
+    depends on the list standing still, and a set that grows quietly makes every number
+    recorded against it mean something different.
+
+    A set belonging to no project is global and is offered to every project, but it cannot
+    be edited here, because it has no single project whose permissions would govern it.
+    """
+
+    queryset = OccurrenceSet.objects.all()
+    serializer_class = OccurrenceSetSerializer
+    permission_classes = [OccurrenceSetPermission]
+    # No put: a full replace would include the occurrences.
+    http_method_names = ["get", "post", "patch", "delete", "head", "options"]
+    ordering_fields = ["name", "created_at", "updated_at"]
+    search_fields = ["name"]
+    ordering = ["name"]
+    # Listing without a project would answer with every set on the platform, so it is
+    # required there. A detail route names one set, which carries its own project, and the
+    # create payload names the project it belongs to.
+    require_project = False
+    require_project_for_list = True
+
+    def create(self, request, *args, **kwargs):
+        """
+        Create without the base class's unsaved-instance permission check.
+
+        ``DefaultViewSet.create`` builds ``Model(**validated_data)`` to check object
+        permissions before saving. A set's project and its occurrences arrive as payload
+        fields rather than columns, so that instance cannot be built. The same check runs
+        earlier instead, in ``OccurrenceSetPermission``, against the project the payload
+        names.
+        """
+        serializer = self.get_serializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        self.perform_create(serializer)
+        headers = self.get_success_headers(serializer.data)
+        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
+
+    @extend_schema(parameters=[project_id_doc_param], responses=OccurrenceSetSerializer(many=True))
+    @action(detail=False, methods=["get"], name="choices")
+    def choices(self, request: Request) -> Response:
+        """
+        Choices for the occurrence-set filter and pickers.
+
+        Follows SourceImageCollectionViewSet.choices: most recently updated first and
+        enough of them that a dropdown never has to page.
+        """
+        self.ordering_fields = ["id", "created_at", "updated_at", "name"]
+        queryset = self.filter_queryset(self.get_queryset())
+        paginator = ChoicesPagination()
+        page = paginator.paginate_queryset(queryset, request, view=self)
+        serializer = self.get_serializer(page, many=True)
+        return paginator.get_paginated_response(serializer.data)
+
+    def get_queryset(self) -> QuerySet["OccurrenceSet"]:
+        qs = super().get_queryset().annotate(annotated_occurrences_count=models.Count("occurrences"))
+        project = self.get_active_project()
+        if project is not None:
+            # A set with no project is global, so it is offered everywhere.
+            return qs.for_project(project)
+        return qs.visible_for_user(self.request.user)
+
+
 class SourceImageCollectionViewSet(DefaultViewSet, ProjectMixin):
     """
     Endpoint for viewing capture sets or samples of captures.
@@ -1496,6 +1571,9 @@ class OccurrenceFilterSet(FilterSet):
     """
 
     detections__source_image = RelatedIdFilter()
+    # Named for what it means to a reader rather than for the reverse accessor, which is
+    # called evaluation_sets because scoring was the first thing to use one.
+    occurrence_set = RelatedIdFilter(field_name="evaluation_sets")
 
     class Meta:
         model = Occurrence

ami/main/migrations/0098_occurrence_set.py

@@ -0,0 +1,102 @@
+# Generated by Django 4.2.10 on 2026-10-08 13:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("main", "0097_detection_and_classification_job_indexes"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="project",
+            options={
+                "ordering": ["-priority", "created_at"],
+                "permissions": [
+                    ("create_identification", "Can create identifications"),
+                    ("update_identification", "Can update identifications"),
+                    ("delete_identification", "Can delete identifications"),
+                    ("create_job", "Can create a job"),
+                    ("update_job", "Can update a job"),
+                    ("run_ml_job", "Can run/retry/cancel ML jobs"),
+                    ("run_populate_captures_collection_job", "Can run/retry/cancel Populate Collection jobs"),
+                    ("run_data_storage_sync_job", "Can run/retry/cancel Data Storage Sync jobs"),
+                    ("run_regroup_events_job", "Can run/retry/cancel Regroup Events jobs"),
+                    ("run_data_export_job", "Can run/retry/cancel Data Export jobs"),
+                    ("run_single_image_ml_job", "Can process a single capture"),
+                    ("run_post_processing_job", "Can run/retry/cancel Post-Processing jobs"),
+                    ("delete_job", "Can delete a job"),
+                    ("create_deployment", "Can create a deployment"),
+                    ("delete_deployment", "Can delete a deployment"),
+                    ("update_deployment", "Can update a deployment"),
+                    ("sync_deployment", "Can sync images to a deployment"),
+                    ("regroup_sessions_deployment", "Can regroup deployment captures into sessions"),
+                    ("create_occurrenceset", "Can create an occurrence set"),
+                    ("update_occurrenceset", "Can rename or describe an occurrence set"),
+                    ("delete_occurrenceset", "Can delete an occurrence set"),
+                    ("create_sourceimagecollection", "Can create a collection"),
+                    ("update_sourceimagecollection", "Can update a collection"),
+                    ("delete_sourceimagecollection", "Can delete a collection"),
+                    ("populate_sourceimagecollection", "Can populate a collection"),
+                    ("create_sourceimage", "Can create a source image"),
+                    ("update_sourceimage", "Can update a source image"),
+                    ("delete_sourceimage", "Can delete a source image"),
+                    ("star_sourceimage", "Can star a source image"),
+                    ("create_sourceimageupload", "Can create a source image upload"),
+                    ("update_sourceimageupload", "Can update a source image upload"),
+                    ("delete_sourceimageupload", "Can delete a source image upload"),
+                    ("create_s3storagesource", "Can create storage"),
+                    ("delete_s3storagesource", "Can delete storage"),
+                    ("update_s3storagesource", "Can update storage"),
+                    ("test_s3storagesource", "Can test storage connection"),
+                    ("create_site", "Can create a site"),
+                    ("delete_site", "Can delete a site"),
+                    ("update_site", "Can update a site"),
+                    ("create_device", "Can create a device"),
+                    ("delete_device", "Can delete a device"),
+                    ("update_device", "Can update a device"),
+                    ("view_userprojectmembership", "Can view project members"),
+                    ("create_userprojectmembership", "Can add a user to the project"),
+                    ("update_userprojectmembership", "Can update a user's project membership and role in the project"),
+                    ("delete_userprojectmembership", "Can remove a user from the project"),
+                    ("create_dataexport", "Can create a data export"),
+                    ("update_dataexport", "Can update a data export"),
+                    ("delete_dataexport", "Can delete a data export"),
+                    ("create_projectpipelineconfig", "Can register pipelines for the project"),
+                    ("update_projectpipelineconfig", "Can update pipeline configurations"),
+                    ("delete_projectpipelineconfig", "Can remove pipelines from the project"),
+                    ("create_taxalist", "Can create a taxa list"),
+                    ("update_taxalist", "Can update a taxa list"),
+                    ("delete_taxalist", "Can delete a taxa list"),
+                    ("view_private_data", "Can view private data"),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="OccurrenceSet",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("name", models.CharField(max_length=255)),
+                ("description", models.TextField(blank=True)),
+                (
+                    "occurrences",
+                    models.ManyToManyField(blank=True, related_name="evaluation_sets", to="main.occurrence"),
+                ),
+                (
+                    "projects",
+                    models.ManyToManyField(
+                        blank=True,
+                        help_text="Projects this set belongs to. A set with none is available everywhere.",
+                        related_name="occurrence_sets",
+                        to="main.project",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["name"],
+            },
+        ),
+    ]