diff --git a/ami/base/pagination.py b/ami/base/pagination.py index 9ebca7b21..ffcbda445 100644 --- a/ami/base/pagination.py +++ b/ami/base/pagination.py @@ -29,3 +29,15 @@ def _get_project(self): if hasattr(view, "get_active_project"): return view.get_active_project() return None + + +class TrainingDataPagination(LimitOffsetPagination): + """ + Paging for embedding rows, which are far bigger than a normal API row. + + A 1024-dimension vector serialises to roughly 20 KB of JSON, so the platform default of + 10 is uselessly small and an unbounded limit would return hundreds of megabytes. + """ + + default_limit = 200 + max_limit = 1000 diff --git a/ami/base/permissions.py b/ami/base/permissions.py index 328aa8384..af2636c94 100644 --- a/ami/base/permissions.py +++ b/ami/base/permissions.py @@ -198,6 +198,35 @@ def has_permission(self, request, view): return super().has_permission(request, view) +class OccurrenceSetPermission(ObjectPermission): + """ + Permission for the occurrence-set route, where list and create have no object yet. + + Listing follows the project: anyone who may see the project may see which sets it has, + since only a set's name and size are exposed. Creating one is gated on the project's + ``create_occurrenceset`` permission. Everything else falls through to the object check, + which refuses a global set because it has no project to check against. + """ + + def has_permission(self, request, view): + from ami.main.models import Project + + if view.action == "list": + # The project id is required for this action, so reaching here means it was + # given and the caller may see it. + return view.get_active_project() is not None + + if view.action == "create": + # Read from the payload rather than the query string: the set names the project + # it will belong to, and that is the project whose permission must allow it. + project = Project.objects.filter(pk=request.data.get("project_id")).first() + if not project: + return False + return request.user.has_perm(Project.Permissions.CREATE_OCCURRENCE_SET, project) + + return super().has_permission(request, view) + + class UserMembershipPermission(ObjectPermission): """ Custom permission for UserProjectMembershipViewSet. diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 1ca588980..5611106ff 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -26,6 +26,7 @@ Event, Identification, Occurrence, + OccurrenceSet, Page, Project, ProjectSettingsMixin, @@ -1418,6 +1419,59 @@ def to_representation(self, instance): return {key: value for key, value in data.items() if value is not None} +class OccurrenceSetSerializer(DefaultSerializer): + """A fixed list of occurrences. ``occurrence_ids`` is accepted only when it is created.""" + + project_id = serializers.PrimaryKeyRelatedField(queryset=Project.objects.all(), write_only=True) + occurrence_ids = serializers.PrimaryKeyRelatedField( + queryset=Occurrence.objects.all(), many=True, write_only=True, allow_empty=False + ) + occurrences_count = serializers.SerializerMethodField() + + class Meta: + model = OccurrenceSet + fields = [ + "id", + "details", + "name", + "description", + "project_id", + "occurrence_ids", + "occurrences_count", + "created_at", + "updated_at", + ] + + def get_occurrences_count(self, obj) -> int: + return getattr(obj, "annotated_occurrences_count", None) or obj.occurrences.count() + + def validate(self, attrs): + # Scores recorded against a set were measured on these occurrences, so moving them + # would change what those numbers mean. Another set is the way to change the list. + if self.instance is not None and "occurrence_ids" in attrs: + raise serializers.ValidationError( + {"occurrence_ids": "A set's occurrences cannot be changed. Create another set instead."} + ) + + project = attrs.get("project_id") + occurrences = attrs.get("occurrence_ids") + if project and occurrences: + outside = sorted(o.pk for o in occurrences if o.project_id != project.pk) + if outside: + raise serializers.ValidationError( + {"occurrence_ids": f"Not occurrences in this project: {outside[:10]}"} + ) + return attrs + + def create(self, validated_data): + project = validated_data.pop("project_id") + occurrences = validated_data.pop("occurrence_ids") + occurrence_set = OccurrenceSet.objects.create(**validated_data) + occurrence_set.projects.add(project) + occurrence_set.occurrences.set(occurrences) + return occurrence_set + + class SourceImageCollectionSerializer(DefaultSerializer): source_images = serializers.SerializerMethodField() kwargs = SourceImageCollectionCommonKwargsSerializer(required=False, partial=True) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 65dde9f4e..b2805d94b 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -29,7 +29,12 @@ from ami.base.metadata import ResponseSchemaMetadata from ami.base.models import BaseQuerySet from ami.base.pagination import LimitOffsetPaginationWithPermissions -from ami.base.permissions import IsActiveStaffOrReadOnly, IsProjectMemberOrReadOnly, ObjectPermission +from ami.base.permissions import ( + IsActiveStaffOrReadOnly, + IsProjectMemberOrReadOnly, + ObjectPermission, + OccurrenceSetPermission, +) from ami.base.serializers import FilterParamsSerializer, SingleParamSerializer from ami.base.views import ProjectMixin from ami.main.api.schemas import limit_doc_param, project_id_doc_param @@ -50,6 +55,7 @@ Event, Identification, Occurrence, + OccurrenceSet, Page, Project, ProjectQuerySet, @@ -85,6 +91,7 @@ ModelAgreementSerializer, OccurrenceListSerializer, OccurrenceSerializer, + OccurrenceSetSerializer, PageListSerializer, PageSerializer, ProjectListSerializer, @@ -954,6 +961,74 @@ class ChoicesPagination(LimitOffsetPaginationWithPermissions): max_limit = 100 +class OccurrenceSetViewSet(DefaultViewSet, ProjectMixin): + """ + Fixed lists of occurrences, so the same data can be used again later. + + Membership is decided when a set is created and cannot be changed afterwards: there is + no endpoint to add or remove an occurrence. Anything that compares results over time + depends on the list standing still, and a set that grows quietly makes every number + recorded against it mean something different. + + A set belonging to no project is global and is offered to every project, but it cannot + be edited here, because it has no single project whose permissions would govern it. + """ + + queryset = OccurrenceSet.objects.all() + serializer_class = OccurrenceSetSerializer + permission_classes = [OccurrenceSetPermission] + # No put: a full replace would include the occurrences. + http_method_names = ["get", "post", "patch", "delete", "head", "options"] + ordering_fields = ["name", "created_at", "updated_at"] + search_fields = ["name"] + ordering = ["name"] + # Listing without a project would answer with every set on the platform, so it is + # required there. A detail route names one set, which carries its own project, and the + # create payload names the project it belongs to. + require_project = False + require_project_for_list = True + + def create(self, request, *args, **kwargs): + """ + Create without the base class's unsaved-instance permission check. + + ``DefaultViewSet.create`` builds ``Model(**validated_data)`` to check object + permissions before saving. A set's project and its occurrences arrive as payload + fields rather than columns, so that instance cannot be built. The same check runs + earlier instead, in ``OccurrenceSetPermission``, against the project the payload + names. + """ + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + + @extend_schema(parameters=[project_id_doc_param], responses=OccurrenceSetSerializer(many=True)) + @action(detail=False, methods=["get"], name="choices") + def choices(self, request: Request) -> Response: + """ + Choices for the occurrence-set filter and pickers. + + Follows SourceImageCollectionViewSet.choices: most recently updated first and + enough of them that a dropdown never has to page. + """ + self.ordering_fields = ["id", "created_at", "updated_at", "name"] + queryset = self.filter_queryset(self.get_queryset()) + paginator = ChoicesPagination() + page = paginator.paginate_queryset(queryset, request, view=self) + serializer = self.get_serializer(page, many=True) + return paginator.get_paginated_response(serializer.data) + + def get_queryset(self) -> QuerySet["OccurrenceSet"]: + qs = super().get_queryset().annotate(annotated_occurrences_count=models.Count("occurrences")) + project = self.get_active_project() + if project is not None: + # A set with no project is global, so it is offered everywhere. + return qs.for_project(project) + return qs.visible_for_user(self.request.user) + + class SourceImageCollectionViewSet(DefaultViewSet, ProjectMixin): """ Endpoint for viewing capture sets or samples of captures. @@ -1496,6 +1571,9 @@ class OccurrenceFilterSet(FilterSet): """ detections__source_image = RelatedIdFilter() + # Named for what it means to a reader rather than for the reverse accessor, which is + # called evaluation_sets because scoring was the first thing to use one. + occurrence_set = RelatedIdFilter(field_name="evaluation_sets") class Meta: model = Occurrence diff --git a/ami/main/migrations/0098_occurrence_set.py b/ami/main/migrations/0098_occurrence_set.py new file mode 100644 index 000000000..26153b846 --- /dev/null +++ b/ami/main/migrations/0098_occurrence_set.py @@ -0,0 +1,102 @@ +# Generated by Django 4.2.10 on 2026-10-08 13:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0097_detection_and_classification_job_indexes"), + ] + + operations = [ + migrations.AlterModelOptions( + name="project", + options={ + "ordering": ["-priority", "created_at"], + "permissions": [ + ("create_identification", "Can create identifications"), + ("update_identification", "Can update identifications"), + ("delete_identification", "Can delete identifications"), + ("create_job", "Can create a job"), + ("update_job", "Can update a job"), + ("run_ml_job", "Can run/retry/cancel ML jobs"), + ("run_populate_captures_collection_job", "Can run/retry/cancel Populate Collection jobs"), + ("run_data_storage_sync_job", "Can run/retry/cancel Data Storage Sync jobs"), + ("run_regroup_events_job", "Can run/retry/cancel Regroup Events jobs"), + ("run_data_export_job", "Can run/retry/cancel Data Export jobs"), + ("run_single_image_ml_job", "Can process a single capture"), + ("run_post_processing_job", "Can run/retry/cancel Post-Processing jobs"), + ("delete_job", "Can delete a job"), + ("create_deployment", "Can create a deployment"), + ("delete_deployment", "Can delete a deployment"), + ("update_deployment", "Can update a deployment"), + ("sync_deployment", "Can sync images to a deployment"), + ("regroup_sessions_deployment", "Can regroup deployment captures into sessions"), + ("create_occurrenceset", "Can create an occurrence set"), + ("update_occurrenceset", "Can rename or describe an occurrence set"), + ("delete_occurrenceset", "Can delete an occurrence set"), + ("create_sourceimagecollection", "Can create a collection"), + ("update_sourceimagecollection", "Can update a collection"), + ("delete_sourceimagecollection", "Can delete a collection"), + ("populate_sourceimagecollection", "Can populate a collection"), + ("create_sourceimage", "Can create a source image"), + ("update_sourceimage", "Can update a source image"), + ("delete_sourceimage", "Can delete a source image"), + ("star_sourceimage", "Can star a source image"), + ("create_sourceimageupload", "Can create a source image upload"), + ("update_sourceimageupload", "Can update a source image upload"), + ("delete_sourceimageupload", "Can delete a source image upload"), + ("create_s3storagesource", "Can create storage"), + ("delete_s3storagesource", "Can delete storage"), + ("update_s3storagesource", "Can update storage"), + ("test_s3storagesource", "Can test storage connection"), + ("create_site", "Can create a site"), + ("delete_site", "Can delete a site"), + ("update_site", "Can update a site"), + ("create_device", "Can create a device"), + ("delete_device", "Can delete a device"), + ("update_device", "Can update a device"), + ("view_userprojectmembership", "Can view project members"), + ("create_userprojectmembership", "Can add a user to the project"), + ("update_userprojectmembership", "Can update a user's project membership and role in the project"), + ("delete_userprojectmembership", "Can remove a user from the project"), + ("create_dataexport", "Can create a data export"), + ("update_dataexport", "Can update a data export"), + ("delete_dataexport", "Can delete a data export"), + ("create_projectpipelineconfig", "Can register pipelines for the project"), + ("update_projectpipelineconfig", "Can update pipeline configurations"), + ("delete_projectpipelineconfig", "Can remove pipelines from the project"), + ("create_taxalist", "Can create a taxa list"), + ("update_taxalist", "Can update a taxa list"), + ("delete_taxalist", "Can delete a taxa list"), + ("view_private_data", "Can view private data"), + ], + }, + ), + migrations.CreateModel( + name="OccurrenceSet", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("name", models.CharField(max_length=255)), + ("description", models.TextField(blank=True)), + ( + "occurrences", + models.ManyToManyField(blank=True, related_name="evaluation_sets", to="main.occurrence"), + ), + ( + "projects", + models.ManyToManyField( + blank=True, + help_text="Projects this set belongs to. A set with none is available everywhere.", + related_name="occurrence_sets", + to="main.project", + ), + ), + ], + options={ + "ordering": ["name"], + }, + ), + ] diff --git a/ami/main/migrations/0099_grant_occurrence_set_permissions.py b/ami/main/migrations/0099_grant_occurrence_set_permissions.py new file mode 100644 index 000000000..f69e2fcda --- /dev/null +++ b/ami/main/migrations/0099_grant_occurrence_set_permissions.py @@ -0,0 +1,98 @@ +""" +Grant the new occurrence-set permissions to the roles that already curate a project's data. + +``create_roles_for_project`` gives new projects their role permissions, and a +``post_migrate`` signal re-syncs every project on migrate, so this backfill is belt and +braces. It is kept so the grant is explicit rather than a side effect of a signal, and so +the reverse is written down. + +Guardian reads **object-level** rows: ``get_perms(user, project)`` and +``user.has_perm("create_occurrenceset", project)`` both look at +``GroupObjectPermission``. Adding the permission to ``group.permissions`` alone would not +show up there, so each project gets a row per role group. + +``ProjectManager`` inherits ``MLDataManager``'s permissions, so both groups are covered. +""" + +from django.db import migrations +from django.db.models import Q + +PERMISSIONS = ("create_occurrenceset", "update_occurrenceset", "delete_occurrenceset") +ROLE_GROUP_SUFFIXES = ("_MLDataManager", "_ProjectManager") + + +def _permissions(apps): + Permission = apps.get_model("auth", "Permission") + ContentType = apps.get_model("contenttypes", "ContentType") + try: + project_ct = ContentType.objects.get(app_label="main", model="project") + except ContentType.DoesNotExist: + return [], None + found = list(Permission.objects.filter(codename__in=PERMISSIONS, content_type=project_ct)) + return found, project_ct + + +def _project_pk_from_group(group): + # Group names are "{project_pk}_{project_name}_{RoleName}"; the pk is the immutable + # leading segment, since a project name may itself contain underscores. + try: + return int(group.name.split("_", 1)[0]) + except (ValueError, IndexError): + return None + + +def _role_groups(apps): + Group = apps.get_model("auth", "Group") + query = Q() + for suffix in ROLE_GROUP_SUFFIXES: + query |= Q(name__endswith=suffix) + return Group.objects.filter(query) + + +def grant(apps, schema_editor): + GroupObjectPermission = apps.get_model("guardian", "GroupObjectPermission") + permissions, project_ct = _permissions(apps) + if not permissions: + return + + for group in _role_groups(apps): + project_pk = _project_pk_from_group(group) + if project_pk is None: + continue + for permission in permissions: + group.permissions.add(permission) + GroupObjectPermission.objects.get_or_create( + permission=permission, + content_type=project_ct, + object_pk=str(project_pk), + group=group, + ) + + +def revoke(apps, schema_editor): + GroupObjectPermission = apps.get_model("guardian", "GroupObjectPermission") + permissions, project_ct = _permissions(apps) + if not permissions: + return + + for group in _role_groups(apps): + project_pk = _project_pk_from_group(group) + if project_pk is None: + continue + for permission in permissions: + group.permissions.remove(permission) + GroupObjectPermission.objects.filter( + permission=permission, + content_type=project_ct, + object_pk=str(project_pk), + group=group, + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0098_occurrence_set"), + ("guardian", "0002_generic_permissions_index"), + ] + + operations = [migrations.RunPython(grant, revoke)] diff --git a/ami/main/migrations/0100_train_classifier_job_permission.py b/ami/main/migrations/0100_train_classifier_job_permission.py new file mode 100644 index 000000000..ccb070eb1 --- /dev/null +++ b/ami/main/migrations/0100_train_classifier_job_permission.py @@ -0,0 +1,77 @@ +# Generated by Django 4.2.10 on 2026-10-08 16:04 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0099_grant_occurrence_set_permissions"), + ] + + operations = [ + migrations.AlterModelOptions( + name="project", + options={ + "ordering": ["-priority", "created_at"], + "permissions": [ + ("create_identification", "Can create identifications"), + ("update_identification", "Can update identifications"), + ("delete_identification", "Can delete identifications"), + ("create_job", "Can create a job"), + ("update_job", "Can update a job"), + ("run_ml_job", "Can run/retry/cancel ML jobs"), + ("run_populate_captures_collection_job", "Can run/retry/cancel Populate Collection jobs"), + ("run_data_storage_sync_job", "Can run/retry/cancel Data Storage Sync jobs"), + ("run_regroup_events_job", "Can run/retry/cancel Regroup Events jobs"), + ("run_data_export_job", "Can run/retry/cancel Data Export jobs"), + ("run_single_image_ml_job", "Can process a single capture"), + ("run_post_processing_job", "Can run/retry/cancel Post-Processing jobs"), + ("run_train_classifier_job", "Can run/retry/cancel Train Classifier jobs"), + ("delete_job", "Can delete a job"), + ("create_deployment", "Can create a deployment"), + ("delete_deployment", "Can delete a deployment"), + ("update_deployment", "Can update a deployment"), + ("sync_deployment", "Can sync images to a deployment"), + ("regroup_sessions_deployment", "Can regroup deployment captures into sessions"), + ("create_occurrenceset", "Can create an occurrence set"), + ("update_occurrenceset", "Can rename or describe an occurrence set"), + ("delete_occurrenceset", "Can delete an occurrence set"), + ("create_sourceimagecollection", "Can create a collection"), + ("update_sourceimagecollection", "Can update a collection"), + ("delete_sourceimagecollection", "Can delete a collection"), + ("populate_sourceimagecollection", "Can populate a collection"), + ("create_sourceimage", "Can create a source image"), + ("update_sourceimage", "Can update a source image"), + ("delete_sourceimage", "Can delete a source image"), + ("star_sourceimage", "Can star a source image"), + ("create_sourceimageupload", "Can create a source image upload"), + ("update_sourceimageupload", "Can update a source image upload"), + ("delete_sourceimageupload", "Can delete a source image upload"), + ("create_s3storagesource", "Can create storage"), + ("delete_s3storagesource", "Can delete storage"), + ("update_s3storagesource", "Can update storage"), + ("test_s3storagesource", "Can test storage connection"), + ("create_site", "Can create a site"), + ("delete_site", "Can delete a site"), + ("update_site", "Can update a site"), + ("create_device", "Can create a device"), + ("delete_device", "Can delete a device"), + ("update_device", "Can update a device"), + ("view_userprojectmembership", "Can view project members"), + ("create_userprojectmembership", "Can add a user to the project"), + ("update_userprojectmembership", "Can update a user's project membership and role in the project"), + ("delete_userprojectmembership", "Can remove a user from the project"), + ("create_dataexport", "Can create a data export"), + ("update_dataexport", "Can update a data export"), + ("delete_dataexport", "Can delete a data export"), + ("create_projectpipelineconfig", "Can register pipelines for the project"), + ("update_projectpipelineconfig", "Can update pipeline configurations"), + ("delete_projectpipelineconfig", "Can remove pipelines from the project"), + ("create_taxalist", "Can create a taxa list"), + ("update_taxalist", "Can update a taxa list"), + ("delete_taxalist", "Can delete a taxa list"), + ("view_private_data", "Can view private data"), + ], + }, + ), + ] diff --git a/ami/main/migrations/0101_grant_train_classifier_job_permission.py b/ami/main/migrations/0101_grant_train_classifier_job_permission.py new file mode 100644 index 000000000..19c5cb8ec --- /dev/null +++ b/ami/main/migrations/0101_grant_train_classifier_job_permission.py @@ -0,0 +1,94 @@ +""" +Grant ``run_train_classifier_job`` to the role that already runs the other ML jobs. + +``create_roles_for_project`` gives new projects their role permissions, and a +``post_migrate`` signal re-syncs every project on migrate, so this backfill is belt and +braces. It is kept so the grant is explicit rather than a side effect of a signal. + +Guardian reads object-level rows, so each project gets a ``GroupObjectPermission``; adding +to ``group.permissions`` alone would not appear in ``get_perms(user, project)``. Mirrors +``0099_grant_occurrence_set_permissions``. +""" + +from django.db import migrations +from django.db.models import Q + +PERMISSION = "run_train_classifier_job" +ROLE_GROUP_SUFFIXES = ("_MLDataManager", "_ProjectManager") + + +def _permission(apps): + Permission = apps.get_model("auth", "Permission") + ContentType = apps.get_model("contenttypes", "ContentType") + try: + project_ct = ContentType.objects.get(app_label="main", model="project") + except ContentType.DoesNotExist: + return None, None + try: + return Permission.objects.get(codename=PERMISSION, content_type=project_ct), project_ct + except Permission.DoesNotExist: + return None, None + + +def _project_pk_from_group(group): + # Group names are "{project_pk}_{project_name}_{RoleName}"; the pk is the immutable + # leading segment, since a project name may itself contain underscores. + try: + return int(group.name.split("_", 1)[0]) + except (ValueError, IndexError): + return None + + +def _role_groups(apps): + Group = apps.get_model("auth", "Group") + query = Q() + for suffix in ROLE_GROUP_SUFFIXES: + query |= Q(name__endswith=suffix) + return Group.objects.filter(query) + + +def grant(apps, schema_editor): + GroupObjectPermission = apps.get_model("guardian", "GroupObjectPermission") + permission, project_ct = _permission(apps) + if permission is None: + return + + for group in _role_groups(apps): + project_pk = _project_pk_from_group(group) + if project_pk is None: + continue + group.permissions.add(permission) + GroupObjectPermission.objects.get_or_create( + permission=permission, + content_type=project_ct, + object_pk=str(project_pk), + group=group, + ) + + +def revoke(apps, schema_editor): + GroupObjectPermission = apps.get_model("guardian", "GroupObjectPermission") + permission, project_ct = _permission(apps) + if permission is None: + return + + for group in _role_groups(apps): + project_pk = _project_pk_from_group(group) + if project_pk is None: + continue + group.permissions.remove(permission) + GroupObjectPermission.objects.filter( + permission=permission, + content_type=project_ct, + object_pk=str(project_pk), + group=group, + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0100_train_classifier_job_permission"), + ("guardian", "0002_generic_permissions_index"), + ] + + operations = [migrations.RunPython(grant, revoke)] diff --git a/ami/main/migrations/0102_project_default_taxa_list.py b/ami/main/migrations/0102_project_default_taxa_list.py new file mode 100644 index 000000000..5634621b9 --- /dev/null +++ b/ami/main/migrations/0102_project_default_taxa_list.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.10 on 2026-10-08 16:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0101_grant_train_classifier_job_permission"), + ] + + operations = [ + migrations.AddField( + model_name="project", + name="default_taxa_list", + field=models.ForeignKey( + blank=True, + help_text="The species this project expects to see. Used as the class list when retraining a classifier head, so the head covers the region rather than only the species someone happened to verify.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="default_for_projects", + to="main.taxalist", + ), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index 41323cb3d..616800277 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -429,6 +429,19 @@ def check_custom_permission(self, user, action: str) -> bool: # Fall back to default permission checking for other actions return super().check_custom_permission(user, action) + default_taxa_list = models.ForeignKey( + "TaxaList", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="default_for_projects", + help_text=( + "The species this project expects to see. Used as the class list when retraining " + "a classifier head, so the head covers the region rather than only the species " + "someone happened to verify." + ), + ) + class Permissions: """CRUD Permission names follow the convention: `create_`, `update_`, `delete_`, `view_`""" @@ -454,6 +467,7 @@ class Permissions: RUN_REGROUP_EVENTS_JOB = "run_regroup_events_job" RUN_DATA_EXPORT_JOB = "run_data_export_job" RUN_POST_PROCESSING_JOB = "run_post_processing_job" + RUN_TRAIN_CLASSIFIER_JOB = "run_train_classifier_job" DELETE_JOB = "delete_job" # Deployment permissions @@ -463,6 +477,11 @@ class Permissions: SYNC_DEPLOYMENT = "sync_deployment" REGROUP_SESSIONS_DEPLOYMENT = "regroup_sessions_deployment" + # Occurrence set permissions + CREATE_OCCURRENCE_SET = "create_occurrenceset" + UPDATE_OCCURRENCE_SET = "update_occurrenceset" + DELETE_OCCURRENCE_SET = "delete_occurrenceset" + # Collection permissions CREATE_COLLECTION = "create_sourceimagecollection" UPDATE_COLLECTION = "update_sourceimagecollection" @@ -537,6 +556,7 @@ class Meta: ("run_data_export_job", "Can run/retry/cancel Data Export jobs"), ("run_single_image_ml_job", "Can process a single capture"), ("run_post_processing_job", "Can run/retry/cancel Post-Processing jobs"), + ("run_train_classifier_job", "Can run/retry/cancel Train Classifier jobs"), ("delete_job", "Can delete a job"), # Deployment permissions ("create_deployment", "Can create a deployment"), @@ -545,6 +565,9 @@ class Meta: ("sync_deployment", "Can sync images to a deployment"), ("regroup_sessions_deployment", "Can regroup deployment captures into sessions"), # Collection permissions + ("create_occurrenceset", "Can create an occurrence set"), + ("update_occurrenceset", "Can rename or describe an occurrence set"), + ("delete_occurrenceset", "Can delete an occurrence set"), ("create_sourceimagecollection", "Can create a collection"), ("update_sourceimagecollection", "Can update a collection"), ("delete_sourceimagecollection", "Can delete a collection"), @@ -4947,6 +4970,74 @@ def html(self) -> str: ] +class OccurrenceSetQuerySet(BaseQuerySet): + def for_project(self, project) -> models.QuerySet: + """Sets this project can use: its own, plus any that belong to no project.""" + return self.filter(models.Q(projects=project) | models.Q(projects__isnull=True)).distinct() + + def visible_for_user(self, user) -> models.QuerySet: + """ + Global sets stay visible; project sets follow their project. + + The inherited filter keeps a row only if it reaches a non-draft project, which a + set belonging to no project never does. Without this a global set is hidden from + everyone but a superuser, which is the opposite of what global means. Only a set's + name and size are exposed, never the occurrences inside it. + """ + visible = super().visible_for_user(user) + return self.filter(models.Q(pk__in=visible.values("pk")) | models.Q(projects__isnull=True)).distinct() + + +@final +class OccurrenceSet(BaseModel): + """ + A fixed list of occurrences, kept so the same ones can be used again later. + + Membership is stored rather than described by a filter, because a filter answers + differently as data arrives. Anything that compares results over time needs the list to + stand still: scoring two classifiers is only meaningful if both saw the same + occurrences. + + A set with no projects is global, which is how one set is shared across the platform; + that follows how TaxaList already treats a list with no project. + """ + + name = models.CharField(max_length=255) + description = models.TextField(blank=True) + occurrences = models.ManyToManyField("main.Occurrence", related_name="evaluation_sets", blank=True) + projects = models.ManyToManyField( + "main.Project", + related_name="occurrence_sets", + blank=True, + help_text="Projects this set belongs to. A set with none is available everywhere.", + ) + + objects = OccurrenceSetQuerySet.as_manager() + + class Meta: + ordering = ["name"] + + def __str__(self) -> str: + return f"{self.name} ({self.occurrences.count()} occurrences)" + + @property + def is_global(self) -> bool: + return not self.projects.exists() + + def get_project(self): + """ + The project whose permissions govern this set. + + BaseModel returns None for a many-to-many project relation, and + ``check_permission`` refuses every action without a project, so a global set cannot + be renamed or deleted through the API at all. That is the behaviour we want: a set + shared across projects changing under one of them would quietly change what every + other project's stored scores were measured on. Global sets stay a deliberate, + out-of-band act. + """ + return self.projects.first() + + class SourceImageCollectionQuerySet(BaseQuerySet): def with_source_images_count(self): return self.annotate( diff --git a/ami/main/test_occurrence_sets.py b/ami/main/test_occurrence_sets.py new file mode 100644 index 000000000..ad84c7c58 --- /dev/null +++ b/ami/main/test_occurrence_sets.py @@ -0,0 +1,254 @@ +""" +Occurrence sets: a fixed list of occurrences, decided once. + +The point of the model is that the list does not move. Anything comparing results over +time — scoring two classifiers, re-running an export — is only meaningful if both runs saw +the same occurrences. So these tests are mostly about what the API refuses. +""" + +import datetime +import uuid + +from rest_framework import status +from rest_framework.test import APITestCase + +from ami.main.models import ( + Classification, + Deployment, + Detection, + Event, + Occurrence, + OccurrenceSet, + Project, + S3StorageSource, + SourceImage, + Taxon, + TaxonRank, +) +from ami.users.models import User +from ami.users.roles import BasicMember, MLDataManager + + +class OccurrenceSetFixture(APITestCase): + """ + Occurrences are created directly rather than through the capture fixtures. + + These tests are about what the endpoint accepts and refuses, and need nothing from a + detection. Going through create_captures/create_occurrences pulls in event grouping and + a storage read for image dimensions, neither of which works in a test, and the failures + land at random on whichever test ran first. + """ + + def setUp(self) -> None: + self.owner = User.objects.create_user(email="owner@example.com", password="testpass123") + self.curator = User.objects.create_user(email="curator@example.com", password="testpass123") + self.member = User.objects.create_user(email="member@example.com", password="testpass123") + + self.project = Project.objects.create(name="Set Project", owner=self.owner) + MLDataManager.assign_user(self.curator, self.project) + BasicMember.assign_user(self.member, self.project) + + self.occurrences = [self.make_occurrence(self.project) for _ in range(3)] + self.url = "/api/v2/occurrences/sets/" + + @staticmethod + def make_occurrence(project: Project) -> Occurrence: + """ + One occurrence the list endpoint will actually return. + + Built here rather than through the capture fixtures, which time their images from + the current clock and group them into events by reading image dimensions from + storage. Neither works in a test, and the failures land at random on whichever test + ran first. The list only shows occurrences with a real detection and a + determination, so both are set. + """ + storage = S3StorageSource.objects.create(name=f"source {uuid.uuid4().hex[:6]}", project=project, bucket="test") + deployment = Deployment.objects.create( + name=f"station {uuid.uuid4().hex[:6]}", project=project, data_source=storage + ) + night = datetime.datetime(2026, 6, 1, 22, 0, 0, tzinfo=datetime.timezone.utc) + event = Event.objects.create(deployment=deployment, project=project, start=night) + image = SourceImage.objects.create( + deployment=deployment, + project=project, + event=event, + timestamp=night, + path=f"test/{uuid.uuid4().hex[:8]}.jpg", + ) + taxon = Taxon.objects.create(name=f"Testus {uuid.uuid4().hex[:6]}", rank=TaxonRank.SPECIES.name) + taxon.projects.add(project) + detection = Detection.objects.create(source_image=image, timestamp=night, bbox=[0.1, 0.1, 0.2, 0.2]) + Classification.objects.create(detection=detection, taxon=taxon, score=0.9, timestamp=night) + occurrence = detection.associate_new_occurrence() + occurrence.refresh_from_db() + return occurrence + + def _create(self, user, **overrides): + payload = { + "name": "Blind set", + "description": "", + "project_id": self.project.pk, + "occurrence_ids": [o.pk for o in self.occurrences], + } + payload.update(overrides) + self.client.force_authenticate(user=user) + return self.client.post(self.url, payload, format="json") + + +class TestCreatingASet(OccurrenceSetFixture): + def test_a_curator_can_create_one(self): + response = self._create(self.curator) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.data) + created = OccurrenceSet.objects.get(pk=response.data["id"]) + self.assertEqual(created.occurrences.count(), 3) + self.assertEqual(list(created.projects.all()), [self.project]) + + def test_a_member_without_the_permission_cannot(self): + response = self._create(self.member) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertFalse(OccurrenceSet.objects.exists()) + + def test_an_anonymous_caller_cannot(self): + response = self._create(None) + + self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) + self.assertFalse(OccurrenceSet.objects.exists()) + + def test_an_occurrence_from_another_project_is_refused(self): + """Otherwise a set quietly reaches into data its project cannot see.""" + other = Project.objects.create(name="Other Project", owner=self.owner) + outsider = self.make_occurrence(other) + + response = self._create(self.curator, occurrence_ids=[self.occurrences[0].pk, outsider.pk]) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("occurrence_ids", response.data) + self.assertFalse(OccurrenceSet.objects.exists()) + + def test_an_empty_set_is_refused(self): + response = self._create(self.curator, occurrence_ids=[]) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + +class TestMembershipDoesNotMove(OccurrenceSetFixture): + def setUp(self) -> None: + super().setUp() + self.set = OccurrenceSet.objects.create(name="Blind set") + self.set.projects.add(self.project) + self.set.occurrences.add(*self.occurrences) + self.detail_url = f"{self.url}{self.set.pk}/" + self.client.force_authenticate(user=self.curator) + + def test_the_occurrences_cannot_be_changed(self): + response = self.client.patch(self.detail_url, {"occurrence_ids": [self.occurrences[0].pk]}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(self.set.occurrences.count(), 3) + + def test_the_name_and_description_can_be_changed(self): + response = self.client.patch(self.detail_url, {"name": "Renamed", "description": "why"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_200_OK, response.data) + self.set.refresh_from_db() + self.assertEqual(self.set.name, "Renamed") + + def test_there_is_no_put(self): + """A full replace would carry the occurrences with it.""" + response = self.client.put(self.detail_url, {"name": "Replaced"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED) + + def test_a_curator_can_delete_one(self): + response = self.client.delete(self.detail_url) + + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertFalse(OccurrenceSet.objects.filter(pk=self.set.pk).exists()) + + +class TestFilteringOccurrencesBySet(OccurrenceSetFixture): + """The occurrence list can be narrowed to one set, which is how a set is reviewed.""" + + def setUp(self) -> None: + super().setUp() + self.set = OccurrenceSet.objects.create(name="Blind set") + self.set.projects.add(self.project) + self.set.occurrences.add(self.occurrences[0], self.occurrences[1]) + self.client.force_authenticate(user=self.curator) + + def _ids(self, **params): + # apply_defaults=false: the list hides occurrences without a determination or a + # score by default, and these fixtures are bare rows made to exercise the filter. + response = self.client.get( + "/api/v2/occurrences/", + {"project_id": self.project.pk, "apply_defaults": "false", **params}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK, response.data) + return {row["id"] for row in response.data["results"]} + + def test_only_the_sets_occurrences_come_back(self): + self.assertEqual( + self._ids(occurrence_set=self.set.pk), + {self.occurrences[0].pk, self.occurrences[1].pk}, + ) + + def test_without_the_filter_every_occurrence_comes_back(self): + self.assertEqual(self._ids(), {o.pk for o in self.occurrences}) + + +class TestGlobalSets(OccurrenceSetFixture): + def setUp(self) -> None: + super().setUp() + self.global_set = OccurrenceSet.objects.create(name="Platform-wide set") + self.client.force_authenticate(user=self.curator) + + def _names(self, response): + return [row["name"] for row in response.data["results"]] + + def test_a_global_set_is_offered_to_every_project(self): + response = self.client.get(self.url, {"project_id": self.project.pk}) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn(self.global_set.name, self._names(response)) + + def test_a_global_set_cannot_be_edited_through_the_api(self): + """It has no single project whose permissions would govern it.""" + response = self.client.patch(f"{self.url}{self.global_set.pk}/", {"name": "Mine now"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.global_set.refresh_from_db() + self.assertEqual(self.global_set.name, "Platform-wide set") + + +class TestListing(OccurrenceSetFixture): + def setUp(self) -> None: + super().setUp() + self.set = OccurrenceSet.objects.create(name="Blind set") + self.set.projects.add(self.project) + self.set.occurrences.add(*self.occurrences) + + def test_a_project_is_required(self): + """Without one the endpoint would list every set on the platform.""" + self.client.force_authenticate(user=self.curator) + response = self.client.get(self.url) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_the_size_is_reported_without_the_occurrences(self): + self.client.force_authenticate(user=self.curator) + response = self.client.get(self.url, {"project_id": self.project.pk}) + + row = next(r for r in response.data["results"] if r["name"] == "Blind set") + self.assertEqual(row["occurrences_count"], 3) + self.assertNotIn("occurrences", row) + + def test_another_project_does_not_see_it(self): + other = Project.objects.create(name="Other Project", owner=self.owner) + MLDataManager.assign_user(self.curator, other) + self.client.force_authenticate(user=self.curator) + + response = self.client.get(self.url, {"project_id": other.pk}) + + self.assertNotIn("Blind set", [row["name"] for row in response.data["results"]]) diff --git a/ami/ml/schemas.py b/ami/ml/schemas.py index 2393dd62b..d1d00a575 100644 --- a/ami/ml/schemas.py +++ b/ami/ml/schemas.py @@ -503,3 +503,61 @@ class AlgorithmTrainingInfo(pydantic.BaseModel): class Config: extra = "allow" + +class NamedReference(pydantic.BaseModel): + """ + A row a training set came from. + + Carries the name as well as the id so a dataset still says where it came from after the + row it points at is deleted. + """ + + id: int + name: str + + +class TrainedAlgorithmReference(AlgorithmReference): + """The algorithm a training set was built for, at the version it was then.""" + + version: int | None = None + + +class TrainingDatasetSettings(pydantic.BaseModel): + """The settings a training set was built under, so a later run can be made to match.""" + + min_per_species: int + split_salt: str + test_fraction: float + split_grouped_by: str = pydantic.Field( + default="occurrence", + description=( + "What the train/test split is grouped by. Splitting per detection puts " + "near-identical crops on both sides." + ), + ) + + +class TrainingDatasetMetadata(pydantic.BaseModel): + """ + What a training set is, written inside the file and echoed back by the service. + + Declared rather than assembled as a dict so the shape is checked once, here, instead of + at each reader. + """ + + url: str + project: NamedReference + algorithm: TrainedAlgorithmReference + dimensions: int = pydantic.Field(description="Width of one feature vector, read from the stored vectors.") + dtype: str + classes: list[str] = pydantic.Field(description="The head's output classes, in order.") + counts: dict[str, int] = pydantic.Field(default_factory=dict, description="Verified crops per class.") + taxa_list: NamedReference | None = None + occurrence_set: NamedReference | None = None + classes_without_verified_data: list[str] = pydantic.Field(default_factory=list) + dropped_species: list[str] = pydantic.Field(default_factory=list) + rows: int + train: int + test: int + verified_detections_without_embedding: int = 0 + settings: TrainingDatasetSettings diff --git a/ami/ml/serializers.py b/ami/ml/serializers.py index 1f39b5b23..0ece73409 100644 --- a/ami/ml/serializers.py +++ b/ami/ml/serializers.py @@ -176,3 +176,21 @@ class PipelineRegistrationSerializer(serializers.Serializer): processing_service_name = serializers.CharField() pipelines = SchemaField(schema=list[PipelineConfigResponse], default=[]) + +class TrainingDataRowSerializer(serializers.Serializer): + """ + One verified crop: its human label and the embedding to train on. + + Not a ModelSerializer — the useful fields live on the related occurrence and on the + pgvector column, and the split is computed, so building the dict directly is clearer. + """ + + def to_representation(self, instance): + from ami.ml import training + + return training.row_as_dict( + instance, + salt=self.context.get("split_salt", training.DEFAULT_SPLIT_SALT), + test_fraction=self.context.get("test_fraction", training.DEFAULT_TEST_FRACTION), + include_features=self.context.get("include_features", True), + ) diff --git a/ami/ml/test_training.py b/ami/ml/test_training.py new file mode 100644 index 000000000..896e7fc7a --- /dev/null +++ b/ami/ml/test_training.py @@ -0,0 +1,260 @@ +""" +Building a training set from an occurrence set. + +A run learns from a named set rather than from whatever happened to be verified that day, +so these tests are mostly about the set deciding what goes in. +""" + +import datetime +import uuid + +from django.test import TestCase +from django.urls import reverse +from rest_framework.test import APIClient + +from ami.main.models import ( + Classification, + Deployment, + Detection, + Event, + Identification, + Occurrence, + OccurrenceSet, + Project, + S3StorageSource, + SourceImage, + Taxon, + TaxonRank, +) +from ami.ml.models import Algorithm, AlgorithmCategoryMap +from ami.ml.models.embedding import DetectionEmbedding +from ami.ml.schemas import NamedReference +from ami.ml.training import NotEnoughVerifiedData, build_training_dataset, verified_training_rows +from ami.users.models import User +from ami.users.roles import ProjectManager + +VECTOR_LENGTH = 8 + + +class TrainingSetFixture(TestCase): + """ + Occurrences, identifications and vectors built directly. + + The capture fixtures time their images from datetime.now() and group events by reading + image dimensions from storage, neither of which is deterministic in a test. + """ + + def setUp(self) -> None: + self.user = User.objects.create_user(email="curator@example.com", password="testpass123") + self.project = Project.objects.create(name="Training Project", owner=self.user) + storage = S3StorageSource.objects.create(name="src", project=self.project, bucket="test") + self.deployment = Deployment.objects.create(name="station", project=self.project, data_source=storage) + night = datetime.datetime(2026, 6, 1, 22, 0, 0, tzinfo=datetime.timezone.utc) + self.event = Event.objects.create(deployment=self.deployment, project=self.project, start=night) + self.night = night + + self.algorithm = Algorithm.objects.create(name="Backbone", key="backbone", trainable=True) + self.taxa = [ + Taxon.objects.create(name=f"Trainus {name}", rank=TaxonRank.SPECIES.name) for name in ("alpha", "beta") + ] + for taxon in self.taxa: + taxon.projects.add(self.project) + self.algorithm.category_map = AlgorithmCategoryMap.objects.create( + labels=[t.name for t in self.taxa], + data=[{"index": i, "label": t.name} for i, t in enumerate(self.taxa)], + version="training-test", + ) + self.algorithm.save() + + def make_occurrence(self, taxon: Taxon, with_vector: bool = True) -> Occurrence: + """One verified occurrence, optionally carrying a feature vector.""" + image = SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + event=self.event, + timestamp=self.night, + path=f"test/{uuid.uuid4().hex[:8]}.jpg", + ) + detection = Detection.objects.create(source_image=image, timestamp=self.night, bbox=[0.1, 0.1, 0.2, 0.2]) + Classification.objects.create(detection=detection, taxon=taxon, score=0.9, timestamp=self.night) + occurrence = detection.associate_new_occurrence() + Identification.objects.create(occurrence=occurrence, taxon=taxon, user=self.user) + if with_vector: + DetectionEmbedding.objects.create( + detection=detection, + algorithm=self.algorithm, + project=self.project, + vector=[0.1] * VECTOR_LENGTH, + ) + occurrence.refresh_from_db() + return occurrence + + def make_set(self, name: str, occurrences: list[Occurrence]) -> OccurrenceSet: + occurrence_set = OccurrenceSet.objects.create(name=name) + occurrence_set.projects.add(self.project) + occurrence_set.occurrences.set(occurrences) + return occurrence_set + + +class TestTheSetDecidesWhatIsLearnedFrom(TrainingSetFixture): + def test_only_the_sets_occurrences_are_used(self): + inside = [self.make_occurrence(self.taxa[0]), self.make_occurrence(self.taxa[1])] + outside = self.make_occurrence(self.taxa[0]) + occurrence_set = self.make_set("Spring review", inside) + + rows = verified_training_rows(self.project, self.algorithm, occurrence_set) + used = {row.detection.occurrence_id for row in rows} + + self.assertEqual(used, {o.pk for o in inside}) + self.assertNotIn(outside.pk, used) + + def test_without_a_set_every_verified_occurrence_is_used(self): + """The helper still answers without one; it is the job that insists on a set.""" + occurrences = [self.make_occurrence(self.taxa[0]), self.make_occurrence(self.taxa[1])] + + rows = verified_training_rows(self.project, self.algorithm) + + self.assertEqual({row.detection.occurrence_id for row in rows}, {o.pk for o in occurrences}) + + def test_the_dataset_says_which_set_it_came_from(self): + occurrences = [self.make_occurrence(self.taxa[0]) for _ in range(4)] + occurrences += [self.make_occurrence(self.taxa[1]) for _ in range(4)] + occurrence_set = self.make_set("Spring review", occurrences) + + # Half held out: the split is a hash of the occurrence, so a handful of rows at the + # usual fifth can land entirely on one side, which the build refuses. + dataset = build_training_dataset( + project=self.project, + algorithm=self.algorithm, + min_per_species=1, + test_fraction=0.5, + occurrence_set=occurrence_set, + ) + + self.assertEqual( + dataset["metadata"].occurrence_set, + NamedReference(id=occurrence_set.pk, name="Spring review"), + ) + self.assertEqual(dataset["metadata"].rows, 8) + + def test_an_empty_set_has_nothing_to_train_on(self): + self.make_occurrence(self.taxa[0]) + empty = self.make_set("Nothing in here", []) + + with self.assertRaises(NotEnoughVerifiedData): + build_training_dataset( + project=self.project, algorithm=self.algorithm, min_per_species=1, occurrence_set=empty + ) + + def test_the_vector_width_comes_from_the_stored_vectors(self): + """Nothing assumes a width: a backbone of another size is read, not truncated.""" + occurrences = [self.make_occurrence(self.taxa[0]) for _ in range(6)] + occurrence_set = self.make_set("Spring review", occurrences) + + dataset = build_training_dataset( + project=self.project, + algorithm=self.algorithm, + min_per_species=1, + test_fraction=0.5, + occurrence_set=occurrence_set, + ) + + self.assertEqual(dataset["metadata"].dimensions, VECTOR_LENGTH) + + +class TestTheSummaryTheFormShows(TrainingSetFixture): + """ + The numbers the training job form puts in front of someone before they start a run. + + A form that asked for a split ratio without saying how many occurrences it would split + would be asking blind, so these cover what changes when the set or the ratio changes. + """ + + def setUp(self) -> None: + super().setUp() + ProjectManager.assign_user(self.user, self.project) + self.client = APIClient() + self.client.force_authenticate(user=self.user) + self.url = reverse("api:training-data-summary") + + def _summary(self, **params): + response = self.client.get( + self.url, + {"project_id": self.project.pk, "algorithm": self.algorithm.key, **params}, + ) + self.assertEqual(response.status_code, 200, response.data) + return response.data + + def test_without_a_set_it_counts_every_verified_occurrence(self): + for taxon in self.taxa: + self.make_occurrence(taxon) + + summary = self._summary() + + self.assertIsNone(summary["occurrence_set"]) + self.assertEqual(summary["occurrences"], 2) + self.assertEqual(summary["rows"], 2) + + def test_a_set_narrows_the_counts_and_is_named_back(self): + mine = self.make_set("Mine", [self.make_occurrence(self.taxa[0])]) + self.make_occurrence(self.taxa[1]) + + summary = self._summary(occurrence_set=mine.pk) + + self.assertEqual(summary["occurrence_set"], {"id": mine.pk, "name": "Mine"}) + self.assertEqual(summary["occurrences"], 1) + + def test_a_set_from_another_project_is_not_counted(self): + other = Project.objects.create(name="Someone else", owner=self.user) + theirs = OccurrenceSet.objects.create(name="Theirs") + theirs.projects.add(other) + + response = self.client.get( + self.url, + {"project_id": self.project.pk, "algorithm": self.algorithm.key, "occurrence_set": theirs.pk}, + ) + + self.assertEqual(response.status_code, 404) + + def test_the_split_ratio_moves_the_counts(self): + for taxon in self.taxa: + self.make_occurrence(taxon) + self.make_occurrence(taxon) + + everything_held_out = self._summary(test_fraction=0.99) + + self.assertEqual(everything_held_out["test"], everything_held_out["rows"]) + self.assertEqual(everything_held_out["train"], 0) + self.assertEqual(everything_held_out["settings"]["test_fraction"], 0.99) + + def test_a_ratio_no_run_could_use_is_refused(self): + response = self.client.get( + self.url, + {"project_id": self.project.pk, "algorithm": self.algorithm.key, "test_fraction": 1.5}, + ) + + self.assertEqual(response.status_code, 400) + + def test_species_without_enough_crops_are_reported_as_dropped(self): + self.make_occurrence(self.taxa[0]) + self.make_occurrence(self.taxa[1]) + self.make_occurrence(self.taxa[1]) + + summary = self._summary(min_per_species=2) + + self.assertEqual(summary["classes"], 2) + self.assertEqual(summary["trainable_classes"], 1) + self.assertEqual(summary["dropped_species"], [self.taxa[0].name]) + + def test_someone_who_cannot_retrain_cannot_read_the_training_set(self): + """ + A non-draft project is readable by anyone, so visibility alone would hand every + verified label and vector to any account. The permission is the gate. + """ + self.make_occurrence(self.taxa[0]) + outsider = User.objects.create_user(email="outsider@example.com", password="testpass123") + self.client.force_authenticate(user=outsider) + + response = self.client.get(self.url, {"project_id": self.project.pk, "algorithm": self.algorithm.key}) + + self.assertEqual(response.status_code, 403) diff --git a/ami/ml/training/__init__.py b/ami/ml/training/__init__.py new file mode 100644 index 000000000..6770457ad --- /dev/null +++ b/ami/ml/training/__init__.py @@ -0,0 +1,41 @@ +""" +Retraining a classifier head from the species people have verified. + +``dataset`` decides what the head learns from and writes it to storage. The names +re-exported here are the ones callers outside this package use; anything else is internal +to it. +""" + +from ami.ml.training.dataset import ( + DEFAULT_SPLIT_SALT, + DEFAULT_TEST_FRACTION, + SPLIT_TEST, + SPLIT_TRAIN, + SPLITS, + NotEnoughVerifiedData, + build_training_dataset, + count_missing_embeddings, + label_counts, + row_as_dict, + species_with_enough_examples, + split_for, + verified_occurrence_ids, + verified_training_rows, +) + +__all__ = [ + "DEFAULT_SPLIT_SALT", + "DEFAULT_TEST_FRACTION", + "SPLITS", + "SPLIT_TEST", + "SPLIT_TRAIN", + "NotEnoughVerifiedData", + "build_training_dataset", + "count_missing_embeddings", + "label_counts", + "row_as_dict", + "species_with_enough_examples", + "split_for", + "verified_occurrence_ids", + "verified_training_rows", +] diff --git a/ami/ml/training/dataset.py b/ami/ml/training/dataset.py new file mode 100644 index 000000000..d81a1e8dd --- /dev/null +++ b/ami/ml/training/dataset.py @@ -0,0 +1,341 @@ +""" +Build a classifier-head training set out of what people verified, and write it to storage. + +Two halves of one job: deciding what counts as training data — which occurrences a person +identified, which species have enough of them, which rows are held out — and turning that +into a single file. + +It is a file because a service is handed a URL, not the rows. A project with a few hundred +thousand verified labels runs to hundreds of megabytes, which is fragile to send in one +request and has to start over if the connection drops. +""" + +import hashlib +import logging +import pathlib +import tempfile +import typing +from typing import TYPE_CHECKING + +import numpy as np +from django.conf import settings +from django.core.files.storage import default_storage +from django.db.models import QuerySet +from django.utils.text import slugify + +from ami.main.models import DEFAULT_EMBEDDING_KEY, Identification, Project +from ami.ml.models.algorithm import Algorithm +from ami.ml.models.embedding import DetectionEmbedding +from ami.ml.schemas import NamedReference, TrainedAlgorithmReference, TrainingDatasetMetadata, TrainingDatasetSettings + +if TYPE_CHECKING: + from ami.jobs.models import Job + from ami.main.models import OccurrenceSet, TaxaList + +logger = logging.getLogger(__name__) + + +DEFAULT_SPLIT_SALT = "antenna-head-v1" +DEFAULT_TEST_FRACTION = 0.2 + +# The two sides of the split. Named so the API, the dataset builder and the export +# cannot drift apart on spelling. +SPLIT_TRAIN = "train" +SPLIT_TEST = "test" +SPLITS = (SPLIT_TRAIN, SPLIT_TEST) + + +def split_for( + occurrence_id: int, + salt: str = DEFAULT_SPLIT_SALT, + test_fraction: float = DEFAULT_TEST_FRACTION, +) -> str: + """ + Assign an occurrence to "train" or "test", the same way every time. + + Grouped by occurrence, not detection: an occurrence is one insect across several + frames, so a detection-level split puts near-identical crops on both sides and + overstates accuracy. Hash-based, not random, so the test set does not move when new + data arrives — an eval set that drifts cannot compare two heads. + """ + digest = hashlib.sha256(f"{salt}:{occurrence_id}".encode()).hexdigest() + return SPLIT_TEST if (int(digest[:8], 16) / 0xFFFFFFFF) < test_fraction else SPLIT_TRAIN + + +def verified_occurrence_ids(project: Project, occurrence_set: "OccurrenceSet | None" = None) -> QuerySet: + """ + Occurrences a person has identified and not taken back. + + Narrowed to an occurrence set when one is given. A set is fixed once created, so a run + that names one can be repeated and says for itself what it learned from; without one the + pool is whatever happened to be verified that day, which no later reader can reconstruct. + """ + identifications = Identification.objects.filter( + withdrawn=False, + occurrence__project=project, + occurrence__determination__isnull=False, + ) + if occurrence_set is not None: + identifications = identifications.filter(occurrence__evaluation_sets=occurrence_set) + return identifications.order_by().values_list("occurrence_id", flat=True).distinct() + + +def verified_training_rows( + project: Project, algorithm: Algorithm, occurrence_set: "OccurrenceSet | None" = None +) -> QuerySet[DetectionEmbedding]: + """ + Embeddings whose detection sits under a verified occurrence. + + Constrained to one algorithm on purpose: vectors from different backbones are in + different spaces and must never be mixed into one training set. + """ + return ( + DetectionEmbedding.objects.filter( + algorithm=algorithm, + key=DEFAULT_EMBEDDING_KEY, + detection__occurrence_id__in=verified_occurrence_ids(project, occurrence_set), + detection__occurrence__determination__isnull=False, + ) + .select_related("detection__occurrence__determination") + .order_by("pk") + ) + + +def count_missing_embeddings( + project: Project, algorithm: Algorithm, occurrence_set: "OccurrenceSet | None" = None +) -> int: + """Verified detections this algorithm has never embedded. They need a pipeline re-run.""" + from ami.main.models import Detection + + return ( + Detection.objects.filter(occurrence_id__in=verified_occurrence_ids(project)) + .exclude(embeddings__algorithm=algorithm) + .count() + ) + + +def label_counts( + project: Project, algorithm: Algorithm, occurrence_set: "OccurrenceSet | None" = None +) -> dict[str, int]: + """Verified crops per species, for the chosen algorithm.""" + from django.db.models import Count + + rows = ( + verified_training_rows(project, algorithm, occurrence_set) + .order_by() + .values("detection__occurrence__determination__name") + .annotate(n=Count("pk")) + ) + return {r["detection__occurrence__determination__name"]: r["n"] for r in rows} + + +def species_with_enough_examples(counts: dict[str, int], minimum: int) -> set[str]: + """A class with a single example cannot be both trained on and evaluated.""" + return {name for name, n in counts.items() if name and n >= minimum} + + +def row_as_dict( + embedding: DetectionEmbedding, + salt: str = DEFAULT_SPLIT_SALT, + test_fraction: float = DEFAULT_TEST_FRACTION, + include_features: bool = True, +) -> dict[str, typing.Any]: + """One training row, in the shape the training-data API returns.""" + occurrence = embedding.detection.occurrence + row = { + "detection_id": embedding.detection_id, + "occurrence_id": occurrence.pk if occurrence else None, + "label": occurrence.determination.name if occurrence and occurrence.determination else None, + "label_id": occurrence.determination_id if occurrence else None, + "split": split_for(occurrence.pk, salt, test_fraction) if occurrence else None, + } + if include_features: + row["features"] = embedding.vector.to_list() + return row + + +# Vectors are written as float16 because that is exactly what Postgres stores (halfvec), +# so nothing is lost. Measured on 10k rows: npz float16 is 21 MB against 227 MB of JSON, +# and 1.1s to build against 29s. +DATASET_DTYPE = np.float16 + +DATASET_DIRECTORY = "training" + +# Rows are pulled from the database in batches so a large project does not have to fit +# every vector in memory at once. +FETCH_BATCH_SIZE = 2000 + + +class NotEnoughVerifiedData(Exception): + """The project does not hold enough verified labels to train and evaluate on.""" + + +def build_training_dataset( + project: Project, + algorithm: Algorithm, + min_per_species: int = 2, + split_salt: str = DEFAULT_SPLIT_SALT, + test_fraction: float = DEFAULT_TEST_FRACTION, + taxa_list: "TaxaList | None" = None, + occurrence_set: "OccurrenceSet | None" = None, + job: "Job | None" = None, +) -> dict[str, typing.Any]: + """ + Collect verified labels and their embeddings, and save them as one npz file. + + Returns the storage path, the URL a service can fetch, and the metadata describing + what went in. Raises NotEnoughVerifiedData rather than writing a file nothing can be + trained on. + """ + counts = label_counts(project, algorithm, occurrence_set) + taxa_list = taxa_list or project.default_taxa_list + + if taxa_list: + # The taxa list decides what the head can predict; the verified crops only decide + # how well it predicts each one. Without this the head shrinks to whatever happened + # to be verified, which silently narrows the pipeline. + classes = sorted(taxa_list.taxa.exclude(name="").values_list("name", flat=True)) + if not classes: + raise NotEnoughVerifiedData(f"Taxa list '{taxa_list}' is empty, so there is nothing to train.") + keep = set(classes) + outside = sorted(name for name in counts if name and name not in keep) + else: + keep = species_with_enough_examples(counts, min_per_species) + if not keep: + raise NotEnoughVerifiedData( + f"No species in '{project.name}' has at least {min_per_species} verified crops with an " + f"embedding from {algorithm.key}. Verify more occurrences, or re-run the pipeline so the " + "verified detections get embeddings." + ) + classes = sorted(keep) + outside = sorted(set(counts) - keep) + + class_index = {name: i for i, name in enumerate(classes)} + + rows = verified_training_rows(project, algorithm, occurrence_set) + total = rows.count() + + # Asked rather than assumed: a vector's width belongs to the model that produced it, and + # the column is unsized so two algorithms may differ. + dimensions = DetectionEmbedding.objects.stored_length(algorithm.pk) + if dimensions is None: + raise NotEnoughVerifiedData( + f"No detection has a feature vector from '{algorithm.key}' yet, so there is nothing to train on." + ) + features = np.zeros((total, dimensions), dtype=DATASET_DTYPE) + labels = np.zeros(total, dtype=np.int64) + detection_ids = np.zeros(total, dtype=np.int64) + occurrence_ids = np.zeros(total, dtype=np.int64) + splits: list[str] = [] + + kept = 0 + for embedding in rows.iterator(chunk_size=FETCH_BATCH_SIZE): + occurrence = embedding.detection.occurrence + if not occurrence or not occurrence.determination: + continue + name = occurrence.determination.name + if name not in class_index: + continue + features[kept] = np.asarray(embedding.vector.to_list(), dtype=DATASET_DTYPE) + labels[kept] = class_index[name] + detection_ids[kept] = embedding.detection_id + occurrence_ids[kept] = occurrence.pk + splits.append(split_for(occurrence.pk, split_salt, test_fraction)) + kept += 1 + + if not kept: + raise NotEnoughVerifiedData("No verified detection has an embedding from this algorithm yet.") + + features = features[:kept] + labels = labels[:kept] + detection_ids = detection_ids[:kept] + occurrence_ids = occurrence_ids[:kept] + split_array = np.array(splits) + + n_train = int((split_array == "train").sum()) + n_test = int((split_array == "test").sum()) + if not n_train or not n_test: + raise NotEnoughVerifiedData( + f"The split left {n_train} training and {n_test} held-out rows. Both sides need rows " + "before a new head can be compared against the current one." + ) + + file_path = dataset_path(project, algorithm, job.pk if job else None) + # Declared, not assembled: the shape is checked here rather than at each reader, and + # the service echoes it back in its result. + metadata = TrainingDatasetMetadata( + url=f"{settings.MEDIA_URL}{file_path}", + project=NamedReference(id=project.pk, name=project.name), + algorithm=TrainedAlgorithmReference(key=algorithm.key, name=algorithm.name, version=algorithm.version), + dimensions=dimensions, + dtype=np.dtype(DATASET_DTYPE).name, + classes=classes, + # .get(): with a taxa list, a class can legitimately have no verified crops yet. + counts={name: counts.get(name, 0) for name in classes}, + taxa_list=NamedReference(id=taxa_list.pk, name=taxa_list.name) if taxa_list else None, + occurrence_set=(NamedReference(id=occurrence_set.pk, name=occurrence_set.name) if occurrence_set else None), + classes_without_verified_data=sorted(name for name in classes if not counts.get(name)), + dropped_species=outside, + rows=kept, + train=n_train, + test=n_test, + verified_detections_without_embedding=count_missing_embeddings(project, algorithm, occurrence_set), + settings=TrainingDatasetSettings( + min_per_species=min_per_species, + split_salt=split_salt, + test_fraction=test_fraction, + ), + ) + + file_path = _save( + file_path=file_path, + arrays={ + "features": features, + "labels": labels, + "detection_ids": detection_ids, + "occurrence_ids": occurrence_ids, + "split": split_array, + "classes": np.array(classes), + }, + metadata=metadata, + ) + + file_url = metadata.url + logger.info(f"Wrote training dataset with {kept} rows over {len(classes)} species to {file_path}") + return {"path": file_path, "url": file_url, "metadata": metadata} + + +def dataset_path(project: Project, algorithm: Algorithm, job_id: int | None) -> str: + """Where this project's training set for this algorithm is written.""" + stem = f"{slugify(project.name)}-{slugify(algorithm.key)}" + suffix = f"job-{job_id}" if job_id else "manual" + return f"{DATASET_DIRECTORY}/{stem}-{suffix}.npz" + + +def _save( + file_path: str, + arrays: dict[str, np.ndarray], + metadata: TrainingDatasetMetadata, +) -> str: + """ + Write the npz to storage and return its path. + + default_storage is the local filesystem in development and the project's S3 bucket in + production, so this follows wherever captures already live without a special case. + """ + with tempfile.TemporaryDirectory() as tmp: + local = pathlib.Path(tmp) / "dataset.npz" + # Uncompressed: embeddings are close to random, so compression buys about 10 per + # cent for real CPU. Metadata rides inside the archive so the file is self-describing. + np.savez(local, metadata=np.array(metadata.json()), **arrays) + if default_storage.exists(file_path): + # A re-run of the same job replaces its dataset instead of piling up copies. + default_storage.delete(file_path) + with open(local, "rb") as f: + saved_path = default_storage.save(file_path, f) + + if saved_path != file_path: + # The metadata inside the archive records the URL it was written to, so a rename + # by the storage backend would leave the file describing somewhere it is not. + logger.warning(f"Storage wrote the training set to {saved_path}, not {file_path}.") + return saved_path diff --git a/ami/ml/views.py b/ami/ml/views.py index 61a63ff7e..516aa3444 100644 --- a/ami/ml/views.py +++ b/ami/ml/views.py @@ -1,24 +1,31 @@ import logging +import pydantic from django.db import transaction from django.db.models import Prefetch from django.db.models.query import QuerySet +from django.http import Http404 from django.utils.text import slugify from drf_spectacular.utils import extend_schema from rest_framework import exceptions as api_exceptions -from rest_framework import mixins, status, viewsets +from rest_framework import mixins, serializers, status, viewsets from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response +from ami.base.pagination import TrainingDataPagination from ami.base.permissions import ProjectPipelineConfigPermission +from ami.base.serializers import SingleParamSerializer from ami.base.views import ProjectMixin from ami.main.api.schemas import project_id_doc_param from ami.main.api.views import DefaultViewSet -from ami.main.models import Project, SourceImage -from ami.ml.schemas import PipelineRegistrationResponse +from ami.main.models import OccurrenceSet, Project, SourceImage +from ami.ml import training +from ami.ml.schemas import AlgorithmTrainingConfig, PipelineRegistrationResponse from .models.algorithm import Algorithm, AlgorithmCategoryMap +from .models.embedding import DetectionEmbedding from .models.pipeline import Pipeline from .models.processing_service import ProcessingService from .models.project_pipeline_config import ProjectPipelineConfig @@ -28,6 +35,7 @@ PipelineRegistrationSerializer, PipelineSerializer, ProcessingServiceSerializer, + TrainingDataRowSerializer, ) logger = logging.getLogger(__name__) @@ -298,3 +306,216 @@ def create(self, request, *args, **kwargs): processing_service.mark_seen(live=True) return Response(response.dict(), status=status.HTTP_201_CREATED) + + +class TrainingDataViewSet(ProjectMixin, mixins.ListModelMixin, viewsets.GenericViewSet): + """ + Verified detections and their embeddings, for retraining a classifier head. + + A head is trained on embeddings, not pixels, and the backbone that produced them is + frozen. So a trainer can pull this and fit a new head without touching the images. + + Requires `project_id` and `algorithm` (an algorithm key). Constraining to one + algorithm is not optional: vectors from different backbones are in different spaces. + + `occurrence_set` narrows this to one saved set. Without it the rows are every verified + occurrence in the project, which is what a retrain does by default. + + GET /api/v2/ml/training-data/?project_id=3&algorithm= + GET /api/v2/ml/training-data/summary/?project_id=3&algorithm=&occurrence_set=7 + """ + + queryset = DetectionEmbedding.objects.none() + serializer_class = TrainingDataRowSerializer + require_project = True + # The project permission is checked per request, in _get_project_for_training(). + # ObjectPermission's has_object_permission only runs for detail actions, and both + # actions here are detail=False, so a viewset-level class would never see the project. + permission_classes = [IsAuthenticated] + filter_backends: list = [] + pagination_class = TrainingDataPagination + + def _get_project_for_training(self) -> Project: + """ + The requested project, if this user may retrain its classifier. + + Project visibility is not enough of a gate here. A non-draft project is readable by + anyone, and what this returns is every verified label in it together with the vector + for each crop: the whole training set, not a page of occurrences. It is gated on the + same permission the training job is, so the people who can start a run are the ones + who can see what it would learn from. + """ + project = self.get_active_project() + if not project: + raise Http404("Project not found.") + if not self.request.user.has_perm(Project.Permissions.RUN_TRAIN_CLASSIFIER_JOB, project): + raise api_exceptions.PermissionDenied("You do not have permission to retrain classifiers in this project.") + return project + + def _get_algorithm(self) -> Algorithm: + # Cached because the split settings fall back to this algorithm's own training + # config, so one request asks for it from more than one place. + if getattr(self, "_algorithm", None) is None: + key = SingleParamSerializer[str].clean( + "algorithm", + serializers.CharField( + required=True, + help_text="Key of the algorithm whose embeddings to train on.", + ), + self.request.query_params, + ) + algorithm = Algorithm.objects.filter(key=key).first() + if not algorithm: + raise api_exceptions.NotFound(f"No algorithm with key '{key}'.") + self._algorithm = algorithm + return self._algorithm + + def _get_occurrence_set(self, project: Project) -> OccurrenceSet | None: + """ + The set to count, or None for every verified occurrence in the project. + + Optional, because retraining on everything verified so far is the common case and + the form shows those numbers before anyone has chosen a set. + """ + occurrence_set_id = SingleParamSerializer[int].clean( + "occurrence_set", + serializers.IntegerField(required=False, allow_null=True, default=None), + self.request.query_params, + ) + if not occurrence_set_id: + return None + # Scoped to the project: a set id from elsewhere must not report this project's + # verified labels, or any other project's. + occurrence_set = OccurrenceSet.objects.for_project(project).filter(pk=occurrence_set_id).first() + if not occurrence_set: + raise api_exceptions.NotFound(f"No occurrence set with id {occurrence_set_id} in this project.") + return occurrence_set + + def _get_split_settings(self) -> AlgorithmTrainingConfig: + """ + The settings these numbers are computed under. + + Defaulted to the algorithm's own training config rather than to the module defaults, + so a form that shows the stats before anything is edited shows what an actual run + would do. Anything the caller passes wins, which is how the form previews a changed + split ratio without starting a job. + """ + config = self._get_algorithm().training_config + params = self.request.query_params + overrides = { + "split_salt": SingleParamSerializer[str].clean( + "split_salt", + serializers.CharField(required=False, allow_null=True, default=None), + params, + ), + "test_fraction": SingleParamSerializer[float].clean( + "test_fraction", + serializers.FloatField(required=False, allow_null=True, default=None), + params, + ), + "min_per_species": SingleParamSerializer[int].clean( + "min_per_species", + serializers.IntegerField(required=False, allow_null=True, default=None), + params, + ), + } + # A parameter that was not sent must not override the config with None. + overrides = {name: value for name, value in overrides.items() if value is not None} + try: + return AlgorithmTrainingConfig.for_run(config, overrides) + except pydantic.ValidationError as e: + # The same bounds a job is held to, refused here rather than reported as stats + # computed under settings no run could use. + raise api_exceptions.ValidationError({"detail": e.errors()}) + + def get_queryset(self) -> QuerySet[DetectionEmbedding]: + project = self._get_project_for_training() + qs = training.verified_training_rows(project, self._get_algorithm(), self._get_occurrence_set(project)) + + self._split_filter = SingleParamSerializer[str].clean( + "split", + serializers.ChoiceField(choices=list(training.SPLITS), required=False, allow_null=True, default=None), + self.request.query_params, + ) + return qs + + def get_serializer_context(self): + context = super().get_serializer_context() + config = self._get_split_settings() + context["split_salt"] = config.split_salt + context["test_fraction"] = config.test_fraction + # Not url_boolean_param: it returns `value or default`, so a default of True can + # never be turned off. + context["include_features"] = SingleParamSerializer[bool].clean( + "include_features", + serializers.BooleanField(required=False, default=True), + self.request.query_params, + ) + return context + + @extend_schema(parameters=[project_id_doc_param]) + def list(self, request, *args, **kwargs): + response = super().list(request, *args, **kwargs) + split = getattr(self, "_split_filter", None) + if split: + # Filtering by split in Python rather than SQL: the assignment is a hash of the + # occurrence id, which Postgres cannot compute. Callers that need whole splits + # should page through everything and group client-side. + results = [row for row in response.data["results"] if row["split"] == split] + response.data["results"] = results + return response + + @extend_schema(parameters=[project_id_doc_param]) + @action(detail=False, methods=["get"]) + def summary(self, request, *args, **kwargs): + """ + Counts only, for what a retrain would learn from right now. + + This is what the training job form shows before anyone starts a run: how much + verified data there is, how many species of it are usable, and how the split would + fall. Cheap enough to re-fetch whenever the chosen set or the split ratio changes. + """ + project = self._get_project_for_training() + algorithm = self._get_algorithm() + occurrence_set = self._get_occurrence_set(project) + config = self._get_split_settings() + + counts = training.label_counts(project, algorithm, occurrence_set) + kept = training.species_with_enough_examples(counts, config.min_per_species) + rows = training.verified_training_rows(project, algorithm, occurrence_set) + + splits = {name: 0 for name in training.SPLITS} + occurrences = set() + for occurrence_id in rows.values_list("detection__occurrence_id", flat=True): + splits[training.split_for(occurrence_id, config.split_salt, config.test_fraction)] += 1 + occurrences.add(occurrence_id) + + return Response( + { + "project": {"id": project.pk, "name": project.name}, + "algorithm": {"key": algorithm.key, "name": algorithm.name, "version": algorithm.version}, + # None means every verified occurrence in the project, which is what a run + # without a set does. + "occurrence_set": ({"id": occurrence_set.pk, "name": occurrence_set.name} if occurrence_set else None), + "dimensions": DetectionEmbedding.objects.stored_length(algorithm.pk), + "rows": sum(counts.values()), + "occurrences": len(occurrences), + "classes": len(counts), + # What the head would actually learn, once species with too few crops are + # dropped. Showing only the raw class count overstates the run. + "trainable_classes": len(kept), + "dropped_species": sorted(name for name in counts if name and name not in kept), + "counts": dict(sorted(counts.items(), key=lambda kv: -kv[1])), + "train": splits[training.SPLIT_TRAIN], + "test": splits[training.SPLIT_TEST], + "verified_detections_without_embedding": training.count_missing_embeddings( + project, algorithm, occurrence_set + ), + "settings": { + "min_per_species": config.min_per_species, + "split_salt": config.split_salt, + "test_fraction": config.test_fraction, + "split_grouped_by": "occurrence", + }, + } + ) diff --git a/ami/users/roles.py b/ami/users/roles.py index 718146e41..d05aa209a 100644 --- a/ami/users/roles.py +++ b/ami/users/roles.py @@ -146,6 +146,9 @@ class MLDataManager(Role): Project.Permissions.SYNC_DEPLOYMENT, Project.Permissions.RUN_REGROUP_EVENTS_JOB, Project.Permissions.RUN_DATA_EXPORT_JOB, + # Retraining a head is ML data work, so it sits with the other run_*_job + # permissions this role already holds. + Project.Permissions.RUN_TRAIN_CLASSIFIER_JOB, Project.Permissions.DELETE_OCCURRENCES, Project.Permissions.CREATE_PROJECT_PIPELINE_CONFIG, Project.Permissions.UPDATE_PROJECT_PIPELINE_CONFIG, @@ -154,6 +157,11 @@ class MLDataManager(Role): Project.Permissions.UPDATE_COLLECTION, Project.Permissions.DELETE_COLLECTION, Project.Permissions.POPULATE_COLLECTION, + # Whoever curates the data a model is measured on is the same person who manages + # the data it learns from, so this sits with the collection permissions. + Project.Permissions.CREATE_OCCURRENCE_SET, + Project.Permissions.UPDATE_OCCURRENCE_SET, + Project.Permissions.DELETE_OCCURRENCE_SET, } diff --git a/config/api_router.py b/config/api_router.py index 56a01c591..790acbc53 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -41,6 +41,7 @@ router.register(r"detections", views.DetectionViewSet) # Register before r"occurrences" — see docs/claude/reference/api-stats-pattern.md (guarded by regression test). router.register(r"occurrences/stats", views.OccurrenceStatsViewSet, basename="occurrence-stats") +router.register(r"occurrences/sets", views.OccurrenceSetViewSet) router.register(r"occurrences", views.OccurrenceViewSet) router.register(r"taxa/lists", views.TaxaListViewSet) # NESTED: /taxa/lists/{taxalist_id}/taxa/ @@ -53,6 +54,7 @@ router.register(r"taxa", views.TaxonViewSet) router.register(r"tags", views.TagViewSet) router.register(r"ml/algorithms", ml_views.AlgorithmViewSet) +router.register(r"ml/training-data", ml_views.TrainingDataViewSet, basename="training-data") router.register(r"ml/labels", ml_views.AlgorithmCategoryMapViewSet) router.register(r"ml/pipelines", ml_views.PipelineViewSet) router.register(r"ml/processing_services", ml_views.ProcessingServiceViewSet) diff --git a/ui/src/components/filtering/filter-control.tsx b/ui/src/components/filtering/filter-control.tsx index e9cc0eed5..552724853 100644 --- a/ui/src/components/filtering/filter-control.tsx +++ b/ui/src/components/filtering/filter-control.tsx @@ -10,6 +10,7 @@ import { DateFilter } from './filters/date-filter' import { DeviceFilter } from './filters/device-filter' import { ImageFilter } from './filters/image-filter' import { OccurrenceFilter } from './filters/occurrence-filter' +import { OccurrenceSetFilter } from './filters/occurrence-set-filter' import { PipelineFilter } from './filters/pipeline-filter' import { SessionFilter } from './filters/session-filter' import { SiteFilter } from './filters/site-filter' @@ -45,6 +46,7 @@ const ComponentMap: { not_algorithm: NotAlgorithmFilter, not_tag_id: TagFilter, not_taxa_list_id: TaxaListFilter, + occurrence_set: OccurrenceSetFilter, pipeline: PipelineFilter, similar_to: OccurrenceFilter, source_image_collection: CaptureSetFilter, diff --git a/ui/src/components/filtering/filters/occurrence-set-filter.tsx b/ui/src/components/filtering/filters/occurrence-set-filter.tsx new file mode 100644 index 000000000..3e6b2c620 --- /dev/null +++ b/ui/src/components/filtering/filters/occurrence-set-filter.tsx @@ -0,0 +1,17 @@ +import { API_ROUTES } from 'data-services/constants' +import { EntityPicker } from 'nova-ui-kit' +import { FilterProps } from './types' + +export const OccurrenceSetFilter = ({ onAdd, onClear, value }: FilterProps) => ( + { + if (value) { + onAdd(value) + } else { + onClear() + } + }} + value={value} + /> +) diff --git a/ui/src/data-services/constants.ts b/ui/src/data-services/constants.ts index 59bae6ede..26891161f 100644 --- a/ui/src/data-services/constants.ts +++ b/ui/src/data-services/constants.ts @@ -3,6 +3,8 @@ export const API_URL = '/api/v2' export const API_ROUTES = { ALGORITHM: 'ml/algorithms', CAPTURE_SET_CHOICES: 'captures/collections/choices', + OCCURRENCE_SETS: 'occurrences/sets', + OCCURRENCE_SET_CHOICES: 'occurrences/sets/choices', CAPTURE_SETS: 'captures/collections', CAPTURES: 'captures', CLASSIFICATIONS: 'classifications', diff --git a/ui/src/data-services/hooks/occurrence-sets/useCreateOccurrenceSet.ts b/ui/src/data-services/hooks/occurrence-sets/useCreateOccurrenceSet.ts new file mode 100644 index 000000000..8d6b6154c --- /dev/null +++ b/ui/src/data-services/hooks/occurrence-sets/useCreateOccurrenceSet.ts @@ -0,0 +1,41 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query' +import axios from 'axios' +import { API_ROUTES, API_URL, SUCCESS_TIMEOUT } from 'data-services/constants' +import { getAuthHeader } from 'data-services/utils' +import { useUser } from 'utils/user/userContext' + +interface OccurrenceSetFieldValues { + name: string + description?: string + projectId: string + occurrenceIds: string[] +} + +const convertToServerFieldValues = (fieldValues: OccurrenceSetFieldValues) => ({ + name: fieldValues.name, + description: fieldValues.description ?? '', + project_id: fieldValues.projectId, + occurrence_ids: fieldValues.occurrenceIds.map((id) => Number(id)), +}) + +export const useCreateOccurrenceSet = (onSuccess?: (id: string) => void) => { + const { user } = useUser() + const queryClient = useQueryClient() + + const { mutateAsync, isLoading, isSuccess, reset, error } = useMutation({ + mutationFn: (fieldValues: OccurrenceSetFieldValues) => + axios.post<{ id: number }>( + `${API_URL}/${API_ROUTES.OCCURRENCE_SETS}/`, + convertToServerFieldValues(fieldValues), + { headers: getAuthHeader(user) } + ), + onSuccess: ({ data }) => { + queryClient.invalidateQueries([API_ROUTES.OCCURRENCE_SETS]) + queryClient.invalidateQueries([API_ROUTES.OCCURRENCE_SET_CHOICES]) + onSuccess?.(`${data.id}`) + setTimeout(reset, SUCCESS_TIMEOUT) + }, + }) + + return { createOccurrenceSet: mutateAsync, isLoading, isSuccess, error } +} diff --git a/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx new file mode 100644 index 000000000..59025acc0 --- /dev/null +++ b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set-popover.tsx @@ -0,0 +1,39 @@ +import { BookmarkPlusIcon } from 'lucide-react' +import { Button, Popover } from 'nova-ui-kit' +import { useState } from 'react' +import { STRING, translate } from 'utils/language' +import { CreateOccurrenceSet } from './create-occurrence-set' + +interface CreateOccurrenceSetPopoverProps { + occurrenceIds: string[] +} + +export const CreateOccurrenceSetPopover = ({ + occurrenceIds, +}: CreateOccurrenceSetPopoverProps) => { + const [open, setOpen] = useState(false) + + return ( + + + + + + setOpen(false)} + /> + + + ) +} diff --git a/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set.tsx b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set.tsx new file mode 100644 index 000000000..f249c36f9 --- /dev/null +++ b/ui/src/pages/occurrences/create-occurrence-set/create-occurrence-set.tsx @@ -0,0 +1,70 @@ +import { FormError } from 'components/form/layout/layout' +import { useCreateOccurrenceSet } from 'data-services/hooks/occurrence-sets/useCreateOccurrenceSet' +import { Loader2Icon } from 'lucide-react' +import { Button, Input } from 'nova-ui-kit' +import { useState } from 'react' +import { useParams } from 'react-router-dom' +import { STRING, translate } from 'utils/language' +import { parseServerError } from 'utils/parseServerError/parseServerError' + +interface CreateOccurrenceSetProps { + occurrenceIds: string[] + onCancel: () => void +} + +export const CreateOccurrenceSet = ({ + occurrenceIds, + onCancel, +}: CreateOccurrenceSetProps) => { + const { projectId } = useParams() + const [name, setName] = useState('') + const { createOccurrenceSet, isLoading, error } = + useCreateOccurrenceSet(onCancel) + const formError = error ? parseServerError(error)?.message : undefined + + return ( + <> + {formError && ( + + )} +
+
+ setName(e.target.value)} + value={name} + /> +
+ + {translate(STRING.MESSAGE_OCCURRENCE_SET_IS_FIXED, { + count: `${occurrenceIds.length}`, + })} + +
+ + +
+
+ + ) +} diff --git a/ui/src/pages/occurrences/occurrence-filters.ts b/ui/src/pages/occurrences/occurrence-filters.ts index e63d5093c..6a79c7741 100644 --- a/ui/src/pages/occurrences/occurrence-filters.ts +++ b/ui/src/pages/occurrences/occurrence-filters.ts @@ -11,6 +11,7 @@ export const FILTERS_TO_OCCURRENCES = [ 'verified', 'verified_by_me', 'collection', + 'occurrence_set', 'date_start', 'date_end', 'deployment', diff --git a/ui/src/pages/occurrences/occurrences.tsx b/ui/src/pages/occurrences/occurrences.tsx index bbffc1e12..1fda091c3 100644 --- a/ui/src/pages/occurrences/occurrences.tsx +++ b/ui/src/pages/occurrences/occurrences.tsx @@ -31,6 +31,7 @@ import { columns } from './occurrence-columns' import { OccurrenceDetailsDialog } from './occurrence-details-dialog' import { OccurrenceGallery } from './occurrence-gallery' import { OccurrenceStats } from './occurrence-stats' +import { CreateOccurrenceSetPopover } from './create-occurrence-set/create-occurrence-set-popover' import { OccurrencesActions } from './occurrences-actions' export const Occurrences = () => { @@ -114,6 +115,7 @@ export const Occurrences = () => { defaultOpen={someActive( [ 'collection', + 'occurrence_set', 'deployment', 'deployment__device', 'deployment__research_site', @@ -127,6 +129,7 @@ export const Occurrences = () => { + @@ -167,6 +170,9 @@ export const Occurrences = () => { Export + {selectedItems.length > 0 && ( + + )} [ }, }, }, + { + label: translate(STRING.FIELD_LABEL_OCCURRENCE_SET), + field: 'occurrence_set', + tooltip: { + text: translate(STRING.TOOLTIP_OCCURRENCE_SET), + }, + }, { label: translate(STRING.FIELD_LABEL_CAPTURE_SET), field: 'collection', // This is for viewing occurrences by capture set. @TODO: Can we update this key to "capture_set_id" to streamline?