From ff99d4bb2a78d1d9e35f481540f7a1fc099fdbb5 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 3 Sep 2026 17:49:05 -0400 Subject: [PATCH 01/31] feat(ml): store embeddings and retrain a classifier head from verified species When someone confirms a species in Antenna, that answer can now be used to improve the model. BioCLIP itself is untouched: the backbone is frozen, so only the small classifier head on top of it is retrained. Because the backbone never changes, a crop's embedding never changes either. It is saved once when the crop is first classified and reused by every later retrain, so the big model never runs over the same image twice. Two new job types drive this. One fills in embeddings for verified crops without re-running the detector. The other collects verified labels and their embeddings, writes them to a file in project storage, and hands a processing service a link to it. The service fits a new head, scores it against the head it is currently serving on the same held-out rows, and reports back. Every retrain is recorded as a new algorithm version with the dataset it was fitted on and the scores it achieved, so any prediction stays traceable to the exact weights that made it. Nothing is promoted automatically. Needs the pgvector extension in the database. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/base/pagination.py | 12 + .../0024_train_classifier_job_type.py | 31 + .../0025_generate_embeddings_job_type.py | 32 + ami/jobs/models.py | 347 +++++++- ami/jobs/serializers.py | 4 + ami/main/models.py | 4 + .../commands/export_verified_training_data.py | 168 ++++ .../0029_detectionembedding_and_more.py | 53 ++ ami/ml/migrations/0030_algorithm_trainable.py | 20 + .../0031_algorithm_training_fields.py | 36 + ami/ml/models/__init__.py | 2 + ami/ml/models/algorithm.py | 28 + ami/ml/models/embedding.py | 60 ++ ami/ml/models/pipeline.py | 80 ++ ami/ml/schemas.py | 87 ++ ami/ml/serializers.py | 27 +- ami/ml/tests.py | 825 +++++++++++++++++- ami/ml/training_data.py | 116 +++ ami/ml/training_dataset.py | 180 ++++ ami/ml/training_dispatch.py | 76 ++ ami/ml/views.py | 117 +++ compose/local/postgres/Dockerfile | 11 + config/api_router.py | 1 + requirements/base.txt | 1 + .../components/filtering/filter-control.tsx | 4 + .../filtering/filters/device-filter.tsx | 17 + .../filtering/filters/site-filter.tsx | 17 + .../useCreateIdentification.ts | 3 + .../useCreateIdentifications.ts | 2 + .../occurrences/stats/useModelAgreement.ts | 87 ++ .../data-services/hooks/species/useSpecies.ts | 16 +- ui/src/data-services/models/species.ts | 39 + ui/src/data-services/types.ts | 1 + ui/src/data-services/utils.ts | 3 + .../components/table/table/table.tsx | 5 +- .../occurrences/occurrence-details-dialog.tsx | 86 ++ .../pages/occurrences/occurrence-filters.ts | 22 + .../occurrences/occurrence-navigation.tsx | 92 +- ui/src/pages/occurrences/occurrence-stats.tsx | 344 ++++++++ ui/src/pages/occurrences/occurrences.tsx | 108 +-- .../pages/species-details/species-details.tsx | 15 +- ui/src/pages/species/species-columns.tsx | 104 ++- ui/src/pages/species/species-filters.ts | 18 + ui/src/pages/species/species.tsx | 163 +++- ui/src/utils/buildCarryOverFilters.ts | 14 + ui/src/utils/carryOverFilters.test.ts | 48 + ui/src/utils/getAppRoute.ts | 6 + ui/src/utils/language.ts | 34 + ui/src/utils/useColumnSettings.tsx | 8 +- ui/src/utils/useFilters.ts | 23 + 50 files changed, 3426 insertions(+), 171 deletions(-) create mode 100644 ami/jobs/migrations/0024_train_classifier_job_type.py create mode 100644 ami/jobs/migrations/0025_generate_embeddings_job_type.py create mode 100644 ami/ml/management/commands/export_verified_training_data.py create mode 100644 ami/ml/migrations/0029_detectionembedding_and_more.py create mode 100644 ami/ml/migrations/0030_algorithm_trainable.py create mode 100644 ami/ml/migrations/0031_algorithm_training_fields.py create mode 100644 ami/ml/models/embedding.py create mode 100644 ami/ml/training_data.py create mode 100644 ami/ml/training_dataset.py create mode 100644 ami/ml/training_dispatch.py create mode 100644 ui/src/components/filtering/filters/device-filter.tsx create mode 100644 ui/src/components/filtering/filters/site-filter.tsx create mode 100644 ui/src/data-services/hooks/occurrences/stats/useModelAgreement.ts create mode 100644 ui/src/pages/occurrences/occurrence-details-dialog.tsx create mode 100644 ui/src/pages/occurrences/occurrence-filters.ts create mode 100644 ui/src/pages/occurrences/occurrence-stats.tsx create mode 100644 ui/src/pages/species/species-filters.ts create mode 100644 ui/src/utils/buildCarryOverFilters.ts create mode 100644 ui/src/utils/carryOverFilters.test.ts diff --git a/ami/base/pagination.py b/ami/base/pagination.py index 9ebca7b21..e3b8e6308 100644 --- a/ami/base/pagination.py +++ b/ami/base/pagination.py @@ -29,3 +29,15 @@ def _get_project(self): if hasattr(view, "get_active_project"): return view.get_active_project() return None + + +class TrainingDataPagination(LimitOffsetPagination): + """ + Paging for embedding rows, which are far bigger than a normal API row. + + A 1024-dimension vector serialises to roughly 20 KB of JSON, so the platform default + of 10 is uselessly small and an unbounded limit would return hundreds of megabytes. + """ + + default_limit = 200 + max_limit = 1000 diff --git a/ami/jobs/migrations/0024_train_classifier_job_type.py b/ami/jobs/migrations/0024_train_classifier_job_type.py new file mode 100644 index 000000000..5434571a1 --- /dev/null +++ b/ami/jobs/migrations/0024_train_classifier_job_type.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.10 on 2026-09-02 13:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("jobs", "0023_alter_job_job_type_key"), + ] + + operations = [ + migrations.AlterField( + model_name="job", + name="job_type_key", + field=models.CharField( + choices=[ + ("ml", "ML pipeline"), + ("populate_captures_collection", "Populate capture set"), + ("data_storage_sync", "Data storage sync"), + ("regroup_events", "Regroup sessions"), + ("unknown", "Unknown"), + ("data_export", "Data Export"), + ("post_processing", "Post Processing"), + ("train_classifier", "Train classifier"), + ], + default="unknown", + max_length=255, + verbose_name="Job Type", + ), + ), + ] diff --git a/ami/jobs/migrations/0025_generate_embeddings_job_type.py b/ami/jobs/migrations/0025_generate_embeddings_job_type.py new file mode 100644 index 000000000..5f6962190 --- /dev/null +++ b/ami/jobs/migrations/0025_generate_embeddings_job_type.py @@ -0,0 +1,32 @@ +# Generated by Django 4.2.10 on 2026-09-03 16:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("jobs", "0024_train_classifier_job_type"), + ] + + operations = [ + migrations.AlterField( + model_name="job", + name="job_type_key", + field=models.CharField( + choices=[ + ("ml", "ML pipeline"), + ("populate_captures_collection", "Populate capture set"), + ("data_storage_sync", "Data storage sync"), + ("regroup_events", "Regroup sessions"), + ("unknown", "Unknown"), + ("data_export", "Data Export"), + ("post_processing", "Post Processing"), + ("train_classifier", "Train classifier"), + ("generate_embeddings", "Generate embeddings"), + ], + default="unknown", + max_length=255, + verbose_name="Job Type", + ), + ), + ] diff --git a/ami/jobs/models.py b/ami/jobs/models.py index ff65f31f2..83ebb2690 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -562,7 +562,14 @@ def run(cls, job: "Job"): cls.process_images(job, images) @classmethod - def process_images(cls, job, images): + def process_images(cls, job, images, reprocess_all_images: bool | None = None): + """ + `reprocess_all_images` defaults to the project flag. An embedding job overrides it, + because it must always send the existing detections: it needs the vectors for crops + that are already detected, not a fresh run of the detector. + """ + if reprocess_all_images is None: + reprocess_all_images = job.project.feature_flags.reprocess_all_images image_count = len(images) # Keep track of sub-tasks for saving results, pair with batch number save_tasks: list[tuple[int, AsyncResult]] = [] @@ -585,7 +592,7 @@ def process_images(cls, job, images): images=chunk, job_id=job.pk, project_id=job.project.pk, - reprocess_all_images=job.project.feature_flags.reprocess_all_images, + reprocess_all_images=reprocess_all_images, ) job.logger.info(f"Processed image batch {i+1} in {time.time() - request_sent:.2f}s") except Exception as e: @@ -920,6 +927,340 @@ def run(cls, job: "Job"): job.save() +class GenerateEmbeddingsJob(JobType): + """ + Fill in the embeddings that a classifier head needs to be retrained. + + Runs the same inference pipeline as an ML job, but only over detections that a person + has verified and that have no embedding yet. The backbone is frozen, so an embedding + is computed once and reused by every later retrain. + + Existing detections are sent with the request, so the detector does not run again and + no new detections are created. Only the vectors are new. + """ + + name = "Generate embeddings" + key = "generate_embeddings" + + @classmethod + def run(cls, job: "Job"): + if not job.pipeline: + raise ValueError("A generate_embeddings job needs a pipeline to run the backbone.") + if not job.project.feature_flags.store_classification_embeddings: + raise ValueError( + f"Project '{job.project}' does not have store_classification_embeddings enabled, " + "so the embeddings this job produces would be thrown away." + ) + + algorithm = cls.target_algorithm(job) + job.progress.add_stage("Finding detections", "collect") + job.progress.add_stage("Computing embeddings", "process") + job.update_status(JobState.STARTED) + job.started_at = datetime.datetime.now() + job.finished_at = None + job.progress.update_stage("collect", status=JobState.STARTED, progress=0) + job.save() + + images = cls.images_needing_embeddings(job, algorithm) + job.progress.add_stage_param("collect", "Algorithm", algorithm.key) + job.progress.add_stage_param("collect", "Captures", len(images)) + job.progress.update_stage("collect", status=JobState.SUCCESS, progress=1, total_images=len(images)) + job.save() + + if not images: + job.logger.info(f"Every verified detection already has an embedding from {algorithm.key}. Nothing to do.") + job.progress.update_stage("process", status=JobState.SUCCESS, progress=1) + job.finished_at = datetime.datetime.now() + job.update_status(JobState.SUCCESS, save=True) + return + + job.logger.info(f"Computing {algorithm.key} embeddings for detections in {len(images)} capture(s)") + before = cls.embedding_count(algorithm) + + # Reuse the ML job's dispatch loop rather than a parallel copy of it: batching, + # progress and result saving are already handled there. + MLJob.process_images(job, images, reprocess_all_images=True) + + created = cls.embedding_count(algorithm) - before + job.logger.info(f"Stored {created} new embedding(s)") + job.progress.add_stage_param("process", "Embeddings stored", created) + job.save() + + @classmethod + def target_algorithm(cls, job: "Job"): + """The pipeline's trainable algorithm. That is the one whose embeddings are worth storing.""" + candidates = list(job.pipeline.algorithms.filter(trainable=True)) if job.pipeline else [] + key = (job.params or {}).get("algorithm_key") + if key: + match = next((a for a in candidates if a.key == key), None) + if not match: + raise ValueError( + f"Algorithm '{key}' is not a trainable algorithm on pipeline '{job.pipeline}'. " + f"Trainable algorithms: {[a.key for a in candidates] or 'none'}" + ) + return match + if not candidates: + raise ValueError( + f"Pipeline '{job.pipeline}' has no algorithm marked trainable, so there is no head " + "whose embeddings are worth storing." + ) + if len(candidates) > 1: + raise ValueError( + f"Pipeline '{job.pipeline}' has several trainable algorithms " + f"({[a.key for a in candidates]}). Pass algorithm_key in the job params to choose one." + ) + return candidates[0] + + @classmethod + def images_needing_embeddings(cls, job: "Job", algorithm) -> list[SourceImage]: + """ + Captures holding at least one verified detection with no embedding from this algorithm. + + Verified only, by decision: an embedding is ~2 KB and only a labelled crop can train + a head, so embedding everything would cost gigabytes to no purpose. + """ + from ami.main.models import Detection + from ami.ml import training_data + + detections = ( + Detection.objects.filter(occurrence_id__in=training_data.verified_occurrence_ids(job.project)) + .exclude(embeddings__algorithm=algorithm) + .order_by() + ) + image_ids = list(detections.values_list("source_image_id", flat=True).distinct()) + if job.limit: + image_ids = image_ids[: job.limit] + return list(SourceImage.objects.filter(pk__in=image_ids)) + + @classmethod + def embedding_count(cls, algorithm) -> int: + from ami.ml.models import DetectionEmbedding + + return DetectionEmbedding.objects.filter(algorithm=algorithm).count() + + +class TrainClassifierJob(JobType): + """ + Retrain a classifier head from the species people have verified in this project. + + Antenna prepares the training set and hands the processing service a URL to it, rather + than posting the rows: a project with a few hundred thousand verified labels runs to + hundreds of megabytes, which is fragile to send in one request and has to start over if + the connection drops. + + The job does not wait for training to finish. It dispatches and leaves the job STARTED; + the service reports back through the job's result endpoint. Waiting inline would hit the + sub-task timeout on anything but a toy dataset. + """ + + name = "Train classifier" + key = "train_classifier" + + STAGE_PREPARE = "prepare" + STAGE_DISPATCH = "dispatch" + STAGE_TRAIN = "train" + + @classmethod + def run(cls, job: "Job"): + from ami.ml.models import Algorithm + from ami.ml.models.processing_service import ProcessingService + from ami.ml.training_dataset import NotEnoughVerifiedData, build_training_dataset + + params = job.params or {} + algorithm_key = params.get("algorithm_key") + if not algorithm_key: + raise ValueError("A train_classifier job needs an 'algorithm_key' in its params.") + + algorithm = Algorithm.objects.filter(key=algorithm_key).first() + if not algorithm: + raise ValueError(f"No algorithm with key '{algorithm_key}'.") + if not algorithm.trainable: + raise ValueError( + f"Algorithm '{algorithm_key}' is not marked trainable by its processing service. " + "Re-register the pipelines if the service has since been updated." + ) + + job.progress.add_stage("Preparing training set", cls.STAGE_PREPARE) + job.progress.add_stage("Sending to processing service", cls.STAGE_DISPATCH) + job.progress.add_stage("Training", cls.STAGE_TRAIN) + job.update_status(JobState.STARTED) + job.started_at = datetime.datetime.now() + job.finished_at = None + job.progress.update_stage(cls.STAGE_PREPARE, status=JobState.STARTED, progress=0) + job.save() + + try: + dataset = build_training_dataset( + project=job.project, + algorithm=algorithm, + min_per_species=params.get("min_per_species", 2), + job_id=job.pk, + ) + except NotEnoughVerifiedData as e: + # A data problem, not a crash. Say so plainly rather than failing with a traceback. + job.logger.error(str(e)) + job.progress.update_stage(cls.STAGE_PREPARE, status=JobState.FAILURE, progress=0) + job.finished_at = datetime.datetime.now() + job.result = {"error": str(e)} + job.update_status(JobState.FAILURE, save=True) + return + + meta = dataset["metadata"] + job.logger.info( + f"Training set: {meta['rows']} verified crops over {len(meta['classes'])} species " + f"({meta['train']} train / {meta['test']} held out)" + ) + if meta["verified_detections_without_embedding"]: + job.logger.warning( + f"{meta['verified_detections_without_embedding']} verified detection(s) have no embedding " + "from this algorithm and were left out. Re-run the pipeline over them to include them." + ) + job.progress.add_stage_param(cls.STAGE_PREPARE, "Rows", meta["rows"]) + job.progress.add_stage_param(cls.STAGE_PREPARE, "Species", len(meta["classes"])) + job.progress.add_stage_param(cls.STAGE_PREPARE, "Dataset", dataset["url"]) + job.progress.update_stage(cls.STAGE_PREPARE, status=JobState.SUCCESS, progress=1) + job.save() + + service = ( + ProcessingService.objects.filter( + projects=job.project, + pipelines__algorithms=algorithm, + endpoint_url__isnull=False, + ) + .exclude(endpoint_url="") + .distinct() + .first() + ) + if not service: + # Pull-mode workers register with a null endpoint_url, so there is nothing to + # send a training request to. Supporting them means routing this through the + # task queue instead. + raise ValueError( + f"No push-mode processing service in this project serves '{algorithm_key}'. " + "Training cannot be dispatched to a pull-mode worker." + ) + + job.progress.update_stage(cls.STAGE_DISPATCH, status=JobState.STARTED, progress=0) + job.save() + cls.dispatch(job=job, service=service, algorithm=algorithm, dataset=dataset) + + @classmethod + def dispatch(cls, job: "Job", service, algorithm, dataset: dict) -> None: + """Hand the service the dataset URL and leave the job running until it reports back.""" + from ami.ml.training_dispatch import send_training_request + + response = send_training_request(job=job, service=service, algorithm=algorithm, dataset=dataset) + job.progress.update_stage(cls.STAGE_DISPATCH, status=JobState.SUCCESS, progress=1) + job.progress.update_stage(cls.STAGE_TRAIN, status=JobState.STARTED, progress=0) + job.logger.info(f"Training request accepted by {service.name}. Waiting for it to report back.") + job.save() + + if response is not None: + # The service answered synchronously, which small datasets do. Record it now + # rather than leaving the job waiting for a callback that already happened. + cls.record_result( + job=job, + payload={ + "result": response, + "dataset": dataset["metadata"], + # Kept alongside the metadata so the registered version can point at the + # exact file it was fitted on, not just describe it. + "dataset_url": dataset["url"], + }, + ) + + @classmethod + def record_result(cls, job: "Job", payload: dict) -> None: + """Store what the service reported, register the new version, and finish the job.""" + result = payload.get("result") or {} + job.result = payload + + for warning in result.get("warnings", []): + job.logger.warning(warning) + + candidate = result.get("candidate_metrics") or {} + incumbent = result.get("incumbent_metrics") or {} + job.progress.add_stage_param(cls.STAGE_TRAIN, "New head top-1", candidate.get("top1")) + job.progress.add_stage_param(cls.STAGE_TRAIN, "Current head top-1", incumbent.get("top1")) + job.progress.add_stage_param(cls.STAGE_TRAIN, "Better", result.get("promote")) + job.logger.info(result.get("reason", "Training finished.")) + + new_version = cls.register_new_version(job=job, payload=payload) + if new_version: + job.progress.add_stage_param(cls.STAGE_TRAIN, "New version", new_version.key) + job.logger.info(f"Registered {new_version} as version {new_version.version}") + + job.progress.update_stage(cls.STAGE_TRAIN, status=JobState.SUCCESS, progress=1) + job.finished_at = datetime.datetime.now() + job.update_status(JobState.SUCCESS, save=True) + job.save() + + @classmethod + def register_new_version(cls, job: "Job", payload: dict): + """ + Record the trained head as a new version of the algorithm it was trained from. + + A Classification points at an Algorithm row, so a version that changed in place + would make past predictions untraceable. Each retrain therefore gets its own row: + same name, next version number, and a new key, because key is unique on its own. + """ + from ami.ml.models import Algorithm, AlgorithmCategoryMap + from ami.ml.schemas import AlgorithmTrainingInfo + + result = payload.get("result") or {} + dataset = payload.get("dataset") or {} + parent_key = (job.params or {}).get("algorithm_key") + parent = Algorithm.objects.filter(key=parent_key).first() + if not parent: + job.logger.warning(f"Cannot register a new version: no algorithm with key '{parent_key}'.") + return None + + labels = result.get("labels") or [] + if not labels: + job.logger.warning("The service returned no class list, so no new version was registered.") + return None + + trained_at = result.get("trained_at") or "" + stamp = str(trained_at).replace(":", "").replace("-", "").replace(".", "")[:15] or f"job{job.pk}" + version = ( + Algorithm.objects.filter(name=parent.name).order_by("-version").values_list("version", flat=True).first() + or parent.version + ) + 1 + + category_map = AlgorithmCategoryMap.objects.create( + labels=labels, + data=[{"label": label, "index": i} for i, label in enumerate(labels)], + version=stamp, + description=( + f"Retrained from {dataset.get('rows', len(labels))} verified crops in " + f"project '{job.project.name}' by job #{job.pk}." + ), + ) + + return Algorithm.objects.create( + name=parent.name, + key=f"{parent.key}-v{version}-{stamp}", + version=version, + version_name=str(trained_at) or stamp, + task_type=parent.task_type, + description=parent.description, + trainable=parent.trainable, + training_config=parent.training_config, + category_map=category_map, + training_info=AlgorithmTrainingInfo( + trained_at=trained_at or None, + dataset_url=payload.get("dataset_url") or dataset.get("url"), + dataset_rows=(result.get("rows") or {}).get("kept"), + dataset_classes=len(labels), + metrics=result.get("candidate_metrics") or {}, + previous_metrics=result.get("incumbent_metrics") or {}, + parent_algorithm_key=parent.key, + job_id=job.pk, + warnings=result.get("warnings") or [], + ), + ) + + class RegroupEventsJob(JobType): """ Regroup a deployment's captures into Events using the project's @@ -985,6 +1326,8 @@ def run(cls, job: "Job"): UnknownJobType, DataExportJob, PostProcessingJob, + TrainClassifierJob, + GenerateEmbeddingsJob, ] diff --git a/ami/jobs/serializers.py b/ami/jobs/serializers.py index f53199e73..b739797cb 100644 --- a/ami/jobs/serializers.py +++ b/ami/jobs/serializers.py @@ -55,6 +55,9 @@ class JobListSerializer(DefaultSerializer): # All jobs created from the Jobs UI are ML jobs (datasync, etc. are created for the user) # @TODO Remove this when the UI is updated pass a job type. This should be a required field. job_type_key = serializers.SlugField(write_only=True, default=MLJob.key) + # Free-form, and read by the job type that consumes it. A train_classifier job + # carries its algorithm_key here; an ML job ignores it entirely. + params = serializers.JSONField(required=False, allow_null=True) project_id = serializers.PrimaryKeyRelatedField( label="Project", @@ -129,6 +132,7 @@ class Meta: "logs", "job_type", "job_type_key", + "params", "data_export", "dispatch_mode", # "duration", diff --git a/ami/main/models.py b/ami/main/models.py index 78e67e275..3d055ff8e 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -279,6 +279,10 @@ class ProjectFeatureFlags(pydantic.BaseModel): # Feature flag for jobs to reprocess all images in the project, even if already processed reprocess_all_images: bool = False async_pipeline_workers: bool = True # Whether to use async pipeline workers that pull tasks from a queue + # Whether to save the feature vectors returned with classifications, for retraining + # classifier heads from verified labels. Off by default: a stored vector costs about + # 5 KB once the vector index is counted, so this is opt-in per project. + store_classification_embeddings: bool = False def get_default_feature_flags() -> ProjectFeatureFlags: diff --git a/ami/ml/management/commands/export_verified_training_data.py b/ami/ml/management/commands/export_verified_training_data.py new file mode 100644 index 000000000..9dc3b44d3 --- /dev/null +++ b/ami/ml/management/commands/export_verified_training_data.py @@ -0,0 +1,168 @@ +""" +Export human-verified detections as a training set for a classifier head. + +The backbone is frozen, so retraining a head means fitting a small matrix over stored +embeddings. This command gathers the labels people have confirmed in the UI, pairs them +with the embeddings saved by the pipeline, and writes them out. + +Usage: + python manage.py export_verified_training_data --project 3 --algorithm bioclip-2-5-nf-749 + +Writes two files next to each other: + .npz embeddings, label indices, detection ids, split assignment + .json label map, counts, and the settings used to produce it + +The split is deterministic and grouped by occurrence. See training_data.split_for() for why that +matters. +""" + +import json +import pathlib + +import numpy as np +from django.core.management.base import BaseCommand, CommandError + +from ami.main.models import Project +from ami.ml import training_data +from ami.ml.models import Algorithm +from ami.ml.models.embedding import EMBEDDING_DIMENSIONS + + +class Command(BaseCommand): + help = "Export human-verified detections and their embeddings as a classifier training set" + + def add_arguments(self, parser): + parser.add_argument("--project", type=int, required=True, help="Project ID to export from") + parser.add_argument( + "--algorithm", + type=str, + required=True, + help=( + "Key of the algorithm whose embeddings to use. Vectors from different " + "algorithms are in different spaces and must never be mixed." + ), + ) + parser.add_argument("--output", type=str, default="verified_training_data", help="Output path, without suffix") + parser.add_argument( + "--test-fraction", + type=float, + default=training_data.DEFAULT_TEST_FRACTION, + help="Share of occurrences held out for evaluation", + ) + parser.add_argument( + "--split-salt", + type=str, + default=training_data.DEFAULT_SPLIT_SALT, + help="Changing this reshuffles the split. Keep it fixed to keep an eval set comparable.", + ) + parser.add_argument( + "--min-per-species", + type=int, + default=2, + help="Drop species with fewer verified crops than this. A class with one example cannot be evaluated.", + ) + + def handle(self, *args, **options): + project = Project.objects.filter(pk=options["project"]).first() + if not project: + raise CommandError(f"No project with id {options['project']}") + + algorithm = Algorithm.objects.filter(key=options["algorithm"]).first() + if not algorithm: + known = list(Algorithm.objects.values_list("key", flat=True)[:20]) + raise CommandError(f"No algorithm with key '{options['algorithm']}'. Known keys: {known}") + + occurrence_ids = list(training_data.verified_occurrence_ids(project)) + self.stdout.write(f"Occurrences with a standing identification: {len(occurrence_ids)}") + if not occurrence_ids: + raise CommandError("Nothing has been verified in this project yet, so there is nothing to export.") + + rows = list( + training_data.verified_training_rows(project, algorithm).values_list( + "detection_id", + "detection__occurrence_id", + "detection__occurrence__determination__name", + "vector", + ) + ) + self.stdout.write(f"Verified detections with an embedding from {algorithm.key}: {len(rows)}") + + missing = training_data.count_missing_embeddings(project, algorithm) + if missing: + self.stdout.write( + self.style.WARNING( + f"{missing} verified detection(s) have no embedding from this algorithm and were skipped. " + "Re-run the pipeline over them with the store_classification_embeddings flag on." + ) + ) + if not rows: + raise CommandError("No verified detection has an embedding yet, so there is nothing to train on.") + + # Drop species too rare to both train and evaluate on. + counts: dict[str, int] = {} + for _, _, name, _ in rows: + counts[name] = counts.get(name, 0) + 1 + min_per_species = options["min_per_species"] + keep = training_data.species_with_enough_examples(counts, min_per_species) + dropped = sorted(set(counts) - keep) + if dropped: + self.stdout.write( + f"Dropped {len(dropped)} species with fewer than {min_per_species} verified crops: " + f"{', '.join(dropped[:8])}{' ...' if len(dropped) > 8 else ''}" + ) + rows = [r for r in rows if r[2] in keep] + + labels = sorted(keep) + label_to_index = {name: i for i, name in enumerate(labels)} + + embeddings = np.zeros((len(rows), EMBEDDING_DIMENSIONS), dtype=np.float32) + y = np.zeros(len(rows), dtype=np.int64) + detection_ids = np.zeros(len(rows), dtype=np.int64) + splits = [] + for i, (detection_id, occurrence_id, name, vector) in enumerate(rows): + embeddings[i] = np.asarray(vector.to_list(), dtype=np.float32) + y[i] = label_to_index[name] + detection_ids[i] = detection_id + splits.append(training_data.split_for(occurrence_id, options["split_salt"], options["test_fraction"])) + split_array = np.array(splits) + + out = pathlib.Path(options["output"]) + np.savez_compressed( + out.with_suffix(".npz"), + embeddings=embeddings, + labels=y, + detection_ids=detection_ids, + split=split_array, + ) + + n_train = int((split_array == "train").sum()) + n_test = int((split_array == "test").sum()) + meta = { + "project": {"id": project.pk, "name": project.name}, + "algorithm": {"key": algorithm.key, "name": algorithm.name, "version": algorithm.version}, + "dimensions": EMBEDDING_DIMENSIONS, + "classes": labels, + "counts": {name: counts[name] for name in labels}, + "rows": len(rows), + "train": n_train, + "test": n_test, + "occurrences": len(occurrence_ids), + "verified_detections_without_embedding": missing, + "settings": { + "test_fraction": options["test_fraction"], + "split_salt": options["split_salt"], + "min_per_species": options["min_per_species"], + "split_grouped_by": "occurrence", + }, + } + out.with_suffix(".json").write_text(json.dumps(meta, indent=2)) + + self.stdout.write( + self.style.SUCCESS( + f"Wrote {len(rows)} rows over {len(labels)} species " + f"(train {n_train} / test {n_test}) to {out.with_suffix('.npz')}" + ) + ) + # Guard against a silently useless export. + if n_test == 0: + self.stdout.write(self.style.WARNING("The test split is empty. Verify more occurrences before training.")) diff --git a/ami/ml/migrations/0029_detectionembedding_and_more.py b/ami/ml/migrations/0029_detectionembedding_and_more.py new file mode 100644 index 000000000..6ab708148 --- /dev/null +++ b/ami/ml/migrations/0029_detectionembedding_and_more.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.10 on 2026-09-01 20:26 + +import django.db.models.deletion +import pgvector.django.halfvec +from django.db import migrations, models +from pgvector.django import VectorExtension + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0095_grant_sync_deployment_to_mldatamanager"), + ("ml", "0028_normalize_empty_endpoint_url_to_null"), + ] + + operations = [ + # Must run before the table is created. makemigrations does not add this, so it + # is written in by hand. Needs a database user with permission to create an + # extension; the local compose user is the database owner, so it can. + VectorExtension(), + migrations.CreateModel( + name="DetectionEmbedding", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "vector", + pgvector.django.halfvec.HalfVectorField(dimensions=1024), + ), + ( + "algorithm", + models.ForeignKey( + help_text="The algorithm whose backbone produced this vector. Vectors from different algorithms live in different spaces and must never be compared to each other.", + on_delete=django.db.models.deletion.CASCADE, + related_name="embeddings", + to="ml.algorithm", + ), + ), + ( + "detection", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="embeddings", to="main.detection" + ), + ), + ], + ), + migrations.AddConstraint( + model_name="detectionembedding", + constraint=models.UniqueConstraint( + fields=("detection", "algorithm"), name="unique_embedding_per_detection_and_algorithm" + ), + ), + ] diff --git a/ami/ml/migrations/0030_algorithm_trainable.py b/ami/ml/migrations/0030_algorithm_trainable.py new file mode 100644 index 000000000..3dc91a747 --- /dev/null +++ b/ami/ml/migrations/0030_algorithm_trainable.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.10 on 2026-09-02 13:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ml", "0029_detectionembedding_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="algorithm", + name="trainable", + field=models.BooleanField( + default=False, + help_text="Whether the processing service reports that this algorithm can be retrained. Mirrored from the service's /info response so the UI does not have to ask it.", + ), + ), + ] diff --git a/ami/ml/migrations/0031_algorithm_training_fields.py b/ami/ml/migrations/0031_algorithm_training_fields.py new file mode 100644 index 000000000..cc2fdc55b --- /dev/null +++ b/ami/ml/migrations/0031_algorithm_training_fields.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.10 on 2026-09-03 17:05 + +import ami.ml.schemas +from django.db import migrations +import django_pydantic_field.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("ml", "0030_algorithm_trainable"), + ] + + operations = [ + migrations.AddField( + model_name="algorithm", + name="training_config", + field=django_pydantic_field.fields.PydanticSchemaField( + blank=True, + config=None, + default=ami.ml.schemas.AlgorithmTrainingConfig, + help_text="Settings used when retraining this algorithm.", + schema=ami.ml.schemas.AlgorithmTrainingConfig, + ), + ), + migrations.AddField( + model_name="algorithm", + name="training_info", + field=django_pydantic_field.fields.PydanticSchemaField( + blank=True, + config=None, + default=ami.ml.schemas.AlgorithmTrainingInfo, + help_text="Where this version's weights came from. Empty on a version that was never retrained.", + schema=ami.ml.schemas.AlgorithmTrainingInfo, + ), + ), + ] diff --git a/ami/ml/models/__init__.py b/ami/ml/models/__init__.py index 5000c7f53..80df44738 100644 --- a/ami/ml/models/__init__.py +++ b/ami/ml/models/__init__.py @@ -1,4 +1,5 @@ from ami.ml.models.algorithm import Algorithm, AlgorithmCategoryMap +from ami.ml.models.embedding import DetectionEmbedding from ami.ml.models.pipeline import Pipeline from ami.ml.models.processing_service import ProcessingService from ami.ml.models.project_pipeline_config import ProjectPipelineConfig @@ -6,6 +7,7 @@ __all__ = [ "Algorithm", "AlgorithmCategoryMap", + "DetectionEmbedding", "Pipeline", "ProcessingService", "ProjectPipelineConfig", diff --git a/ami/ml/models/algorithm.py b/ami/ml/models/algorithm.py index 605f90861..9df7a8fc1 100644 --- a/ami/ml/models/algorithm.py +++ b/ami/ml/models/algorithm.py @@ -12,8 +12,10 @@ from django.contrib.postgres.fields import ArrayField from django.db import models from django.utils.text import slugify +from django_pydantic_field import SchemaField from ami.base.models import BaseModel, BaseQuerySet +from ami.ml.schemas import AlgorithmTrainingConfig, AlgorithmTrainingInfo @typing.final @@ -249,6 +251,32 @@ class Algorithm(BaseModel): help_text=("A URI to the weights or model details. Could be a public web URL or object store path."), ) + trainable = models.BooleanField( + default=False, + help_text=( + "Whether the processing service reports that this algorithm can be retrained. " + "Mirrored from the service's /info response so the UI does not have to ask it." + ), + ) + # Seeded from the service on first registration, then owned here so an admin's edits + # are not overwritten every time /info is read. + training_config = SchemaField( + AlgorithmTrainingConfig, + default=AlgorithmTrainingConfig, + null=False, + blank=True, + help_text="Settings used when retraining this algorithm.", + ) + # Written by the training job. Every retrain creates a new version, so this records + # where one particular set of weights came from. + training_info = SchemaField( + AlgorithmTrainingInfo, + default=AlgorithmTrainingInfo, + null=False, + blank=True, + help_text="Where this version's weights came from. Empty on a version that was never retrained.", + ) + category_map = models.ForeignKey( AlgorithmCategoryMap, on_delete=models.CASCADE, diff --git a/ami/ml/models/embedding.py b/ami/ml/models/embedding.py new file mode 100644 index 000000000..7aafc29a7 --- /dev/null +++ b/ami/ml/models/embedding.py @@ -0,0 +1,60 @@ +import logging + +from django.db import models +from pgvector.django import HalfVectorField + +from ami.base.models import BaseModel + +logger = logging.getLogger(__name__) + +# Width of the vectors this table holds. BioCLIP 2.5 (ViT-H/14) emits 1024 dimensions. +# pgvector needs a fixed width per column in order to index it, so a backbone with a +# different width (BioCLIP 2 ViT-L/14 is 768) needs its own column or its own table. +EMBEDDING_DIMENSIONS = 1024 + + +class DetectionEmbedding(BaseModel): + """ + The feature vector a classifier's backbone produced for one detection crop. + + The backbone is frozen, so a crop's embedding never changes. Storing it lets a + classifier head be retrained from verified labels without re-running the backbone. + + Kept off Detection deliberately: the vector is ~2 KB, Detection is scanned constantly + by the list views, and Postgres would TOAST a column that wide anyway. + """ + + project_accessor = "detection__source_image__project" + + detection = models.ForeignKey( + "main.Detection", + on_delete=models.CASCADE, + related_name="embeddings", + ) + algorithm = models.ForeignKey( + "ml.Algorithm", + on_delete=models.CASCADE, + related_name="embeddings", + help_text=( + "The algorithm whose backbone produced this vector. Vectors from different " + "algorithms live in different spaces and must never be compared to each other." + ), + ) + # halfvec (2 bytes/dim) rather than vector (4). Measured at 20k rows, a row plus its + # share of the HNSW index costs ~5 KB; float32 would roughly double that. + vector = HalfVectorField(dimensions=EMBEDDING_DIMENSIONS) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["detection", "algorithm"], + name="unique_embedding_per_detection_and_algorithm", + ) + ] + # Deliberately no vector index. These are read in bulk to train a head, never + # searched by nearest neighbour, and an HNSW index measured at roughly the size of + # the rows themselves (45 MB against 54 MB at 20k rows). Add one if similarity + # search is ever wanted; it can be built on the existing data. + + def __str__(self) -> str: + return f"Embedding of Detection #{self.detection_id} by {self.algorithm}" diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index 8ebaf6d6d..62d8d2f84 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -40,9 +40,11 @@ ) from ami.ml.exceptions import PipelineNotConfigured from ami.ml.models.algorithm import Algorithm, AlgorithmCategoryMap +from ami.ml.models.embedding import EMBEDDING_DIMENSIONS, DetectionEmbedding from ami.ml.schemas import ( AlgorithmConfigResponse, AlgorithmReference, + AlgorithmTrainingConfig, ClassificationResponse, DetectionRequest, DetectionResponse, @@ -461,6 +463,9 @@ def get_or_create_algorithm_and_category_map( "version_name": algorithm_config.version_name, "uri": algorithm_config.uri, "category_map": None, + # Seeded once. Deliberately not in fields_to_update below: an admin who tunes + # these settings must not have them reset the next time /info is read. + "training_config": algorithm_config.training_config or AlgorithmTrainingConfig(), }, ) if _created: @@ -503,7 +508,11 @@ def get_or_create_algorithm_and_category_map( fields_to_update = { "task_type": algorithm_config.task_type, "uri": algorithm_config.uri, + "trainable": algorithm_config.trainable, } + if algorithm_config.training_info: + # A fact about where these weights came from, so the service always wins. + fields_to_update["training_info"] = algorithm_config.training_info for field in fields_to_update: new_value = fields_to_update[field] if getattr(algo, field) != new_value: @@ -901,6 +910,66 @@ def create_classifications( return existing_classifications + new_classifications +def create_detection_embeddings( + detections: list[Detection], + detection_responses: list[DetectionResponse], + algorithms_known: dict[str, Algorithm], + logger: logging.Logger = logger, +) -> int: + """ + Save the feature vectors a processing service returned with its classifications. + + Returns the number offered to the database. Existing rows are ignored rather than + raised on, so re-running a pipeline over the same detections stays safe. + """ + to_create: list[DetectionEmbedding] = [] + seen: set[tuple[int, int]] = set() + wrong_size = 0 + + for detection, detection_resp in zip(detections, detection_responses): + if not detection.pk: + # A detection that failed to save has nothing to attach an embedding to. + continue + for classification_resp in detection_resp.classifications: + features = classification_resp.features + if not features: + continue + algorithm = algorithms_known.get(classification_resp.algorithm.key) + if not algorithm: + # create_classification already raises on unknown algorithms; skip quietly. + continue + if len(features) != EMBEDDING_DIMENSIONS: + wrong_size += 1 + continue + # One vector per (detection, algorithm). A pipeline that returns several + # classifications from the same algorithm sends the same vector each time. + key = (detection.pk, algorithm.pk) + if key in seen: + continue + seen.add(key) + to_create.append( + DetectionEmbedding( + detection=detection, + algorithm=algorithm, + vector=features, + ) + ) + + if wrong_size: + logger.warning( + f"Skipped {wrong_size} feature vector(s) that were not {EMBEDDING_DIMENSIONS} dimensions. " + "DetectionEmbedding.vector has a fixed width, so a backbone of a different width " + "needs its own column or table." + ) + + if not to_create: + return 0 + + DetectionEmbedding.objects.bulk_create(to_create, batch_size=500, ignore_conflicts=True) + logger.info(f"Saved {len(to_create)} detection embedding(s), ignoring any that already existed") + return len(to_create) + + def create_and_update_occurrences_for_detections( detections: list[Detection], logger: logging.Logger = logger, @@ -1084,6 +1153,17 @@ def save_results( logger=job_logger, ) + # Opt-in per project: a stored vector costs roughly 5 KB once the vector index is + # counted, so this is only worth paying for on projects that will retrain a head. + project = next((image.project for image in source_images if image.project_id), None) + if project and project.feature_flags.store_classification_embeddings: + create_detection_embeddings( + detections=detections, + detection_responses=results.detections, + algorithms_known=algorithms_known, + logger=job_logger, + ) + # Create a new occurrence for each detection (no tracking yet) # @TODO remove when we implement tracking! create_and_update_occurrences_for_detections( diff --git a/ami/ml/schemas.py b/ami/ml/schemas.py index 9322e4116..8f536e8d3 100644 --- a/ami/ml/schemas.py +++ b/ami/ml/schemas.py @@ -92,6 +92,69 @@ class AlgorithmCategoryMapResponse(pydantic.BaseModel): ) +class AlgorithmTrainingConfig(pydantic.BaseModel): + """ + How to retrain an algorithm. Declared by the processing service, editable afterwards. + + Split across two sides on purpose: Antenna reads the dataset settings when it builds + the training set, and passes the rest to the service, which owns the fitting. + """ + + # Dataset settings, used by Antenna. + min_per_species: int = pydantic.Field( + default=2, + description="Drop species with fewer verified crops than this. One example cannot be evaluated.", + ) + test_fraction: float = pydantic.Field(default=0.2, description="Share of occurrences held out for evaluation.") + split_salt: str = pydantic.Field( + default="antenna-head-v1", + description="Changing this reshuffles the held-out set, which makes old and new heads incomparable.", + ) + + # Fitting settings, used by the processing service. + head_type: str = pydantic.Field( + default="linear", + description="Shape of the head to fit, e.g. 'linear' or 'mlp1'.", + examples=["linear", "mlp1"], + ) + epochs: int = 300 + learning_rate: float = 0.01 + weight_decay: float = 1e-4 + min_improvement: float = pydantic.Field( + default=0.0, + description="A new head must beat the current one by more than this to be worth swapping in.", + ) + + class Config: + extra = "allow" + + +class AlgorithmTrainingInfo(pydantic.BaseModel): + """ + What actually happened when this version was trained. Written by the service, read-only. + + Every retrain produces a new algorithm version, so this is the record of where a + particular set of weights came from. + """ + + trained_at: datetime.datetime | None = None + dataset_url: str | None = pydantic.Field(default=None, description="The exact training set this was fitted on.") + dataset_rows: int | None = None + dataset_classes: int | None = None + metrics: dict = pydantic.Field(default_factory=dict, description="Scores on the held-out split.") + previous_metrics: dict = pydantic.Field( + default_factory=dict, description="What the version it was compared against scored on the same rows." + ) + parent_algorithm_key: str | None = pydantic.Field( + default=None, description="The version this one was trained to beat." + ) + job_id: int | None = None + warnings: list[str] = pydantic.Field(default_factory=list) + + class Config: + extra = "allow" + + class AlgorithmConfigResponse(pydantic.BaseModel): name: str key: str = pydantic.Field( @@ -115,6 +178,22 @@ class AlgorithmConfigResponse(pydantic.BaseModel): default=None, description="A URI to the weight or model details, could be a public web URL or object store path.", ) + trainable: bool = pydantic.Field( + default=False, + description=( + "Whether this algorithm can be retrained from labelled data. A service usually hosts " + "several algorithms and only some of them, typically a classifier head over a frozen " + "backbone, are cheap enough to retrain." + ), + ) + training_config: AlgorithmTrainingConfig | None = pydantic.Field( + default=None, + description="The service's default retraining settings. Antenna seeds its own copy from this.", + ) + training_info: AlgorithmTrainingInfo | None = pydantic.Field( + default=None, + description="Where this version's weights came from. Only set on a version that was retrained.", + ) category_map: AlgorithmCategoryMapResponse | None = None class Config: @@ -133,6 +212,14 @@ class ClassificationResponse(pydantic.BaseModel): ) scores: list[float] = [] logits: list[float] | None = None + features: list[float] | None = pydantic.Field( + default=None, + description=( + "The embedding the model's backbone produced for this crop, taken before the " + "classification head. Optional, and only useful if every value comes from the " + "same backbone." + ), + ) inference_time: float | None = None algorithm: AlgorithmReference terminal: bool = True diff --git a/ami/ml/serializers.py b/ami/ml/serializers.py index e7e9e6aaf..3d50f5461 100644 --- a/ami/ml/serializers.py +++ b/ami/ml/serializers.py @@ -2,12 +2,13 @@ from rest_framework import serializers from ami.main.api.serializers import DefaultSerializer, MinimalNestedModelSerializer +from ami.ml import training_data from .models.algorithm import Algorithm, AlgorithmCategoryMap from .models.pipeline import Pipeline, PipelineStage from .models.processing_service import ProcessingService from .models.project_pipeline_config import ProjectPipelineConfig -from .schemas import PipelineConfigResponse +from .schemas import AlgorithmTrainingConfig, AlgorithmTrainingInfo, PipelineConfigResponse class AlgorithmCategoryMapSerializer(DefaultSerializer): @@ -30,6 +31,10 @@ class Meta: class AlgorithmSerializer(DefaultSerializer): category_map = MinimalCategoryMapNestedSerializer(read_only=True, source="category_map_id") + # Pydantic-backed model fields have to be declared, the way the job serializers do. + # A bare name in Meta.fields raises ImproperlyConfigured at import time. + training_config = SchemaField(schema=AlgorithmTrainingConfig, required=False) + training_info = SchemaField(schema=AlgorithmTrainingInfo, read_only=True) class Meta: model = Algorithm @@ -43,6 +48,9 @@ class Meta: "version", "version_name", "task_type", + "trainable", + "training_config", + "training_info", "category_map", "category_count", "created_at", @@ -168,3 +176,20 @@ def get_projects(self, obj): class PipelineRegistrationSerializer(serializers.Serializer): processing_service_name = serializers.CharField() pipelines = SchemaField(schema=list[PipelineConfigResponse], default=[]) + + +class TrainingDataRowSerializer(serializers.Serializer): + """ + One verified crop: its human label and the embedding to train on. + + Not a ModelSerializer — the useful fields live on the related occurrence and on the + pgvector column, and the split is computed, so building the dict directly is clearer. + """ + + def to_representation(self, instance): + return training_data.row_as_dict( + instance, + salt=self.context.get("split_salt", training_data.DEFAULT_SPLIT_SALT), + test_fraction=self.context.get("test_fraction", training_data.DEFAULT_TEST_FRACTION), + include_features=self.context.get("include_features", True), + ) diff --git a/ami/ml/tests.py b/ami/ml/tests.py index bd92bb02f..d1f0a22f5 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -1,9 +1,13 @@ import concurrent.futures import datetime +import io +import json import pathlib import unittest import uuid +import numpy as np +from django.core.files.storage import default_storage from django.test import TestCase from rest_framework.test import APIRequestFactory, APITestCase @@ -22,7 +26,9 @@ TaxonRank, group_images_into_events, ) -from ami.ml.models import Algorithm, Pipeline, ProcessingService +from ami.ml import training_data +from ami.ml.models import Algorithm, DetectionEmbedding, Pipeline, ProcessingService +from ami.ml.models.embedding import EMBEDDING_DIMENSIONS from ami.ml.models.pipeline import collect_images, get_or_create_algorithm_and_category_map, save_results from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask from ami.ml.schemas import ( @@ -2288,3 +2294,820 @@ def test_used_lookup_is_deduplicated_in_the_database(self): len(list(lookup.order_by().distinct())), 1, "Deduplicating collapses them to the one algorithm" ) self.assertIn("Chatty Masked Classifier", self._choice_names(self.project.pk)) + + +class TestDetectionEmbeddings(TestCase): + """ + Feature vectors returned with a classification are stored as DetectionEmbedding rows, + so that a classifier head can be retrained from verified labels later without running + the backbone over every crop again. + """ + + def setUp(self): + self.project = Project.objects.create(name="Embedding Test Project") + self.test_images = [ + SourceImage.objects.create(path="embed1-20240101000000.jpg", project=self.project), + SourceImage.objects.create(path="embed2-20240101001000.jpg", project=self.project), + ] + self.pipeline = Pipeline.objects.create(name="Embedding Test Pipeline") + self.algorithms = { + key: get_or_create_algorithm_and_category_map(val) for key, val in ALGORITHM_CHOICES.items() + } + self.pipeline.algorithms.set( + [ + self.algorithms["random-detector"], + self.algorithms["random-species-classifier"], + ] + ) + + def _results(self, features: list[float] | None) -> PipelineResultsResponse: + detector = ALGORITHM_CHOICES["random-detector"] + classifier = ALGORITHM_CHOICES["random-species-classifier"] + assert classifier.category_map + return PipelineResultsResponse( + pipeline=self.pipeline.slug, + total_time=0.01, + source_images=[SourceImageResponse(id=image.pk, url=image.path) for image in self.test_images], + detections=[ + DetectionResponse( + source_image_id=image.pk, + bbox=BoundingBox(x1=0.0, y1=0.0, x2=1.0, y2=1.0), + algorithm=AlgorithmReference(name=detector.name, key=detector.key), + timestamp=datetime.datetime.now(), + classifications=[ + ClassificationResponse( + classification=classifier.category_map.labels[0], + labels=classifier.category_map.labels, + scores=[0.77], + features=features, + algorithm=AlgorithmReference(name=classifier.name, key=classifier.key), + timestamp=datetime.datetime.now(), + terminal=True, + ), + ], + ) + for image in self.test_images + ], + ) + + def _enable_flag(self): + self.project.feature_flags.store_classification_embeddings = True + self.project.save() + + def test_embeddings_are_not_saved_when_the_flag_is_off(self): + save_results(self._results(features=[0.5] * EMBEDDING_DIMENSIONS)) + self.assertEqual(DetectionEmbedding.objects.count(), 0, "Storing embeddings must be opt-in") + + def test_embeddings_are_saved_when_the_flag_is_on(self): + self._enable_flag() + save_results(self._results(features=[0.5] * EMBEDDING_DIMENSIONS)) + + self.assertEqual(DetectionEmbedding.objects.count(), len(self.test_images)) + embedding = DetectionEmbedding.objects.first() + assert embedding + self.assertEqual(embedding.algorithm, self.algorithms["random-species-classifier"]) + self.assertEqual(len(embedding.vector.to_list()), EMBEDDING_DIMENSIONS) + + def test_a_classification_without_features_stores_nothing(self): + self._enable_flag() + save_results(self._results(features=None)) + self.assertEqual(DetectionEmbedding.objects.count(), 0) + + def test_a_vector_of_the_wrong_width_is_skipped_not_fatal(self): + """A backbone of a different width must not break the job.""" + self._enable_flag() + save_results(self._results(features=[0.5] * 768)) + self.assertEqual(DetectionEmbedding.objects.count(), 0) + self.assertEqual( + Classification.objects.filter(detection__source_image__project=self.project).count(), + len(self.test_images), + "Classifications are still saved", + ) + + def test_reprocessing_the_same_detections_does_not_raise(self): + """ + The unique constraint on (detection, algorithm) must be ignored, not raised, + so re-running a pipeline over processed captures stays safe. + """ + self._enable_flag() + results = self._results(features=[0.5] * EMBEDDING_DIMENSIONS) + save_results(results) + self.project.feature_flags.reprocess_all_images = True + self.project.save() + save_results(results) + self.assertEqual(DetectionEmbedding.objects.count(), len(self.test_images), "No duplicate embeddings") + + def test_nearest_neighbour_search_finds_the_closest_vector(self): + """The point of storing these: find similar crops without leaving the database.""" + from pgvector.django import CosineDistance + + self._enable_flag() + save_results(self._results(features=[0.5] * EMBEDDING_DIMENSIONS)) + + near = [0.5] * EMBEDDING_DIMENSIONS + far = [0.5] * (EMBEDDING_DIMENSIONS - 1) + [-40.0] + # Constrained to one algorithm: vectors from different backbones are not comparable. + results = ( + DetectionEmbedding.objects.filter(algorithm=self.algorithms["random-species-classifier"]) + .annotate(near=CosineDistance("vector", near)) + .annotate(far=CosineDistance("vector", far)) + .first() + ) + assert results + self.assertLess(results.near, results.far, "An identical vector must be closer than a different one") + + +class TestExportVerifiedTrainingData(TestCase): + """ + The export command turns what people verified in the UI into a training set for a + classifier head: embeddings in, labels from the human identifications. + """ + + def setUp(self): + self.project = Project.objects.create(name="Export Test Project") + self.project.feature_flags.store_classification_embeddings = True + self.project.save() + + self.user = User.objects.create_user(email="verifier@example.com", password="testpass123") + self.images = [ + SourceImage.objects.create(path=f"export{i}-2024010100{i:02d}00.jpg", project=self.project) + for i in range(6) + ] + self.pipeline = Pipeline.objects.create(name="Export Test Pipeline") + self.algorithms = { + key: get_or_create_algorithm_and_category_map(val) for key, val in ALGORITHM_CHOICES.items() + } + self.classifier = self.algorithms["random-species-classifier"] + self.pipeline.algorithms.set([self.algorithms["random-detector"], self.classifier]) + + save_results(self._results()) + self.taxa = [ + Taxon.objects.create(name="Testus unus", rank=TaxonRank.SPECIES.name), + Taxon.objects.create(name="Testus duo", rank=TaxonRank.SPECIES.name), + ] + + def _results(self) -> PipelineResultsResponse: + detector = ALGORITHM_CHOICES["random-detector"] + classifier = ALGORITHM_CHOICES["random-species-classifier"] + assert classifier.category_map + return PipelineResultsResponse( + pipeline=self.pipeline.slug, + total_time=0.01, + source_images=[SourceImageResponse(id=image.pk, url=image.path) for image in self.images], + detections=[ + DetectionResponse( + source_image_id=image.pk, + bbox=BoundingBox(x1=0.0, y1=0.0, x2=1.0, y2=1.0), + algorithm=AlgorithmReference(name=detector.name, key=detector.key), + timestamp=datetime.datetime.now(), + classifications=[ + ClassificationResponse( + classification=classifier.category_map.labels[0], + labels=classifier.category_map.labels, + scores=[0.5], + # A different vector per image, so the classes are separable. + features=[float(i)] * EMBEDDING_DIMENSIONS, + algorithm=AlgorithmReference(name=classifier.name, key=classifier.key), + timestamp=datetime.datetime.now(), + terminal=True, + ), + ], + ) + for i, image in enumerate(self.images) + ], + ) + + def _verify_all(self): + """A human confirms a species for every occurrence, alternating between two taxa.""" + for i, occurrence in enumerate(Occurrence.objects.filter(project=self.project).order_by("pk")): + Identification.objects.create( + occurrence=occurrence, + taxon=self.taxa[i % len(self.taxa)], + user=self.user, + ) + + def _run_export(self, **kwargs) -> dict: + import tempfile + + from django.core.management import call_command + + with tempfile.TemporaryDirectory() as tmp: + out = pathlib.Path(tmp) / "export" + call_command( + "export_verified_training_data", + project=self.project.pk, + algorithm=self.classifier.key, + output=str(out), + stdout=io.StringIO(), + **kwargs, + ) + data = np.load(out.with_suffix(".npz")) + meta = json.loads(out.with_suffix(".json").read_text()) + return {"npz": {k: data[k] for k in data.files}, "meta": meta} + + def test_export_requires_something_verified(self): + from django.core.management import call_command + from django.core.management.base import CommandError + + with self.assertRaises(CommandError): + call_command( + "export_verified_training_data", + project=self.project.pk, + algorithm=self.classifier.key, + stdout=io.StringIO(), + ) + + def test_export_pairs_embeddings_with_human_labels(self): + self._verify_all() + result = self._run_export(min_per_species=1) + + self.assertEqual(result["npz"]["embeddings"].shape, (len(self.images), EMBEDDING_DIMENSIONS)) + self.assertEqual(sorted(result["meta"]["classes"]), ["Testus duo", "Testus unus"]) + self.assertEqual(result["meta"]["rows"], len(self.images)) + self.assertEqual(result["meta"]["algorithm"]["key"], self.classifier.key) + + def test_a_withdrawn_identification_is_not_exported(self): + self._verify_all() + Identification.objects.all().update(withdrawn=True) + + from django.core.management import call_command + from django.core.management.base import CommandError + + with self.assertRaises(CommandError): + call_command( + "export_verified_training_data", + project=self.project.pk, + algorithm=self.classifier.key, + stdout=io.StringIO(), + ) + + def test_the_split_is_stable_across_runs(self): + """An eval set that moves between runs cannot be used to compare two heads.""" + self._verify_all() + first = self._run_export(min_per_species=1) + second = self._run_export(min_per_species=1) + self.assertTrue((first["npz"]["split"] == second["npz"]["split"]).all()) + self.assertEqual(first["meta"]["settings"]["split_grouped_by"], "occurrence") + + def test_rare_species_are_dropped(self): + self._verify_all() + # Give one occurrence a species nothing else has. + rare = Taxon.objects.create(name="Testus rarus", rank=TaxonRank.SPECIES.name) + occurrence = Occurrence.objects.filter(project=self.project).order_by("pk").first() + assert occurrence + Identification.objects.create(occurrence=occurrence, taxon=rare, user=self.user) + + result = self._run_export(min_per_species=2) + self.assertNotIn("Testus rarus", result["meta"]["classes"]) + + +class TestTrainingDataAPI(APITestCase): + """ + The endpoint a trainer pulls from: verified labels paired with their embeddings. + """ + + def setUp(self): + self.project = Project.objects.create(name="Training API Project") + self.project.feature_flags.store_classification_embeddings = True + self.project.save() + self.user = User.objects.create_user(email="trainer@example.com", password="testpass123") + self.project.members.add(self.user) + + self.images = [ + SourceImage.objects.create(path=f"train{i}-2024010100{i:02d}00.jpg", project=self.project) + for i in range(4) + ] + self.pipeline = Pipeline.objects.create(name="Training API Pipeline") + self.algorithms = { + key: get_or_create_algorithm_and_category_map(val) for key, val in ALGORITHM_CHOICES.items() + } + self.classifier = self.algorithms["random-species-classifier"] + self.pipeline.algorithms.set([self.algorithms["random-detector"], self.classifier]) + + detector = ALGORITHM_CHOICES["random-detector"] + classifier = ALGORITHM_CHOICES["random-species-classifier"] + assert classifier.category_map + save_results( + PipelineResultsResponse( + pipeline=self.pipeline.slug, + total_time=0.01, + source_images=[SourceImageResponse(id=i.pk, url=i.path) for i in self.images], + detections=[ + DetectionResponse( + source_image_id=image.pk, + bbox=BoundingBox(x1=0.0, y1=0.0, x2=1.0, y2=1.0), + algorithm=AlgorithmReference(name=detector.name, key=detector.key), + timestamp=datetime.datetime.now(), + classifications=[ + ClassificationResponse( + classification=classifier.category_map.labels[0], + labels=classifier.category_map.labels, + scores=[0.5], + features=[float(i)] * EMBEDDING_DIMENSIONS, + algorithm=AlgorithmReference(name=classifier.name, key=classifier.key), + timestamp=datetime.datetime.now(), + terminal=True, + ) + ], + ) + for i, image in enumerate(self.images) + ], + ) + ) + self.taxon = Taxon.objects.create(name="Trainicus testus", rank=TaxonRank.SPECIES.name) + for occurrence in Occurrence.objects.filter(project=self.project): + Identification.objects.create(occurrence=occurrence, taxon=self.taxon, user=self.user) + + self.url = reverse_with_params("api:training-data-list") + self.summary_url = reverse_with_params("api:training-data-summary") + self.params = {"project_id": self.project.pk, "algorithm": self.classifier.key} + + def test_anonymous_users_get_nothing(self): + response = self.client.get(self.url, self.params) + self.assertEqual(response.status_code, 401) + + def test_rows_carry_the_human_label_and_the_embedding(self): + self.client.force_authenticate(user=self.user) + response = self.client.get(self.url, self.params) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], len(self.images)) + row = response.json()["results"][0] + self.assertEqual(row["label"], self.taxon.name) + self.assertEqual(len(row["features"]), EMBEDDING_DIMENSIONS) + self.assertIn(row["split"], ("train", "test")) + + def test_features_can_be_left_out(self): + """Callers deciding whether a retrain is worth it should not pull megabytes.""" + self.client.force_authenticate(user=self.user) + response = self.client.get(self.url, {**self.params, "include_features": "false"}) + self.assertEqual(response.status_code, 200) + self.assertNotIn("features", response.json()["results"][0]) + + def test_algorithm_is_required(self): + """Mixing backbones would produce a meaningless training set, so it cannot be omitted.""" + self.client.force_authenticate(user=self.user) + response = self.client.get(self.url, {"project_id": self.project.pk}) + self.assertEqual(response.status_code, 400) + + def test_unknown_algorithm_is_404(self): + self.client.force_authenticate(user=self.user) + response = self.client.get(self.url, {**self.params, "algorithm": "does-not-exist"}) + self.assertEqual(response.status_code, 404) + + def test_project_is_required(self): + self.client.force_authenticate(user=self.user) + response = self.client.get(self.url, {"algorithm": self.classifier.key}) + self.assertEqual(response.status_code, 400) + + def test_unverified_occurrences_are_not_returned(self): + self.client.force_authenticate(user=self.user) + Identification.objects.all().update(withdrawn=True) + response = self.client.get(self.url, self.params) + self.assertEqual(response.json()["count"], 0) + + def test_summary_reports_counts_without_sending_vectors(self): + self.client.force_authenticate(user=self.user) + response = self.client.get(self.summary_url, self.params) + self.assertEqual(response.status_code, 200) + body = response.json() + self.assertEqual(body["rows"], len(self.images)) + self.assertEqual(body["counts"][self.taxon.name], len(self.images)) + self.assertEqual(body["dimensions"], EMBEDDING_DIMENSIONS) + self.assertEqual(body["train"] + body["test"], len(self.images)) + self.assertEqual(body["settings"]["split_grouped_by"], "occurrence") + + def test_the_split_matches_the_export_command(self): + """The API and the export must not disagree about what is held out.""" + self.client.force_authenticate(user=self.user) + response = self.client.get(self.url, {**self.params, "include_features": "false"}) + for row in response.json()["results"]: + self.assertEqual(row["split"], training_data.split_for(row["occurrence_id"])) + + +class TestTrainableFlag(TestCase): + """A service declares which of its algorithms can be retrained; Antenna mirrors that.""" + + def test_trainable_is_mirrored_from_the_service_config(self): + config = ALGORITHM_CHOICES["random-species-classifier"].copy(update={"trainable": True}) + algorithm = get_or_create_algorithm_and_category_map(config) + self.assertTrue(algorithm.trainable) + + def test_algorithms_are_not_trainable_by_default(self): + algorithm = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES["random-detector"]) + self.assertFalse(algorithm.trainable) + + def test_the_flag_follows_the_service_when_it_changes(self): + """A service that gains training support must not need its algorithm deleted.""" + key = "random-species-classifier" + get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES[key]) + updated = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES[key].copy(update={"trainable": True})) + self.assertTrue(updated.trainable) + + +class TestTrainingDatasetAndJob(TestCase): + """ + Antenna prepares the training set as a file and hands the service a URL to it. + """ + + def setUp(self): + self.project = Project.objects.create(name="Training Job Project") + self.project.feature_flags.store_classification_embeddings = True + self.project.save() + self.user = User.objects.create_user(email="trainjob@example.com", password="testpass123") + + self.images = [ + SourceImage.objects.create(path=f"tj{i}-2024010100{i:02d}00.jpg", project=self.project) for i in range(12) + ] + self.pipeline = Pipeline.objects.create(name="Training Job Pipeline") + self.algorithms = { + key: get_or_create_algorithm_and_category_map(val) for key, val in ALGORITHM_CHOICES.items() + } + self.classifier = self.algorithms["random-species-classifier"] + self.classifier.trainable = True + self.classifier.save() + self.pipeline.algorithms.set([self.algorithms["random-detector"], self.classifier]) + + detector = ALGORITHM_CHOICES["random-detector"] + classifier = ALGORITHM_CHOICES["random-species-classifier"] + assert classifier.category_map + save_results( + PipelineResultsResponse( + pipeline=self.pipeline.slug, + total_time=0.01, + source_images=[SourceImageResponse(id=i.pk, url=i.path) for i in self.images], + detections=[ + DetectionResponse( + source_image_id=image.pk, + bbox=BoundingBox(x1=0.0, y1=0.0, x2=1.0, y2=1.0), + algorithm=AlgorithmReference(name=detector.name, key=detector.key), + timestamp=datetime.datetime.now(), + classifications=[ + ClassificationResponse( + classification=classifier.category_map.labels[0], + labels=classifier.category_map.labels, + scores=[0.5], + features=[float(i)] * EMBEDDING_DIMENSIONS, + algorithm=AlgorithmReference(name=classifier.name, key=classifier.key), + timestamp=datetime.datetime.now(), + terminal=True, + ) + ], + ) + for i, image in enumerate(self.images) + ], + ) + ) + self.taxa = [ + Taxon.objects.create(name="Datasetus unus", rank=TaxonRank.SPECIES.name), + Taxon.objects.create(name="Datasetus duo", rank=TaxonRank.SPECIES.name), + ] + + def _verify_all(self): + for i, occurrence in enumerate(Occurrence.objects.filter(project=self.project).order_by("pk")): + Identification.objects.create(occurrence=occurrence, taxon=self.taxa[i % 2], user=self.user) + + def test_dataset_holds_the_embeddings_and_the_human_labels(self): + from ami.ml.training_dataset import build_training_dataset + + self._verify_all() + result = build_training_dataset( + project=self.project, algorithm=self.classifier, min_per_species=1, test_fraction=0.5 + ) + + with default_storage.open(result["path"], "rb") as f: + archive = np.load(f, allow_pickle=True) + features = archive["features"] + classes = [str(c) for c in archive["classes"]] + metadata = json.loads(str(archive["metadata"])) + + self.assertEqual(features.shape, (len(self.images), EMBEDDING_DIMENSIONS)) + self.assertEqual(sorted(classes), sorted(t.name for t in self.taxa)) + self.assertEqual(metadata["rows"], len(self.images)) + self.assertEqual(metadata["train"] + metadata["test"], len(self.images)) + default_storage.delete(result["path"]) + + def test_vectors_are_stored_as_float16(self): + """float16 is what Postgres holds, so anything wider ships bytes that carry nothing.""" + from ami.ml.training_dataset import build_training_dataset + + self._verify_all() + result = build_training_dataset( + project=self.project, algorithm=self.classifier, min_per_species=1, test_fraction=0.5 + ) + with default_storage.open(result["path"], "rb") as f: + self.assertEqual(np.load(f, allow_pickle=True)["features"].dtype, np.float16) + default_storage.delete(result["path"]) + + def test_nothing_verified_means_no_file_is_written(self): + from ami.ml.training_dataset import NotEnoughVerifiedData, build_training_dataset + + with self.assertRaises(NotEnoughVerifiedData): + build_training_dataset(project=self.project, algorithm=self.classifier) + + def test_the_job_fails_cleanly_when_there_is_nothing_to_train_on(self): + """A data problem should read as a message, not a traceback.""" + from ami.jobs.models import Job, JobState, TrainClassifierJob + + job = Job.objects.create( + project=self.project, + name="Retrain", + job_type_key=TrainClassifierJob.key, + params={"algorithm_key": self.classifier.key}, + ) + job.run() + job.refresh_from_db() + self.assertEqual(job.status, JobState.FAILURE.name) + self.assertIn("error", job.result) + + def test_the_job_refuses_an_algorithm_the_service_cannot_train(self): + from ami.jobs.models import Job, TrainClassifierJob + + self._verify_all() + job = Job.objects.create( + project=self.project, + name="Retrain a detector", + job_type_key=TrainClassifierJob.key, + params={"algorithm_key": self.algorithms["random-detector"].key}, + ) + with self.assertRaises(ValueError): + job.run() + + def test_the_job_refuses_when_only_a_pull_worker_serves_the_algorithm(self): + """Antenna cannot POST to a worker with no endpoint, so it must say so, not hang.""" + from ami.jobs.models import Job, TrainClassifierJob + + self._verify_all() + self.project.processing_services.clear() + worker = ProcessingService.objects.create(name="Pull worker", endpoint_url=None) + worker.projects.add(self.project) + worker.pipelines.add(self.pipeline) + + job = Job.objects.create( + project=self.project, + name="Retrain", + job_type_key=TrainClassifierJob.key, + params={"algorithm_key": self.classifier.key, "min_per_species": 1}, + ) + with self.assertRaises(ValueError) as ctx: + job.run() + self.assertIn("pull-mode", str(ctx.exception).lower()) + + +class TestAbsoluteMediaURL(TestCase): + def test_an_absolute_url_is_left_alone(self): + """In production MEDIA_URL is already an S3 URL.""" + from ami.ml.training_dispatch import absolute_media_url + + url = "https://bucket.s3.amazonaws.com/uploads/training/set.npz" + self.assertEqual(absolute_media_url(url), url) + + def test_a_relative_path_gets_a_base(self): + from ami.ml.training_dispatch import absolute_media_url + + self.assertEqual( + absolute_media_url("/media/training/set.npz", "http://antenna:8000"), + "http://antenna:8000/media/training/set.npz", + ) + + +class TestGenerateEmbeddingsJob(TestCase): + """ + The job that fills in embeddings for verified crops, so a head can be retrained + without running the backbone over the same image twice. + """ + + def setUp(self): + self.project = Project.objects.create(name="Embedding Job Project") + self.project.feature_flags.store_classification_embeddings = True + self.project.save() + self.user = User.objects.create_user(email="embedder@example.com", password="testpass123") + self.images = [ + SourceImage.objects.create(path=f"ej{i}-2024010100{i:02d}00.jpg", project=self.project) for i in range(3) + ] + self.pipeline = Pipeline.objects.create(name="Embedding Job Pipeline") + self.algorithms = { + key: get_or_create_algorithm_and_category_map(val) for key, val in ALGORITHM_CHOICES.items() + } + self.classifier = self.algorithms["random-species-classifier"] + self.classifier.trainable = True + self.classifier.save() + self.detector = self.algorithms["random-detector"] + self.pipeline.algorithms.set([self.detector, self.classifier]) + self.taxon = Taxon.objects.create(name="Embeddicus testus", rank=TaxonRank.SPECIES.name) + + def _job(self, **params): + from ami.jobs.models import GenerateEmbeddingsJob, Job + + return Job.objects.create( + project=self.project, + name="Generate embeddings", + job_type_key=GenerateEmbeddingsJob.key, + pipeline=self.pipeline, + params=params or None, + ) + + def _detections_with_occurrences(self, verified: bool): + for image in self.images: + detection = Detection.objects.create( + source_image=image, bbox=[0, 0, 10, 10], detection_algorithm=self.detector + ) + occurrence = detection.associate_new_occurrence() + if verified: + Identification.objects.create(occurrence=occurrence, taxon=self.taxon, user=self.user) + + def test_it_refuses_when_the_project_would_discard_the_result(self): + """Running the backbone to throw the vectors away is pure waste.""" + from ami.jobs.models import GenerateEmbeddingsJob + + self.project.feature_flags.store_classification_embeddings = False + self.project.save() + with self.assertRaises(ValueError) as ctx: + GenerateEmbeddingsJob.run(self._job()) + self.assertIn("store_classification_embeddings", str(ctx.exception)) + + def test_it_refuses_a_pipeline_with_no_trainable_algorithm(self): + from ami.jobs.models import GenerateEmbeddingsJob + + self.classifier.trainable = False + self.classifier.save() + with self.assertRaises(ValueError) as ctx: + GenerateEmbeddingsJob.run(self._job()) + self.assertIn("trainable", str(ctx.exception)) + + def test_it_targets_the_trainable_algorithm(self): + from ami.jobs.models import GenerateEmbeddingsJob + + self.assertEqual(GenerateEmbeddingsJob.target_algorithm(self._job()), self.classifier) + + def test_an_unknown_algorithm_key_is_rejected(self): + from ami.jobs.models import GenerateEmbeddingsJob + + with self.assertRaises(ValueError): + GenerateEmbeddingsJob.target_algorithm(self._job(algorithm_key="not-on-this-pipeline")) + + def test_only_verified_detections_are_collected(self): + """Embedding everything would cost gigabytes; only a labelled crop can train a head.""" + from ami.jobs.models import GenerateEmbeddingsJob + + self._detections_with_occurrences(verified=False) + job = self._job() + self.assertEqual(GenerateEmbeddingsJob.images_needing_embeddings(job, self.classifier), []) + + self._detections_with_occurrences(verified=True) + images = GenerateEmbeddingsJob.images_needing_embeddings(job, self.classifier) + self.assertEqual(len(images), len(self.images)) + + def test_detections_that_already_have_an_embedding_are_skipped(self): + """The whole point is not to compute the same vector twice.""" + from ami.jobs.models import GenerateEmbeddingsJob + + self._detections_with_occurrences(verified=True) + job = self._job() + for detection in Detection.objects.filter(source_image__project=self.project): + DetectionEmbedding.objects.create( + detection=detection, algorithm=self.classifier, vector=[0.1] * EMBEDDING_DIMENSIONS + ) + self.assertEqual(GenerateEmbeddingsJob.images_needing_embeddings(job, self.classifier), []) + + def test_nothing_to_do_finishes_successfully(self): + from ami.jobs.models import GenerateEmbeddingsJob, JobState + + job = self._job() + GenerateEmbeddingsJob.run(job) + job.refresh_from_db() + self.assertEqual(job.status, JobState.SUCCESS.name) + + +class TestAlgorithmVersioning(TestCase): + """ + Every retrain produces a new algorithm version, so a prediction can always be traced + back to the exact weights that made it. + """ + + def setUp(self): + from ami.jobs.models import Job, TrainClassifierJob + + self.project = Project.objects.create(name="Versioning Project") + self.parent = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES["random-species-classifier"]) + self.parent.trainable = True + self.parent.save() + self.job = Job.objects.create( + project=self.project, + name="Retrain", + job_type_key=TrainClassifierJob.key, + params={"algorithm_key": self.parent.key}, + ) + self.payload = { + "result": { + "labels": ["Alpha one", "Beta two", "Gamma three"], + "rows": {"total": 30, "kept": 30, "train": 24, "test": 6}, + "candidate_metrics": {"top1": 0.9, "macro_recall": 0.88, "n": 6}, + "incumbent_metrics": {"top1": 0.7, "n": 6}, + "trained_at": "2026-09-03T21:08:03", + "warnings": ["only 6 held-out rows"], + "promote": True, + }, + "dataset": {"rows": 30}, + "dataset_url": "/media/training/set-job-1.npz", + } + + def _register(self): + from ami.jobs.models import TrainClassifierJob + + return TrainClassifierJob.register_new_version(job=self.job, payload=self.payload) + + def test_a_retrain_creates_a_new_row_not_an_edit(self): + """A Classification points at an Algorithm row, so a version must never change in place.""" + before = Algorithm.objects.count() + new = self._register() + assert new + self.assertEqual(Algorithm.objects.count(), before + 1) + self.assertNotEqual(new.pk, self.parent.pk) + self.parent.refresh_from_db() + self.assertEqual(self.parent.version, 1) + + def test_the_new_version_keeps_the_name_and_bumps_the_number(self): + new = self._register() + assert new + self.assertEqual(new.name, self.parent.name) + self.assertEqual(new.version, self.parent.version + 1) + self.assertNotEqual(new.key, self.parent.key) + + def test_versions_keep_incrementing(self): + first = self._register() + second = self._register() + assert first and second + self.assertEqual(second.version, first.version + 1) + + def test_provenance_is_recorded(self): + new = self._register() + assert new + info = new.training_info + self.assertEqual(info.dataset_url, "/media/training/set-job-1.npz") + self.assertEqual(info.dataset_classes, 3) + self.assertEqual(info.dataset_rows, 30) + self.assertEqual(info.metrics["top1"], 0.9) + self.assertEqual(info.previous_metrics["top1"], 0.7) + self.assertEqual(info.parent_algorithm_key, self.parent.key) + self.assertEqual(info.job_id, self.job.pk) + self.assertEqual(info.warnings, ["only 6 held-out rows"]) + + def test_the_new_version_carries_its_own_class_list(self): + """The retrained head predicts a different set of species than its parent.""" + new = self._register() + assert new + assert new.category_map + self.assertEqual(new.category_map.labels, ["Alpha one", "Beta two", "Gamma three"]) + self.assertNotEqual(new.category_map_id, self.parent.category_map_id) + + def test_no_class_list_means_no_version(self): + """Registering a head that cannot say what it predicts would be untraceable.""" + self.payload["result"]["labels"] = [] + self.assertIsNone(self._register()) + + def test_an_unknown_parent_means_no_version(self): + self.job.params = {"algorithm_key": "does-not-exist"} + self.job.save() + self.assertIsNone(self._register()) + + +class TestAlgorithmTrainingConfig(TestCase): + """Settings are seeded from the service once, then owned by Antenna.""" + + def test_config_is_seeded_from_the_service(self): + from ami.ml.schemas import AlgorithmTrainingConfig + + # A key nothing else registers, so this exercises the create path rather than + # finding an algorithm some other fixture already made. + config = ALGORITHM_CHOICES["random-species-classifier"].copy( + update={ + # Unique name as well as key: Algorithm is unique on (name, version) too. + "name": "Seeded Config Classifier", + "key": "seeded-config-classifier", + "training_config": AlgorithmTrainingConfig(head_type="mlp1", epochs=50), + } + ) + algorithm = get_or_create_algorithm_and_category_map(config) + self.assertEqual(algorithm.training_config.head_type, "mlp1") + self.assertEqual(algorithm.training_config.epochs, 50) + + def test_re_registering_does_not_overwrite_an_edited_config(self): + """An admin who tunes these must not have them reset on the next /info read.""" + from ami.ml.schemas import AlgorithmTrainingConfig + + base = ALGORITHM_CHOICES["random-species-classifier"].copy( + update={"name": "Edited Config Classifier", "key": "edited-config-classifier"} + ) + algorithm = get_or_create_algorithm_and_category_map(base) + algorithm.training_config = AlgorithmTrainingConfig(head_type="mlp1", epochs=999) + algorithm.save() + + again = get_or_create_algorithm_and_category_map( + base.copy(update={"training_config": AlgorithmTrainingConfig(epochs=300)}) + ) + self.assertEqual(again.training_config.epochs, 999) + + def test_defaults_exist_without_the_service_sending_any(self): + algorithm = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES["random-detector"]) + self.assertEqual(algorithm.training_config.min_per_species, 2) + self.assertEqual(algorithm.training_info.trained_at, None) diff --git a/ami/ml/training_data.py b/ami/ml/training_data.py new file mode 100644 index 000000000..61cca7276 --- /dev/null +++ b/ami/ml/training_data.py @@ -0,0 +1,116 @@ +""" +Building a classifier-head training set out of what people verified in the UI. + +Shared by the export management command and the training-data API so both agree on what +counts as a label and which rows land in the test split. +""" + +import hashlib +import typing + +from django.db.models import QuerySet + +from ami.main.models import Identification, Project +from ami.ml.models.algorithm import Algorithm +from ami.ml.models.embedding import DetectionEmbedding + +DEFAULT_SPLIT_SALT = "antenna-head-v1" +DEFAULT_TEST_FRACTION = 0.2 + + +def split_for( + occurrence_id: int, + salt: str = DEFAULT_SPLIT_SALT, + test_fraction: float = DEFAULT_TEST_FRACTION, +) -> str: + """ + Assign an occurrence to "train" or "test", the same way every time. + + Grouped by occurrence, not detection: an occurrence is one insect across several + frames, so a detection-level split puts near-identical crops on both sides and + overstates accuracy. Hash-based, not random, so the test set does not move when new + data arrives — an eval set that drifts cannot compare two heads. + """ + digest = hashlib.sha256(f"{salt}:{occurrence_id}".encode()).hexdigest() + return "test" if (int(digest[:8], 16) / 0xFFFFFFFF) < test_fraction else "train" + + +def verified_occurrence_ids(project: Project) -> QuerySet: + """Occurrences a person has identified and not taken back.""" + return ( + Identification.objects.filter( + withdrawn=False, + occurrence__project=project, + occurrence__determination__isnull=False, + ) + .order_by() + .values_list("occurrence_id", flat=True) + .distinct() + ) + + +def verified_training_rows(project: Project, algorithm: Algorithm) -> QuerySet[DetectionEmbedding]: + """ + Embeddings whose detection sits under a verified occurrence. + + Constrained to one algorithm on purpose: vectors from different backbones are in + different spaces and must never be mixed into one training set. + """ + return ( + DetectionEmbedding.objects.filter( + algorithm=algorithm, + detection__occurrence_id__in=verified_occurrence_ids(project), + detection__occurrence__determination__isnull=False, + ) + .select_related("detection__occurrence__determination") + .order_by("pk") + ) + + +def count_missing_embeddings(project: Project, algorithm: Algorithm) -> int: + """Verified detections this algorithm has never embedded. They need a pipeline re-run.""" + from ami.main.models import Detection + + return ( + Detection.objects.filter(occurrence_id__in=verified_occurrence_ids(project)) + .exclude(embeddings__algorithm=algorithm) + .count() + ) + + +def label_counts(project: Project, algorithm: Algorithm) -> dict[str, int]: + """Verified crops per species, for the chosen algorithm.""" + from django.db.models import Count + + rows = ( + verified_training_rows(project, algorithm) + .order_by() + .values("detection__occurrence__determination__name") + .annotate(n=Count("pk")) + ) + return {r["detection__occurrence__determination__name"]: r["n"] for r in rows} + + +def species_with_enough_examples(counts: dict[str, int], minimum: int) -> set[str]: + """A class with a single example cannot be both trained on and evaluated.""" + return {name for name, n in counts.items() if name and n >= minimum} + + +def row_as_dict( + embedding: DetectionEmbedding, + salt: str = DEFAULT_SPLIT_SALT, + test_fraction: float = DEFAULT_TEST_FRACTION, + include_features: bool = True, +) -> dict[str, typing.Any]: + """One training row, in the shape the training-data API returns.""" + occurrence = embedding.detection.occurrence + row = { + "detection_id": embedding.detection_id, + "occurrence_id": occurrence.pk if occurrence else None, + "label": occurrence.determination.name if occurrence and occurrence.determination else None, + "label_id": occurrence.determination_id if occurrence else None, + "split": split_for(occurrence.pk, salt, test_fraction) if occurrence else None, + } + if include_features: + row["features"] = embedding.vector.to_list() + return row diff --git a/ami/ml/training_dataset.py b/ami/ml/training_dataset.py new file mode 100644 index 000000000..fb4f03137 --- /dev/null +++ b/ami/ml/training_dataset.py @@ -0,0 +1,180 @@ +""" +Write a classifier-head training set to storage, for a processing service to collect. + +The service is handed a URL rather than the rows themselves. A project with a few hundred +thousand verified labels is hundreds of megabytes; that is fragile to send in one request +and has to start over if the connection drops. A file in storage is small to hand over, +cheap to retry, and the service already downloads capture images from the same place. +""" + +import json +import logging +import pathlib +import tempfile +import typing + +import numpy as np +from django.conf import settings +from django.core.files.storage import default_storage +from django.utils.text import slugify + +from ami.main.models import Project +from ami.ml import training_data +from ami.ml.models.algorithm import Algorithm +from ami.ml.models.embedding import EMBEDDING_DIMENSIONS + +logger = logging.getLogger(__name__) + +# Vectors are written as float16 because that is exactly what Postgres stores (halfvec), +# so nothing is lost. Measured on 10k rows: npz float16 is 21 MB against 227 MB of JSON, +# and 1.1s to build against 29s. +DATASET_DTYPE = np.float16 + +DATASET_DIRECTORY = "training" + +# Rows are pulled from the database in batches so a large project does not have to fit +# every vector in memory at once. +FETCH_BATCH_SIZE = 2000 + + +class NotEnoughVerifiedData(Exception): + """The project does not hold enough verified labels to train and evaluate on.""" + + +def build_training_dataset( + project: Project, + algorithm: Algorithm, + min_per_species: int = 2, + split_salt: str = training_data.DEFAULT_SPLIT_SALT, + test_fraction: float = training_data.DEFAULT_TEST_FRACTION, + job_id: int | None = None, +) -> dict[str, typing.Any]: + """ + Collect verified labels and their embeddings, and save them as one npz file. + + Returns the storage path, the URL a service can fetch, and the metadata describing + what went in. Raises NotEnoughVerifiedData rather than writing a file nothing can be + trained on. + """ + counts = training_data.label_counts(project, algorithm) + keep = training_data.species_with_enough_examples(counts, min_per_species) + if not keep: + raise NotEnoughVerifiedData( + f"No species in '{project.name}' has at least {min_per_species} verified crops with an " + f"embedding from {algorithm.key}. Verify more occurrences, or re-run the pipeline so the " + "verified detections get embeddings." + ) + + classes = sorted(keep) + class_index = {name: i for i, name in enumerate(classes)} + + rows = training_data.verified_training_rows(project, algorithm) + total = rows.count() + + features = np.zeros((total, EMBEDDING_DIMENSIONS), dtype=DATASET_DTYPE) + labels = np.zeros(total, dtype=np.int64) + detection_ids = np.zeros(total, dtype=np.int64) + occurrence_ids = np.zeros(total, dtype=np.int64) + splits: list[str] = [] + + kept = 0 + for embedding in rows.iterator(chunk_size=FETCH_BATCH_SIZE): + occurrence = embedding.detection.occurrence + if not occurrence or not occurrence.determination: + continue + name = occurrence.determination.name + if name not in class_index: + continue + features[kept] = np.asarray(embedding.vector.to_list(), dtype=DATASET_DTYPE) + labels[kept] = class_index[name] + detection_ids[kept] = embedding.detection_id + occurrence_ids[kept] = occurrence.pk + splits.append(training_data.split_for(occurrence.pk, split_salt, test_fraction)) + kept += 1 + + if not kept: + raise NotEnoughVerifiedData("No verified detection has an embedding from this algorithm yet.") + + features = features[:kept] + labels = labels[:kept] + detection_ids = detection_ids[:kept] + occurrence_ids = occurrence_ids[:kept] + split_array = np.array(splits) + + n_train = int((split_array == "train").sum()) + n_test = int((split_array == "test").sum()) + if not n_train or not n_test: + raise NotEnoughVerifiedData( + f"The split left {n_train} training and {n_test} held-out rows. Both sides need rows " + "before a new head can be compared against the current one." + ) + + metadata = { + "project": {"id": project.pk, "name": project.name}, + "algorithm": {"key": algorithm.key, "name": algorithm.name, "version": algorithm.version}, + "dimensions": EMBEDDING_DIMENSIONS, + "dtype": np.dtype(DATASET_DTYPE).name, + "classes": classes, + "counts": {name: counts[name] for name in classes}, + "dropped_species": sorted(set(counts) - keep), + "rows": kept, + "train": n_train, + "test": n_test, + "verified_detections_without_embedding": training_data.count_missing_embeddings(project, algorithm), + "settings": { + "min_per_species": min_per_species, + "split_salt": split_salt, + "test_fraction": test_fraction, + "split_grouped_by": "occurrence", + }, + } + + file_path = _save( + project=project, + algorithm=algorithm, + job_id=job_id, + arrays={ + "features": features, + "labels": labels, + "detection_ids": detection_ids, + "occurrence_ids": occurrence_ids, + "split": split_array, + "classes": np.array(classes), + }, + metadata=metadata, + ) + + file_url = f"{settings.MEDIA_URL}{file_path}" + logger.info(f"Wrote training dataset with {kept} rows over {len(classes)} species to {file_path}") + return {"path": file_path, "url": file_url, "metadata": metadata} + + +def _save( + project: Project, + algorithm: Algorithm, + job_id: int | None, + arrays: dict[str, np.ndarray], + metadata: dict[str, typing.Any], +) -> str: + """ + Write the npz to storage and return its path. + + default_storage is the local filesystem in development and the project's S3 bucket in + production, so this follows wherever captures already live without a special case. + """ + stem = f"{slugify(project.name)}-{slugify(algorithm.key)}" + suffix = f"job-{job_id}" if job_id else "manual" + file_path = f"{DATASET_DIRECTORY}/{stem}-{suffix}.npz" + + with tempfile.TemporaryDirectory() as tmp: + local = pathlib.Path(tmp) / "dataset.npz" + # Uncompressed: embeddings are close to random, so compression buys about 10 per + # cent for real CPU. Metadata rides inside the archive so the file is self-describing. + np.savez(local, metadata=np.array(json.dumps(metadata)), **arrays) + if default_storage.exists(file_path): + # A re-run of the same job replaces its dataset instead of piling up copies. + default_storage.delete(file_path) + with open(local, "rb") as f: + file_path = default_storage.save(file_path, f) + + return file_path diff --git a/ami/ml/training_dispatch.py b/ami/ml/training_dispatch.py new file mode 100644 index 000000000..813518af4 --- /dev/null +++ b/ami/ml/training_dispatch.py @@ -0,0 +1,76 @@ +""" +Hand a training request to a processing service. + +Antenna sends a URL, not the rows. See ami/ml/training_dataset.py for why. +""" + +import logging +import typing +from urllib.parse import urljoin + +from django.conf import settings + +from ami.utils.requests import create_session, extract_error_message_from_response + +logger = logging.getLogger(__name__) + +# How long to wait for the service to acknowledge the request. This is not how long +# training takes: a service that answers within this window returns its result inline, +# and one that does not is expected to report back through the job's result endpoint. +DISPATCH_TIMEOUT_SECONDS = 600 + + +def absolute_media_url(url: str, base_url: str | None = None) -> str: + """ + Turn a stored file's URL into one a processing service can fetch. + + In production MEDIA_URL is already an absolute S3 URL and this is a no-op. Locally it is + the relative /media/ path, so it needs a base in front. EXTERNAL_BASE_URL points at the + UI, which is not always where media is served from, so a job may override it. + """ + if url.startswith("http://") or url.startswith("https://"): + return url + base = base_url or getattr(settings, "EXTERNAL_BASE_URL", "") + if not base: + raise ValueError( + "The training set is stored at a relative URL and no base URL is configured, so the " + "processing service has no way to download it. Set EXTERNAL_BASE_URL, or pass " + "media_base_url in the job params." + ) + return urljoin(base.rstrip("/") + "/", url.lstrip("/")) + + +def send_training_request(job, service, algorithm, dataset: dict) -> dict | None: + """ + Ask a processing service to retrain a head. + + Returns the service's result if it answered inline, or None if it accepted the work and + will report back later. Raises if the service refused the request. + """ + endpoint = urljoin(service.endpoint_url.rstrip("/") + "/", "train") + payload: dict[str, typing.Any] = { + "dataset_url": absolute_media_url(dataset["url"], (job.params or {}).get("media_base_url")), + "algorithm_key": algorithm.key, + "job_id": job.pk, + "name": f"{algorithm.key}-job-{job.pk}", + "min_per_species": (job.params or {}).get("min_per_species", 2), + } + + callback = (job.params or {}).get("callback_url") + if callback: + payload["callback_url"] = callback + payload["callback_token"] = (job.params or {}).get("callback_token") + + job.logger.info(f"Sending training request to {endpoint} for {algorithm.key}") + session = create_session() + response = session.post(endpoint, json=payload, timeout=DISPATCH_TIMEOUT_SECONDS) + + if not response.ok: + message = extract_error_message_from_response(response) + raise ValueError(f"The processing service refused the training request: {message}") + + try: + return response.json() + except ValueError: + # Accepted, but nothing useful in the body. The service will report back. + return None diff --git a/ami/ml/views.py b/ami/ml/views.py index 63e460af6..2ff9fa7a5 100644 --- a/ami/ml/views.py +++ b/ami/ml/views.py @@ -8,17 +8,21 @@ from rest_framework import exceptions as api_exceptions from rest_framework import mixins, status, viewsets from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response +from ami.base.pagination import TrainingDataPagination from ami.base.permissions import ProjectPipelineConfigPermission from ami.base.views import ProjectMixin from ami.main.api.schemas import project_id_doc_param from ami.main.api.views import DefaultViewSet from ami.main.models import Project, SourceImage +from ami.ml import training_data from ami.ml.schemas import PipelineRegistrationResponse from .models.algorithm import Algorithm, AlgorithmCategoryMap +from .models.embedding import EMBEDDING_DIMENSIONS, DetectionEmbedding from .models.pipeline import Pipeline from .models.processing_service import ProcessingService from .models.project_pipeline_config import ProjectPipelineConfig @@ -28,6 +32,7 @@ PipelineRegistrationSerializer, PipelineSerializer, ProcessingServiceSerializer, + TrainingDataRowSerializer, ) logger = logging.getLogger(__name__) @@ -298,3 +303,115 @@ def create(self, request, *args, **kwargs): processing_service.mark_seen(live=True) return Response(response.dict(), status=status.HTTP_201_CREATED) + + +class TrainingDataViewSet(ProjectMixin, mixins.ListModelMixin, viewsets.GenericViewSet): + """ + Verified detections and their embeddings, for retraining a classifier head. + + A head is trained on embeddings, not pixels, and the backbone that produced them is + frozen. So a trainer can pull this and fit a new head without touching the images. + + Requires `project_id` and `algorithm` (an algorithm key). Constraining to one + algorithm is not optional: vectors from different backbones are in different spaces. + + GET /api/v2/ml/training-data/?project_id=3&algorithm= + GET /api/v2/ml/training-data/summary/?project_id=3&algorithm= + """ + + queryset = DetectionEmbedding.objects.none() + serializer_class = TrainingDataRowSerializer + require_project = True + permission_classes = [IsAuthenticated] + filter_backends: list = [] + pagination_class = TrainingDataPagination + + def _get_algorithm(self) -> Algorithm: + key = self.request.query_params.get("algorithm") + if not key: + raise api_exceptions.ValidationError( + {"algorithm": "Required. The algorithm key whose embeddings to train on."} + ) + algorithm = Algorithm.objects.filter(key=key).first() + if not algorithm: + raise api_exceptions.NotFound(f"No algorithm with key '{key}'.") + return algorithm + + def _get_split_settings(self) -> tuple[str, float]: + params = self.request.query_params + salt = params.get("split_salt", training_data.DEFAULT_SPLIT_SALT) + raw = params.get("test_fraction", training_data.DEFAULT_TEST_FRACTION) + try: + fraction = float(raw) + except (TypeError, ValueError): + raise api_exceptions.ValidationError({"test_fraction": "Must be a number between 0 and 1."}) + if not 0 <= fraction < 1: + raise api_exceptions.ValidationError({"test_fraction": "Must be between 0 and 1."}) + return salt, fraction + + def get_queryset(self) -> QuerySet[DetectionEmbedding]: + project = self.get_active_project() + assert project # require_project=True + self.check_object_permissions(self.request, project) + qs = training_data.verified_training_rows(project, self._get_algorithm()) + + split = self.request.query_params.get("split") + if split and split not in ("train", "test"): + raise api_exceptions.ValidationError({"split": "Must be 'train' or 'test'."}) + self._split_filter = split + return qs + + def get_serializer_context(self): + context = super().get_serializer_context() + salt, fraction = self._get_split_settings() + context["split_salt"] = salt + context["test_fraction"] = fraction + context["include_features"] = self.request.query_params.get("include_features", "true").lower() != "false" + return context + + @extend_schema(parameters=[project_id_doc_param]) + def list(self, request, *args, **kwargs): + response = super().list(request, *args, **kwargs) + split = getattr(self, "_split_filter", None) + if split: + # Filtering by split in Python rather than SQL: the assignment is a hash of the + # occurrence id, which Postgres cannot compute. Callers that need whole splits + # should page through everything and group client-side. + results = [row for row in response.data["results"] if row["split"] == split] + response.data["results"] = results + return response + + @extend_schema(parameters=[project_id_doc_param]) + @action(detail=False, methods=["get"]) + def summary(self, request, *args, **kwargs): + """Counts only. Cheap enough to poll before deciding whether a retrain is worth it.""" + project = self.get_active_project() + assert project + self.check_object_permissions(request, project) + algorithm = self._get_algorithm() + salt, fraction = self._get_split_settings() + + counts = training_data.label_counts(project, algorithm) + rows = training_data.verified_training_rows(project, algorithm) + splits = {"train": 0, "test": 0} + for occurrence_id in rows.values_list("detection__occurrence_id", flat=True): + splits[training_data.split_for(occurrence_id, salt, fraction)] += 1 + + return Response( + { + "project": {"id": project.pk, "name": project.name}, + "algorithm": {"key": algorithm.key, "name": algorithm.name, "version": algorithm.version}, + "dimensions": EMBEDDING_DIMENSIONS, + "rows": sum(counts.values()), + "classes": len(counts), + "counts": dict(sorted(counts.items(), key=lambda kv: -kv[1])), + "train": splits["train"], + "test": splits["test"], + "verified_detections_without_embedding": training_data.count_missing_embeddings(project, algorithm), + "settings": { + "split_salt": salt, + "test_fraction": fraction, + "split_grouped_by": "occurrence", + }, + } + ) diff --git a/compose/local/postgres/Dockerfile b/compose/local/postgres/Dockerfile index 5f864a4a0..fe2f457da 100644 --- a/compose/local/postgres/Dockerfile +++ b/compose/local/postgres/Dockerfile @@ -1,6 +1,17 @@ FROM postgres:16 # FROM esgn/pgtuned:latest +# pgvector, required by ami.ml.models.DetectionEmbedding. +# +# Installed into the stock image rather than switching the base to +# pgvector/pgvector:pg16. That image is built on Debian 12 (glibc 2.36) while +# postgres:16 is on Debian 13 (glibc 2.41), so swapping it under an existing PGDATA +# volume makes Postgres report a collation version mismatch and would need a full +# REINDEX. Same extension version (0.8.6) either way. +RUN apt-get update \ + && apt-get install -y --no-install-recommends postgresql-16-pgvector \ + && rm -rf /var/lib/apt/lists/* + COPY ./compose/local/postgres/maintenance /usr/local/bin/maintenance RUN chmod +x /usr/local/bin/maintenance/* RUN mv /usr/local/bin/maintenance/* /usr/local/bin \ diff --git a/config/api_router.py b/config/api_router.py index 56a01c591..93af2afa0 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -56,6 +56,7 @@ router.register(r"ml/labels", ml_views.AlgorithmCategoryMapViewSet) router.register(r"ml/pipelines", ml_views.PipelineViewSet) router.register(r"ml/processing_services", ml_views.ProcessingServiceViewSet) +router.register(r"ml/training-data", ml_views.TrainingDataViewSet, basename="training-data") router.register(r"classifications", views.ClassificationViewSet) router.register(r"identifications", views.IdentificationViewSet) router.register(r"jobs", job_views.JobViewSet) diff --git a/requirements/base.txt b/requirements/base.txt index be5d24b67..849214152 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -55,6 +55,7 @@ Werkzeug[watchdog]==2.3.6 # https://github.com/pallets/werkzeug ipdb==0.13.13 # https://github.com/gotcha/ipdb psycopg[binary]==3.1.9 # https://github.com/psycopg/psycopg # psycopg==3.1.9 # https://github.com/psycopg/psycopg # the non-binary version is needed for some platforms +pgvector==0.4.2 # https://github.com/pgvector/pgvector-python # requires the pgvector extension in postgres watchfiles==0.19.0 # https://github.com/samuelcolvin/watchfiles # Testing diff --git a/ui/src/components/filtering/filter-control.tsx b/ui/src/components/filtering/filter-control.tsx index 0e0944919..64dafd9cc 100644 --- a/ui/src/components/filtering/filter-control.tsx +++ b/ui/src/components/filtering/filter-control.tsx @@ -6,9 +6,11 @@ import { AlgorithmFilter, NotAlgorithmFilter } from './filters/algorithm-filter' import { BooleanFilter } from './filters/boolean-filter' import { CaptureSetFilter } from './filters/capture-set-filter' import { DateFilter } from './filters/date-filter' +import { DeviceFilter } from './filters/device-filter' import { ImageFilter } from './filters/image-filter' import { PipelineFilter } from './filters/pipeline-filter' import { SessionFilter } from './filters/session-filter' +import { SiteFilter } from './filters/site-filter' import { StationFilter } from './filters/station-filter' import { StatusFilter } from './filters/status-filter' import { TagFilter } from './filters/tag-filter' @@ -29,6 +31,8 @@ const ComponentMap: { date_end: DateFilter, date_start: DateFilter, deployment: StationFilter, + deployment__device: DeviceFilter, + deployment__research_site: SiteFilter, detections__source_image: ImageFilter, event: SessionFilter, processed: ProcessingStatusFilter, diff --git a/ui/src/components/filtering/filters/device-filter.tsx b/ui/src/components/filtering/filters/device-filter.tsx new file mode 100644 index 000000000..eb4b025a5 --- /dev/null +++ b/ui/src/components/filtering/filters/device-filter.tsx @@ -0,0 +1,17 @@ +import { API_ROUTES } from 'data-services/constants' +import { EntityPicker } from 'nova-ui-kit' +import { FilterProps } from './types' + +export const DeviceFilter = ({ onAdd, onClear, value }: FilterProps) => ( + { + if (value) { + onAdd(value) + } else { + onClear() + } + }} + value={value} + /> +) diff --git a/ui/src/components/filtering/filters/site-filter.tsx b/ui/src/components/filtering/filters/site-filter.tsx new file mode 100644 index 000000000..27cc7f05e --- /dev/null +++ b/ui/src/components/filtering/filters/site-filter.tsx @@ -0,0 +1,17 @@ +import { API_ROUTES } from 'data-services/constants' +import { EntityPicker } from 'nova-ui-kit' +import { FilterProps } from './types' + +export const SiteFilter = ({ onAdd, onClear, value }: FilterProps) => ( + { + if (value) { + onAdd(value) + } else { + onClear() + } + }} + value={value} + /> +) diff --git a/ui/src/data-services/hooks/identifications/useCreateIdentification.ts b/ui/src/data-services/hooks/identifications/useCreateIdentification.ts index ace5c984c..cb27ad33b 100644 --- a/ui/src/data-services/hooks/identifications/useCreateIdentification.ts +++ b/ui/src/data-services/hooks/identifications/useCreateIdentification.ts @@ -35,6 +35,9 @@ export const useCreateIdentification = ( if (invalidate) { queryClient.invalidateQueries([API_ROUTES.IDENTIFICATIONS]) queryClient.invalidateQueries([API_ROUTES.OCCURRENCES]) + // Refresh the taxa list so verified counts and the example thumbnail + // reflect the new identification. + queryClient.invalidateQueries([API_ROUTES.SPECIES]) } onSuccess?.() }, diff --git a/ui/src/data-services/hooks/identifications/useCreateIdentifications.ts b/ui/src/data-services/hooks/identifications/useCreateIdentifications.ts index 004848c41..0f68fe8d8 100644 --- a/ui/src/data-services/hooks/identifications/useCreateIdentifications.ts +++ b/ui/src/data-services/hooks/identifications/useCreateIdentifications.ts @@ -55,6 +55,8 @@ export const useCreateIdentifications = ( setLastAttempt({ failed, total: submitted.length }) queryClient.invalidateQueries([API_ROUTES.IDENTIFICATIONS]) queryClient.invalidateQueries([API_ROUTES.OCCURRENCES]) + // Keep the taxa list in sync (verified counts + example thumbnail). + queryClient.invalidateQueries([API_ROUTES.SPECIES]) onSuccess?.() if (!failed.length) { successResetTimeout.current = setTimeout(() => reset(), SUCCESS_TIMEOUT) diff --git a/ui/src/data-services/hooks/occurrences/stats/useModelAgreement.ts b/ui/src/data-services/hooks/occurrences/stats/useModelAgreement.ts new file mode 100644 index 000000000..8db718f23 --- /dev/null +++ b/ui/src/data-services/hooks/occurrences/stats/useModelAgreement.ts @@ -0,0 +1,87 @@ +import { API_ROUTES, API_URL } from 'data-services/constants' +import { useAuthorizedQuery } from '../../auth/useAuthorizedQuery' + +export interface ModelAgreementResponse { + project_id: number + total_occurrences: number + verified_count: number + verified_pct: number + verified_with_prediction_count: number + no_prediction_count: number + verified_without_taxon_count: number + // Denominator for every agreed_*_pct and CI: verified occurrences that have + // both a model prediction and a human taxon. + comparable_count: number + agreed_exact_count: number + agreed_exact_pct: number + agreed_exact_ci_low: number | null + agreed_exact_ci_high: number | null + agreed_any_rank_count: number + agreed_any_rank_pct: number + agreed_any_rank_ci_low: number | null + agreed_any_rank_ci_high: number | null + // Cohen's kappa (exact-taxon) — agreement beyond chance. Range [-1, 1]; + // null when denominator is 0 or expected agreement is 1.0. + cohens_kappa: number | null + // Only populated when the caller passes ?agreement_coarsest_rank=. + agreement_coarsest_rank: string | null + agreed_coarser_rank_count: number | null + agreed_coarser_rank_pct: number | null +} + +type FilterPrimitive = string | number | boolean +type FilterValue = FilterPrimitive | FilterPrimitive[] | null | undefined + +// Accepts an arbitrary filter map so the occurrence list page's filter state +// can be threaded through unchanged (deployment, event, taxon, score +// thresholds, apply_defaults, etc). Arrays are appended as repeated query +// params so multi-select filters (e.g. `algorithm`, `not_algorithm`, which +// the backend reads via `request.query_params.getlist(...)`) survive. +export const useModelAgreement = ( + projectId?: string, + filters?: Record, + enabled?: boolean +) => { + const url = `${API_URL}/${API_ROUTES.OCCURRENCES}/stats/model-agreement/` + + const params = new URLSearchParams() + if (projectId) params.set('project_id', projectId) + if (filters) { + Object.entries(filters).forEach(([key, value]) => { + if (value === undefined || value === null || value === '') return + if (Array.isArray(value)) { + value.forEach((item) => { + if (item !== undefined && item !== null && item !== '') { + params.append(key, String(item)) + } + }) + return + } + params.set(key, String(value)) + }) + } + // Sort so two equivalent filter maps produce one query string, and so one + // react-query cache key. + params.sort() + const queryString = params.toString() + + const { data, isLoading, isFetching, error } = + useAuthorizedQuery({ + enabled: !!projectId && enabled !== false, + queryKey: [ + API_ROUTES.OCCURRENCES, + 'stats', + 'model-agreement', + projectId, + queryString, + ], + url: `${url}?${queryString}`, + }) + + return { + data, + isLoading, + isFetching, + error, + } +} diff --git a/ui/src/data-services/hooks/species/useSpecies.ts b/ui/src/data-services/hooks/species/useSpecies.ts index 27f5fbd5f..283ea6fbd 100644 --- a/ui/src/data-services/hooks/species/useSpecies.ts +++ b/ui/src/data-services/hooks/species/useSpecies.ts @@ -16,7 +16,21 @@ export const useSpecies = ( isFetching: boolean error?: unknown } => { - const fetchUrl = getFetchUrl({ collection: API_ROUTES.SPECIES, params }) + // Only the caller that renders the Example column asks for example occurrences, and + // never under a capture-set (collection) filter: on that path the example subqueries + // degrade to per-row scans, which is why the backend keeps them opt-in. + const hasCollectionFilter = params?.filters?.some( + (filter) => filter.field === 'collection' && filter.value + ) + const fetchParams = { + ...params, + withExampleOccurrences: + !!params?.withExampleOccurrences && !hasCollectionFilter, + } + const fetchUrl = getFetchUrl({ + collection: API_ROUTES.SPECIES, + params: fetchParams, + }) const { data, isLoading, isFetching, error } = useAuthorizedQuery<{ results: ServerSpecies[] diff --git a/ui/src/data-services/models/species.ts b/ui/src/data-services/models/species.ts index b4f766b66..37d6d622c 100644 --- a/ui/src/data-services/models/species.ts +++ b/ui/src/data-services/models/species.ts @@ -17,6 +17,41 @@ export class Species extends Taxon { return `https://api.antenna.insectai.org/bereich/main/taxon/${this.id}` // TODO: Use dynamic admin URL based on environment? } + get bestScoringOccurrenceId(): number | undefined { + return this._species.best_scoring_occurrence_id ?? undefined + } + + // One occurrence chosen by the backend to verify this taxon's presence: the + // best-scoring unverified occurrence for unverified taxa, or the most recent + // occurrence for already-verified taxa. Populated only when the list is + // fetched with the example-occurrences flag; undefined otherwise. + // + // Named to avoid colliding with SpeciesDetails.exampleOccurrence, which is a + // different concept (a cover-style example image on the taxon detail model). + get verificationExample(): + | { + id: number + detectionId: number | null + imageUrl: string | null + score: number | null + verified: boolean + } + | undefined { + const example = this._species.example_occurrence + + if (!example) { + return undefined + } + + return { + id: example.id, + detectionId: example.detection_id ?? null, + imageUrl: example.image_url ?? null, + score: example.score ?? null, + verified: !!example.verified, + } + } + get coverImage(): { url: string; caption?: string } | undefined { if (!this._species.cover_image_url) { return undefined @@ -64,6 +99,10 @@ export class Species extends Taxon { : undefined } + get lastDetectedOccurrenceId(): number | undefined { + return this._species.last_detected_occurrence_id ?? undefined + } + get lastSeen() { if (!this._species.last_detected) { return undefined diff --git a/ui/src/data-services/types.ts b/ui/src/data-services/types.ts index c534c3607..49eec8291 100644 --- a/ui/src/data-services/types.ts +++ b/ui/src/data-services/types.ts @@ -4,6 +4,7 @@ export interface FetchParams { sort?: { field: string; order: 'asc' | 'desc' } filters?: { field: string; value?: string; error?: string }[] withCounts?: boolean + withExampleOccurrences?: boolean } export interface APIValidationError { diff --git a/ui/src/data-services/utils.ts b/ui/src/data-services/utils.ts index c123b48cf..9d017b984 100644 --- a/ui/src/data-services/utils.ts +++ b/ui/src/data-services/utils.ts @@ -35,6 +35,9 @@ export const getFetchUrl = ({ if (params?.withCounts) { queryParams.with_counts = String(true) } + if (params?.withExampleOccurrences) { + queryParams.with_example_occurrences = String(true) + } const baseUrl = `${API_URL}/${collection}` const queryString = new URLSearchParams(queryParams).toString() diff --git a/ui/src/nova-ui-kit/components/table/table/table.tsx b/ui/src/nova-ui-kit/components/table/table/table.tsx index 987a21b4f..47b85e5dd 100644 --- a/ui/src/nova-ui-kit/components/table/table/table.tsx +++ b/ui/src/nova-ui-kit/components/table/table/table.tsx @@ -29,6 +29,8 @@ interface TableProps { items?: T[] onSelectedItemsChange?: (selectedItems: string[]) => void onSortSettingsChange?: (sortSettings?: TableSortSettings) => void + // Optional per-row class hook, e.g. to de-emphasize already-processed rows. + rowClassName?: (item: T) => string | undefined selectable?: boolean selectedItems?: string[] sortable?: boolean @@ -43,6 +45,7 @@ export const Table = ({ items = [], onSelectedItemsChange, onSortSettingsChange, + rowClassName, selectable, selectedItems = [], sortable, @@ -125,7 +128,7 @@ export const Table = ({ {items.map((item, rowIndex) => ( - + {selectable && ( diff --git a/ui/src/pages/occurrences/occurrence-details-dialog.tsx b/ui/src/pages/occurrences/occurrence-details-dialog.tsx new file mode 100644 index 000000000..8362cdfd9 --- /dev/null +++ b/ui/src/pages/occurrences/occurrence-details-dialog.tsx @@ -0,0 +1,86 @@ +import { useOccurrenceDetails } from 'data-services/hooks/occurrences/useOccurrenceDetails' +import { Dialog } from 'nova-ui-kit' +import { + OccurrenceDetails, + TABS, +} from 'pages/occurrence-details/occurrence-details' +import { useContext, useEffect } from 'react' +import { useLocation } from 'react-router-dom' +import { BreadcrumbContext } from 'utils/breadcrumbContext' +import { STRING, translate } from 'utils/language' +import { useSelectedView } from 'utils/useSelectedView' +import { OccurrenceNavigation } from './occurrence-navigation' + +// Occurrence identification modal. Rendered over a list (occurrences or taxa); +// the parent owns which occurrence is shown and how closing updates the URL. +export const OccurrenceDetailsDialog = ({ + id, + occurrences, + onClose, + onNavigate, + defaultTab = TABS.FIELDS, +}: { + id: string + // Ordered items the prev/next buttons page through. Only the id is used. + occurrences?: { id: string }[] + onClose: () => void + // How prev/next switches occurrence. When omitted, navigation routes to the + // occurrence detail page; the taxa list passes this to swap ?verifyOccurrence in place. + onNavigate?: (id: string) => void + // Tab to open on when no ?tab= is set. The taxa list opens on Identification so + // verifying is the immediate action; the occurrences list keeps Fields. + defaultTab?: string +}) => { + const { state } = useLocation() + const { selectedView, setSelectedView } = useSelectedView(defaultTab, 'tab') + const { setDetailBreadcrumb } = useContext(BreadcrumbContext) + const { occurrence, isLoading, error } = useOccurrenceDetails(id) + + useEffect(() => { + // If a default tab is set from router state, set this as active + if (state?.defaultTab) { + setSelectedView(state.defaultTab) + } + }, [state?.defaultTab]) + + useEffect(() => { + setDetailBreadcrumb( + occurrence ? { title: occurrence.displayName } : undefined + ) + + return () => { + setDetailBreadcrumb(undefined) + } + }, [occurrence]) + + return ( + { + if (!open) { + setSelectedView(undefined) + onClose() + } + }} + > + + {occurrence ? ( + + ) : null} + + + + ) +} diff --git a/ui/src/pages/occurrences/occurrence-filters.ts b/ui/src/pages/occurrences/occurrence-filters.ts new file mode 100644 index 000000000..61b9e9443 --- /dev/null +++ b/ui/src/pages/occurrences/occurrence-filters.ts @@ -0,0 +1,22 @@ +// Filter fields another view may carry into the occurrence list (see useCarryOverFilters). +// A field belongs here only if the occurrence list backend honors it (keep in sync with +// OCCURRENCE_FILTERSET_FIELDS by hand) and the occurrence filter panel can display it, so a +// carried filter is always visible and clearable on arrival. See #1347. +export const FILTERS_TO_OCCURRENCES = [ + 'detections__source_image', + 'event', + 'taxon', + 'taxa_list_id', + 'not_taxa_list_id', + 'verified', + 'verified_by_me', + 'collection', + 'date_start', + 'date_end', + 'deployment', + 'deployment__device', + 'deployment__research_site', + 'algorithm', + 'not_algorithm', + 'apply_defaults', +] diff --git a/ui/src/pages/occurrences/occurrence-navigation.tsx b/ui/src/pages/occurrences/occurrence-navigation.tsx index 055679b66..ce56a3eeb 100644 --- a/ui/src/pages/occurrences/occurrence-navigation.tsx +++ b/ui/src/pages/occurrences/occurrence-navigation.tsx @@ -1,4 +1,3 @@ -import { Occurrence } from 'data-services/models/occurrence' import { ChevronLeftIcon, ChevronRightIcon } from 'lucide-react' import { Button } from 'nova-ui-kit' import { useCallback, useEffect } from 'react' @@ -7,46 +6,50 @@ import { APP_ROUTES } from 'utils/constants' import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' -const useOccurrenceNavigation = (occurrences?: Occurrence[]) => { - const { projectId, id } = useParams() - const navigate = useNavigate() - const currentIndex = occurrences?.findIndex((o) => o.id === id) - const prevId = - currentIndex !== undefined ? occurrences?.[currentIndex - 1]?.id : undefined - const nextId = - currentIndex !== undefined ? occurrences?.[currentIndex + 1]?.id : undefined - - const goToPrev = useCallback(() => { - if (!prevId) { - return - } +// Ordered items the modal can page through. Only the id is needed: on the occurrences +// list these are Occurrence models; on the taxa list they are the per-row example +// occurrences, so paging steps to the next taxon's example. +type NavItem = { id: string } - navigate( - getAppRoute({ - to: APP_ROUTES.OCCURRENCE_DETAILS({ - projectId: projectId as string, - occurrenceId: prevId, - }), - keepSearchParams: true, - }) - ) - }, [nextId]) +const useOccurrenceNavigation = ( + items?: NavItem[], + currentId?: string, + onNavigate?: (id: string) => void +) => { + const { projectId, id: routeId } = useParams() + const navigate = useNavigate() + const activeId = currentId ?? routeId + const currentIndex = items?.findIndex((o) => o.id === activeId) + const hasCurrent = currentIndex !== undefined && currentIndex >= 0 + const prevId = hasCurrent ? items?.[currentIndex - 1]?.id : undefined + const nextId = hasCurrent ? items?.[currentIndex + 1]?.id : undefined - const goToNext = useCallback(() => { - if (!nextId) { - return - } + const goTo = useCallback( + (targetId?: string) => { + if (!targetId) { + return + } + // The taxa list keeps the modal open and swaps the ?verifyOccurrence id in place; + // the occurrences list routes to that occurrence's detail page. + if (onNavigate) { + onNavigate(targetId) + return + } + navigate( + getAppRoute({ + to: APP_ROUTES.OCCURRENCE_DETAILS({ + projectId: projectId as string, + occurrenceId: targetId, + }), + keepSearchParams: true, + }) + ) + }, + [navigate, onNavigate, projectId] + ) - navigate( - getAppRoute({ - to: APP_ROUTES.OCCURRENCE_DETAILS({ - projectId: projectId as string, - occurrenceId: nextId, - }), - keepSearchParams: true, - }) - ) - }, [nextId]) + const goToPrev = useCallback(() => goTo(prevId), [goTo, prevId]) + const goToNext = useCallback(() => goTo(nextId), [goTo, nextId]) return { prevId, @@ -58,11 +61,18 @@ const useOccurrenceNavigation = (occurrences?: Occurrence[]) => { export const OccurrenceNavigation = ({ occurrences, + currentId, + onNavigate, }: { - occurrences?: Occurrence[] + occurrences?: NavItem[] + currentId?: string + onNavigate?: (id: string) => void }) => { - const { prevId, nextId, goToPrev, goToNext } = - useOccurrenceNavigation(occurrences) + const { prevId, nextId, goToPrev, goToNext } = useOccurrenceNavigation( + occurrences, + currentId, + onNavigate + ) // Listen to key down events useEffect(() => { diff --git a/ui/src/pages/occurrences/occurrence-stats.tsx b/ui/src/pages/occurrences/occurrence-stats.tsx new file mode 100644 index 000000000..65dd8849c --- /dev/null +++ b/ui/src/pages/occurrences/occurrence-stats.tsx @@ -0,0 +1,344 @@ +import { + ModelAgreementResponse, + useModelAgreement, +} from 'data-services/hooks/occurrences/stats/useModelAgreement' +import { ChevronsUpDown } from 'lucide-react' +import { Box, Button, Collapsible, InfoTooltip } from 'nova-ui-kit' +import { useState } from 'react' +import { STRING, translate } from 'utils/language' + +interface OccurrenceStatsProps { + projectId?: string + filters: { field: string; value?: string; error?: string }[] +} + +const clampPct = (value: number) => + Math.round(Math.min(Math.max(value, 0), 1) * 100) + +// "<1%" reads better than "0%" when the count is non-zero but rounds down. +const pctText = (value: number, count: number) => { + const pct = clampPct(value) + return pct === 0 && count ? '<1%' : `${pct}%` +} + +const ciRangeText = ( + pct: number, + ciLow?: number | null, + ciHigh?: number | null +) => + ciLow != null && ciHigh != null + ? `${clampPct(ciLow)}–${clampPct(ciHigh)}%` + : `${clampPct(pct)}%` + +const StatLabel = ({ label, tooltip }: { label: string; tooltip: string }) => ( +
+ + {label} + + +
+) + +const Bar = ({ + label, + tooltip, + fill, + valueText, +}: { + label: string + tooltip: string + fill: number + valueText: string +}) => ( +
+ +
+
+
+
+ + {valueText} + +
+
+) + +const AgreementBar = ({ + label, + tooltip, + value, + ciLow, + ciHigh, + valueText, +}: { + label: string + tooltip: string + value: number + ciLow?: number | null + ciHigh?: number | null + valueText: string +}) => { + const hasCi = ciLow != null && ciHigh != null + const lowPct = ciLow != null ? clampPct(ciLow) : 0 + const highPct = ciHigh != null ? clampPct(ciHigh) : 0 + + return ( +
+ +
+
+ {hasCi ? ( + <> +
+ {/* The hatch marking the confidence interval is drawn over the + gray track, not the solid fill, so it stays visible when the + estimate sits near 100%. See #1308. */} +
+ + ) : ( +
+ )} +
+ + {valueText} + +
+
+ ) +} + +const SignedBar = ({ + label, + tooltip, + value, +}: { + label: string + tooltip: string + value: number | null +}) => { + const clamped = value === null ? null : Math.min(Math.max(value, -1), 1) + const widthPct = clamped === null ? 0 : Math.abs(clamped) * 50 + const leftPct = + clamped === null ? 50 : clamped >= 0 ? 50 : 50 - Math.abs(clamped) * 50 + const valueText = + clamped === null + ? translate(STRING.VALUE_NOT_AVAILABLE) + : clamped.toFixed(2) + + return ( +
+ +
+
+
+ {clamped !== null ? ( +
+ ) : null} +
+ {valueText} +
+
+ ) +} + +// Live verified / agreement stats for the occurrence list. Threads the same +// filter array the list view sends so the numbers always match the result set. +// Collapsed by default, and the query only runs while it is open. +export const OccurrenceStats = ({ + projectId, + filters, +}: OccurrenceStatsProps) => { + const [open, setOpen] = useState(false) + + const activeFilters = filters.reduce>( + (acc, { field, value, error }) => { + if (value?.length && !error) { + acc[field] = value + } + return acc + }, + {} + ) + + const { data, error } = useModelAgreement(projectId, activeFilters, open) + + return ( + + +
+
+ + {translate(STRING.STATS)} + + +
+ + + +
+ + + + + + ) +} + +const StatsContent = ({ + data, + error, +}: { + data?: ModelAgreementResponse + error?: unknown +}) => { + if (error) { + return ( + + {translate(STRING.UNKNOWN_ERROR)} + + ) + } + + if (!data) { + return ( + <> +
+
+ + ) + } + + const comparable = data.comparable_count + + return ( + <> + + + {comparable === 0 ? ( + + {translate(STRING.MESSAGE_STATS_NO_COMPARABLE)} + + ) : ( + <> + + + + + + + + + + + + + )} + + ) +} diff --git a/ui/src/pages/occurrences/occurrences.tsx b/ui/src/pages/occurrences/occurrences.tsx index a527c04ed..af53cb098 100644 --- a/ui/src/pages/occurrences/occurrences.tsx +++ b/ui/src/pages/occurrences/occurrences.tsx @@ -2,16 +2,13 @@ import { DefaultFiltersControl } from 'components/filtering/default-filter-contr import { FilterControl } from 'components/filtering/filter-control' import { FilterSection } from 'components/filtering/filter-section' import { someActive } from 'components/filtering/utils' -import { useOccurrenceDetails } from 'data-services/hooks/occurrences/useOccurrenceDetails' import { useOccurrences } from 'data-services/hooks/occurrences/useOccurrences' import { useTaxaLists } from 'data-services/hooks/taxa-lists/useTaxaLists' -import { Occurrence } from 'data-services/models/occurrence' import { DownloadIcon, Grid2X2Icon, TableIcon } from 'lucide-react' import { BulkActionBar, buttonVariants, ColumnSettings, - Dialog, PageFooter, PageHeader, PaginationBar, @@ -19,13 +16,8 @@ import { Table, ToggleGroup, } from 'nova-ui-kit' -import { - OccurrenceDetails, - TABS, -} from 'pages/occurrence-details/occurrence-details' -import { useContext, useEffect, useState } from 'react' -import { Link, useLocation, useNavigate, useParams } from 'react-router-dom' -import { BreadcrumbContext } from 'utils/breadcrumbContext' +import { useEffect, useState } from 'react' +import { Link, useNavigate, useParams } from 'react-router-dom' import { APP_ROUTES, DOCS_LINKS } from 'utils/constants' import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' @@ -36,12 +28,14 @@ import { useUser } from 'utils/user/userContext' import { useSelectedView } from 'utils/useSelectedView' import { useSort } from 'utils/useSort' import { columns } from './occurrence-columns' +import { OccurrenceDetailsDialog } from './occurrence-details-dialog' import { OccurrenceGallery } from './occurrence-gallery' -import { OccurrenceNavigation } from './occurrence-navigation' +import { OccurrenceStats } from './occurrence-stats' import { OccurrencesActions } from './occurrences-actions' export const Occurrences = () => { const { user } = useUser() + const navigate = useNavigate() const { projectId, id } = useParams() const { columnSettings, setColumnSettings } = useColumnSettings( 'occurrences', @@ -96,6 +90,7 @@ export const Occurrences = () => { <>
+ @@ -111,9 +106,16 @@ export const Occurrences = () => { @@ -121,6 +123,8 @@ export const Occurrences = () => { + + @@ -221,73 +225,19 @@ export const Occurrences = () => { ) : null} {id ? ( - + + navigate( + getAppRoute({ + to: APP_ROUTES.OCCURRENCES({ projectId: projectId as string }), + keepSearchParams: true, + }) + ) + } + /> ) : null} ) } - -const OccurrenceDetailsDialog = ({ - id, - occurrences, -}: { - id: string - occurrences?: Occurrence[] -}) => { - const navigate = useNavigate() - const { state } = useLocation() - const { selectedView, setSelectedView } = useSelectedView(TABS.FIELDS, 'tab') - const { projectId } = useParams() - const { setDetailBreadcrumb } = useContext(BreadcrumbContext) - const { occurrence, isLoading, error } = useOccurrenceDetails(id) - - useEffect(() => { - // If a default tab is set from router state, set this as active - if (state?.defaultTab) { - setSelectedView(state.defaultTab) - } - }, [state?.defaultTab]) - - useEffect(() => { - setDetailBreadcrumb( - occurrence ? { title: occurrence.displayName } : undefined - ) - - return () => { - setDetailBreadcrumb(undefined) - } - }, [occurrence]) - - return ( - { - if (!open) { - setSelectedView(undefined) - } - - navigate( - getAppRoute({ - to: APP_ROUTES.OCCURRENCES({ projectId: projectId as string }), - keepSearchParams: true, - }) - ) - }} - > - - {occurrence ? ( - - ) : null} - - - - ) -} diff --git a/ui/src/pages/species-details/species-details.tsx b/ui/src/pages/species-details/species-details.tsx index 6d4adc5f1..81e8c5f53 100644 --- a/ui/src/pages/species-details/species-details.tsx +++ b/ui/src/pages/species-details/species-details.tsx @@ -20,6 +20,9 @@ import { APP_ROUTES } from 'utils/constants' import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' +import { useCarryOverFilters } from 'utils/useFilters' +import { FILTERS_TO_OCCURRENCES } from 'pages/occurrences/occurrence-filters' +import { FILTERS_TO_TAXA } from 'pages/species/species-filters' import { UserPermission } from 'utils/user/types' import styles from './species-details.module.scss' @@ -40,6 +43,8 @@ export const SpeciesDetails = ({ const { projectId } = useParams() const navigate = useNavigate() const { project } = useProjectDetails(projectId as string, true) + const occurrenceFilters = useCarryOverFilters(FILTERS_TO_OCCURRENCES) + const taxaFilters = useCarryOverFilters(FILTERS_TO_TAXA) const canUpdate = species.userPermissions.includes(UserPermission.Update) const hasChildren = species.rank !== 'SPECIES' @@ -139,7 +144,7 @@ export const SpeciesDetails = ({ to: APP_ROUTES.TAXA({ projectId: projectId as string, }), - filters: { taxon: species.id }, + filters: { ...taxaFilters, taxon: species.id }, })} /> @@ -154,7 +159,7 @@ export const SpeciesDetails = ({ to: APP_ROUTES.OCCURRENCES({ projectId: projectId as string, }), - filters: { taxon: species.id }, + filters: { ...occurrenceFilters, taxon: species.id }, })} /> @@ -165,7 +170,11 @@ export const SpeciesDetails = ({ to: APP_ROUTES.OCCURRENCES({ projectId: projectId as string, }), - filters: { taxon: species.id, verified: 'true' }, + filters: { + ...occurrenceFilters, + taxon: species.id, + verified: 'true', + }, })} /> diff --git a/ui/src/pages/species/species-columns.tsx b/ui/src/pages/species/species-columns.tsx index b7f7a33ec..dd4e2e779 100644 --- a/ui/src/pages/species/species-columns.tsx +++ b/ui/src/pages/species/species-columns.tsx @@ -2,6 +2,7 @@ import { DeterminationScore } from 'components/determination-score' import { TaxonDetails } from 'components/taxon-details/taxon-details' import { Tag } from 'components/taxon-tags/tag' import { Species } from 'data-services/models/species' +import { ShieldCheckIcon } from 'lucide-react' import { BasicTableCell, CellTheme, @@ -19,7 +20,15 @@ import { STRING, translate } from 'utils/language' export const columns: (project: { projectId: string featureFlags?: { [key: string]: boolean } -}) => TableColumn[] = ({ projectId, featureFlags }) => [ + // Active taxa-list filters (station, verified, device, site, …) carried over + // when drilling into a taxon's occurrences so the occurrence list stays scoped + // to the same selection instead of showing every occurrence of the taxon. + carryFilters?: Record +}) => TableColumn[] = ({ + projectId, + featureFlags, + carryFilters = {}, +}) => [ { id: 'cover-image', name: translate(STRING.FIELD_LABEL_IMAGE), @@ -37,6 +46,30 @@ export const columns: (project: { ) }, }, + { + id: 'example', + name: translate(STRING.FIELD_LABEL_EXAMPLE), + tooltip: translate(STRING.TOOLTIP_VERIFY_EXAMPLE), + renderCell: (item: Species) => { + const example = item.verificationExample + + return ( + + ) + }, + }, { id: 'name', sortField: 'name', @@ -75,7 +108,22 @@ export const columns: (project: { id: 'last-seen', sortField: 'last_detected', name: 'Last seen', - renderCell: (item: Species) => , + renderCell: (item: Species) => + item.lastDetectedOccurrenceId ? ( + + + + ) : ( + + ), }, { id: 'occurrences', @@ -88,7 +136,7 @@ export const columns: (project: { @@ -106,10 +154,18 @@ export const columns: (project: { - +
+ {item.numVerified > 0 ? ( + + ) : null} + +
), }, @@ -117,17 +173,35 @@ export const columns: (project: { id: 'best-determination-score', name: translate(STRING.FIELD_LABEL_BEST_SCORE), sortField: 'best_determination_score', - renderCell: (item: Species) => ( - - { + const cell = ( + + + + ) + + return item.bestScoringOccurrenceId ? ( + - - ), + > + {cell} + + ) : ( + cell + ) + }, }, { id: 'created-at', diff --git a/ui/src/pages/species/species-filters.ts b/ui/src/pages/species/species-filters.ts new file mode 100644 index 000000000..aa16d6b0f --- /dev/null +++ b/ui/src/pages/species/species-filters.ts @@ -0,0 +1,18 @@ +// Filter fields another view may carry into the taxa list (see useCarryOverFilters). A field +// belongs here only if the taxa list backend honors it (keep in sync with +// TaxonViewSet.get_occurrence_filters by hand) and the taxa filter panel can display it, so +// a carried filter is always visible and clearable on arrival. See #1347. +export const FILTERS_TO_TAXA = [ + 'event', + 'taxon', + 'taxa_list_id', + 'not_taxa_list_id', + 'verified', + 'include_unobserved', + 'deployment', + 'deployment__device', + 'deployment__research_site', + 'tag_id', + 'not_tag_id', + 'apply_defaults', +] diff --git a/ui/src/pages/species/species.tsx b/ui/src/pages/species/species.tsx index 60b8e460f..51d2f1083 100644 --- a/ui/src/pages/species/species.tsx +++ b/ui/src/pages/species/species.tsx @@ -1,6 +1,7 @@ import { DefaultFiltersControl } from 'components/filtering/default-filter-control' import { FilterControl } from 'components/filtering/filter-control' import { FilterSection } from 'components/filtering/filter-section' +import { someActive } from 'components/filtering/utils' import { useProjectDetails } from 'data-services/hooks/projects/useProjectDetails' import { useSpecies } from 'data-services/hooks/species/useSpecies' import { useSpeciesDetails } from 'data-services/hooks/species/useSpeciesDetails' @@ -17,15 +18,18 @@ import { Table, ToggleGroup, } from 'nova-ui-kit' +import { OccurrenceDetailsDialog } from 'pages/occurrences/occurrence-details-dialog' +import { TABS as OCCURRENCE_TABS } from 'pages/occurrence-details/occurrence-details' import { SpeciesDetails, TABS } from 'pages/species-details/species-details' -import { useContext, useEffect, useMemo } from 'react' -import { useNavigate, useParams } from 'react-router-dom' +import { useContext, useEffect, useMemo, useRef } from 'react' +import { useNavigate, useParams, useSearchParams } from 'react-router-dom' import { BreadcrumbContext } from 'utils/breadcrumbContext' import { APP_ROUTES } from 'utils/constants' import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' import { useColumnSettings } from 'utils/useColumnSettings' -import { useFilters } from 'utils/useFilters' +import { useCarryOverFilters, useFilters } from 'utils/useFilters' +import { FILTERS_TO_OCCURRENCES } from 'pages/occurrences/occurrence-filters' import { usePagination } from 'utils/usePagination' import { useSelectedView } from 'utils/useSelectedView' import { useSort } from 'utils/useSort' @@ -34,9 +38,14 @@ import { SpeciesGallery } from './species-gallery' export const Species = () => { const { projectId, id } = useParams() + const [searchParams, setSearchParams] = useSearchParams() + // Occurrence to verify in a modal over the taxa list. Keyed off a search + // param (not the :id path segment, which already means taxon detail). + const verifyOccurrenceId = searchParams.get('verifyOccurrence') ?? undefined const { project } = useProjectDetails(projectId as string, true) const { columnSettings, setColumnSettings } = useColumnSettings('species', { 'cover-image': true, + example: true, name: true, rank: false, 'last-seen': true, @@ -48,16 +57,79 @@ export const Species = () => { }) const { sort, setSort } = useSort({ field: 'name', order: 'asc' }) const { pagination, setPage } = usePagination() - const { filters } = useFilters() + const { activeFilters, filters } = useFilters() const { species, total, isLoading, isFetching, error } = useSpecies({ projectId, sort, pagination, filters, + // This list renders the Example column and links Last-seen / Best-score to an occurrence. + withExampleOccurrences: true, }) + // Ordered example occurrences, one per taxon row that has one, so the modal's + // prev/next steps to the next taxon's example (rows without an example are skipped). + const exampleNavItems = useMemo( + () => + (species ?? []).flatMap((item) => + item.verificationExample + ? [{ id: String(item.verificationExample.id) }] + : [] + ), + [species] + ) + // Remember where the open example sits in the list so the sweep can continue if it + // drops out. After verifying, that row's example rolls to a different occurrence (or, + // under ?verified=false, the row leaves the list), so the open ?verifyOccurrence id is + // no longer in exampleNavItems. Advance to whatever example now occupies that position + // instead of dead-ending with both nav buttons disabled. + const verifyIndexRef = useRef(-1) + useEffect(() => { + if (!verifyOccurrenceId) { + return + } + if (exampleNavItems.length === 0) { + // The list emptied after a verification (not a fetch in flight or a fresh deep + // link), so the sweep is finished: close the modal instead of leaving it open on + // an occurrence that is no longer listed. + if (!isFetching && verifyIndexRef.current >= 0) { + verifyIndexRef.current = -1 + setSearchParams( + (prev) => { + const next = new URLSearchParams(prev) + next.delete('verifyOccurrence') + return next + }, + { replace: true } + ) + } + return + } + const index = exampleNavItems.findIndex( + (item) => item.id === verifyOccurrenceId + ) + if (index >= 0) { + verifyIndexRef.current = index + return + } + const nextId = + exampleNavItems[ + Math.min(verifyIndexRef.current, exampleNavItems.length - 1) + ]?.id + if (nextId) { + setSearchParams( + (prev) => { + const next = new URLSearchParams(prev) + next.set('verifyOccurrence', nextId) + return next + }, + { replace: true } + ) + } + }, [exampleNavItems, verifyOccurrenceId, isFetching, setSearchParams]) const { selectedView, setSelectedView } = useSelectedView('table') const { taxaLists = [] } = useTaxaLists({ projectId: projectId as string }) const { tags = [] } = useTags({ projectId: projectId as string }) + const carryFilters = useCarryOverFilters(FILTERS_TO_OCCURRENCES) const pageTitle = useMemo(() => { const taxaListFilter = filters.find( (filter) => filter.field === 'taxa_list_id' @@ -74,26 +146,44 @@ export const Species = () => { return ( <>
- - - - - {taxaLists.length > 0 && ( - <> - - - - )} - - - {project?.featureFlags.tags ? ( - <> - - - - ) : null} - - +
+ + + + {taxaLists.length > 0 && ( + <> + + + + )} + + + + + + + + + {project?.featureFlags.tags ? ( + <> + + + + ) : null} + +
{ columns={columns({ projectId: projectId as string, featureFlags: project?.featureFlags, + carryFilters, }).filter((column) => !!columnSettings[column.id])} error={error} - isLoading={!id && isLoading} + isLoading={!id && !verifyOccurrenceId && isLoading} items={species} onSortSettingsChange={setSort} + rowClassName={(item) => + item.numVerified > 0 ? 'opacity-50' : undefined + } sortable sortSettings={sort} /> @@ -147,7 +241,7 @@ export const Species = () => { {selectedView === 'gallery' && ( )} @@ -163,6 +257,23 @@ export const Species = () => { ) : null} {id ? : null} + {verifyOccurrenceId ? ( + { + const nextParams = new URLSearchParams(searchParams) + nextParams.set('verifyOccurrence', occurrenceId) + setSearchParams(nextParams) + }} + onClose={() => { + const nextParams = new URLSearchParams(searchParams) + nextParams.delete('verifyOccurrence') + setSearchParams(nextParams) + }} + /> + ) : null} ) } diff --git a/ui/src/utils/buildCarryOverFilters.ts b/ui/src/utils/buildCarryOverFilters.ts new file mode 100644 index 000000000..4f875253d --- /dev/null +++ b/ui/src/utils/buildCarryOverFilters.ts @@ -0,0 +1,14 @@ +// Carry the source view's active filters into a destination list. `fields` is the +// destination's carry contract (e.g. FILTERS_TO_OCCURRENCES), so source-only state such as +// sort order or page number never reaches the destination URL. Kept dependency-free so it +// can be unit-tested without loading the filter registry. See #1347. +export const buildCarryOverFilters = ( + filters: { field: string; value?: string }[], + fields: string[] +): Record => + filters.reduce>((acc, filter) => { + if (filter.value && fields.includes(filter.field)) { + acc[filter.field] = filter.value + } + return acc + }, {}) diff --git a/ui/src/utils/carryOverFilters.test.ts b/ui/src/utils/carryOverFilters.test.ts new file mode 100644 index 000000000..4844816af --- /dev/null +++ b/ui/src/utils/carryOverFilters.test.ts @@ -0,0 +1,48 @@ +import { FILTERS_TO_OCCURRENCES } from 'pages/occurrences/occurrence-filters' +import { FILTERS_TO_TAXA } from 'pages/species/species-filters' +import { buildCarryOverFilters } from 'utils/buildCarryOverFilters' + +describe('buildCarryOverFilters', () => { + it('carries only active filters whose field is in the destination set', () => { + const filters = [ + { field: 'deployment', value: '5' }, + { field: 'verified', value: 'false' }, + // active, but not part of the occurrence carry contract -> dropped + { field: 'include_unobserved', value: 'true' }, + // part of the set, but inactive -> dropped + { field: 'taxon', value: undefined }, + ] + + expect(buildCarryOverFilters(filters, FILTERS_TO_OCCURRENCES)).toEqual({ + deployment: '5', + verified: 'false', + }) + }) + + it('returns an empty object when no active filter is in the set', () => { + const filters = [{ field: 'page', value: '3' }] + + expect(buildCarryOverFilters(filters, FILTERS_TO_OCCURRENCES)).toEqual({}) + }) +}) + +describe('carry-over contracts', () => { + // Source-only state must never carry into a destination URL, regardless of destination. + const SOURCE_ONLY = ['page', 'ordering'] + + it.each([ + ['FILTERS_TO_OCCURRENCES', FILTERS_TO_OCCURRENCES], + ['FILTERS_TO_TAXA', FILTERS_TO_TAXA], + ])( + '%s carries no pagination or sort state and has no duplicates', + (_n, fields) => { + expect(fields.filter((f) => SOURCE_ONLY.includes(f))).toEqual([]) + expect(new Set(fields).size).toBe(fields.length) + } + ) + + it('keeps "show unobserved taxa" a taxa-only filter, never carried to occurrences', () => { + expect(FILTERS_TO_TAXA).toContain('include_unobserved') + expect(FILTERS_TO_OCCURRENCES).not.toContain('include_unobserved') + }) +}) diff --git a/ui/src/utils/getAppRoute.ts b/ui/src/utils/getAppRoute.ts index ca4012f94..bafc8290c 100644 --- a/ui/src/utils/getAppRoute.ts +++ b/ui/src/utils/getAppRoute.ts @@ -4,6 +4,8 @@ type FilterType = | 'collection' | 'collections' | 'deployment' + | 'deployment__device' + | 'deployment__research_site' | 'detections__source_image' | 'event' | 'include_unobserved' @@ -14,6 +16,10 @@ type FilterType = | 'taxon' | 'timestamp' | 'verified' + // Not a data filter: the id of the occurrence whose verification modal should + // open over the current list. Kept here so getAppRoute can set it as a search + // param alongside the active filters. + | 'verifyOccurrence' export const getAppRoute = ({ to, diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 7ad27a6da..feb83bed6 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -24,6 +24,7 @@ export enum STRING { LOGOUT, MANAGE_ACCESS, MORE, + MORE_FILTERS, NEXT, POPULATE, PREVIOUS, @@ -108,6 +109,7 @@ export enum STRING { FIELD_LABEL_EMAIL, FIELD_LABEL_ENDPOINT, FIELD_LABEL_ERRORS, + FIELD_LABEL_EXAMPLE, FIELD_LABEL_FILE_SIZE, FIELD_LABEL_FILENAME, FIELD_LABEL_FINISHED_AT, @@ -217,6 +219,7 @@ export enum STRING { MESSAGE_RESET_INSTRUCTIONS_SENT, MESSAGE_RESULT_RANGE, MESSAGE_SIGNED_UP, + MESSAGE_STATS_NO_COMPARABLE, MESSAGE_SYNC_ALL_CONFIRM, MESSAGE_SYNC_CONFIRM, MESSAGE_VALUE_INVALID, @@ -285,12 +288,21 @@ export enum STRING { TOOLTIP_SCORE, TOOLTIP_SESSION, TOOLTIP_SITE, + TOOLTIP_STATS, + TOOLTIP_STATS_AGREEMENT_ANY_RANK, + TOOLTIP_STATS_AGREEMENT_EXACT, + TOOLTIP_STATS_KAPPA, + TOOLTIP_STATS_VERIFIED, TOOLTIP_STORAGE, + TOOLTIP_VERIFY_EXAMPLE, TOOLTIP_VIEW_SOURCE_FILE, /* OTHER */ ABOUT_ROLE, ABOUT_ROLES, + AGREEMENT_ANY_RANK, + AGREEMENT_EXACT, + AGREEMENT_KAPPA, ALGORITHMS, ANONYMOUS_USER, APPLY_ID_SHORT, @@ -350,6 +362,7 @@ export enum STRING { SORT_OCCURRENCE_UPDATES, SORT_RECENT_CAPTURES, STAGES, + STATS, SUMMARY, TABLE_COLUMNS, TERMINAL_CLASSIFICATION, @@ -361,6 +374,7 @@ export enum STRING { USER_INFO, VALUE_NOT_AVAILABLE, VERIFIED_BY, + VERIFIED_OCCURRENCES, VERIFIED, VIEW_IN_SESSION, VIEW_SETTINGS, @@ -393,6 +407,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.LOGOUT]: 'Logout', [STRING.MANAGE_ACCESS]: 'Manage access', [STRING.MORE]: 'More', + [STRING.MORE_FILTERS]: 'More filters', [STRING.NEXT]: 'Next', [STRING.POPULATE]: 'Populate', [STRING.PREVIOUS]: 'Previous', @@ -456,6 +471,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.FIELD_LABEL_EMAIL]: 'Email', [STRING.FIELD_LABEL_ENDPOINT]: 'Endpoint URL', [STRING.FIELD_LABEL_ERRORS]: 'Errors', + [STRING.FIELD_LABEL_EXAMPLE]: 'Example', [STRING.FIELD_LABEL_FILE_SIZE]: 'File size', [STRING.FIELD_LABEL_FILENAME]: 'Filename', [STRING.FIELD_LABEL_FINISHED_AT]: 'Finished at', @@ -613,6 +629,8 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { 'Reset intructions has been sent to {{email}}!', [STRING.MESSAGE_RESULT_RANGE]: 'Showing {{start}}-{{end}} of {{total}} result(s)', + [STRING.MESSAGE_STATS_NO_COMPARABLE]: + 'None of the verified occurrences matching the current filters have both a model prediction and a confirmed taxon, so there is nothing to compare.', [STRING.MESSAGE_SIGNED_UP]: 'Signed up successfully!', [STRING.MESSAGE_SYNC_ALL_CONFIRM]: 'This starts a background sync job for each of the {{count}} station(s) with a storage source.', @@ -697,13 +715,27 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { 'A session is a fixed period of time of monitoring for one station. The period is typically one night.', [STRING.TOOLTIP_SITE]: 'A site is a physical location where monitoring is taking place. One or many stations can be connected to a site.', + [STRING.TOOLTIP_STATS]: + "How closely the model agrees with people, for the occurrences currently shown by the filters. Agreement is given as a 95% confidence range rather than a single number, so a wide range means there are still too few verifications to be sure. Two things push these numbers up: confirmations made by clicking Agree on the model's own suggestion match by definition, and people tend to verify the striking or unusual detections first, so the verified set is not a random sample of the project.", + [STRING.TOOLTIP_STATS_AGREEMENT_ANY_RANK]: + 'The model landed on the same branch of the tree of life as the confirmed taxon, for example the right genus or family even when the species differs, for {{count}} of {{comparable}} comparable occurrences ({{pct}}%).', + [STRING.TOOLTIP_STATS_AGREEMENT_EXACT]: + 'The model predicted exactly the taxon a person confirmed, for {{count}} of {{comparable}} comparable occurrences ({{pct}}%).', + [STRING.TOOLTIP_STATS_KAPPA]: + "Cohen's kappa adjusts exact agreement for luck. Where one species dominates a project, a person and the model agree often just by both picking the common one. 1 means perfect agreement, 0 means no better than chance, and below 0 means worse than chance.", + [STRING.TOOLTIP_STATS_VERIFIED]: + '{{verified}} of {{total}} occurrences matching the current filters have been verified by a person. {{comparable}} of those can be compared against a model prediction, which is what the agreement numbers are measured on.', [STRING.TOOLTIP_STORAGE]: 'A storage is a place where captures are kept, for example a S3 bucket. One or many stations can be connected to a storage.', + [STRING.TOOLTIP_VERIFY_EXAMPLE]: 'Verify one occurrence of this taxon.', [STRING.TOOLTIP_VIEW_SOURCE_FILE]: 'View source file', /* OTHER */ [STRING.ABOUT_ROLE]: 'About role', [STRING.ABOUT_ROLES]: 'About roles', + [STRING.AGREEMENT_ANY_RANK]: 'Agreement (any rank)', + [STRING.AGREEMENT_EXACT]: 'Agreement (exact taxon)', + [STRING.AGREEMENT_KAPPA]: "Cohen's kappa (beyond chance)", [STRING.ALGORITHMS]: 'Algorithms', [STRING.ANONYMOUS_USER]: 'Anonymous user', [STRING.APPLY_ID_SHORT]: 'Apply', @@ -762,6 +794,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.SORT_OCCURRENCE_UPDATES]: 'Occurrence updates', [STRING.SORT_RECENT_CAPTURES]: 'Recent captures', [STRING.STAGES]: 'Stages', + [STRING.STATS]: 'Stats', [STRING.SUMMARY]: 'Summary', [STRING.TABLE_COLUMNS]: 'Table columns', [STRING.TERMINAL_CLASSIFICATION]: 'Terminal classification', @@ -773,6 +806,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.USER_INFO]: 'User info', [STRING.VALUE_NOT_AVAILABLE]: 'n/a', [STRING.VERIFIED_BY]: 'Verified by\n{{name}}', + [STRING.VERIFIED_OCCURRENCES]: 'Verified occurrences', [STRING.VERIFIED]: 'Verified', [STRING.VIEW_IN_SESSION]: 'View in session', [STRING.VIEW_SETTINGS]: 'View settings', diff --git a/ui/src/utils/useColumnSettings.tsx b/ui/src/utils/useColumnSettings.tsx index e39dd4cee..2cd2c5463 100644 --- a/ui/src/utils/useColumnSettings.tsx +++ b/ui/src/utils/useColumnSettings.tsx @@ -7,7 +7,13 @@ export const useColumnSettings = ( const { userPreferences, setUserPreferences } = useUserPreferences() return { - columnSettings: userPreferences.columnSettings[tableKey] ?? defaultSettings, + // Merge persisted choices over the defaults so a column added after a user last + // customized this table (whose key is absent from their saved settings) still + // picks up its default visibility instead of being silently dropped. + columnSettings: { + ...defaultSettings, + ...userPreferences.columnSettings?.[tableKey], + }, setColumnSettings: (settings: { [columnKey: string]: boolean }) => { setUserPreferences({ ...userPreferences, diff --git a/ui/src/utils/useFilters.ts b/ui/src/utils/useFilters.ts index b028a2592..8e9dd3336 100644 --- a/ui/src/utils/useFilters.ts +++ b/ui/src/utils/useFilters.ts @@ -1,5 +1,7 @@ import { isBefore, isValid } from 'date-fns' +import { useMemo } from 'react' import { useParams, useSearchParams } from 'react-router-dom' +import { buildCarryOverFilters } from './buildCarryOverFilters' import { APP_ROUTES } from './constants' import { STRING, translate } from './language' import { SEARCH_PARAM_KEY_PAGE } from './usePagination' @@ -76,6 +78,14 @@ export const AVAILABLE_FILTERS = (projectId: string): FilterConfig[] => [ }, }, }, + { + label: 'Device', + field: 'deployment__device', + }, + { + label: 'Site', + field: 'deployment__research_site', + }, { label: 'End date', field: 'date_end', @@ -275,3 +285,16 @@ export const useFilters = (defaultFilters?: { [field: string]: string }) => { filters, } } + +// Hook form of buildCarryOverFilters: pass the destination's carry contract (e.g. +// FILTERS_TO_OCCURRENCES). Reads the active filters of the current view, so any link into +// that destination — from any source view — carries a consistent set. +export const useCarryOverFilters = ( + fields: string[] +): Record => { + const { filters } = useFilters() + return useMemo( + () => buildCarryOverFilters(filters, fields), + [filters, fields] + ) +} From c8999fab089b9b1095e94d098d9d6f11de35c712 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 3 Sep 2026 17:49:14 -0400 Subject: [PATCH 02/31] feat(ml): return embeddings from the BioCLIP service and let it retrain its own head The service already computed a 1024-dimension embedding for every crop before naming a species, then discarded it. It now returns it alongside the classification, which is what makes retraining cheap on the Antenna side. Adds a training endpoint that downloads the dataset Antenna prepared, fits a new head, and scores it against the head currently in service on the same held-out rows. It never swaps the running head: an automatic swap would let one bad training run quietly degrade every later classification. Each algorithm now declares whether it can be retrained and what settings to use, so Antenna only offers retraining where it is cheap. Only the classifier heads are marked trainable; the detector and the zero-shot classifiers are not. The minimal test service returns stand-in embeddings so the path can be exercised without a GPU. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- processing_services/bioclip/Dockerfile | 7 + processing_services/bioclip/api/__init__.py | 0 processing_services/bioclip/api/algorithms.py | 748 ++++++++++++++++++ processing_services/bioclip/api/api.py | 405 ++++++++++ processing_services/bioclip/api/pipelines.py | 438 ++++++++++ processing_services/bioclip/api/schemas.py | 385 +++++++++ processing_services/bioclip/api/test.py | 64 ++ processing_services/bioclip/api/training.py | 300 +++++++ processing_services/bioclip/api/utils.py | 149 ++++ .../bioclip/docker-compose.yml | 25 + processing_services/bioclip/main.py | 4 + processing_services/bioclip/requirements.txt | 12 + .../bioclip/scripts/export_logreg_head.py | 144 ++++ .../bioclip/scripts/run_on_remote_gpu.sh | 49 ++ .../bioclip25_logreg_categories.json | 62 ++ .../weights_test/bioclip25_logreg_head.pt | Bin 0 -> 43093 bytes processing_services/minimal/api/pipelines.py | 20 + processing_services/minimal/api/schemas.py | 8 + 18 files changed, 2820 insertions(+) create mode 100644 processing_services/bioclip/Dockerfile create mode 100644 processing_services/bioclip/api/__init__.py create mode 100644 processing_services/bioclip/api/algorithms.py create mode 100644 processing_services/bioclip/api/api.py create mode 100644 processing_services/bioclip/api/pipelines.py create mode 100644 processing_services/bioclip/api/schemas.py create mode 100644 processing_services/bioclip/api/test.py create mode 100644 processing_services/bioclip/api/training.py create mode 100644 processing_services/bioclip/api/utils.py create mode 100644 processing_services/bioclip/docker-compose.yml create mode 100644 processing_services/bioclip/main.py create mode 100644 processing_services/bioclip/requirements.txt create mode 100644 processing_services/bioclip/scripts/export_logreg_head.py create mode 100755 processing_services/bioclip/scripts/run_on_remote_gpu.sh create mode 100644 processing_services/bioclip/weights_test/bioclip25_logreg_categories.json create mode 100644 processing_services/bioclip/weights_test/bioclip25_logreg_head.pt diff --git a/processing_services/bioclip/Dockerfile b/processing_services/bioclip/Dockerfile new file mode 100644 index 000000000..3e0781f92 --- /dev/null +++ b/processing_services/bioclip/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.11-slim + +# Set up ml backend FastAPI +WORKDIR /app +COPY . /app +RUN pip install -r ./requirements.txt +CMD ["python", "/app/main.py"] diff --git a/processing_services/bioclip/api/__init__.py b/processing_services/bioclip/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/processing_services/bioclip/api/algorithms.py b/processing_services/bioclip/api/algorithms.py new file mode 100644 index 000000000..76846743b --- /dev/null +++ b/processing_services/bioclip/api/algorithms.py @@ -0,0 +1,748 @@ +import datetime +import json +import logging +import math +import os +import random + +import torch + +from .schemas import ( + AlgorithmCategoryMapResponse, + AlgorithmConfigResponse, + AlgorithmTrainingConfig, + AlgorithmReference, + BoundingBox, + ClassificationResponse, + Detection, + SourceImage, +) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +SAVED_MODELS = {} + + +def get_best_device() -> str: + """ + Returns the best available device for running the model. + + MPS is not supported by the current algoritms. + """ + if torch.cuda.is_available(): + return f"cuda:{torch.cuda.current_device()}" + else: + return "cpu" + + +class Algorithm: + algorithm_config_response: AlgorithmConfigResponse + + def compile(self): + raise NotImplementedError("Subclasses must implement the compile method") + + def run(self, inputs: list[SourceImage] | list[Detection]) -> list[Detection]: + raise NotImplementedError("Subclasses must implement the run method") + + def _build_categories(self, classes, label_map) -> list[dict]: + """ + Map each head row to an Antenna category. + + The label map is keyed by the class value, which is the iNat taxon id for the + Newfoundland head. + """ + return [ + { + "index": index, + "label": label_map[str(source_class)]["species_name"], + "taxon_rank": "SPECIES", + "source_class": str(source_class), + "inat_taxon_id": label_map[str(source_class)]["inat_taxon_id"], + } + for index, source_class in enumerate(classes) + ] + + def get_category_map(self) -> AlgorithmCategoryMapResponse: + return AlgorithmCategoryMapResponse( + data=[], + labels=[], + version="v1", + description="A model without labels.", + uri=None, + ) + + def get_algorithm_config_response(self) -> AlgorithmConfigResponse: + return AlgorithmConfigResponse( + name="Base Algorithm", + key="base", + task_type="base", + description="A base class for all algorithms.", + version=1, + version_name="v1", + category_map=self.get_category_map(), + ) + + def __init__(self): + self.algorithm_config_response = self.get_algorithm_config_response() + + +class ZeroShotObjectDetector(Algorithm): + """ + Huggingface Zero-Shot Object Detection model. + Produces both a bounding box and a classification for each detection. + The classification is based on the candidate labels. + """ + + candidate_labels: list[str] = ["insect"] + + def compile(self, device: str | None = None): + saved_models_key = "zero_shot_object_detector" # generate a key for each uniquely compiled algorithm + + if saved_models_key not in SAVED_MODELS: + from transformers import pipeline + + device_choice = device or get_best_device() + device_index = int(device_choice.split(":")[-1]) if ":" in device_choice else -1 + logger.info(f"Compiling {self.algorithm_config_response.name} on device {device_choice}...") + checkpoint = "google/owlv2-base-patch16-ensemble" + self.model = pipeline( + model=checkpoint, + task="zero-shot-object-detection", + use_fast=True, + device=device_index, + ) + SAVED_MODELS[saved_models_key] = self.model + else: + logger.info(f"Using saved model for {self.algorithm_config_response.name}...") + self.model = SAVED_MODELS[saved_models_key] + + def run(self, source_images: list[SourceImage], intermediate=False) -> list[Detection]: + detector_responses: list[Detection] = [] + for source_image in source_images: + if source_image.width and source_image.height and source_image._pil: + start_time = datetime.datetime.now() + logger.info("Predicting...") + if not self.candidate_labels: + raise ValueError("No candidate labels are provided during inference.") + logger.info(f"Predicting with candidate labels: {self.candidate_labels}") + predictions = self.model(source_image._pil, candidate_labels=self.candidate_labels) + end_time = datetime.datetime.now() + elapsed_time = (end_time - start_time).total_seconds() + + for prediction in predictions: + logger.info("Prediction: %s", prediction) + bbox = BoundingBox( + x1=prediction["box"]["xmin"], + x2=prediction["box"]["xmax"], + y1=prediction["box"]["ymin"], + y2=prediction["box"]["ymax"], + ) + cropped_image_pil = source_image._pil.crop((bbox.x1, bbox.y1, bbox.x2, bbox.y2)) + detection = Detection( + id=f"{source_image.id}-crop-{bbox.x1}-{bbox.y1}-{bbox.x2}-{bbox.y2}", + url=source_image.url, # @TODO: ideally, should save cropped image at separate url + width=cropped_image_pil.width, + height=cropped_image_pil.height, + timestamp=datetime.datetime.now(), + source_image=source_image, + bbox=bbox, + inference_time=elapsed_time, + algorithm=AlgorithmReference( + name=self.algorithm_config_response.name, + key=self.algorithm_config_response.key, + ), + classifications=[ + ClassificationResponse( + classification=prediction["label"], + labels=[prediction["label"]], + scores=[prediction["score"]], + logits=[prediction["score"]], + inference_time=elapsed_time, + timestamp=datetime.datetime.now(), + algorithm=AlgorithmReference( + name=self.algorithm_config_response.name, + key=self.algorithm_config_response.key, + ), + terminal=not intermediate, + ) + ], + ) + detection._pil = cropped_image_pil + detector_responses.append(detection) + else: + raise ValueError(f"Source image {source_image.id} does not have width and height attributes.") + + return detector_responses + + def get_category_map(self) -> AlgorithmCategoryMapResponse: + return AlgorithmCategoryMapResponse( + data=[{"index": i, "label": label} for i, label in enumerate(self.candidate_labels)], + labels=self.candidate_labels, + version="v1", + description="Candidate labels used for zero-shot object detection.", + uri=None, + ) + + def get_algorithm_config_response(self) -> AlgorithmConfigResponse: + return AlgorithmConfigResponse( + name="Zero Shot Object Detector", + key="zero-shot-object-detector", + task_type="detection", + description=( + "Huggingface Zero Shot Object Detection model." + "Produces both a bounding box and a candidate label classification for each detection." + ), + version=1, + version_name="v1", + category_map=self.get_category_map(), + ) + + +class HFImageClassifier(Algorithm): + """ + A local classifier that uses the Hugging Face pipeline to classify images. + """ + + model_name: str = "google/vit-base-patch16-224" # Vision Transformer model trained on ImageNet-1k + + def compile(self): + saved_models_key = "hf_image_classifier" # generate a key for each uniquely compiled algorithm + + if saved_models_key not in SAVED_MODELS: + from transformers import pipeline + + logger.info(f"Compiling {self.algorithm_config_response.name} from scratch...") + self.model = pipeline("image-classification", model=self.model_name, device=get_best_device()) + SAVED_MODELS[saved_models_key] = self.model + else: + logger.info(f"Using saved model for {self.algorithm_config_response.name}...") + self.model = SAVED_MODELS[saved_models_key] + + def run(self, detections: list[Detection]) -> list[Detection]: + detections_to_return: list[Detection] = [] + start_time = datetime.datetime.now() + + opened_cropped_images = [detection._pil for detection in detections] # type: ignore + + # Process the entire batch of cropped images at once + results = self.model(images=opened_cropped_images) + + end_time = datetime.datetime.now() + elapsed_time = (end_time - start_time).total_seconds() + + for detection, preds in zip(detections, results): + labels = [pred["label"] for pred in preds] + scores = [pred["score"] for pred in preds] + max_score_index = scores.index(max(scores)) + classification = labels[max_score_index] + logger.info(f"Classification: {classification}") + logger.info(f"labels: {labels}") + logger.info(f"scores: {scores}") + + existing_classifications = detection.classifications + + detection_with_classification = detection.copy(deep=True) + detection_with_classification.classifications = existing_classifications + [ + ClassificationResponse( + classification=classification, + labels=labels, + scores=scores, + logits=scores, + inference_time=elapsed_time, + timestamp=datetime.datetime.now(), + algorithm=AlgorithmReference( + name=self.algorithm_config_response.name, key=self.algorithm_config_response.key + ), + terminal=True, + ) + ] + + detections_to_return.append(detection_with_classification) + + return detections_to_return + + def get_category_map(self) -> AlgorithmCategoryMapResponse: + """ + Extract the category map from the model. + Returns an AlgorithmCategoryMapResponse with labels, data, and model information. + """ + from transformers.models.auto.configuration_auto import AutoConfig + + logger.info(f"Loading configuration for {self.model_name}") + config = AutoConfig.from_pretrained(self.model_name) + + # Extract label information + if not hasattr(config, "id2label") or not config.id2label: + raise ValueError( + f"Cannot create category map for model {self.model_name}, no id2label mapping found in config" + ) + else: + # Sort labels by index + # Ensure keys are strings for consistent access + id2label: dict[str, str] = {str(k): v for k, v in config.id2label.items()} + indices = sorted([int(k) for k in id2label.keys()]) + + # Create labels and data + labels = [id2label[str(i)] for i in indices] + data = [{"label": label, "index": idx} for idx, label in zip(indices, labels)] + + # Build description + description_text = ( + f"Vision Transformer model trained on ImageNet-1k. " + f"Contains {len(labels)} object classes. Model: {self.model_name}" + ) + + return AlgorithmCategoryMapResponse( + data=data, + labels=labels, + version="ImageNet-1k", + description=description_text, + uri=f"https://huggingface.co/{self.model_name}", + ) + + def get_algorithm_config_response(self) -> AlgorithmConfigResponse: + return AlgorithmConfigResponse( + name="HF Image Classifier", + key="hf-image-classifier", + task_type="classification", + description="HF ViT for image classification.", + version=1, + version_name="v1", + category_map=self.get_category_map(), + ) + + +class RandomSpeciesClassifier(Algorithm): + """ + A local classifier that produces random butterfly species classifications. + """ + + def compile(self): + pass + + def _make_random_prediction( + self, + terminal: bool = True, + max_labels: int = 2, + ) -> ClassificationResponse: + assert self.algorithm_config_response.category_map is not None + category_labels = self.algorithm_config_response.category_map.labels + logits = [random.random() for _ in category_labels] + softmax = [math.exp(logit) / sum([math.exp(logit) for logit in logits]) for logit in logits] + top_class = category_labels[softmax.index(max(softmax))] + return ClassificationResponse( + classification=top_class, + labels=category_labels if len(category_labels) <= max_labels else None, + scores=softmax, + logits=logits, + timestamp=datetime.datetime.now(), + algorithm=AlgorithmReference( + name=self.algorithm_config_response.name, + key=self.algorithm_config_response.key, + ), + terminal=terminal, + ) + + def run(self, detections: list[Detection]) -> list[Detection]: + detections_to_return: list[Detection] = [] + for detection in detections: + detection_with_classification = detection.copy(deep=True) + detection_with_classification.classifications = [self._make_random_prediction(terminal=True)] + detections_to_return.append(detection_with_classification) + return detections_to_return + + algorithm_config_response = AlgorithmConfigResponse( + name="Random species classifier", + key="random-species-classifier", + task_type="classification", + description="A random species classifier", + version=1, + version_name="v1", + uri="https://huggingface.co/RolnickLab/random-species-classifier", + category_map=AlgorithmCategoryMapResponse( + data=[ + { + "index": 0, + "gbif_key": "1234", + "label": "Vanessa atalanta", + "source": "manual", + "taxon_rank": "SPECIES", + }, + { + "index": 1, + "gbif_key": "4543", + "label": "Vanessa cardui", + "source": "manual", + "taxon_rank": "SPECIES", + }, + { + "index": 2, + "gbif_key": "7890", + "label": "Vanessa itea", + "source": "manual", + "taxon_rank": "SPECIES", + }, + ], + labels=["Vanessa atalanta", "Vanessa cardui", "Vanessa itea"], + version="v1", + description="A simple species classifier", + uri="https://huggingface.co/RolnickLab/random-species-classifier", + ), + ) + + +class ConstantClassifier(Algorithm): + """ + A local classifier that always returns a constant species classification. + """ + + def compile(self): + pass + + def _make_constant_prediction( + self, + terminal: bool = True, + ) -> ClassificationResponse: + assert self.algorithm_config_response.category_map is not None + labels = self.algorithm_config_response.category_map.labels + return ClassificationResponse( + classification=labels[0], + labels=labels, + scores=[0.9], # Constant score for each detection + timestamp=datetime.datetime.now(), + algorithm=AlgorithmReference( + name=self.algorithm_config_response.name, + key=self.algorithm_config_response.key, + ), + terminal=terminal, + ) + + def run(self, detections: list[Detection]) -> list[Detection]: + detections_to_return: list[Detection] = [] + for detection in detections: + detection_with_classification = detection.copy(deep=True) + detection_with_classification.classifications = [self._make_constant_prediction(terminal=True)] + detections_to_return.append(detection_with_classification) + return detections_to_return + + algorithm_config_response = AlgorithmConfigResponse( + name="Constant classifier", + key="constant-classifier", + task_type="classification", + description="Always return a classification of 'Moth'", + version=1, + version_name="v1", + uri="https://huggingface.co/RolnickLab/constant-classifier", + category_map=AlgorithmCategoryMapResponse( + data=[ + { + "index": 0, + "gbif_key": "1234", + "label": "Moth", + "source": "manual", + "taxon_rank": "SUPERFAMILY", + } + ], + labels=["Moth"], + version="v1", + description="A classifier that always returns 'Moth'", + uri="https://huggingface.co/RolnickLab/constant-classifier", + ), + ) + + +class BioCLIPWithLinearHead(torch.nn.Module): + """ + A frozen BioCLIP image encoder with a linear classification head on top. + + Keeps the encoder, the head and the image transform in a single module so that the + whole classifier can be cached in SAVED_MODELS as one object. + """ + + def __init__(self, encoder, head, preprocess): + super().__init__() + self.encoder = encoder + self.head = head + self.preprocess = preprocess + + def forward(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Returns (logits, features). The features are what the head consumes, so they + are the right thing to store for retraining a head later.""" + features = self.encoder.encode_image(images).float() + # The head was fit on L2-normalised embeddings, so normalise here too. + features = features / features.norm(dim=-1, keepdim=True) + return self.head(features), features + + +class BioCLIP25LogRegClassifier(Algorithm): + """ + A local classifier that uses a frozen BioCLIP 2.5 image encoder with a linear + logistic-regression head. + + The head is a single Linear layer whose weights come from an sklearn + LogisticRegression fit on L2-normalised BioCLIP embeddings, so softmax over the + linear output reproduces sklearn's multinomial predict_proba exactly. See + scripts/export_logreg_head.py for the conversion. + """ + + model_name: str = "hf-hub:imageomics/bioclip-2.5-vith14" # BioCLIP 2.5 ViT-H/14, loaded by open_clip + # The head and its label map are read from the Hugging Face repo that already serves + # them to the Newfoundland trap classifier demo, so there is a single source of truth + # for the weights rather than a second copy that can drift. + head_repo_id: str = "mohammedelabbas/newfoundland-leps-trap-classifier" + head_repo_type: str = "space" + head_filename: str = "logreg_head_antenna.npz" + categories_filename: str = "label_map.json" + # Read the head from this directory instead of the hub, for offline or local testing. + # The directory must contain the same two filenames. + head_local_dir: str | None = os.environ.get("BIOCLIP_HEAD_DIR") or None + # One cache slot per head. Two heads sharing a slot would silently serve the wrong + # weights, because the encoder is identical and only the Linear layer differs. + saved_models_key: str = "bioclip_25_logreg_nf749" + # Antenna's schema allows omitting the label list per classification when the class + # list is large. The full list always travels in the category map instead. + max_labels_in_response: int = 100 + + @property + def head_uri(self) -> str: + prefix = "spaces/" if self.head_repo_type == "space" else "" + return f"https://huggingface.co/{prefix}{self.head_repo_id}" + + def _head_file(self, filename: str) -> str: + if self.head_local_dir: + logger.info(f"Loading {filename} from local directory {self.head_local_dir}") + return f"{self.head_local_dir}/{filename}" + + from huggingface_hub import hf_hub_download + + logger.info(f"Loading {filename} from {self.head_repo_id} ({self.head_repo_type})") + return hf_hub_download(repo_id=self.head_repo_id, filename=filename, repo_type=self.head_repo_type) + + def _load_head_arrays(self): + """ + Load the exported sklearn LogisticRegression as (weight, bias, classes). + + `classes` holds the label-map key for each row of the weight matrix, so the row + order of the head and the order of the category map stay tied together. + """ + import numpy as np + + checkpoint = np.load(self._head_file(self.head_filename)) + return checkpoint["W"], checkpoint["b"], checkpoint["classes"] + + def _load_head(self, embed_dim: int) -> torch.nn.Module: + weight, bias, _classes = self._load_head_arrays() + + if weight.shape[1] != embed_dim: + raise ValueError( + f"Head was fit on {weight.shape[1]}-dim embeddings but {self.model_name} " + f"produces {embed_dim}-dim embeddings." + ) + + head = torch.nn.Linear(embed_dim, weight.shape[0]) + head.weight.data = torch.from_numpy(weight).float() + head.bias.data = torch.from_numpy(bias).float() + return head + + def compile(self): + saved_models_key = self.saved_models_key + + self.device = get_best_device() + if saved_models_key not in SAVED_MODELS: + import open_clip + + logger.info(f"Compiling {self.algorithm_config_response.name} from scratch...") + encoder, _, preprocess = open_clip.create_model_and_transforms(self.model_name) + head = self._load_head(encoder.visual.output_dim) + self.model = BioCLIPWithLinearHead(encoder, head, preprocess).eval().to(self.device) + SAVED_MODELS[saved_models_key] = self.model + else: + logger.info(f"Using saved model for {self.algorithm_config_response.name}...") + self.model = SAVED_MODELS[saved_models_key] + + def run(self, detections: list[Detection]) -> list[Detection]: + detections_to_return: list[Detection] = [] + start_time = datetime.datetime.now() + + opened_cropped_images = [detection._pil for detection in detections] # type: ignore + + # Process the entire batch of cropped images at once + crops = torch.stack([self.model.preprocess(image.convert("RGB")) for image in opened_cropped_images]) + with torch.inference_mode(): + batch_logits, batch_features = self.model(crops.to(self.device)) + batch_scores = torch.nn.functional.softmax(batch_logits, dim=-1) + + end_time = datetime.datetime.now() + elapsed_time = (end_time - start_time).total_seconds() + + assert self.algorithm_config_response.category_map is not None + category_labels = self.algorithm_config_response.category_map.labels + + for detection, logits, scores, features in zip( + detections, batch_logits.cpu(), batch_scores.cpu(), batch_features.cpu() + ): + classification = category_labels[int(scores.argmax())] + logger.info(f"Classification: {classification}") + + existing_classifications = detection.classifications + + detection_with_classification = detection.copy(deep=True) + detection_with_classification.classifications = existing_classifications + [ + ClassificationResponse( + classification=classification, + labels=category_labels if len(category_labels) <= self.max_labels_in_response else None, + scores=scores.tolist(), + logits=logits.tolist(), + features=features.tolist(), + inference_time=elapsed_time, + timestamp=datetime.datetime.now(), + algorithm=AlgorithmReference( + name=self.algorithm_config_response.name, key=self.algorithm_config_response.key + ), + terminal=True, + ) + ] + + detections_to_return.append(detection_with_classification) + + return detections_to_return + + def get_category_map(self) -> AlgorithmCategoryMapResponse: + """ + Load the category map exported alongside the head. + Returns an AlgorithmCategoryMapResponse with labels, data, and model information. + """ + categories_path = self._head_file(self.categories_filename) + with open(categories_path) as f: + label_map = json.load(f) + + if not label_map: + raise ValueError( + f"Cannot create category map for model {self.model_name}, " + f"no categories found in {self.categories_filename}" + ) + + # Row i of the head predicts classes[i], so walk the classes in row order to keep + # the head and the labels aligned. + _weight, _bias, classes = self._load_head_arrays() + categories = self._build_categories(classes, label_map) + labels = [category["label"] for category in categories] + + # Build description + description_text = ( + f"Logistic-regression head over frozen {self.model_name} embeddings. " + f"Contains {len(labels)} classes." + ) + + return AlgorithmCategoryMapResponse( + data=categories, + labels=labels, + version="v1", + description=description_text, + uri=self.head_uri, + ) + + def training_config(self) -> AlgorithmTrainingConfig: + """ + Defaults for retraining this head. Antenna seeds its own copy once, so changing + these afterwards only affects algorithms registered from here on. + """ + return AlgorithmTrainingConfig( + # A linear head matches what is deployed, so a retrained one is a drop-in. + # An MLP-1 head scores far better on rare species and is the likely next default. + head_type="linear", + epochs=300, + learning_rate=0.01, + min_per_species=2, + ) + + def get_algorithm_config_response(self) -> AlgorithmConfigResponse: + return AlgorithmConfigResponse( + name="BioCLIP 2.5 + LogReg head (Newfoundland, 749 species)", + key="bioclip-2-5-vith14-logreg-nf749", + task_type="classification", + description="Frozen BioCLIP 2.5 (ViT-H/14) encoder with a linear logistic-regression head.", + # A different head means a different key, not a new version. Antenna looks + # algorithms up with get_or_create(key=..., version=...) but has a unique + # constraint on key alone, so bumping the version of an existing key raises + # an IntegrityError instead of registering the new category map. + version=1, + version_name="v1", + uri=self.head_uri, + # The encoder is frozen, so only this linear head is retrained. The zero-shot + # detector in stage 0 is not trainable and stays False. + trainable=True, + training_config=self.training_config(), + category_map=self.get_category_map(), + ) + + +class BioCLIPPanamaLogRegClassifier(BioCLIP25LogRegClassifier): + """ + The Panama (BCI + Mount Totumas) head over the same frozen BioCLIP 2.5 encoder. + + Same architecture as the Newfoundland head, but the export carries its label + vocabulary inside the npz instead of a separate label map, and the vocabulary is + larger than the number of trained rows: 1,095 names, 900 output classes. The 195 + species with no training data can never be predicted, so they are not categories. + """ + + head_repo_id: str | None = None # not published; read from a local directory + head_filename: str = "head_combined.npz" + categories_filename: str | None = None # labels travel inside the npz + head_local_dir: str | None = os.environ.get("BIOCLIP_PANAMA_HEAD_DIR") or None + saved_models_key: str = "bioclip_25_logreg_panama900" + + @property + def head_uri(self) -> str | None: + return None + + def _load_label_vocabulary(self): + import numpy as np + + return np.load(self._head_file(self.head_filename), allow_pickle=True)["labels"] + + def _build_categories(self, classes, label_map) -> list[dict]: + # `label_map` is the vocabulary array here, indexed by class value rather than + # keyed by it. + return [ + { + "index": index, + "label": str(label_map[int(source_class)]), + "taxon_rank": "SPECIES", + "source_class": str(int(source_class)), + } + for index, source_class in enumerate(classes) + ] + + def get_category_map(self) -> AlgorithmCategoryMapResponse: + _weight, _bias, classes = self._load_head_arrays() + vocabulary = self._load_label_vocabulary() + categories = self._build_categories(classes, vocabulary) + labels = [category["label"] for category in categories] + + return AlgorithmCategoryMapResponse( + data=categories, + labels=labels, + version="v1", + description=( + f"Logistic-regression head over frozen {self.model_name} embeddings. " + f"Contains {len(labels)} classes from a {len(vocabulary)}-species vocabulary." + ), + uri=self.head_uri, + ) + + def get_algorithm_config_response(self) -> AlgorithmConfigResponse: + return AlgorithmConfigResponse( + name="BioCLIP 2.5 + LogReg head (Panama, 900 species)", + key="bioclip-2-5-vith14-logreg-panama900", + task_type="classification", + description="Frozen BioCLIP 2.5 (ViT-H/14) encoder with a linear logistic-regression head.", + version=1, + version_name="v1", + uri=self.head_uri, + trainable=True, + training_config=self.training_config(), + category_map=self.get_category_map(), + ) diff --git a/processing_services/bioclip/api/api.py b/processing_services/bioclip/api/api.py new file mode 100644 index 000000000..9973c98da --- /dev/null +++ b/processing_services/bioclip/api/api.py @@ -0,0 +1,405 @@ +""" +Fast API interface for processing images through the localization and classification pipelines. +""" + +import datetime +import inspect +import logging +import os +import pathlib + +import fastapi +import pydantic +import requests + +from .pipelines import ( + BioCLIP25LogRegPipeline, + BioCLIPPanamaPipeline, + Pipeline, + ZeroShotHFClassifierPipeline, + ZeroShotObjectDetectorPipeline, + ZeroShotObjectDetectorWithConstantClassifierPipeline, + ZeroShotObjectDetectorWithRandomSpeciesClassifierPipeline, +) +from .schemas import ( + AlgorithmConfigResponse, + Detection, + DetectionRequest, + PipelineRequest, + PipelineRequestConfigParameters, + PipelineResultsResponse, + ProcessingServiceInfoResponse, + SourceImage, +) +from . import algorithms, training +from .utils import is_base64, is_url + +# Configure root logger +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S" +) + +# Get the root logger +logger = logging.getLogger(__name__) + +# Where retrained heads are written. Kept off the model cache so a training run +# cannot overwrite the head the service is currently serving. +TRAINED_HEADS_DIR = os.environ.get("BIOCLIP_TRAINED_HEADS_DIR", "/data/bioclip-service/trained_heads") + +app = fastapi.FastAPI() + + +pipelines: list[type[Pipeline]] = [ + BioCLIP25LogRegPipeline, + BioCLIPPanamaPipeline, + ZeroShotHFClassifierPipeline, + ZeroShotObjectDetectorPipeline, + ZeroShotObjectDetectorWithConstantClassifierPipeline, + ZeroShotObjectDetectorWithRandomSpeciesClassifierPipeline, +] +pipeline_choices: dict[str, type[Pipeline]] = {pipeline.config.slug: pipeline for pipeline in pipelines} +algorithm_choices: dict[str, AlgorithmConfigResponse] = { + algorithm.key: algorithm for pipeline in pipelines for algorithm in pipeline.config.algorithms +} + +# ----------- +# API endpoints +# ----------- + + +@app.get("/") +async def root(): + return fastapi.responses.RedirectResponse("/docs") + + +@app.get("/info", tags=["services"]) +async def info() -> ProcessingServiceInfoResponse: + info = ProcessingServiceInfoResponse( + name="BioCLIP ML Backend", + description=("BioCLIP 2.5 with a logistic-regression classification head."), + pipelines=[pipeline.config for pipeline in pipelines], + # algorithms=list(algorithm_choices.values()), + ) + return info + + +# Check if the server is online +@app.get("/livez", tags=["health checks"]) +async def livez(): + return fastapi.responses.JSONResponse(status_code=200, content={"status": True}) + + +# Check if the pipelines are ready to process data +@app.get("/readyz", tags=["health checks"]) +async def readyz(): + """ + Check if the server is ready to process data. + + Returns a list of pipeline slugs that are online and ready to process data. + @TODO may need to simplify this to just return True/False. Pipeline algorithms will likely be loaded into memory + on-demand when the pipeline is selected. + """ + if pipeline_choices: + return fastapi.responses.JSONResponse(status_code=200, content={"status": list(pipeline_choices.keys())}) + else: + return fastapi.responses.JSONResponse(status_code=503, content={"status": []}) + + +@app.post("/process", tags=["services"]) +async def process(data: PipelineRequest) -> PipelineResultsResponse: + pipeline_slug = data.pipeline + request_config = data.config + + source_images = [SourceImage(**img.model_dump()) for img in data.source_images] + # Open source images once before processing + for img in source_images: + img.open(raise_exception=True) + + detections = create_detections( + source_images=source_images, + detection_requests=data.detections, + ) + + try: + Pipeline = pipeline_choices[pipeline_slug] + except KeyError: + raise fastapi.HTTPException(status_code=422, detail=f"Invalid pipeline choice: {pipeline_slug}") + + pipeline_request_config = PipelineRequestConfigParameters(**dict(request_config)) if request_config else {} + try: + pipeline = Pipeline( + source_images=source_images, + request_config=pipeline_request_config, + existing_detections=detections, + ) + pipeline.compile() + except Exception as e: + logger.error(f"Error compiling pipeline: {e}") + raise fastapi.HTTPException(status_code=422, detail=f"{e}") + + try: + response = pipeline.run() + except Exception as e: + logger.error(f"Error running pipeline: {e}") + raise fastapi.HTTPException(status_code=422, detail=f"{e}") + + return response + + +# ----------- +# Helper functions +# ----------- + + +def create_detections( + source_images: list[SourceImage], + detection_requests: list[DetectionRequest] | None, +): + if not detection_requests: + return [] + + # Group detection requests by source image id + source_image_map = {img.id: img for img in source_images} + grouped_detection_requests = {} + for request in detection_requests: + if request.source_image.id not in grouped_detection_requests: + grouped_detection_requests[request.source_image.id] = [] + grouped_detection_requests[request.source_image.id].append(request) + + # Process each source image and its detection requests + detections = [] + for source_image_id, requests in grouped_detection_requests.items(): + if source_image_id not in source_image_map: + raise ValueError( + f"A detection request for source image {source_image_id} was received, " + "but no source image with that ID was provided." + ) + + logger.info(f"Processing existing detections for source image {source_image_id}.") + + for request in requests: + source_image = source_image_map[source_image_id] + cropped_image_id = ( + f"{source_image.id}-crop-{request.bbox.x1}-{request.bbox.y1}-{request.bbox.x2}-{request.bbox.y2}" + ) + if not request.crop_image_url: + logger.info("Detection request does not have a crop_image_url, crop the original source image.") + assert source_image._pil is not None, "Source image must be opened before cropping." + cropped_image_pil = source_image._pil.crop( + (request.bbox.x1, request.bbox.y1, request.bbox.x2, request.bbox.y2) + ) + else: + try: + logger.info(f"Opening existing cropped image from {request.crop_image_url}.") + if is_url(request.crop_image_url): + cropped_image = SourceImage( + id=cropped_image_id, + url=request.crop_image_url, + ) + elif is_base64(request.crop_image_url): + logger.info("Decoding base64 cropped image.") + cropped_image = SourceImage( + id=cropped_image_id, + b64=request.crop_image_url, + ) + else: + # Must be a filepath + cropped_image = SourceImage( + id=cropped_image_id, + filepath=request.crop_image_url, + ) + cropped_image.open(raise_exception=True) + cropped_image_pil = cropped_image._pil + except Exception as e: + logger.warning(f"Error opening cropped image: {e}") + logger.info(f"Falling back to cropping the original source image {source_image_id}.") + assert source_image._pil is not None, "Source image must be opened before cropping." + cropped_image_pil = source_image._pil.crop( + (request.bbox.x1, request.bbox.y1, request.bbox.x2, request.bbox.y2) + ) + + # Create a Detection object + det = Detection( + source_image=SourceImage( + id=source_image.id, + url=source_image.url, + ), + bbox=request.bbox, + id=cropped_image_id, + url=request.crop_image_url or source_image.url, + algorithm=request.algorithm, + ) + # Set the _pil attribute to the cropped image + det._pil = cropped_image_pil + detections.append(det) + logger.info(f"Created detection {det.id} for source image {source_image_id}.") + + return detections + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=2000) + + +class TrainRequest(pydantic.BaseModel): + """Retrain a classifier head from a dataset Antenna has already prepared.""" + + dataset_url: str = pydantic.Field( + description="URL of the npz training set Antenna wrote to storage." + ) + algorithm_key: str = pydantic.Field( + description="Which head to retrain. Its current weights are the baseline the new head must beat." + ) + job_id: int | None = pydantic.Field( + default=None, + description="Antenna job to report back to. Without it the result is only returned in this response.", + ) + callback_url: str | None = pydantic.Field( + default=None, + description="Where to post the result when training finishes.", + ) + callback_token: str | None = pydantic.Field( + default=None, + description="Token for the callback, so Antenna can tell a real result from a forged one.", + ) + name: str | None = pydantic.Field(default=None, description="Name for the produced head.") + min_per_species: int = 2 + min_improvement: float = 0.0 + save: bool = pydantic.Field( + default=True, + description="Write the head to disk. It is never loaded into the running service automatically.", + ) + + +class TrainResponse(pydantic.BaseModel): + promote: bool + reason: str + warnings: list[str] + rows: dict + counts: dict + dropped_species: list[str] + candidate_metrics: dict + incumbent_metrics: dict | None + labels: list[str] + saved: dict[str, str] | None + trained_at: str + reported_to_antenna: bool = False + + +@app.post("/train", tags=["training"]) +async def train(data: TrainRequest) -> TrainResponse: + """ + Retrain a classifier head from human-verified labels. + + Antenna prepares the dataset and hands over a URL; this downloads it, fits a new head, + and scores it against the head currently in service on the same held-out rows. It never + swaps the running head: promoting is a separate, deliberate step, because an automatic + swap would let one bad training run quietly degrade every later classification. + """ + try: + rows, dataset_metadata = training.fetch_dataset(data.dataset_url) + except requests.HTTPError as e: + raise fastapi.HTTPException(status_code=502, detail=f"Could not download the training set: {e}") + except Exception as e: + raise fastapi.HTTPException(status_code=422, detail=f"Could not read the training set: {e}") + + if not rows: + raise fastapi.HTTPException(status_code=422, detail="The training set is empty.") + + incumbent = _incumbent_head(data.algorithm_key) + + try: + result = training.retrain( + rows=rows, + incumbent=incumbent, + min_per_species=data.min_per_species, + min_improvement=data.min_improvement, + ) + except training.NotEnoughData as e: + raise fastapi.HTTPException(status_code=422, detail=str(e)) + + saved = None + if data.save: + name = data.name or f"head-{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}" + saved = training.save_head(result, pathlib.Path(TRAINED_HEADS_DIR), name) + + response = TrainResponse( + promote=result["promote"], + reason=result["reason"], + warnings=result["warnings"], + rows=result["rows"], + counts=result["counts"], + dropped_species=result["dropped_species"], + candidate_metrics=result["candidate_metrics"], + incumbent_metrics=result["incumbent_metrics"], + labels=result["labels"], + saved=saved, + trained_at=result["trained_at"], + ) + + if data.callback_url: + response.reported_to_antenna = _report_to_antenna(data, response, dataset_metadata) + + return response + + +def _report_to_antenna(data: TrainRequest, response: "TrainResponse", dataset_metadata: dict) -> bool: + """ + Tell the Antenna job how the run went. + + Reported rather than raised on failure: the training itself succeeded, and losing the + callback should not make the caller think it did not. + """ + payload = { + "job_id": data.job_id, + "algorithm_key": data.algorithm_key, + "dataset": dataset_metadata, + "result": response.model_dump(), + } + headers = {"Content-Type": "application/json"} + if data.callback_token: + headers["Authorization"] = f"Token {data.callback_token}" + try: + reply = requests.post(data.callback_url, json=payload, headers=headers, timeout=60) + reply.raise_for_status() + return True + except Exception as e: + logger.error(f"Trained successfully but could not report to Antenna at {data.callback_url}: {e}") + return False + + +def _incumbent_head(algorithm_key: str) -> dict | None: + """ + Load the weights of the head currently in service, so a new head can be compared to it. + + Scans the algorithm classes rather than a pipeline's `stages`, because stages only + exist once a pipeline is instantiated. Returns None when the key names no local head; + the caller then refuses to promote rather than promoting something it could not compare. + """ + for candidate in vars(algorithms).values(): + if not inspect.isclass(candidate) or not issubclass(candidate, algorithms.Algorithm): + continue + if not hasattr(candidate, "_load_head_arrays"): + continue + try: + instance = candidate() + if instance.algorithm_config_response.key != algorithm_key: + continue + weights, bias, classes = instance._load_head_arrays() + # The npz's `classes` holds column indices, not names. The species names come + # from the category map, so use that; otherwise every label comparison against + # Antenna's taxon names silently finds nothing in common. + labels = [str(label) for label in instance.get_category_map().labels] + if len(labels) != weights.shape[0]: + logger.warning( + f"{candidate.__name__}: category map has {len(labels)} labels but the head has " + f"{weights.shape[0]} outputs. Falling back to the names stored in the npz." + ) + labels = [str(c) for c in classes] + except Exception as e: + logger.warning(f"Could not load the current head for {candidate.__name__}: {e}") + continue + return {"weights": weights, "bias": bias, "labels": labels} + return None diff --git a/processing_services/bioclip/api/pipelines.py b/processing_services/bioclip/api/pipelines.py new file mode 100644 index 000000000..f15e21d14 --- /dev/null +++ b/processing_services/bioclip/api/pipelines.py @@ -0,0 +1,438 @@ +import datetime +import logging +from typing import final + +from .algorithms import ( + Algorithm, + BioCLIP25LogRegClassifier, + BioCLIPPanamaLogRegClassifier, + ConstantClassifier, + HFImageClassifier, + RandomSpeciesClassifier, + ZeroShotObjectDetector, +) +from .schemas import ( + Detection, + DetectionResponse, + PipelineConfigResponse, + PipelineRequestConfigParameters, + PipelineResultsResponse, + SourceImage, + SourceImageResponse, +) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class Pipeline: + """ + A base class for defining and running a pipeline consisting of multiple stages. + Each stage is represented by an algorithm that processes inputs and produces + outputs. The pipeline is designed to handle batch processing using custom batch + sizes for each stage. + + Attributes: + stages (list[Algorithm]): A list of algorithms representing the stages of + the pipeline in order of execution. Typically [Detector(), Classifier()]. + batch_sizes (list[int]): A list of integers specifying the batch size for + each stage. For example, [1, 1] means that the detector can process 1 + source image a time and the classifier can process 1 detection at a time. + config (PipelineConfigResponse): Pipeline metadata. + """ + + stages: list[Algorithm] + batch_sizes: list[int] + request_config: dict + config: PipelineConfigResponse + + stages = [] + batch_sizes = [] + config = PipelineConfigResponse( + name="Base Pipeline", + slug="base", + description="A base class for all pipelines.", + version=1, + algorithms=[], + ) + + def __init__( + self, + source_images: list[SourceImage], + request_config: PipelineRequestConfigParameters | dict = {}, + existing_detections: list[Detection] = [], + custom_batch_sizes: list[int] = [], + ): + self.source_images = source_images + self.request_config = request_config if isinstance(request_config, dict) else request_config.model_dump() + self.existing_detections = existing_detections + + logger.info("Initializing algorithms....") + self.stages = self.stages or self.get_stages() + self.batch_sizes = custom_batch_sizes or self.batch_sizes or [1] * len(self.stages) + assert len(self.batch_sizes) == len(self.stages), "Number of batch sizes must match the number of stages." + + def get_stages(self) -> list[Algorithm]: + """ + An optional function to initialize and return a list of algorithms/stages. + Any pipeline config values relevant to a particular algorithm should be passed or set here. + """ + return [] + + @final + def compile(self): + logger.info("Compiling algorithms....") + for stage_idx, stage in enumerate(self.stages): + logger.info(f"[{stage_idx+1}/{len(self.stages)}] Compiling {stage.algorithm_config_response.name}...") + stage.compile() + + def run(self) -> PipelineResultsResponse: + """ + This function must always return a PipelineResultsResponse object. + """ + raise NotImplementedError("Subclasses must implement") + + @final + def _batchify_inputs(self, inputs: list, batch_size: int) -> list[list]: + """ + Helper function to split the inputs into batches of the specified size. + """ + batched_inputs = [] + for i in range(0, len(inputs), batch_size): + start_id = i + end_id = i + batch_size + batched_inputs.append(inputs[start_id:end_id]) + return batched_inputs + + @final + def _get_detections( + self, algorithm: Algorithm, inputs: list[SourceImage] | list[Detection], batch_size: int, **kwargs + ) -> list[Detection]: + """A single stage, step, or algorithm in a pipeline. Batchifies inputs and produces Detections as outputs.""" + outputs: list[Detection] = [] + batched_inputs = self._batchify_inputs(inputs, batch_size) + for batch in batched_inputs: + outputs.extend(algorithm.run(batch, **kwargs)) + return outputs + + @final + def _get_pipeline_response(self, detections: list[Detection], elapsed_time: float) -> PipelineResultsResponse: + """ + Final stage of the pipeline to format the detections. + """ + detection_responses = [ + DetectionResponse( + source_image_id=detection.source_image.id, + bbox=detection.bbox, + inference_time=detection.inference_time, + algorithm=detection.algorithm, + timestamp=datetime.datetime.now(), + classifications=detection.classifications, + ) + for detection in detections + ] + source_image_responses = [SourceImageResponse(**image.model_dump()) for image in self.source_images] + + return PipelineResultsResponse( + pipeline=self.config.slug, # type: ignore + # algorithms={algorithm.key: algorithm for algorithm in self.config.algorithms}, + total_time=elapsed_time, + source_images=source_image_responses, + detections=detection_responses, + ) + + +class ZeroShotHFClassifierPipeline(Pipeline): + """ + A pipeline that uses the Zero Shot Object Detector to produce bounding boxes + and then applies the HuggingFace image classifier. + """ + + batch_sizes = [1, 1] + config = PipelineConfigResponse( + name="Zero Shot HF Classifier Pipeline", + slug="zero-shot-hf-classifier-pipeline", + description=("Zero Shot Object Detector with HF image classifier."), + version=1, + algorithms=[ + ZeroShotObjectDetector().algorithm_config_response, + HFImageClassifier().algorithm_config_response, + ], + ) + + def get_stages(self) -> list[Algorithm]: + zero_shot_object_detector = ZeroShotObjectDetector() + if "candidate_labels" in self.request_config: + logger.info( + "Setting candidate labels for zero shot object detector to %s", self.request_config["candidate_labels"] + ) + zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] + self.config.algorithms = [ + zero_shot_object_detector.algorithm_config_response, + HFImageClassifier().algorithm_config_response, + ] + + return [zero_shot_object_detector, HFImageClassifier()] + + def run(self) -> PipelineResultsResponse: + start_time = datetime.datetime.now() + detections_with_candidate_labels: list[Detection] = [] + if self.existing_detections: + logger.info("[1/2] Skipping the localizer, use existing detections...") + detections_with_candidate_labels = self.existing_detections + else: + logger.info("[1/2] No existing detections, generating detections...") + detections_with_candidate_labels: list[Detection] = self._get_detections( + self.stages[0], self.source_images, self.batch_sizes[0], intermediate=True + ) + + logger.info("[2/2] Running the classifier...") + detections_with_classifications: list[Detection] = self._get_detections( + self.stages[1], detections_with_candidate_labels, self.batch_sizes[1] + ) + end_time = datetime.datetime.now() + elapsed_time = (end_time - start_time).total_seconds() + + pipeline_response: PipelineResultsResponse = self._get_pipeline_response( + detections_with_classifications, elapsed_time + ) + logger.info(f"Successfully processed {len(detections_with_classifications)} detections.") + + return pipeline_response + + +class ZeroShotObjectDetectorPipeline(Pipeline): + """ + A pipeline that uses the HuggingFace zero shot object detector. + Produces both a bounding box and a classification for each detection. + The classification is based on the candidate labels provided in the request. + """ + + batch_sizes = [1] + config = PipelineConfigResponse( + name="Zero Shot Object Detector Pipeline", + slug="zero-shot-object-detector-pipeline", + description=("Zero shot object detector (bbox and classification)."), + version=1, + algorithms=[ZeroShotObjectDetector().algorithm_config_response], + ) + + def get_stages(self) -> list[Algorithm]: + zero_shot_object_detector = ZeroShotObjectDetector() + if "candidate_labels" in self.request_config: + logger.info( + "Setting candidate labels for zero shot object detector to %s", self.request_config["candidate_labels"] + ) + zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] + self.config.algorithms = [zero_shot_object_detector.algorithm_config_response] + + return [zero_shot_object_detector] + + def run(self) -> PipelineResultsResponse: + start_time = datetime.datetime.now() + logger.info("[1/1] Running the zero shot object detector...") + detections_with_classifications: list[Detection] = self._get_detections( + self.stages[0], self.source_images, self.batch_sizes[0] + ) + end_time = datetime.datetime.now() + elapsed_time = (end_time - start_time).total_seconds() + pipeline_response: PipelineResultsResponse = self._get_pipeline_response( + detections_with_classifications, elapsed_time + ) + logger.info(f"Successfully processed {len(detections_with_classifications)} detections.") + + return pipeline_response + + +class ZeroShotObjectDetectorWithRandomSpeciesClassifierPipeline(Pipeline): + """ + A pipeline that uses the HuggingFace zero shot object detector and a random species classifier. + """ + + batch_sizes = [1, 1] + config = PipelineConfigResponse( + name="Zero Shot Object Detector With Random Species Classifier Pipeline", + slug="zero-shot-object-detector-with-random-species-classifier-pipeline", + description=("HF zero shot object detector with random species classifier."), + version=1, + algorithms=[ + ZeroShotObjectDetector().algorithm_config_response, + RandomSpeciesClassifier().algorithm_config_response, + ], + ) + + def get_stages(self) -> list[Algorithm]: + zero_shot_object_detector = ZeroShotObjectDetector() + if "candidate_labels" in self.request_config: + zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] + + self.config.algorithms = [ + zero_shot_object_detector.algorithm_config_response, + RandomSpeciesClassifier().algorithm_config_response, + ] + + return [zero_shot_object_detector, RandomSpeciesClassifier()] + + def run(self) -> PipelineResultsResponse: + start_time = datetime.datetime.now() + detections: list[Detection] = [] + if self.existing_detections: + logger.info("[1/2] Skipping the localizer, use existing detections...") + detections = self.existing_detections + else: + logger.info("[1/2] No existing detections, generating detections...") + detections = self._get_detections(self.stages[0], self.source_images, self.batch_sizes[0]) + + logger.info("[2/2] Running the classifier...") + detections_with_classifications: list[Detection] = self._get_detections( + self.stages[1], detections, self.batch_sizes[1] + ) + end_time = datetime.datetime.now() + elapsed_time = (end_time - start_time).total_seconds() + pipeline_response: PipelineResultsResponse = self._get_pipeline_response( + detections_with_classifications, elapsed_time + ) + logger.info(f"Successfully processed {len(detections_with_classifications)} detections.") + + return pipeline_response + + +class ZeroShotObjectDetectorWithConstantClassifierPipeline(Pipeline): + """ + A pipeline that uses the HuggingFace zero shot object detector and a constant classifier. + """ + + batch_sizes = [1, 1] + config = PipelineConfigResponse( + name="Zero Shot Object Detector With Constant Classifier Pipeline", + slug="zero-shot-object-detector-with-constant-classifier-pipeline", + description=("HF zero shot object detector with constant classifier."), + version=1, + algorithms=[ + ZeroShotObjectDetector().algorithm_config_response, + ConstantClassifier().algorithm_config_response, + ], + ) + + def get_stages(self) -> list[Algorithm]: + zero_shot_object_detector = ZeroShotObjectDetector() + if "candidate_labels" in self.request_config: + zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] + + self.config.algorithms = [ + zero_shot_object_detector.algorithm_config_response, + ConstantClassifier().algorithm_config_response, + ] + + return [zero_shot_object_detector, ConstantClassifier()] + + def run(self) -> PipelineResultsResponse: + start_time = datetime.datetime.now() + detections: list[Detection] = [] + if self.existing_detections: + logger.info("[1/2] Skipping the localizer, use existing detections...") + detections = self.existing_detections + else: + logger.info("[1/2] No existing detections, generating detections...") + detections = self._get_detections(self.stages[0], self.source_images, self.batch_sizes[0]) + + logger.info("[2/2] Running the classifier...") + detections_with_classifications: list[Detection] = self._get_detections( + self.stages[1], detections, self.batch_sizes[1] + ) + end_time = datetime.datetime.now() + elapsed_time = (end_time - start_time).total_seconds() + pipeline_response: PipelineResultsResponse = self._get_pipeline_response( + detections_with_classifications, elapsed_time + ) + logger.info(f"Successfully processed {len(detections_with_classifications)} detections.") + + return pipeline_response + + +class BioCLIP25LogRegPipeline(Pipeline): + """ + Zero-shot object detector for bounding boxes, then BioCLIP 2.5 + logistic-regression + head for the terminal species classification. + + The detector is a placeholder for local testing. Swap stage 0 for the production + moth detector before using this on real deployments. + """ + + batch_sizes = [1, 1] + config = PipelineConfigResponse( + name="BioCLIP 2.5 + LogReg Head Pipeline", + slug="bioclip-2-5-logreg-pipeline", + description="Zero shot object detector with a BioCLIP 2.5 + logistic-regression species classifier.", + version=1, + algorithms=[ + ZeroShotObjectDetector().algorithm_config_response, + BioCLIP25LogRegClassifier().algorithm_config_response, + ], + ) + + def get_stages(self) -> list[Algorithm]: + zero_shot_object_detector = ZeroShotObjectDetector() + if "candidate_labels" in self.request_config: + zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] + + classifier = BioCLIP25LogRegClassifier() + self.config.algorithms = [ + zero_shot_object_detector.algorithm_config_response, + classifier.algorithm_config_response, + ] + return [zero_shot_object_detector, classifier] + + def run(self) -> PipelineResultsResponse: + start_time = datetime.datetime.now() + + if self.existing_detections: + logger.info("[1/2] Skipping the localizer, use existing detections...") + detections = self.existing_detections + else: + logger.info("[1/2] No existing detections, generating detections...") + detections = self._get_detections( + self.stages[0], self.source_images, self.batch_sizes[0], intermediate=True + ) + + logger.info("[2/2] Running the BioCLIP 2.5 classifier...") + detections_with_classifications = self._get_detections(self.stages[1], detections, self.batch_sizes[1]) + + elapsed_time = (datetime.datetime.now() - start_time).total_seconds() + pipeline_response = self._get_pipeline_response(detections_with_classifications, elapsed_time) + logger.info(f"Successfully processed {len(detections_with_classifications)} detections.") + + return pipeline_response + + +@final +class BioCLIPPanamaPipeline(BioCLIP25LogRegPipeline): + """ + Same two stages as the Newfoundland pipeline, with the Panama head as the terminal + classifier. Both pipelines share the frozen encoder through SAVED_MODELS, so running + both on one GPU costs one copy of the backbone plus two small Linear heads. + """ + + batch_sizes = [1, 1] + config = PipelineConfigResponse( + name="BioCLIP 2.5 + Panama LogReg Head Pipeline", + slug="bioclip-2-5-panama-pipeline", + description="Zero shot object detector with a BioCLIP 2.5 + Panama logistic-regression species classifier.", + version=1, + algorithms=[ + ZeroShotObjectDetector().algorithm_config_response, + BioCLIPPanamaLogRegClassifier().algorithm_config_response, + ], + ) + + def get_stages(self) -> list[Algorithm]: + zero_shot_object_detector = ZeroShotObjectDetector() + if "candidate_labels" in self.request_config: + zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] + + classifier = BioCLIPPanamaLogRegClassifier() + self.config.algorithms = [ + zero_shot_object_detector.algorithm_config_response, + classifier.algorithm_config_response, + ] + return [zero_shot_object_detector, classifier] diff --git a/processing_services/bioclip/api/schemas.py b/processing_services/bioclip/api/schemas.py new file mode 100644 index 000000000..a30ea0d42 --- /dev/null +++ b/processing_services/bioclip/api/schemas.py @@ -0,0 +1,385 @@ +# Can these be imported from the OpenAPI spec yaml? +import datetime +import logging +import pathlib +import typing + +import PIL.Image +import pydantic + +from .utils import get_image + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class BoundingBox(pydantic.BaseModel): + x1: float + y1: float + x2: float + y2: float + + @classmethod + def from_coords(cls, coords: list[float]): + return cls(x1=coords[0], y1=coords[1], x2=coords[2], y2=coords[3]) + + def to_string(self): + return f"{self.x1},{self.y1},{self.x2},{self.y2}" + + def to_path(self): + return "-".join([str(int(x)) for x in [self.x1, self.y1, self.x2, self.y2]]) + + def to_tuple(self): + return (self.x1, self.y1, self.x2, self.y2) + + +class BaseImage(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="ignore", arbitrary_types_allowed=True) + + id: str + url: str | None = None + b64: str | None = None + filepath: str | pathlib.Path | None = None + _pil: PIL.Image.Image | None = None + width: int | None = None + height: int | None = None + timestamp: datetime.datetime | None = None + + # Validate that there is at least one of the following fields + @pydantic.model_validator(mode="after") + def validate_source(self): + if not any([self.url, self.b64, self.filepath, self._pil]): + raise ValueError("At least one of the following fields must be provided: url, b64, filepath, pil") + return self + + def open(self, raise_exception=False) -> PIL.Image.Image | None: + if not self._pil: + logger.warn(f"Opening image {self.id} for the first time") + self._pil = get_image( + url=self.url, + b64=self.b64, + filepath=self.filepath, + raise_exception=raise_exception, + ) + else: + logger.info(f"Using already loaded image {self.id}") + if self._pil: + self.width, self.height = self._pil.size + return self._pil + + +class SourceImage(BaseImage): + pass + + +class AlgorithmReference(pydantic.BaseModel): + name: str + key: str + + +class ClassificationResponse(pydantic.BaseModel): + classification: str + labels: list[str] | None = pydantic.Field( + default=None, + description=( + "A list of all possible labels for the model, in the correct order. " + "Omitted if the model has too many labels to include for each classification in the response. " + "Use the category map from the algorithm to get the full list of labels and metadata." + ), + ) + scores: list[float] = pydantic.Field( + default_factory=list, + description="The calibrated probabilities for each class label, most commonly the softmax output.", + ) + logits: list[float] = pydantic.Field( + default_factory=list, + description="The raw logits output by the model, before any calibration or normalization.", + ) + features: list[float] | None = pydantic.Field( + default=None, + description=( + "The embedding the backbone produced for this crop, taken before the " + "classification head. Optional, and only useful if every value comes from " + "the same backbone." + ), + ) + inference_time: float | None = None + algorithm: AlgorithmReference + terminal: bool = True + timestamp: datetime.datetime + + +class SourceImageRequest(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="ignore") + + id: str + url: str + # b64: str | None = None + # @TODO bring over new SourceImage & b64 validation from the lepsAI repo + + +class SourceImageResponse(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="ignore") + + id: str + url: str + + +class DetectionRequest(pydantic.BaseModel): + source_image: SourceImageRequest # the 'original' image + bbox: BoundingBox + crop_image_url: str | None = None + algorithm: AlgorithmReference + + +class DetectionResponse(pydantic.BaseModel): + # these fields are populated with values from a Detection, excluding source_image details + source_image_id: str + bbox: BoundingBox + inference_time: float | None = None + algorithm: AlgorithmReference + timestamp: datetime.datetime + crop_image_url: str | None = None + classifications: list[ClassificationResponse] = [] + + +class Detection(BaseImage): + """ + An internal representation of a detection with reference to a source image instance. + """ + + source_image: SourceImage # the 'original' uncropped image + bbox: BoundingBox + inference_time: float | None = None + algorithm: AlgorithmReference + classifications: list[ClassificationResponse] = [] + + +class AlgorithmCategoryMapResponse(pydantic.BaseModel): + data: list[dict] = pydantic.Field( + default_factory=dict, + description="Complete data for each label, such as id, gbif_key, explicit index, source, etc.", + examples=[ + [ + {"label": "Moth", "index": 0, "gbif_key": 1234}, + {"label": "Not a moth", "index": 1, "gbif_key": 5678}, + ] + ], + ) + labels: list[str] = pydantic.Field( + default_factory=list, + description="A simple list of string labels, in the correct index order used by the model.", + examples=[["Moth", "Not a moth"]], + ) + version: str | None = pydantic.Field( + default=None, + description="The version of the category map. Can be a descriptive string or a version number.", + examples=["LepNet2021-with-2023-mods"], + ) + description: str | None = pydantic.Field( + default=None, + description="A description of the category map used to train. e.g. source, purpose and modifications.", + examples=["LepNet2021 with Schmidt 2023 corrections. Limited to species with > 1000 observations."], + ) + uri: str | None = pydantic.Field( + default=None, + description="A URI to the category map file, could be a public web URL or object store path.", + ) + + +class AlgorithmTrainingConfig(pydantic.BaseModel): + """ + Default settings for retraining this algorithm. Antenna seeds its own copy from this + and an admin can then override it, so these are defaults, not rules. + """ + + # Used by Antenna when it builds the training set. + min_per_species: int = 2 + test_fraction: float = 0.2 + split_salt: str = "antenna-head-v1" + + # Used here when fitting. + head_type: str = "linear" + epochs: int = 300 + learning_rate: float = 0.01 + weight_decay: float = 1e-4 + min_improvement: float = 0.0 + + model_config = pydantic.ConfigDict(extra="allow") + + +class AlgorithmTrainingInfo(pydantic.BaseModel): + """Where a retrained version's weights came from.""" + + trained_at: datetime.datetime | None = None + dataset_url: str | None = None + dataset_rows: int | None = None + dataset_classes: int | None = None + metrics: dict = pydantic.Field(default_factory=dict) + previous_metrics: dict = pydantic.Field(default_factory=dict) + parent_algorithm_key: str | None = None + job_id: int | None = None + warnings: list[str] = pydantic.Field(default_factory=list) + + model_config = pydantic.ConfigDict(extra="allow") + + +class AlgorithmConfigResponse(pydantic.BaseModel): + name: str + key: str = pydantic.Field( + description=("A unique key for an algorithm to lookup the category map (class list) and other metadata."), + ) + description: str | None = None + task_type: str | None = pydantic.Field( + default=None, + description="The type of task the model is trained for. e.g. 'detection', 'classification', 'embedding', etc.", + examples=["detection", "classification", "segmentation", "embedding"], + ) + version: int = pydantic.Field( + default=1, + description="A sortable version number for the model. Increment this number when the model is updated.", + ) + version_name: str | None = pydantic.Field( + default=None, + description="A complete version name e.g. '2021-01-01', 'LepNet2021'.", + ) + uri: str | None = pydantic.Field( + default=None, + description="A URI to the weights or model details, could be a public web URL or object store path.", + ) + trainable: bool = pydantic.Field( + default=False, + description=( + "Whether this algorithm can be retrained from labelled data. Only the classifier head " + "over the frozen backbone is cheap enough to retrain; the backbone and the detector are not." + ), + ) + training_config: AlgorithmTrainingConfig | None = None + training_info: AlgorithmTrainingInfo | None = None + category_map: AlgorithmCategoryMapResponse | None = None + + class Config: + extra = "ignore" + + +PipelineChoice = typing.Literal[ + "bioclip-2-5-logreg-pipeline", + "bioclip-2-5-panama-pipeline", + "zero-shot-hf-classifier-pipeline", + "zero-shot-object-detector-pipeline", + "zero-shot-object-detector-with-constant-classifier-pipeline", + "zero-shot-object-detector-with-random-species-classifier-pipeline", +] + + +class PipelineRequestConfigParameters(pydantic.BaseModel): + """Parameters used to configure a pipeline request. + + Accepts any serializable key-value pair. + Example: {"force_reprocess": True, "auth_token": "abc123"} + + Supported parameters are defined by the pipeline in the processing service + and should be published in the Pipeline's info response. + """ + + force_reprocess: bool = pydantic.Field( + default=False, + description="Force reprocessing of the image, even if it has already been processed.", + ) + auth_token: str | None = pydantic.Field( + default=None, + description="An optional authentication token to use for the pipeline.", + ) + candidate_labels: list[str] | None = pydantic.Field( + default=None, + description="A list of candidate labels to use for the zero-shot object detector.", + ) + + +class PipelineRequest(pydantic.BaseModel): + pipeline: PipelineChoice + source_images: list[SourceImageRequest] + detections: list[DetectionRequest] | None = None + config: PipelineRequestConfigParameters | dict | None = None + + # Example for API docs: + class Config: + json_schema_extra = { + "example": { + "pipeline": "random", + "source_images": [ + { + "id": "123", + "url": "https://archive.org/download/mma_various_moths_and_butterflies_54143/54143.jpg", + } + ], + "config": {"force_reprocess": True, "auth_token": "abc123"}, + } + } + + +class PipelineResultsResponse(pydantic.BaseModel): + pipeline: PipelineChoice + total_time: float + algorithms: dict[str, AlgorithmConfigResponse] = pydantic.Field( + default_factory=dict, + description=( + "A dictionary of all algorithms used in the pipeline, including their class list and other " + "metadata, keyed by the algorithm key. " + "DEPRECATED: Algorithms should only be provided in the ProcessingServiceInfoResponse." + ), + depreciated=True, + ) + source_images: list[SourceImageResponse] + detections: list[DetectionResponse] + errors: list | str | None = None + + +class PipelineStageParam(pydantic.BaseModel): + """A configurable parameter of a stage of a pipeline.""" + + name: str + key: str + category: str = "default" + + +class PipelineStage(pydantic.BaseModel): + """A configurable stage of a pipeline.""" + + key: str + name: str + params: list[PipelineStageParam] = [] + description: str | None = None + + +class PipelineConfigResponse(pydantic.BaseModel): + """Details about a pipeline, its algorithms and category maps.""" + + name: str + slug: str + version: int + description: str | None = None + algorithms: list[AlgorithmConfigResponse] = [] + stages: list[PipelineStage] = [] + + +class ProcessingServiceInfoResponse(pydantic.BaseModel): + """Information about the processing service.""" + + name: str = pydantic.Field(example="Mila Research Lab - Moth AI Services") + description: str | None = pydantic.Field( + default=None, + examples=["Algorithms developed by the Mila Research Lab for analysis of moth images."], + ) + pipelines: list[PipelineConfigResponse] = pydantic.Field( + default=list, + examples=[ + [ + PipelineConfigResponse(name="Random Pipeline", slug="random", version=1, algorithms=[]), + ] + ], + ) + # algorithms: list[AlgorithmConfigResponse] = pydantic.Field( + # default=list, + # examples=[RANDOM_BINARY_CLASSIFIER], + # ) diff --git a/processing_services/bioclip/api/test.py b/processing_services/bioclip/api/test.py new file mode 100644 index 000000000..b5b1b5f7c --- /dev/null +++ b/processing_services/bioclip/api/test.py @@ -0,0 +1,64 @@ +import unittest + +from fastapi.testclient import TestClient + +from .api import app +from .pipelines import CustomPipeline +from .schemas import PipelineRequest, SourceImage, SourceImageRequest + + +class TestPipeline(unittest.TestCase): + def test_custom_pipeline(self): + # @TODO: Load actual antenna images? + pipeline = CustomPipeline( + source_images=[ + SourceImage( + id="1001", + url=( + "https://huggingface.co/datasets/huggingface/" + "documentation-images/resolve/main/pipeline-cat-chonk.jpeg" + ), + ), + SourceImage(id="1002", url="https://cdn.britannica.com/79/191679-050-C7114D2B/Adult-capybara.jpg"), + ], + detector_batch_size=2, + classifier_batch_size=2, + ) + detections = pipeline.run() + + self.assertEqual(len(detections), 20) + expected_labels = ["lynx, catamount", "beaver"] + for detection_id, detection in enumerate(detections): + self.assertEqual(detection.source_image_id, pipeline.source_images[detection_id].id) + self.assertIsNotNone(detection.bbox) + self.assertEqual(len(detection.classifications), 1) + classification = detection.classifications[0] + self.assertEqual(classification.classification, expected_labels[detection_id]) + self.assertGreaterEqual(classification.scores[0], 0.0) + self.assertLessEqual(classification.scores[0], 1.0) + + +class TestAPI(unittest.TestCase): + def setUp(self): + self.client = TestClient(app) + + def test_root(self): + response = self.client.get("/") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.url, "http://testserver/docs") + + def test_process(self): + source_images = [ + SourceImage(id="1", url="https://example.com/image1.jpg"), + SourceImage(id="2", url="https://example.com/image2.jpg"), + ] + source_image_requests = [SourceImageRequest(**image.dict()) for image in source_images] + request = PipelineRequest(pipeline="local-pipeline", source_images=source_image_requests, config={}) + response = self.client.post("/process", json=request.dict()) + + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data["pipeline"], "local-pipeline") + self.assertEqual(len(data["source_images"]), 2) + self.assertEqual(len(data["detections"]), 2) + self.assertGreater(data["total_time"], 0.0) diff --git a/processing_services/bioclip/api/training.py b/processing_services/bioclip/api/training.py new file mode 100644 index 000000000..06db6a81a --- /dev/null +++ b/processing_services/bioclip/api/training.py @@ -0,0 +1,300 @@ +""" +Retrain the classifier head from labels people verified in Antenna. + +The backbone is frozen, so a head is just a linear layer over stored embeddings. Antenna +already holds those embeddings, so nothing here touches an image: pull rows, fit a head, +score it against the one in service, and keep the new one only if it wins. +""" + +import dataclasses +import datetime +import json +import logging +import pathlib +import tempfile +import typing + +import numpy as np +import requests +import torch + +logger = logging.getLogger(__name__) + +# A new head must beat the incumbent STRICTLY, by more than this margin. A tie must not +# promote: swapping heads is not free (it invalidates comparisons against past results), so +# an equal score is not a reason to change. +DEFAULT_MIN_IMPROVEMENT = 0.0 + +# Below this many held-out rows the comparison is noise, not evidence. The run still +# reports its numbers, but says plainly that they cannot support a decision. +MIN_MEANINGFUL_TEST_ROWS = 30 + + +class NotEnoughData(Exception): + """Raised when the verified data cannot support both training and evaluation.""" + + +@dataclasses.dataclass +class TrainingRow: + detection_id: int + label: str + split: str + features: np.ndarray + + +def fetch_dataset(url: str, timeout: int = 300, session: requests.Session | None = None) -> tuple[list[TrainingRow], dict]: + """ + Download the training set Antenna prepared and unpack it. + + An npz of float16 vectors rather than JSON rows: the values are stored as two bytes in + Postgres, so float16 loses nothing, and it is roughly ten times smaller and far faster + to parse than the equivalent JSON. + """ + session = session or requests.Session() + response = session.get(url, timeout=timeout, stream=True) + response.raise_for_status() + + with tempfile.NamedTemporaryFile(suffix=".npz") as handle: + for chunk in response.iter_content(chunk_size=1 << 20): + handle.write(chunk) + handle.flush() + archive = np.load(handle.name, allow_pickle=True) + + metadata = json.loads(str(archive["metadata"])) + classes = [str(c) for c in archive["classes"]] + features = archive["features"] + labels = archive["labels"] + detection_ids = archive["detection_ids"] + splits = archive["split"] + + rows = [ + TrainingRow( + detection_id=int(detection_id), + label=classes[int(label)], + split=str(split), + features=vector, + ) + for detection_id, label, split, vector in zip(detection_ids, labels, splits, features) + ] + logger.info(f"Loaded {len(rows)} training rows over {len(classes)} species from {url}") + return rows, metadata + + +def _matrices(rows: list[TrainingRow], labels: list[str]) -> tuple[np.ndarray, np.ndarray]: + index = {name: i for i, name in enumerate(labels)} + x = np.asarray([r.features for r in rows], dtype=np.float32) + y = np.asarray([index[r.label] for r in rows], dtype=np.int64) + return x, y + + +def train_linear_head( + x: np.ndarray, + y: np.ndarray, + num_classes: int, + epochs: int = 300, + learning_rate: float = 0.01, + weight_decay: float = 1e-4, + device: str = "cpu", +) -> torch.nn.Linear: + """ + Fit a single linear layer over frozen embeddings. + + Deliberately the same shape as the head already in service (a Linear over L2-normalised + BioCLIP features), so a retrained head is a drop-in replacement rather than a different + kind of model that would need its own serving path. + """ + head = torch.nn.Linear(x.shape[1], num_classes).to(device) + optimizer = torch.optim.AdamW(head.parameters(), lr=learning_rate, weight_decay=weight_decay) + loss_fn = torch.nn.CrossEntropyLoss() + + inputs = torch.from_numpy(x).to(device) + targets = torch.from_numpy(y).to(device) + + head.train() + for epoch in range(epochs): + optimizer.zero_grad() + loss = loss_fn(head(inputs), targets) + loss.backward() + optimizer.step() + if epoch % 50 == 0: + logger.debug(f"epoch {epoch} loss {loss.item():.4f}") + head.eval() + return head + + +def evaluate( + weights: np.ndarray, + bias: np.ndarray, + x: np.ndarray, + y: np.ndarray, +) -> dict[str, float]: + """Top-1 and macro recall. Macro matters because trap data is heavily long-tailed.""" + logits = x @ weights.T + bias + predicted = logits.argmax(axis=1) + correct = predicted == y + + per_class = [] + for cls in np.unique(y): + mask = y == cls + per_class.append(float(correct[mask].mean())) + + return { + "top1": float(correct.mean()), + "macro_recall": float(np.mean(per_class)) if per_class else 0.0, + "n": int(len(y)), + "classes": int(len(np.unique(y))), + } + + +def score_incumbent( + incumbent_weights: np.ndarray, + incumbent_bias: np.ndarray, + incumbent_labels: list[str], + rows: list[TrainingRow], +) -> dict[str, float] | None: + """ + Score the head currently in service on the same held-out rows. + + Only rows whose species the incumbent can actually predict are counted. Scoring it on + species it was never trained to output would understate it and make any new head look + better than it is. + """ + index = {name: i for i, name in enumerate(incumbent_labels)} + usable = [r for r in rows if r.label in index] + if not usable: + return None + x = np.asarray([r.features for r in usable], dtype=np.float32) + y = np.asarray([index[r.label] for r in usable], dtype=np.int64) + result = evaluate(incumbent_weights, incumbent_bias, x, y) + result["skipped_unknown_species"] = len(rows) - len(usable) + return result + + +def retrain( + rows: list[TrainingRow], + incumbent: dict[str, typing.Any] | None = None, + min_per_species: int = 2, + min_improvement: float = DEFAULT_MIN_IMPROVEMENT, + device: str = "cpu", +) -> dict[str, typing.Any]: + """ + Fit a new head and decide whether it deserves to replace the current one. + + Returns the metrics, the decision, and the new head's weights. It does not write + anything: publishing is a separate, deliberate step. + """ + counts: dict[str, int] = {} + for row in rows: + counts[row.label] = counts.get(row.label, 0) + 1 + labels = sorted(name for name, n in counts.items() if n >= min_per_species) + if not labels: + raise NotEnoughData( + f"No species has at least {min_per_species} verified crops. " + f"Verify more occurrences before retraining. Counts: {counts}" + ) + + kept = [r for r in rows if r.label in labels] + train_rows = [r for r in kept if r.split == "train"] + test_rows = [r for r in kept if r.split == "test"] + if not train_rows: + raise NotEnoughData("The train split is empty.") + if not test_rows: + raise NotEnoughData( + "The test split is empty, so a new head cannot be compared against the current one. " + "Verify more occurrences." + ) + + x_train, y_train = _matrices(train_rows, labels) + x_test, y_test = _matrices(test_rows, labels) + + head = train_linear_head(x_train, y_train, num_classes=len(labels), device=device) + weights = head.weight.detach().cpu().numpy() + bias = head.bias.detach().cpu().numpy() + + candidate = evaluate(weights, bias, x_test, y_test) + incumbent_metrics = None + if incumbent: + incumbent_metrics = score_incumbent( + incumbent["weights"], incumbent["bias"], incumbent["labels"], test_rows + ) + + if incumbent_metrics is None: + promote = False + reason = "No incumbent head was scored, so the new head is not promoted automatically." + elif candidate["top1"] > incumbent_metrics["top1"] + min_improvement: + promote = True + reason = ( + f"New head top-1 {candidate['top1']:.3f} beats current {incumbent_metrics['top1']:.3f} " + f"by more than {min_improvement:.3f}." + ) + else: + promote = False + reason = ( + f"New head top-1 {candidate['top1']:.3f} does not beat current " + f"{incumbent_metrics['top1']:.3f} by more than {min_improvement:.3f}." + ) + + warnings: list[str] = [] + if len(test_rows) < MIN_MEANINGFUL_TEST_ROWS: + warnings.append( + f"Only {len(test_rows)} held-out row(s). Anything under {MIN_MEANINGFUL_TEST_ROWS} is too " + "few to tell two heads apart, so treat this result as a smoke test, not evidence." + ) + if incumbent_metrics and incumbent_metrics.get("skipped_unknown_species"): + warnings.append( + f"{incumbent_metrics['skipped_unknown_species']} held-out row(s) name species the current " + "head cannot predict, so they were left out of its score. The two heads were not scored on " + "identical rows." + ) + if len(labels) < 5: + warnings.append( + f"The new head covers only {len(labels)} species. The head in service covers far more, so " + "promoting this one would narrow what the pipeline can predict." + ) + + return { + "labels": labels, + "weights": weights, + "bias": bias, + "counts": {name: counts[name] for name in labels}, + "dropped_species": sorted(set(counts) - set(labels)), + "rows": {"total": len(rows), "kept": len(kept), "train": len(train_rows), "test": len(test_rows)}, + "candidate_metrics": candidate, + "incumbent_metrics": incumbent_metrics, + "promote": promote, + "reason": reason, + "warnings": warnings, + "trained_at": datetime.datetime.now().isoformat(), + } + + +def save_head(result: dict[str, typing.Any], directory: pathlib.Path, name: str) -> dict[str, str]: + """ + Write the head in the same format the service already loads (npz + label_map.json). + + Written under its own name rather than overwriting the current head: Antenna keys + algorithms by key, and a head that is swapped in place makes past classifications + impossible to attribute. + """ + directory.mkdir(parents=True, exist_ok=True) + head_path = directory / f"{name}.npz" + labels_path = directory / f"{name}.label_map.json" + + np.savez( + head_path, + W=result["weights"], + b=result["bias"], + classes=np.array(result["labels"], dtype=object), + ) + labels_path.write_text( + json.dumps( + { + "labels": result["labels"], + "counts": result["counts"], + "metrics": result["candidate_metrics"], + "trained_at": result["trained_at"], + }, + indent=2, + ) + ) + return {"head": str(head_path), "labels": str(labels_path)} diff --git a/processing_services/bioclip/api/utils.py b/processing_services/bioclip/api/utils.py new file mode 100644 index 000000000..a7fcb6a75 --- /dev/null +++ b/processing_services/bioclip/api/utils.py @@ -0,0 +1,149 @@ +import base64 +import binascii +import io +import logging +import pathlib +import re +import tempfile +from urllib.parse import urlparse + +import PIL.Image +import PIL.ImageFile +import requests + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +PIL.ImageFile.LOAD_TRUNCATED_IMAGES = True + +# This is polite and required by some hosts +# see: https://foundation.wikimedia.org/wiki/Policy:User-Agent_policy +USER_AGENT = "AntennaInsectDataPlatform/1.0 (https://insectai.org)" + +# ----------- +# File handling functions +# ----------- + + +def is_url(path: str) -> bool: + return path.startswith("http://") or path.startswith("https://") + + +def is_base64(s: str) -> bool: + try: + # Check if string can be decoded from base64 + return base64.b64encode(base64.b64decode(s)).decode() == s + except Exception: + return False + + +def get_or_download_file(path_or_url, tempdir_prefix="antenna") -> pathlib.Path: + """ + Fetch a file from a URL or local path. If the path is a URL, download the file. + If the URL has already been downloaded, return the existing local path. + If the path is a local path, return the path. + + >>> filepath = get_or_download_file("https://example.uk/images/31-20230919033000-snapshot.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=451d406b7eb1113e1bb05c083ce51481%2F20240429%2F") # noqa: E501 + >>> filepath.name + '31-20230919033000-snapshot.jpg' + >>> filepath = get_or_download_file("/home/user/images/31-20230919033000-snapshot.jpg") + >>> filepath.name + '31-20230919033000-snapshot.jpg' + """ + if not path_or_url: + raise Exception("Specify a URL or path to fetch file from.") + + # If path is a local path instead of a URL then urlretrieve will just return that path + + destination_dir = pathlib.Path(tempfile.mkdtemp(prefix=tempdir_prefix)) + fname = pathlib.Path(urlparse(path_or_url).path).name + if not destination_dir.exists(): + destination_dir.mkdir(parents=True, exist_ok=True) + local_filepath = pathlib.Path(destination_dir) / fname + + if local_filepath and local_filepath.exists(): + logger.info(f"Using existing {local_filepath}") + return local_filepath + + else: + logger.info(f"Downloading {path_or_url} to {local_filepath}") + headers = {"User-Agent": USER_AGENT} + response = requests.get(path_or_url, stream=True, headers=headers) + response.raise_for_status() # Raise an exception for HTTP errors + + with open(local_filepath, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + + resulting_filepath = pathlib.Path(local_filepath).resolve() + logger.info(f"Downloaded to {resulting_filepath}") + return resulting_filepath + + +def open_image(fp: str | bytes | pathlib.Path | io.BytesIO, raise_exception: bool = True) -> PIL.Image.Image | None: + """ + Wrapper from PIL.Image.open that handles errors and converts to RGB. + """ + img = None + try: + img = PIL.Image.open(fp) + except PIL.UnidentifiedImageError: + logger.warn(f"Unidentified image: {str(fp)[:100]}...") + if raise_exception: + raise + except OSError: + logger.warn(f"Could not open image: {str(fp)[:100]}...") + if raise_exception: + raise + else: + # Convert to RGB if necessary + if img.mode != "RGB": + img = img.convert("RGB") + + return img + + +def decode_base64_string(string) -> io.BytesIO: + image_data = re.sub("^data:image/.+;base64,", "", string) + decoded = base64.b64decode(image_data) + buffer = io.BytesIO(decoded) + buffer.seek(0) + return buffer + + +def get_image( + url: str | None = None, + filepath: str | pathlib.Path | None = None, + b64: str | None = None, + raise_exception: bool = True, +) -> PIL.Image.Image | None: + """ + Given a URL, local file path or base64 image, return a PIL image. + """ + + if url: + logger.info(f"Fetching image from URL: {url}") + tempdir = tempfile.TemporaryDirectory(prefix="ami_images") + img_path = get_or_download_file(url, tempdir_prefix=tempdir.name) + return open_image(img_path, raise_exception=raise_exception) + + elif filepath: + logger.info(f"Loading image from local filesystem: {filepath}") + return open_image(filepath, raise_exception=raise_exception) + + elif b64: + logger.info(f"Loading image from base64 string: {b64[:30]}...") + try: + buffer = decode_base64_string(b64) + except binascii.Error as e: + logger.warn(f"Could not decode base64 image: {e}") + if raise_exception: + raise + else: + return None + else: + return open_image(buffer, raise_exception=raise_exception) + + else: + raise Exception("Specify a URL, path or base64 image.") diff --git a/processing_services/bioclip/docker-compose.yml b/processing_services/bioclip/docker-compose.yml new file mode 100644 index 000000000..90e6897aa --- /dev/null +++ b/processing_services/bioclip/docker-compose.yml @@ -0,0 +1,25 @@ +services: + ml_backend_bioclip: + build: + context: . + volumes: + - ./:/app:z + - ./huggingface_cache:/root/.cache/huggingface + - ./pytorch_cache:/root/.cache/torch + ports: + - "2004:2000" + extra_hosts: + - minio:host-gateway + networks: + - antenna_network + # deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: 1 + # capabilities: [ gpu ] + +networks: + antenna_network: + name: antenna_network diff --git a/processing_services/bioclip/main.py b/processing_services/bioclip/main.py new file mode 100644 index 000000000..2ed50004d --- /dev/null +++ b/processing_services/bioclip/main.py @@ -0,0 +1,4 @@ +if __name__ == "__main__": + import uvicorn + + uvicorn.run("api.api:app", host="0.0.0.0", port=2000, reload=True) diff --git a/processing_services/bioclip/requirements.txt b/processing_services/bioclip/requirements.txt new file mode 100644 index 000000000..5f0a86dd0 --- /dev/null +++ b/processing_services/bioclip/requirements.txt @@ -0,0 +1,12 @@ +fastapi==0.116.0 +uvicorn==0.35.0 +pydantic==2.11.7 +Pillow==11.3.0 +requests==2.32.4 +transformers==4.50.3 +torch==2.6.0 +torchvision==0.21.0 +scipy==1.16.0 +open_clip_torch==2.32.0 +huggingface_hub==0.34.4 +numpy==2.2.6 diff --git a/processing_services/bioclip/scripts/export_logreg_head.py b/processing_services/bioclip/scripts/export_logreg_head.py new file mode 100644 index 000000000..ca3d5713a --- /dev/null +++ b/processing_services/bioclip/scripts/export_logreg_head.py @@ -0,0 +1,144 @@ +""" +Convert a trained sklearn LogisticRegression probe into the two files this service needs: + + 1. an npz holding the linear head, as W, b and classes + 2. a label map JSON, keyed by class value, with species_name and inat_taxon_id + +Upload both to the Hugging Face repo named by BioCLIP25LogRegClassifier.head_repo_id. +The service downloads them from there with huggingface_hub.hf_hub_download. + +This is the same layout the Newfoundland trap classifier demo publishes, so a head +exported here can be dropped in next to the existing one without a second format. + +The head is fit on L2-normalised BioCLIP image embeddings, so softmax(x @ W.T + b) +reproduces sklearn's multinomial predict_proba exactly. This script verifies that +numerically before writing anything. + +Example: + + python scripts/export_logreg_head.py \ + --classifier probe.joblib \ + --labels species_labels.csv \ + --model-name hf-hub:imageomics/bioclip-2.5-vith14 \ + --out-head logreg_head.npz \ + --out-labels label_map.json + +--labels is a CSV with a `class` column matching the classifier's own class values +(clf.classes_, e.g. iNat taxon ids) and a `label` column with the name Antenna should +show. An optional `inat_taxon_id` column is passed through; it defaults to the class +value, which is already the iNat taxon id in the existing heads. +""" + +import argparse +import csv +import json +from pathlib import Path + +import numpy as np + + +def load_classifier(path: Path): + if path.suffix == ".joblib": + import joblib + + return joblib.load(path) + + import pickle + + with open(path, "rb") as f: + return pickle.load(f) + + +def load_label_rows(path: Path) -> dict[str, dict]: + """Map the classifier's class value (as a string) to its category metadata.""" + if path.suffix == ".json": + rows = json.loads(path.read_text()) + else: + with open(path, newline="") as f: + rows = list(csv.DictReader(f)) + + label_rows = {} + for row in rows: + if "class" not in row or "label" not in row: + raise ValueError(f"Each row of {path} needs at least a 'class' and a 'label' column, got: {row}") + label_rows[str(row["class"])] = row + return label_rows + + +def verify_softmax_equivalence(clf, embed_dim: int, tolerance: float = 1e-6) -> None: + """ + sklearn's one-vs-rest mode produces per-class sigmoids that are then renormalised, + which a single softmax cannot reproduce. Catch that here rather than in production. + """ + rng = np.random.default_rng(0) + probe = rng.normal(size=(8, embed_dim)) + probe = probe / np.linalg.norm(probe, axis=1, keepdims=True) + + sklearn_proba = clf.predict_proba(probe) + + scores = probe @ clf.coef_.T + clf.intercept_ + scores = scores - scores.max(axis=1, keepdims=True) + softmax_proba = np.exp(scores) / np.exp(scores).sum(axis=1, keepdims=True) + + max_difference = float(np.abs(sklearn_proba - softmax_proba).max()) + if max_difference > tolerance: + raise ValueError( + f"softmax(x @ W.T + b) differs from predict_proba by {max_difference:.2e}. " + "The head is probably one-vs-rest, not multinomial. Refit with a multinomial " + "solver (lbfgs or saga) so a single Linear layer can reproduce it." + ) + print(f"Verified softmax equivalence with sklearn (max diff {max_difference:.2e})") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--classifier", type=Path, required=True, help="Pickled/joblib LogisticRegression") + parser.add_argument("--labels", type=Path, required=True, help="CSV or JSON mapping class value -> label") + parser.add_argument("--model-name", default="hf-hub:imageomics/bioclip-2.5-vith14") + parser.add_argument("--out-head", type=Path, default=Path("logreg_head.npz")) + parser.add_argument("--out-labels", type=Path, default=Path("label_map.json")) + args = parser.parse_args() + + clf = load_classifier(args.classifier) + num_classes, embed_dim = clf.coef_.shape + print(f"Loaded classifier with {num_classes} classes over {embed_dim}-dim embeddings") + + if num_classes != len(clf.classes_): + raise ValueError( + f"Classifier has {len(clf.classes_)} classes but {num_classes} coefficient rows. " + "Binary logistic regression is not supported; this service expects a multinomial head." + ) + + verify_softmax_equivalence(clf, embed_dim) + + label_rows = load_label_rows(args.labels) + missing = [str(c) for c in clf.classes_ if str(c) not in label_rows] + if missing: + raise ValueError(f"{len(missing)} classes have no label, e.g. {missing[:5]}") + + # The label map is keyed by class value, so the service can walk `classes` in head + # row order and look each one up. + label_map = {} + for class_value in clf.classes_: + row = label_rows[str(class_value)] + label_map[str(class_value)] = { + "species_name": row["label"], + "inat_taxon_id": int(row.get("inat_taxon_id") or class_value), + } + + args.out_head.parent.mkdir(parents=True, exist_ok=True) + np.savez( + args.out_head, + W=clf.coef_.astype("float32"), + b=clf.intercept_.astype("float32"), + classes=clf.classes_, + ) + print(f"Wrote head to {args.out_head} (W {clf.coef_.shape}, {num_classes} classes)") + + args.out_labels.parent.mkdir(parents=True, exist_ok=True) + args.out_labels.write_text(json.dumps(label_map, indent=2)) + print(f"Wrote {len(label_map)} labels to {args.out_labels}") + + +if __name__ == "__main__": + main() diff --git a/processing_services/bioclip/scripts/run_on_remote_gpu.sh b/processing_services/bioclip/scripts/run_on_remote_gpu.sh new file mode 100755 index 000000000..8d6001af3 --- /dev/null +++ b/processing_services/bioclip/scripts/run_on_remote_gpu.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Tunnel the bioclip processing service running on a remote GPU host into a local +# Antenna stack. +# +# Usage: ./run_on_remote_gpu.sh [local-port] [remote-port] +# +# Opens two forwards over one SSH connection: +# -L local:2004 -> remote:2000 Antenna calls the service (sync/push mode) +# -R remote:9000 -> local:9000 the service downloads captures from minio +# -R remote:8000 -> local:8000 the ADC worker calls Antenna (async/pull mode) +# +# The reverse forward is required because Antenna signs capture URLs as +# http://minio:9000/... and the host name is covered by the S3 signature, so it +# cannot be rewritten. The remote host needs "127.0.0.1 minio" in /etc/hosts. +# +# Register the service in Antenna as http://host.docker.internal:2004 +# +# The connection is reopened automatically if it drops, which it will over a long +# job or an idle period. Ctrl-C to stop for good. +set -uo pipefail + +SSH_HOST="${1:?usage: $0 [local-port] [remote-port]}" +LOCAL_PORT="${2:-2004}" +REMOTE_PORT="${3:-2000}" +MINIO_PORT=9000 +API_PORT=8000 +RETRY_DELAY=5 + +echo "Antenna -> http://host.docker.internal:${LOCAL_PORT} -> ${SSH_HOST}:${REMOTE_PORT}" +echo "service -> http://minio:${MINIO_PORT} -> local minio-proxy" +echo "worker -> http://127.0.0.1:${API_PORT}/api/v2 -> local Antenna API" +echo "Ctrl-C to close the tunnel." + +trap 'echo; echo "tunnel closed."; exit 0' INT TERM + +while true; do + ssh -N -T \ + -o ExitOnForwardFailure=yes \ + -o ServerAliveInterval=15 \ + -o ServerAliveCountMax=3 \ + -o TCPKeepAlive=yes \ + -L "127.0.0.1:${LOCAL_PORT}:127.0.0.1:${REMOTE_PORT}" \ + -R "127.0.0.1:${MINIO_PORT}:127.0.0.1:${MINIO_PORT}" \ + -R "127.0.0.1:${API_PORT}:127.0.0.1:${API_PORT}" \ + "$SSH_HOST" + echo "$(date '+%H:%M:%S') tunnel dropped, reconnecting in ${RETRY_DELAY}s..." >&2 + sleep "$RETRY_DELAY" +done diff --git a/processing_services/bioclip/weights_test/bioclip25_logreg_categories.json b/processing_services/bioclip/weights_test/bioclip25_logreg_categories.json new file mode 100644 index 000000000..fe5eb3800 --- /dev/null +++ b/processing_services/bioclip/weights_test/bioclip25_logreg_categories.json @@ -0,0 +1,62 @@ +[ + { + "index": 0, + "label": "Actias luna", + "taxon_rank": "SPECIES", + "source_class": "0" + }, + { + "index": 1, + "label": "Automeris io", + "taxon_rank": "SPECIES", + "source_class": "1" + }, + { + "index": 2, + "label": "Hyalophora cecropia", + "taxon_rank": "SPECIES", + "source_class": "2" + }, + { + "index": 3, + "label": "Malacosoma disstria", + "taxon_rank": "SPECIES", + "source_class": "3" + }, + { + "index": 4, + "label": "Orgyia leucostigma", + "taxon_rank": "SPECIES", + "source_class": "4" + }, + { + "index": 5, + "label": "Lymantria dispar", + "taxon_rank": "SPECIES", + "source_class": "5" + }, + { + "index": 6, + "label": "Spilosoma virginica", + "taxon_rank": "SPECIES", + "source_class": "6" + }, + { + "index": 7, + "label": "Halysidota tessellaris", + "taxon_rank": "SPECIES", + "source_class": "7" + }, + { + "index": 8, + "label": "Datana ministra", + "taxon_rank": "SPECIES", + "source_class": "8" + }, + { + "index": 9, + "label": "Euchaetes egle", + "taxon_rank": "SPECIES", + "source_class": "9" + } +] \ No newline at end of file diff --git a/processing_services/bioclip/weights_test/bioclip25_logreg_head.pt b/processing_services/bioclip/weights_test/bioclip25_logreg_head.pt new file mode 100644 index 0000000000000000000000000000000000000000..8abbe4eb3d65753d1b50cfb3d2de6c2a2ddbf56f GIT binary patch literal 43093 zcmbrlc~p+k_cz=;sZb;pNrNV-DC)lUiBJ?0N=S*K(xAzZB+Zkelu`(VGFOJWuf1gm zMF`0pNlGF^=6LS!^LyUE-t}GUdDnZbb?$quYhCA@&)MgkefID<^PMH7#Kh#}#QtB; zXfY|Vl>wovf&$iAT6qVBt_k;BKpt5 z*7!$;iLZ(b4PWJN<{cFo5EP-{9qzX>Dj>+$JJK&CA~f84y`@5!grT8_tf)qW=u4k9 zeqoaTQ$@icDAXr%;s3l;2$S*<7ky(MChZ|9`d_s!Dokdsm|3X zM1(2(i!k_0`u|TK7GZ<^WtkI*TWTsigV;r><&b*suj*!#(6hb^U@@ z`uTeM1_XzxiF)=JBzhGR73{q#$R{GgFCuJ&vx0|$=v{EAuV0XNh)=Lzn7W6a=!O64 z@%~XOZ3BWuqYVuXSQRnh|44ScrJ2?E^#PIo7L&q8dJOwdd!gaMK0yJ|e%}APGv9!a zH4$MNsZnP0on__LYdm=zD3Z#5%KZO2bNXV_|I5sot^QBzk>2b5!XreIa;K;&v!3)H0CpR~x+O%N z|EK(S0NP>}{{_IxO_6?~|5-E+_T&HW`u%^T78DQ}8RYjL3`y{Mqt8s^Y9-2 za39R1en``#o$74Z-7Gfp>`pkJ-HaOohoRibuP~`4+GbVhGS1~k3?I40Sh)T53YaNe z%--04$6NV?UfPZnw4(_B%>Ba6U-%GI_xPiVxDi{E6$3i2H-f%=IJ#)falXFttpL8m!pF4zs%GmpdMdF@b{@sZE3*}`TY%?1B~`*@vFK+Px?lbe<4 zrNd3gzkPkF|nE&G$`nr{(+dDHhti>76?he7n3twLyn6{54qFY=Ap8^VrBkj7@2>5xmZ^-I}L+iL6{ zmItqYrGsm_4V7ls;01?x7HmHoJzNy<*?b|qco@gcQiZNb%#1Pog&EoFbzo}sZxmGPQROLpE<^1X z)VSs_^FtXJ&E}J-Mi~}Vo6}CscD|q`gvJQ3!?9PAEPB;-*pwH9xwSp``(Fbzk9u7@ z!(EdlzwhUL)q>d6B~5(BlW3aix|EYu`^~o-B=DNYZsSR*JKQ0sHX$56%31Auj)y-l zWlneYgY-8II_jp)+|);M?#IKphY`x8d-*PZw9fsya)_iQ7eRUQTj;19O_Q(eg7xREnB=`Ij2q!bMlaX0dXtq@6Pv^fzP{qi zoVP*OzdyXwCpjjUy936VHQ~$~%^=aML*M(C(u!%?5PmWppM6wg>gBcg&G9f?^;9Ht z=^mk4$uQiA55Y7t1O;ygvzcm|Y_*jxB^bU3<*>78|8YEPQcr>BFH^vEj5J9HiqZN* zS0E`SfK^QCgNx3ow!6=dqsAq(s9~xnJ!-4OhI704{sn!o{P_r|XOnQS<8fSUd<71i zFr#~x`S@e;J>in&BbnIvXPDG7oa}#?bMX&5xk-sVc#n9{xSm3P=bhw&uRjI<^3(7$ zu^&$D{Rak8Q{c_=?XbbPO)y)umbJF4(e_$v{C3Qcoz9rRc0Ca1N9>qR)8RFrSUr~A zRzC%w_w9g2r>$80)tR{jZNu}}0^{`7hf zlf4N8-?p*^dpjZXSq3NRT?E!yn<-+A0ZIHg%Mwt&Ra+DR@=zd2APn-+cnTRua*1N!gJT( zWzgYA2OurzCDiv?L*;_iY<)#LUv55y9OneF-&`)P7tUhYPd>t)@u5uNz9T(jT2$Pe z0E&O|@a(v8Z2qzgxY%)#{V8i;dKE%!^p{7i$OPz+oXC#rb>Z}8J8Vk02#Pawsn?B1 z;rzkS)8fRoitCV>v@wNgHt~u4SuRd(33G`0$IV_@54uhR=$bb(yE5KGiTzTgb>!>X+Sw6EJ=(& zscP%-w)R=>HLcmnlEdEf8yu$FI^CYlzQ=qLmdf0LPP=SM z$kJfDHdn(ks}P)et_s2~Ps4?i6sTdY7pI%(ReOuG<#{{$7SEJkkX=J&%M zmu7JF3t?}-k>u~s#J5i?F}`yGds|S2ex2cvI%gTizL3Y#Ti%%6G>tl}ZOPxy7lrc_ zm~w;dk=1kY@aFH)mUQCSjr0v!sWa2NpkoA%Bo$5bG|whs-DDZ!e#Ke z^&-~u`5acxRv>|$2J1O&4@$q&*aF82{-ODOf#tt_TxmoNKdM+c~g=P}gyrV-b;l5S`2=8`)-u=2||Y&2iQp33h74KX#|HQ*t@lTqyV$3k|)(GQ%S zYOraOGuh0-gZ%RF@k}$|4_+OwWLuH83(g-~PL1BT@!hK{IL4X7#qXwraJLL?m{f+| zK3(9aSPh9|18KNU3=Zqj#*Nv2tY5z!29!5-<6@1H*l@WBcHZ+3S5t78zcJz+PX9Q7*GhG0(VJ|1 z*#Mw*E%d;)nhBP-T4SWjIG2Nq%_cB<)tuT>AfzfY1bO?{9n{RW*Ers~ePlg6Q z*5IyNEwEeO1Fy&zv9RM(?BV=hyn63^x_UDUrZqnS8|64wm%kqxW{qS|49BtuR)c8m zULjq5`V2!~9Kzq>Zurg33~mpIahsby@O~)~?CGr-Fkc@*lONPzcFtCqIP)Ej8d@)O zKUG2(|CF$hVa<3=yAJi^7jRF@ck!__1{GiDu<;)?$Xsp*1)Ms`suql=EL@7-FBRZN zKnfHaNwT5S!kOyTdVw%-0&~<~!6dKz!{}LMe6c_thmKI73VSn}vq+JaIF-_D*_AA( ztdB1fmn2o!c&cm_kVV7X+LR|UxIBCs#DwTm-{MH>dl|>)Y*nIbjs;jhs+FIe`W1t$ zZgW=71>}>lmpiaQi#hu9oY*oM_Cp^Xx+RN&SGfZ)Dn(5YBWu2*Nl$Mt7nl2{e@)%GT)efY+YahV0X^Asq`T9$u! z{S7y0vjTOcw&Cpa%lMnwVobg}1|JXGi^({gtIOu`bes0iQ*$OTy zVJ!5?cUV2y7INqQ#jM;!<{fHACaKxDq$Qcw31i{>R$W*Srprb+81UiubMe%u#W45b zBsL}QG>kqx1!mmM$7;bBo96DQ&2ij#2fr6HvF z{((Wgqv?L1CHu477BgSXQ&<`mfzX~Oci$xJ714DG#{!;jKQAg7>cXt>Y{A(n;A zrpJcOzmmZ@9N9@bYc8U+W7r9UcH|TPHl(k z^8DI6Mw(37M@tZ#@dhd+-{3N~44R+*m3vr za38rD_uj)@1APjgC`LykGGX^hJ4h~Di?$~??#sz@utC)k29F(uM>@~ryue0$R`C?~ zyG+ zxZnbw*qFqurDP!f;BQp$o&XjP`-O)Zq%eMWJuG`H3IA>@u!EUS)S;1vY^^tyUww%k zwCQobcMu(1782C93&$+!q2h}rh@wx{$I7wBI(T>@+Rr%%*S=<=Oqu)`SBXh6??Da1y`TC~Zp0e6?y!$f$QwuX4bq&$ z;2YT6a*NyZITC6Py~b|Cu~gx045kkXSlsntY~jYEn64H>$)dCTcKSWOpO)H2RV@+8 z)|$lfJ9vJ+1*>p84qAGi__N>zw&-M|e{&B%{a`+3PD&w*+;oYApe=ic-6C{F`)=|#Lotgt7q8h zGwabTw-p>ePhoX-oAJbGQ||E|X>dz^jGL;nvCHcdDrzY*_t$lF;Z!qDZNGzul^Vdw z$&_|?S~8v7(_FA^6S*7QfXdAseDTp8n6qDW_CELoS#9gs?klRab#nrm_~mhL&C}T& z5MqkS9FWx4Vs8&j#ks5RqeoC|EgfISCMf0#mswxMUB@n9^qhdIP-2Ym6hi5`$J=Jdzl_p zD~uFqR~Jo4H8Ud^nXqgYjoWsj)?gcH5l8k!g2vZOtZ{g3c)M;s~}}wE=#X zs>qKTPG1#x^e8Ra$M37NaWYb@e?bCN1r>sl;WvEpyMm3ongnOvCQ@B`J6;v$;Dg;#Snsow z`5pcWGaO6NXSpkN9>~FujdHa1$!EyfnFQB7E3t1@FIU~D!{m3)fS=!=;}5|^x)F0l z5c68JKsr~mo~t#+2bMrc4vG4cW${9o&5<0cDI(Vipm~?8(c2;JY*sKc+UohxNa4(spCY zHj$&Va~|=xj771?zM%{=2eZ=RTew889j7cj%T<*tvO1OH%&;nlMU5MP6~^P3X8Uda ziPd08an``ImfiSwPC4HZWz23TE#T7+ucl2jhB*6GsH3Mr-%4L%i`ss`_iy0)4|muS zB~KM2oDt(c!m&wOwD{9sNKd>a(7zMMo2XBwv6YH!_piHr-yavI{$L01_duE@p8g^H zdh;gNUmFKjmlo4$D8#H)@t_s{hOa#=#<;tWICJqXEP1+>CN-Xen)M+NmF-N0d-l+x zbEdfMLLZ#&S;#F2DZr>@z;61E#uZDCQdE62gcp3}8`@vt*1SlF$S=ZMMWvM9Rt{gc z7~+AskKuPgDs;44(+yeTFD6|B@d1}w*@As+&YKW+WScrO+GW74Z@r2!X_HaT{WZF8 zJV7ZEVq_q1iG8o)F?Q`);V0*MeD$#m{?SIAuLy9H;VI3gMpt;>;Lslnbo26kzf z$Zo%Q15Z5`C-+Os=w-xensKWR&YkGO^xCN`c>EL4%KFS-NHoH=M;=37Z~>pX_9s8a zb`eJ9&Eq|DQn9woo6ndi&wjkgrCO^Vm{VoSUVS;t{q~K4{Rdyc7G)iplYANuj8Vs{ zAc4aN{DhmnSkdA=`qXxG9EJW&#HO#E+$P6IFv#1KDXNU8wXHG`kYkB$dFlk$vT*|Y z1PLGV;j+$NGP&3x*konNYi2&dm*?U@p>rMjy*dw%{w*P6sRTT$H3K#;O^4;}P1qYb zh#a0(!Ths4^kxpF4Jxt1&3qz^94U`0EOy|Q8cQBr<%L3n!&pD@Xl-xzTexy)Itw-` z<2FtdXDjRy(R0u_s`*(?d+!9%j)<#Zw7wB9ZTN=q!6T^4@+Vh)avQysoek}i2Gh1_ zbGhtOx@zFRdWtx*sjI{CPLie;!De?$+6_5GITe13`4~) z^6@JHmjxBzt>Y&6Bcz^FtCE3Xv+9^nbsualzl}58{h)K`PaNO!92GlWgC!)B+;%af z$D+OXxe~kzM#Dw74hWoC&q>5x2Fv1c6zwvTYG>_WzY?@pp2I^DUH02pB((y)Su$qj$eDJadZv@YMkNK)(v782cDq4h82x# z3KLB0Qzo(R&u}v(kf91kYy4{1R%KrpIH1iQKiSRdq=InKb2nx$+>O1{;yH)(W9-4J zDlpocj&e!GoQ*PLrL`m|YaY!!N3}zfpEENM=tE3r0vuLZ58wKi(*l=ZDEH3-hX<-S z^T;SBcOjE5Epn;l>VM)ix(lBltFSp=MswU0245Fe@h_Z*flBobTokM;kX7gLOKUhr ze<{QZeX6wfUas)1m=v{*O~*U4BMI$eDEePNPPPc^+y=l0P3*7tQ_MqQ|ML8*%I}9rjUcHU`HRvDOiGEaprHoNYB@*HgxGFFyZ*-KjnBwaJkc zHN59vnUx45rJul}7n^AE!(}Y{+W>^WISmb)w=!uHT?$^j63;$Ev^Kg1RmW4IX6JMM z^P~ZsU^aqMHm<^hcKacE{u^kz5C8!p|IFDcoqQ98psKMN>{?W*dHYSid-E6Wr%MT6 zq4AINGqUBZ&<;M#FsD#48}7S*EQRhLN(HkkIms0lIJvV^Nv&UL`sRg$W`-a+gOj*Cve3%~bg}-flmG?cYMrRra(Ki%I0&)ybsbB@{SBiqp(5Vf5tHe&{sL2b<$__>?1m@$SFbG~)bD?q#A8m^IqbzHtxv z^EtzqeBf|0cdP{6i@TY%!w#l7XFTeOe7W3Pcksh|Z|KXlVPocZ;;Xpv=veUsqQ+X% zHXmg&vde~;-Uax{MFn3*X0k`NQk3bv034fhh&h$Q38mp|X{H301X{!VE7M`4bu|Vx zPbdA4s+7l@aI@zPf<6AaRO~XJyPG!Pi`f#{Wq#w><`^RzYN<9_)pd3}U$YZ1+SL`FiSc&%f$jNNhl()8SF~02np;DB0Z{QXR74hL| zP4-1}E-Pv7K;`CE{BC;@j*J1!>oTA_{qKc$l2=md&tY`n;1G(5TnU@Bcff`%bx;pJ z{EQpAC^#U?yS`b*DBut8EAqvM8l~{Lsc~SD>qRynRoL#iQ`odY1>m^PfL&bm7ZTNS zQ8|sLTl!z1eZ^b8q*#vmbY`&4haaKa@-m!V^9Qg6wAqpC+gatbFs3~z z4XUo?GqE;Va=tEy?;F;mv{40~xsZ+5hCD`3zhe60d<2dBifxY{zXK90PSom4?|})+w~W zd&5DH-EoUIDG?Kx&&p=YOU^RsYLV`2XyX(k&%w5SIY^Bhi(b}|n5d^k`A0s$5t)me z<(q5@ifctN%Lg#){3^J0daS@Ny$Lg9o$!k0N4QBz+{z6~@Mha#>Z%ckT6=fEf;5s& zsNgm^uLYZ}RrqpoHQyrCC$V4KY46wP{JxQWs8e;2YDddapy_HFeefaoOfEp6erFY< z21C>}3}l0oa``nUQ?an@Dfjl2Ib?^VV~>c(^~iDuMM|pDu}78o%S4iGEYZW$w*pac z;}awqP9dl9(X_+00PjC4V*QK6NI6S_h2363zmEL@&4ZFObbAKo*?BO_MWu8(_5~{4 zH~~XL>udMVK7tmVDHz+@hEsnZ#oJ~N!K!y6NEAu2XP=LttzI%4?VJYAuC8!+unAo- zH>B@Liso4@OAHG{@(53c(+)L*4~{+mbbfXJAPc_H=fF5gOp{s>8+7)+c|Eez}1$>}DIr@0}iTtZ8XC~g6TjCE*uXf#u5V({hbQErdxI@q{Kz#3CmQC^xYb>TSnA!RjL zzPDkTdU0f>aYrz=XHu=%>CLQe7?0;;<7?HAbi$gfhoB{`MSA0ovA?_gnfX~CI&{sC z@g08!@!swvJNP~hHG2dhTDDyKjze&K$!ySBrVcwoBFSn^rifP-!rV76Wzs$qN#UIY zyWi4+a@K>H!4OT9ODqA)s29AMUlzt}^`l~SIj-%=evp+eg-uM03vdnLLR0U6*s5dv z&ztTPStP^E=H7-qtNx%vR|0pfR+mCks-WfaNDzLV%z0%0#=&9V;9~DPRD7t$Tv~rJ zK6EB7T(8bvZ+neKh8kGXSb%e1UPPbdLR=SHg!iw$#)rKcOv?26iNw;!1!YbtCqbAdD4f`!R0bsR(T*4CRpJ) z?@1K7Dg*GsQEtP)9Tc08%PXGU1n0{V*|n8>P)JprNZK^V`3LbF;`~ z{1s_DGx-zuKuI3v)-}N6zauDQe>s2i@HD)7J_QdRZR6}F?Sm`7qba_(l~>-}gA-EJ zD0an3+t*)`7~YCz#g(`Cx!+5`Vr2*BUQ%U!5CvXo$>i`&h5D~m5VBY2doh1qF?1gc5!r_0S^L=>&@-Hj5BSM= z)Vq;C=r#tbkk@5i4rufe$mvnk}gAv{=?AlUj_oo(2sOk0%p z(voG#`22|#{q(qoQ*$I(o0$NQ>ZC!&tm(W`ZZWp&-4VQ<;>)%hbEK;(#*%8b^XDu5 zP$FUoboDy3x??R!;Gerhzi!jID2MZ>Rr)n4>n76nOU+2tX4iZhg z)5>7FWM&JtXL@1gXeZRk@8YKX9z{=G-lNcr-6I*4IY#VBj;U39x7 z&hCVN!`sqB$>XxL?dHu>$ev$>Qp=~)!ki<_N%k==j9AIEzb>SuJM1xGj5gWqN@dn* z2{7jBcF-|Xg_-R+;B$Hu3p~-x*?v|bJ7&j9M7bax&s=QCwV@}oIkxWk5IP|iK>g#Q zuzmXiI^QAzy1B3U&9^7f&fNpt?_yVaQya%rd$Y++YZBG$Jp|6*^XTTSm;7B(zLi>4 zz@%as?d{#kHVqYWmJKtQ#`8lsqqh4%unPxTILd@NFJ@Q!|95CSMX-Z;)c`mFd*6#TJxvCo&ro zNs7C#g*`*f$ibu(Qw$eVdxkyT3!aYOALo)ah{pIeoR(K6uzkm6NoDaK&PsDPZaK~i z-OLME(Y;gf>{=eAEZKAEN`OU zl2lyW`~U_zv&jF;7)p07M2B_R{3P>OHf5rKmFho4C-bFLGw&83Vl!TJZcoO6MYlCoLM-LwKiO!x{f~FZQ;VY7c63{GSBjNwCA(N$>sdff&*;Y*AhH_HHXD@ zPvtz<@fh8+54GRjgPN5_WD%swz7#ow_VZ%!^1lM7D{`=+Z$I32{fWwU$-Hfx8X8g{ zWPdy^viZ({X-*x4eo-VlQEc>MQyKTv+n9X`9|w*;jx6+3Jvzm{<|HYf{lz4v_?9jT9_zS z|LZEZUAju3w{0;q9qR&7Du)HbZS+7lbSfT`ZlIO(Rrwp4q;+81YV`|EsYEf~WZA8%ue zrcY*lH?~o=@)@o)eU z%U7ZKvn}ZJ!S%w0@#!#zbLZo3eCHM!uA}iA#i=9Af<Dx@P_Qyb+!fm(h?@%x@vq5m;Y<~?jEw_s*6DUQs*(Qp>*Bx`vY zS6PU_O(@@)N}ok_uiq+xb4ezwL?IshcebFa-VSKmYs>%f-^YtLsv*8;13MirmgI2( zMsCX^t5xN^#NPc})3Ru$HTo*LjwqqW%L7Q$J(+gu-p4UtNAtf!!D>Jo0|vAxH1GRt{I_ad??v-3)z12xu6Zcp3vm>E?Njv=cfx=#|ClVvnN91g%{{Hzm_?- zM6*M!*RW|wA@|~U1Dr^iN)}hW$=2Xl0% zqv!8J)?p`yy1I&}mE+FhvSxDcJYCqAYh`?zFaXwwwc!AFlHG5f$*<^_gf|!WvTa3s z_+trGXudp=HbgH4*T-@&f6p)oIH*pZ8{R%i z4Px4r7*{35zu0St6Q{OdL#!36*v_L%mB^3ET}#U~hO*JuTkzahO_86I0MjZS!k1zT z`m6d2H@(YXj@{>?G7Irm#DzBqB8lK;T@9GLo4 zo+*6wWc9JPFf&vE_FNu;f3K9l-{G?CY||ZHp;evrJksD#iR;peIeDD+l$|U-*cac7 z72RjJv|Pk*(qX2hL+C)uDoFf@FkrluX0H*z*uke!A^Sc*Vv{_Kf0vAj(R=9K%urgT z_zu=fPiH@w4R!>$uu<8|FjRaZtdY)vvJIa+yC>N2#0C@Q z)K(?-%d=>g{xuH0K5^AS zEkeV~Dr{eyH0>Pmo~wFSB~Y300w!j!V9OyL=Zo&2)W>WCl|eNmbIyu%-|WNpTIrCp z{y6G8CBT4!Cat;V#5!AL+2@LLu&U02=C*f3eZL%S?^I3=2DH5QMYdm6zo2R z(JK_NNGSvCK8)f+#2Zn5x-n_1Jw&zQ*|catj9R|df_bb4e$|#j=V308I?J7O1sOR7&kqyaT-e=l zdf!^ab$uNOsB1 zo<+oOK+_kqnfY4@d>~>yAMdZA_|nr*-G2+0CoY3(lRPL8#a)JPF=J+W7PP^{i0xEM zq8hV*HmxTY(#?5KVNInrt+^it8z&8i?_-=u`O|dTQazU@7EWe<_948y)*SXLWEo62 z){8>jiR|FDTo@Ltf%mmUHoamDd)I6W$3pi(!u}1Y(DR$e<~dx>rhd5m>nWt%God-J zm00Vc*`O?`N3NR_`KB&Sc6)jR`5m5)4ue**K-CrW;c@5Q7-hr-fP^32GylFRBU1Yu7GXuLLMM=i&q@!lM6!dyN0dWXlAB70#(<`ic7 z{Uxk!(q<*k8@OsuH8v&OisDShGednzn)*9~x~5h#X}4u8_NzX9T_nn#w^nl3o_dMy z!5C4d^bn@5Q3}DH%D7}#7yf!t%1V!jbnC1j{HZ;Oj^tTVd6OG;s%-|=AH&XM*}-7# zK{V|9YhDojmG_*w1RGthqqK`5OCG(L&aZQ0zMcp8@e>!)#bC3kfP)h+i<0~A|>6Q%YaFd z{uNOiWb0aD!xX6b$$30FOpPy%_hMqr>sW#8N`e2m5=dCB!4`bh2cr|G;i=9)?n!tF zNq!##y8>(RkyH;Dgzshh!Zx5!y($BVDm*F50rXY9<&OTcp-lok_!4P_eKLPw)H5SU z&edSUpY-FnkQ%UQyvne80~qWWN7f5Nva3+q#eNZnb~lqYaS-vyv-kpwSrwTJL%g& zYwp75A#84;B>YK{h7G1_bouRf{>0lC@X&S_X}8G&>Wu>5-^R?`+Ke{oi1OS!HnF$A zHAzO-nPvB?(uk09I50RHj(ca|vxVc~L8Usmx?ICaxh`B=j01fO@Wqscr7&qg6$ZyV z<>Wm~Sysy=Hujw>2Bp6MjWKrgXTJ^GK6?|kybGbQH^nGzmB!k4xzhgh4Vbs$2o6~8 zWzL#w=zR4;GL7Ae^MsFZ@X$)UB~^ho&mHNJvkkc9sxkk6v!JFr84ITQ!Q|KLS#7@~ zGtcv7-RH)z4C6qiWjBg-WEb*VBpWfosTHy&7m-wT6`MCy7h;R1QKa8B=szX~<~iTF zWamZP(XO4fuVy^u&#aE)PG%=l`KFVwfXZ3!=e791C^`>+EZ;AVD?1YGvt;utr zBZQ)DR)n-jX(&obiO5cUHH?H%R3zlN&lOE2Dh(Pc4W*%_LF;$_{sOOOxS#7fpZDi| z*3aQZTO}s59B*Ifo0Q16|2&8Wc!En>5sERnmiSFW4R1@tf&HX+;BKCQUCEb0yRMd$ z7R{x?zj|mgJ&@Tv7IK302VCOdU;Lv`M|MUsfhJ#Tpjmb=srp%>`1jl|Abl*JJV=gd z2Q8*klHIWRj?i<_MI7YaLj6j^@W0e>SR&B}3xeYCnRY5=q$}cE;mq9PI}5jpK9lv6 zUh-=^MY4&3bn>4DojKP_+qLgN(koYZS-1nD8f4kA@D|?U>QksJZKGYkB(YHOAAjaT zE$rQVfc;>Pp=j7lHl}B%c#e+)Ho8UOf@kmWM&U=gS*XTLjpgm!5~j0}=O>_E)IZwg z><0d>Pswo6A1WWE4FhVAa1{nR?Bvy3ut#SzI5s^Jr|b&CDRvB#zAEC6E(NHa<-k=> zd<9vK!65$J+OR#5oJwt z$Dw2X@XO3bv6A!pto@^_Fqyy8`!S#wt8)@Rs>(p%c;$LtJGsRzr!iB<+%LCJ9recitX+>OJ0*c!L@=R?CWT0e3iCQ zJod{Tbp7Qd{=8x&JEyP9-_;$$M%dh?$yW&+Hw$dXq5*VUMF~zWIzu{DDI|YoEynxB zvT-Smu%Z1fIexHVbJ9A6Ipzkj+sMV76sGY51Gn&MZ(SgK=T1&7Y&$=uEJ_@6XBDef zn~e?C5kRkwi!RBiz>o+T>`*Uc_73g*i-)hNEXx@?BxA66$xx7NUI#x0?q*T9J?PuU zbapdGOh<;4Qul3XywtN5zP%pECMU|^y1f=C9W5vBd|yMeWcP?}w65l!E}6>xQYZ$u z33K=()t#`?Bc0MF=5t^F1XI%cHMlg^0t@%{LuStraas0mo9icZ!0nAQlO8vY9Y|~9 zn;m`W%hxM3y6y-c@$mwcE*(o9rXw*T=of7~`H{1Ww4)ARfr~n{A1*$%fjhVV@rxec z1&YzZdvh!izsIm850dHAGzk`H(g((`-_g&ztNDL-Y;m!gD_OZ5fz59ls6ShS8YVk) zDfh!@m4h$4JWq@Dmi2M_+@JFHcKNtEodeB#TzG^TYeHTN9pMAk;LA=ndZsIij9r*1gFUWZ24bEfy z8pz1#pv(>9g^bvQ-Tx^DL)yUx6*O~60Sz>5aU0y9HW6|Yl6jG$2j6&jDz6(qj)m`- zPeIXFC^IFF<-7O56~XFRg|v5uFrQS3!5$OqgP~H!WW-@X*H%;Y-~ucwFy;QDwTQ zeZre6mOiD6n8a&t6S{EyY8Xp;8asrU zm&6#lZTblw!ZkP<{)_%L`*UVvIl6OrF&GBs!1w12I(;VbPR^HTn^hkCY|?-MyAM*~ z>vHthssR1)<-C9ESZ>>q7u=D&p;Q#4fm#)bY)H5nG!zZRw~lX!n_mK^8{Bc!m#?C! z0b{VY(pO~PHUXZDT}f;Bv;4`*JUE(h+;+8L5xlaQ#}^srko$>78e7=Li<5#7|9G%M zm%aR2&9nT|Ic8j5Vm^AwS!2ubgY53a43zL3k2R^e@Lv5qH0Qpd_LFtcptl)f=uYORV~gwEyI_dX3!X2%FM65wkbmNFm>e`zkZ<0A{htt- zco|iO#fq}j_OU7c_1voca;$e$2ELHjf?B&T;O%OK4_pNRZec$p`u(DO|8sm_ge-d- ze1_k!J`^%uU*Igi)yH5_o+2FutZY`u0y^5$XSQUqN1+x_; ztGI=OELnJZjL^kaV3rOE@NT~Z20Txrom1SQO|KjB!`pe$*)Euxnoa?iyQwboDaQ}~ z406eetmJScE(+d7Rm%qAedQAF&gyKe2(w|LCv{A+pcJga)!EvY3otKo3(hpxLF-9( zY2!fyIvVmF7HQ0fd*SOiWgA8O;-$?_+Ue4upN(+$b*8BG=tYR^Ea$ov&cNw&{V;rT z7N4>DBs7j2hy9D1>6mf=os@Nh#6UkbZe0N7yTplVwGZ(pP1Z40IxU_ zdytAV68Nmr$83mu8Mm;!fS2BR5;Uf~5$&+mW0Ik+n4R28YcdBjt6?^*FLEYMlQP1W zV{Gx#J9#j5y~&l?88f|^vh2V$chIC9(i5G>XGsIlHGB$t>pzeiDr5y0wtuFkLArE6 z{2!|b6ga4g0c_UmdTMhs;6yRE!0w9*K6|C4htnF`ONW4U>aB+OotNukA_)V6IW;|1k>dc-+CYGnznK9tY)Ep=x5*68A~ z8ZG8}=>q;fH4=SGglECi9zMsRRk(|klJa;;3^6Ly%>#o`!t?o}tbrGBPwfwNHO z-6~pleFC+N4F$uq9LrvHQMC7UFjhbLLTanJxwi8u7(RRqhID_TO&?}5`=5opeo7lX zU+M^-4y>iGlBY3H?E$y1X%F-*RNxzW>*1Qv8FlC$!ta05$RclKijofl*3BXDs<+*o z>$;(MNui|f&z}S2AjNaXZe-)}KcOgj^e8m*(GX~}iSowoVXXnl{Hikn6ssG;I@4;Q z@V+n8ixE2aK05fY>j2C)6y}bJ=6FO|mwa97L2a7>=)1Miz{r7QIn140zkBeHc8^B= z*H^*%=V%NxXyUS_)I!49Y_R*@52lxHQ_J}{)cTqS*87k1Q{0m2PD3^YOg+Me-gpmZ zW@%Er@pYPWYBzp4>cg%b3c`VN9WZISH=F*qk9va2`SpW5Ao;5BjG4HO{a)10J~*C-Md9W~^9k8H;SGgL1Xiq&&9_(|x1pYn=t7V6Mu z_!p7&P`ff5t$QbO?PdqacAz4=rBcpy1`Cd*mVPK7D8f+hMGckclNH4VYkHIi)gmkeH7WI&u_EDLz6!zVC*nyJPYD;lp&W@37cQ=85>o z^1s|3w*<(w$R}E7&H}u(*qcay{50MYrrP*2eU+=A{Zs>shHR$kCXe`CLRWZfV+B>o zOR})djr{&M!r7>MF(p>V;RTI;>WO*@lFv&Z-aVQ5XzRi;J1GkLW=I;RQ#k!(W#$#1 z$Sw~^r)Sz(U^gaCeC&nbKRCG(E>G&IGuot0ZLRGzd}SVcTPee=2UPG+&t}1iGZR^t z)-fp29fJM15*=>@V~}Ypl>9biCp*(f(Qh9q>*!$2)MAQTrAh6=UA|@gAxKm@4wsFt z@Y~io!M!u)SRi30^o5T-kfwB!p^+(j7j=yS^5zIWo8fG&(_R=^If1oV z=%b|WR+9hnn`|wYvM7rHyf^tIdR5=!>Z&8?^(%ik8*G6k#kXm$w-?QRr^2?SD6knG z>J+P`iY3wJ?5gW7-2OQmy1a$##@dj*^>${_l0#s(<`$M!Cc{21(?he8?R5N}4i-e0 zL(9ZjELXb@io3jtc{dOwB9K3wLmNHCx z!V^;c?T94l2kI6LBB$6(qBZ?9Xzh`0Fq;aYd-()5b$1c7FF22Rc9&>i=s4bU+G}!q zJPIG#I#K*Pj^)M;;D1bzzz@uhO3dY9P`o2oByozXkS`O>U$qB3%-mS&>+Mu5^^fbg zxst`mIIz>5+E{p~hI@cRF}r047=%t^?srFs58jNT>gLP*^d=uDv=Urd6aXa|@A+Q_ zH=wK4nGjCE8BQKwhNkc=wL#n>>5Jyi}LlO?U{noVK_GI2&`F_Nxi-<5T<#G z-8Emzj?Z&M|Bxu!y?Y-86)OlVX*3M{e26{0Fj%;Qq=3_CdAJl+NpDrHITv4l?t?>N z-I%op$>gjF4p_gG)>z!Ou`Ni)dmc6{t8FNkVLbrmuC%5jLN92#whih1v|{DsQ(;qx zIm8Niy|_pnoANW^#vOZB_hJvTDb1p@HlIY$DK$Y()fw(md;#Ie!~8PAVNlyTlr8pG zKyQJ6u({(1*Kg-?T@Jn!M2QvPWg85eya-&o{M4G1sRsSxIWdn`$PO7ABoaf+k7 zSj4sx7Vo%x^88|ZF#WU-Of9a{{$;EA_NW{D++pvi)c*sQX)qEmjG2M5X4BcT#}T;k zO%MyY5yuby`;B5po8qypRs3N4*~~`MjV-X$B)^$@`0Vr|7ILy5)`5_1TBy>K`2}Pq zWQ=<&UAQraSMtMOh*+8Q1Ul>Q5Bh%(vaEL};K9T?QDVv%w6LwDd14d1=)E0g69cjv2AocbP9j4UJVsS?Pn#$wVa<(E7v_o6*c1=c%P6pkWwa3`yWrnLydx;Qoaoix%hy0 zk7Tin28_-uM$Mj5u-hhiU3sKy|M(>t|XIhKN?3N0@S>S4BCYhs& z=O}jJ=6ssC^c6(}3ioW8kzCyc9cJWr6Rb=Xpu<(@7z=kkhwH~k^h_VdhGlb?m51W9 zsYdL<)k0`WdrHl5KPge^8vi$Y8>gFW4@b{=lE!CuY%p7cQ#Snvvdj12w9}?wP&pYj zh6uc_1i>^7dFHcX1{?U*lbv}b%%w}8vdsQnXe9i;>M7KK=Wt2(kjyD~pE0wUO#IuQ zX7Jf>1beeFf}w zg&Q&JymOqs&ULQgTCeS2BjA;TD)^j=cKSIki>2-q7bKfHKhAH=^tL241Ir2X2I zlmGsSTYV;mTP8IQ$689^ej$q~*mZy;^EKFk5h1jpw?-5=e=ttXJHqLG(MI_#&$$!5 z6F6hXli>8FoC?+AVU~smehKR2=A6_;o2swGd#`6!VRIn)%YDds_M6U3SxB{q^I2?M zG~1%R3R9nW(f*H@MW@tevbJAB#+%{`9~KT{bEoK_Qn3oYuk4`}ztq{B-d4)?y+^k- zlvwF$4|rKD&!!peXE938cnPOs60DzKeC;u(TUJ01)rVl9qL7C>3y!4la+?ODonq@9 zbEsqaR+9GgVs=y4V84eMUe8UVq^DW@q?6ld{ly@#sgb6h+x7gc(21CNOaqleeW34F zAg=qn58ns7;Mid!SigTJ{m8$_o%pev4kz4(YcsyV%y-ivY}88x4+<-|( zF2KluKCX7@Q~qVA8$W2>ZirfUlXtD#B|f0{9{T1buw#u?(A5d+sBKl%Pov`Ktv2(tX%Di7@(Ow2vwe)>3Tl z7ix4S$i2S}5|77{j9CIx{u#p}dxoIH_GPHwo(Of8Q?Z-hM^+i>Y{t}hx-eOW3toMd zTNp5sE)A1|ia*;#|0y++fx8y|`>2P147Kp|t1aZSIF#?dtVee~yoS(+jufsge6Pko zg#{LK&}%0!zps0_Gn%?M%jOdp%+dto3DfY&JtL-8JeUSJ*@MFS`B2(607uM~qY*!j z!x(3ICR_5IB7FAK(w!UF4YgdDKGO;7j4aq+GodG{B8Ml1*Ui&!I=*EYdcIu5ma2I$ zrPCAeuf|?J%w;O`$=-#LMe=NAz9ZAfn8am?q}kuh?+|fN3Y+=fx@|^r_)XPV)N1zv zblOMK%I#O^=Pm35G zxJH3B(bm^w_AfdhdP^)coNnQ~lgeQF<(+iQ`626RR}@&(>-6u(Q4Eq=#Fm={vZq;t z;E<&j{HZMg_t8=0C*eUa_pip=+neb0dU;;8tOZ6tosYc%Kk3!@H?TcO=zN_wM91l4 z_^Vq02j-7N6N7(rJKLHmj~zw_XbG#?H5Aoc5d6$-ahYa4t>&6p`Q2$yIQs~UOK*Ud zz45eP`yZ^bp2DK~{z1INPOj~uFb7bPrE61WkkOKHX#7eUV~uoJ8quDyV=!q!6OHnlh$lV@Gb`76(mmnNhi)}sXGFH*tX>`VV6q$gD82+U zO4@A4q{Tv4@jj?u62>f_-G!O@0W9~jHEesNjHfRhhL&`Wsz?>5DsI853m2ir{Q-ES zda}=$#Ikes*`AhEOcQkUST;CgNv*aT~XU70OSMPub- zFk#yucE+e5lpOOh;_yP6Ju!lHsFcD0-Qn1nv{^J%?mzr2`vms+-G_RUOuDo*jLLf=&@)%J71>7xz$f+D|9MJA|#tv z4By3`=d0o+6KT|1`k9`Ws?oOqcQ~&63NE%8!w;VgY(|0v3-(MQ^;ZTk;YJjU+~&yS zmhNQZOT1b9S|gU2r-ZKtv~hXuxh$@F6228Oy9Q+)P*%Go^eTxTem)k**9)!zjYY85 zP>fs0^g&qUX%K%?!`aynESr%z;Ka3~e z-hgnCA?uEK#82`tX4jgIL)^@@(0s`lmZxgLvYelEZ1`@>yzmdMAKS$ex3^Q?TH(E` zy$4Fd#55%532c>OQH7X!D1WHD%K!71?ODQ zg)@qySY*4ZFdJ>+#zd^+mL+N61*!8~iF*u_8|I9@$rs`I0X;6~D-W@Yp3OUpeHTWWL&|=0OrYl2Hvj1RXVg(7B@qmUT@< zm+PYiuXi?VI9AK2W#^z}2cTS-Dht{>l*X+cPnXT)s2kpbPSIsrd*Bwx2=kZCHw^(>X&w?wq*^hH!K>1XD zWzHgIHERmf-_Z+h?N01YYc!^>U4(O8r&DiH8PlD{bA}NW6t&k9o1VYqCG1~NdDsEi z*c}W(H)F9Y&>E|&=HeCmpXA=N5^S@65!bnet-o}KU-H#%8FnIJJ_4Gi;{&z3p}&()PHxB07~_Qx>uGOM3fH6d5VFF%DWGNq`2T&xpDkIA?{!pI!c;G)STUIMb@1h6 zUdnJDC4&EMdLHccY2l;V`)T5?{VXnZ88esDf)CD3R36hwX0eyJdt)!bAuU-}cI+

tDds3_n|57s>lGzr;@bml@*%5e#vR7Ea8{-6~cqS+w|{66Fm(zr5{0u+2@

s+MvxTOif1h@L+sf9mB-V2l?id zRcPBX9+&*5!n{NY7#{SMTAmfKXU*2^VUxgLE^8o9#f`jlUMJtxX3anD34o0z@^pA^ zAhtfzV7B}AgWuJqWTSD96X8StTE@e=<+8^iJ9h(Tb*Kc&w1%=MaTNPJ+k-yHXQ6#! zDSfW6pxHa;gPY1xrYZOUBR?S4mTtlKGfRL)M!^QXe(JWgW})|Gnc*#hze@6OH9^dq zms;bBg!7E82A+~eDAsMa=#UDgrq`9%c~dG;~9mL4sfhyRLmy5zB9xj8O~ z(jn8yJA`a`F{eLf108x`PZxBox%S0d@Q3LV$S7jW#C0;;(OO4YjisP?M4OH;wHGpJ z3)=H4JTzbfcX>QVF>iQF4geZ&#OG#ttB}Kk0B8^F#D&w`( z)zH$fMbc~=y*1~Ub7nc!{0U`>`)1LBc1hGt$ReW$A7IG6lN6OW7Wkn@`A-kUqOVVg zcKJ%;X{Ttm`}{!Wn|%{ZJDk{1#ahz(Zy&ANB<8$U48tvgb8J_tHg~W7hsb@@0qB0^ z#CmRihB3>&LUwXEWjxIloJcdo0}a=cdgW;t{qQCHmc9e#$)CB`Yn3>&{YCJ7+g11$ z-3pfiCX@4oEjXqd-?m^ zA4ps_jg4^FjC1V;*SC5MbTmDr_dmyqa=uK(Y$ttwX+;eeAra1wpH>8?#n~`@Oc7M; z6~c+{L8!SUlZD)l6Mf0p0WS_@(6(3MWOl@q76z!Yxbzb7`!q*T_87@z)E)Wh%Gbfs zDp6!K{|!KvE50^Z1WwikjI*?7*R+P>iu_mj=uQh4H)0suwoRQWtXnVcYmUS>?xmc2 zL=z{YpJw|^V>W(o41tYjWoWU6t@za0uXJL!4hwlxkA?~-XwI^+IOEqd?%Re+Zcb4K zOgvTtSzdb~clt=wx<3_G$D}~P*E(>x_?UVYIT zPV*ZQNOEf)ck#6*s>se}4vBXlSvQqLJ@TY;#ezkq4g>8C=Cok>8D7Kj0F3;#9qo)f z1?E&0i;q8}Cudf$qMu$kF$ehj^HyP;hv4+v)IiU7UZSkGmmsPwi>e200<0g6Dl>;r zNP#o0SMK6M<6PON%!lwI_9U1c`U+nMKc|-ZO`O`7y~I!72vyEs;E2gqwoS{6t{yL= z9ox5&wjNTj-)Bl!e#JSm^{i#4FAl3U#uNiR7Lk_?!%7oq*~yhmrv3zeJvxqUy;?2S zI60k}8+MAEx%I5nsFjMg__6lccWI)GB(@5Ux~eRW$@}i7t(LPuX3RLeSO1EC`+Gd= zT4=~xT@G^-7x>dw$8=cn!)G=lkUS^^=f?VUWDbb=eaQ>XVcIYo}|O(V{EUGS8~qWfs%R{_+F2VIRA*8j@9Ci zf|>YlUnu)5Ig{pAdC;A0C&543mbC`QvKf!mQTl2JEnjm67MIN>Kluvqy4nGW{{**` zMKwIPHe>Wem8DgGg}(N8ppjc6`mH!0<<+a8<|e_SAZNx6PljhIGgwr*9jVL6;rCZA zRB3XV^D!F1tkip9{uD2~78-{yS8QTA+8bHl@)69$(2C6sz0K$6AaCuPg8aJ}v4p;%l3`=46sZ;c4;z*_ux0~f<3JkogCr!~$Mc=03BwhTT zen%UFe@Hb*UGia5kG_M@m>49#`K(d(Al^4IVq|cQe9I;Y`JMvm7`M`j4{}U*ST_ew zlGrT%#hsmTk)si4U)#G%EQ4H@6prVrTi7;N;n;D%F1RNviT0yIq-pN&dFvl#bTeR+J z8ijZ7W+iDq_@|n$;V54za?0!hv*W5ECg-=c-o0pFg&^%V z1?=rIFkD)S220yuXk8inQ-2O~A4IZu+2e84AAx_dxFS*?smS*JxWn7ZZQ^eioDrwb z9K;?34QEAbHsa(a34A2I8*7S2(EV*u?E2{cxYu3j6noi7U~J=2w#grIi;|dKK@6-Z z$OAX!X7P(`ceHK~sH-WO%@jPxu#G}ilQ3a4Gp)`>tp+&^8WF-S?M-GeKh-gM-UzPN zOOg{UHN#;`K5=Cq<7w9Rd{Q3yjEqh?v1NV81|6Hk-`F;p8Ll6W-^~2jPK$h)uC9+_ zgCz1Pj)#eHM@j3v(CL|70}K3I87OKaGZ>7wj6Rc9yc8bGI>4K2?c_fHdgP@rSRhXDb7R*SN-;jeM!eMujA?xfcC!XkjWu*sWszM&%rDR4!5} z7P59dHKax-l%5$yIo$`Kxl;)ptdNB+JHV|RyBrIYq+z&(1Y0ThT=*PDvz+%+xH0pM zz*S-wg+6kC?hn)9#LG&&we%XD37iB8l7-wgh2L;ZYbMg1YA8+|#PW^L;|Pm~)Uqjn z_{YI)NVWysx_FcK`Sp+A_F)LaFDcL>IhVCfYM>wAqQwS21b1F)5execG7EBz zj^9s%}fXBd2;K9BlOw3Hw9!Fcmgi= zC4tnmtN6jiPc+SL2Smiku*8i5$D?2iJIyb`gF`Eshw@H1A@P{5?eW2uu;q--nZfUq zUj;w5D=Tr)XAczSutx@NFy~+e#YOkh5@&(WbMnDI8%D8OA>WWRX5uBfE9maT9rQ9I z4XjrWVQuH_Q0dSZc2_b{9QC7%o~F;iM~mF}BT^gi+x{!yW~joZ9Qwld?cKnnMl@2a zZ#^j$yy3ToFepz(R7rmgtA%Y7la79eTSslsD(@M6>XxR0jS2MpbT;SxB$TC%_hS4V zFKD+p$Nx896)QVs;752H6>j`QwE~+ke924_3axC9_hN7r4Pv?iul%a*FLlUIrhW}Z zl|#e0M^_!$g-PYu%Aer}jh5l7jhwJ!-C^VteBE{+g9-zR9)QDX)|AR( z1b*UO=6>W?shF|H??!MdUuKYZVtVl^?U~Hm`A&YL}!lMan{3=Et-^k$-iPbPUqz_)^TcergIWoN@ z&rFSC=uE+Je$;>wP&q9YVYo3)N?44^Bhx{~H46i+-@>QQTflYsM6$nN4rSN0nQFa1 zJJ{gD8I=jo_aFXj{PhWNXhREZ3|)&~b6UagPc|uPAp6gu7%untU=R0^KYUJ~k7?52 zYFAp4w7>@hD7~PEt5=HZdTKdO8pO)D zrPs0MHTK*tpWD>Dz!xSgj3s>VTU4XBfVG?MhTg3c*~%3^NaE=z`Yi1K_$spo77R<~ z&AQt8?CJBlYEOHp?K~ycFH6ERzf-aC))^4>jKxiP)2OL`4{O_yD`WsCguBQx?(Lkr zkn3&0S@M@b$+H&}tQ)vub#vIaDhQv;RKwWpHo8CSE=esu11?1qV2!Cf+r1-{X~2JlP?|5Vl9?+&Szlf=QOepqu^^ zu1ubW#WPGIb+)|S7wviPfeGUn5N}a(T6LJ&{5D% zN$X^p1O5=NRteyS-OmC2_EOs8vKwu=>pW&3q;r}FaCBNESaAe?X6|gCrvb6%b98^k zEpE$+9$SB5C!La3E)2OMcmPjdhl_iKZuI?yXn*MqNY|g>sw-E3#Ix(<-Q7Z6^G30> zYu1pr`R%bhmBB zfYGE@(*sK*_JL-qun+KeFv7A_Hu{s`fa!QbQ4+)1b8Z6$>Y1bFV?WjsUC#SR?Z@yv zmuS2<$D;Ed(0fZamXq|6bH932{4LZLOoOs$!2vJmu}cMwqH0iiG=bE;c#t-9#TQ?d znDvzwE;f50#+rG+_w9wG`}r4*zimcP8UaK4j*x$O7&;lehEK0dnL}m~J?=dPzao}U zp;8+a%k6`*W+OVNGZ4>o-=}xE_Q+;b&D{ec&h`Dd*saGDwW%~$w=AMhL zUs1zl-N%{RuUA}};J^!G(ik;vB)-c6E^D+r-O?J#JD1IXhRlon^7X5D!`Q1(FL00A zseib2wzl|6>kh>x{{a`s!W~X>yrYy0GgvqShh(R79?|>Qjg|XI-TN!oX>^F6`7w_h zv2YI`w`&Bmk8Oi1d+q4TTnqN)N(1M4y#zM?InMX3NQX6Hgr!Dv@S2kd)8~|d&hsn$ zgNtfdJaZ|0aok4l9$a8Demt%H5JnawO5s|=A2Q9;Mekyz^Uk$V?E6h`7Vq`0qDxnvVji#y$PpkKuh4MtDE12ygpcgz(ZxigU6 z>^en_l`c$sq6{-AdngL-nTaa7%i-DmbWqt6!wUD7(Yz(KpnvrTd}@(oCmjrM>Wn(J ztSXjiu06mNS!~92?oxDbdosVm^D2Z-I>+To>r{G>^cp~*2op21{{$$IuuEmp3-KYoR!cD>9?`W=e z#suh_Fo_*Y%&hw{Ruxs-{@Lt|Oa+m}Dn7EjOjQ2(6j=u3QJbVP4vcvS*L(Ma>n#~} zI^+w1 z+ze}f==-O@#<_cPTWozyPr5J09{_+-8gHr(c=7_H$d{nZ2GxekXdeA*p3RP)cmT6Y=AlUc0RJzvmSpBO^CQi#LC#oNHflvM z3|hCH&3`w5C5~GUOnA;}C

q;kyNRzvrE1%GGnATE6ABoM9rP0g=tus-52{4qWt zy0Ux*>mO!~x58r4ah4ss^xG205M0~|tX1GrTk=M*$sW?I>0Tymi5dzkPH{N+d^&vouZ1qk1(DHJZPBpxclfH8 zgW=BRmvGw5oAl-UxsZonIJMi;Aw>BUD%%g|M)wtv)kHPcf8iHB=pa-dIvJ8*o`>6i zHQ4j@ZzxMwf^8_?!?u;>!0kX6AqyQ3Ph;vSdcUw+E-VKJ-4j!}%G7hp&J(yst}JBx-6=tbwi?eyMT3vLO{g2&~8y~?DGF7Hc#j9;4Uv-mb?)r#2MRY{zY1yIN3 z9Za?K6IVC>Eky7?Y4y{5?s54n(6GqD(8XmmMz4e)Gg_5B59#0@Wyi1rU0wVuHGoaH zwTMI?)!C7`5-1-skGXH|B5&C+JijLi4PS{kpM%0K&QFK2{NZNyG5;1fD|Qx&HudsD zoSXRME8pQ;Gt$n)fAldk2X49sQlW;6l@HT1P9()uAK*x>_2q4eq3sBLzO>{68A(cL!4U z4aQ%rnp_1ZV9(4$e1caoH(-D<{9bkl~@BO6mok{R0Ujcmbo`_CXXtS&H(xBEQfJJvMF4PcW_+~9L>3v+-|Pf0f4hFu$U zhL&#ZqpdfNL&pa@h$<5tjVuo9M?40rO*)tr8g*{wcRQBzEfH1w&2XBo7n|L`8GkJv zjVnFX@yPtaO!4fw}-*0GRUN3q1KBr?=n`rqCo`Txr(6TiR z#~rSO(H4R)W1=g`e-9$L`Mcofu)|cBHJZ(FD1(CyR!pa_lm9RICeWn}7^XUgZ!oiG z`$s&4m7W%`>ho0o>_0PZa`reDxio-3xIBnO)SiIJX0bHYNt$gMxQZ*&+)45a&U2+v z_DokXhW&WGg6m9?W_w1)(Eas#+})v7Vt?CPqQSFt+0spK#nR0qgc-CnmoIP1uDgbj zRAK{dQmLlNGSQa!}>Q2YT@89NddI3*JkzH@D6?bk zIZ$&x03ke)w$vD4%<$`6Fmq+&#^=zLpF>a=64}%{o_J4DIA8YiRMMZuMy|_;A@T!> zkNZcrCT4-j;5qmyUmt6y4Z*N+izvr15z`8DpmCxS{inH(JG8~0r92seb4L}z2KQkY zx8fZ63i~_D1O1uzr^n(kA3t%sJ#{EfUCccTj^JmD=5Y6PbeMebE)G`eL2cCw`gb~$ z>|RM>ai0?tu9J$)wf*4nA%C)<~EgL=@&V0Jg&xSXm zhZpuSSX{}qtbWP0Yg|J!fkSQ76&%ak7I0sD-;hqQCY_wN8n?L1Fwc-T+;HPX&{yN3$QVGCqN)rgPy+k!)57mYHO!Fi=)*|oM@I=ApBd=7TNPxG@V z|GX{vE!fWm45M5`w2Ao4h$kSpPDeUex~LHT@Zv zGq(u64Mwt)H}uJ8_h33=FaiC0ZJC1ZEO@)Kk(WGaL|XH8z+2(~HS#(5^K&e=oJoKn zH4z)7;)%nrN3l`aK=-T)VemUSobK_B_dlk>u9>}on@%n`y0075WpdcsnZwv|lc9L| zq#61gyC)j@_c`4>^Ask}R~Mgj(#F0!*0h~7=i(z~!U^X><{;)FxV(Z7Ur|D7-d1$s zK@K%8E#UM1o&v?i%h^T8{UBL2hIuS$r)BR9z)CuR2B1QqCL|TChF_LRwl~bCF=^wE(7ex9{6#XIET<-sS9~Sac{cJ})*PXz)~kGo zaG$t1>o}|$*h$}24~U8_25}t*GuX$@8L;8rZvOejJa{19$Umse;?)B^#daD0tL#g_ zsqEUe&5>k?NQQ(m&m#N2RtOnNA|gp-ZXk(5Whz5x5K$^+ra>~qzOU7!o>FNJ&8nw) zP$?ykzPtDRzVCgX_xSpcHLN$Bv$Dd%= zWC))P-=M0b6B)mvgq)wT2sjD>dHDKc&ZpQuPD)6P~ zC-XSH3cmOp!mHYc7>LgY4V@pjq+^GT&=hYvbleoCdA2YU#u%}F&J&op$6Mj#D|?QN ze*hhVtWfj(W-au~=H!}B8E!qqS zf#`Z~a1J@b+)VN&rOxFvVg6}s(-Z;o9R_5k#3K|N??eZbjakL^n_NF?v4gL5e|dzE zBPoS&-h3(k<$IW}IPeBn?)#2G8As6!|I z5PV%a!s>DPU5ni_*k!q^u&BbCb)0UB+SlaiPI*b{mhJ$wSdGf;+emGM+~~NM0^Axg ziM`+K2WmI=!jp5$XwfZ)UsX27n{fR^^MTz}zWP}~3 zREyGY#Ng!mlhkLmKJ_X;&MV`0T+nW`}>M zfbAcPK=qC?eRn~Np3HyD>Swf}+&&rR!?<)@nk+^h`XZxQXGu;6?xvHc)KaphmS)Zj zBO;dXVf!u{Qk!N8onxn%({&+tZA+aX}Ue0tPu)@bsN$c70cU9w9XZ5vB+Xvpc~b^PGeqNB&!gw zMe3g;m=A71Ez7;MWmuY48uFO)rIX(M}RPj7pdj1En1$%eG<@r+5gTU(8cHmMzd2s-^ZPVJpD zd^^`Mup?HSxHfyEsfGcp=u9Jm??TDL##^BF@Gaoh?ee?OnUSVsMg0CGW*)PP1m=OZa482~AZ)5C@8+RB8ZH#3bmf#0cUsz{1owqlr;mIqB-x zAs$>#=UCe`dgSU%nj;-egk((d&9^*e+L9bc0x#M{b~x({nL$sW92t zC<{vqhH;6B2s3f;I^-BnU_;l5Ge7mW(TNv-z$DQGQZd~Eu3u~i7iFO5xR~0H`)YVU z*Pb3ZaTUMMNJPnuT+ZM58Xn0MBT~CEiNlU1v~Q&y8%s;bK@TsKcPwQSUYmkIyAo)h z6Q`mgf<(GF6jaS3`Jze_KxC5@FJgBGGwwt@4QWuPQ3bD9s~HS&tS#kD5H`f8^b(Wv zR~sG^x1kvWZ<&jK+EdePZ%EjyKp#9h10^zT%)o*SG*(64=EI+1pkXbipL7&X9*n0C zM;nm9*9OFOjwjsnwBmd`Z0MY2t>~eg&E>XiqVMb&dQz^0+}_m0Jku$Jm0YgMdUaqWwFF%hnritIc znJmSR+`41)!DP}WE==lk2T=EM4Q0;CL+q&xl4v5$3_f!qT@#PPt&P)3LW421W_poU z6BmZ9)TG}BCF#`UEqqIEK34YJy-irF4K&qvD%Rxpq3QgWjw_ggcIjh6Qk9}(C%wWzl`z(izAm&|I$IW zA^1G9eTrnhzC4xOo&r;jjipmPO|WKGCi~~$bXq4}3kNHApzM}r42{cTo?IxRy$b~K zTW=QTsO=<8HhcNgdZv=LZ#{Iex(pd=G9Wi6Hj|hneYWUfB&2Q2V#~NSZJm$PV#as@ zDtCB0DU44fJNI%P8U0Ie=e%&dyGk5;1qG>TMjsyYcA|5F9^>I)3;bN^46Sb~Agi{P z4PKB$p0#cv3Rp#S&Yj@7H}~P9?Wb_;#3FodqDwBB3(`-)VZ@QsQ!bI~=h1D~Al_FM zML%5u(d<0T3Dl$`-+ZX%6bbUwaz6y*AE3rY{kXzfo*4Fu;Gy9sc$rBAi?Oj#RBph{ z$rdrKcn3Ug?uB-oL8n+RZT69SuEtNjPhoIF0e6n0@#;%KnTdmp+#B)n_&0t7==yu0F#y9lC~#!*0TC z;YjAaQ6ZBRC5Zl85z_DNVuPADlJk=eQKgW*q@L4_JF?phJgax(-M0N0x%DFWg;Gp06$;UBPa5^z&k=1qQ-AvlsVqX*wi2@m7)MUyOnXF8zH@aIHQK7 z5S40aWb7wRq6cnmCx&PCfdt=&`t0H2rr80|GHO5lzNZ`5G&{EWu?F6?`oiSx4nv*y zMl|?sKZd5Qps@je;dm~`I^nYsYsdk&-JL*P^KQe98w9RJt23RAC1kP5ItUs%$4Xsm zhK#|>ti9JDZ?a1^m!q#reI$y}Z|f*hyI=x-e|`Y_?>kUib|L-HGlQu?HJU8c#YD}h zqg(8JNz9u@{A1F&*3x(;az$CuL&`t=y0!XLYs z+mEauMPuYhjCCz+f4qe3@8II@mVC|^!->;_Y((v+dr(39CA%$iKlXHWvgy`FL|n;) z$UK!G1(H|D=MQ0I-i3M4rWw!Hx*L=%cIKpUWY$bL{=^VdGoa`_9$a||O zPxpc$eGZ%_N!ivV z1t$(HW_ZVYS+$rxUR@`l&d1~k=9kjU$P@^zk!E(D8G!-i&CK|WcU?DbS86)fVKz?;iC)9cxaRuefx;h z+ZD|K!Sr1y`9g`T)RLi{$|bn>hZ?2Qc!9f$Mc??M~rnVu%S+t?H!{+ z?)pw39ZCmydtwIhQh^=0?Ux3_0{7WVpOmSCTM#+6M~h0FosT}3twFfVg%Lbh4HrZw zlk355%>2E{Fiq_zJ6U8CWR-7&NOvv{dhR1&wm(aq#@t_Jl@)xde`oyQ%#;SinjSUTgQ|Qu;(0R}6=L^O zPFuSbW_&DRcDcU7#LjLUU0lG<-p(+?tsC*$r3bjKTZe9I5+Jwr=g@0oa^OShHe&ao z0T1lUM4QttG`VIUS!Wpl=OXQ>Xatx0r+EnUXAPrQ#!d8kbhb^Yb3IBWy2IFy$MCd| zC_@aj$%E@YkYgGMcVDzaZQ%_(8k5Urb$ntHZe9Vwu2*RJV>CIVJ()>wID@G*kq~`; zHEPVRAnsaA_^!o^$i^-EVeQ@lNE8hx2OF2MqO)Xapn?&(n)@e4f8UBkNdu1-o6V+q%->xC*~h)<)$WrJu*(*7LR{#()&wf_&K~OztMJ4hErN|u z0|)(lK;m{9Tzn8nFNSOa#}rTM1Zx34dQW6NLK z%;)yE%a1BiF7XsZmaU@`^2F&MVLqVzd?$6E9x*+$v=D740#Z`?vPR4v$_ZIbNzy^CpKTQ9FldLpj&Pr-{MP3^*#|t75ZdA5(}`2 z&0!{gm8L^l6_6b817h5|SaUL&Jj)+Tysb6KzAP4|RlmWcN}Kl9m4i^#E=6sgzJ(>% zequ>~Wc!@4>Qtj8hsuS_r)?wwchpGZbcfT-X{)<%IwX@iWv4Qs?+=6S&`r?IUxD@P zPjF`9iPOb%u$|NM$q@2^zNnda@3JhJSl$6fS57eQ;>);wjYlL&GYEQQz7jp*3$QNP zg0-)`)GnhOMt5sv^4F&B2it``Ov=gx>O5N$Wf_kDb8<7iR341A_M7=TE*8qhmS#@r;#Pp#1eET0NgbXGfcpnCNW~WR-%|qfAKU z-Cii_+l^vV(_xFbCp~0fO*9|o!=SwnGxwk|6_oqQx^L~kTfSAyxMB-xGVDZ81v#+l zcg#V-^#K1+s4}sBtxwfr-hg{zKJ1^R#H+~ZLJRw;r2oA<5fv__)k~8|&Id)Xvq+_f zZ-0cIpjuSk;00mp<;mCLk8Hr3cX()E3yk|R31t5gz+BI{m@$nq4kuh_#1mDzFc8@f zWvwVFt_qSZGIYi0L|pp59=xekPS;>9JmZ+J^5a)cpg7yY~{Gv$hz!5s0_dCN5Wj)AG%uDH1MP%i zjPJ@}SF|*+*BjJGv78H7TsLCt>LjUdk^)^Er%v4+3flNwJm&bT4KVJOIC>_fkTnaDIdoD1ufv4xCpDKG(xAo9}(Z6)&5Ro2`Sbyq(b%gaIU2z3D^CGwyWc@ zgxfPp4-lc}r&p8A$O*JLupM$7HECV`UK`WJ)?~xwsCOiKQ@&gOkwOq=miMVn}6=Nz%U# z>*5_Tefs7gr%82Hm0K@Nr(ptvysA1iDySv}y{~o>i`5C_M`Hx8Gy81g&3PUgz852Q z><@NJTL9E)=}>)1R}gN_q<`($f$J9ulB25^lFrgSWWG=VzR;DWLhK7j-!PG~J|!6S zZ8TluokVW=8q!evGGd(74h`eF!7Ok##}!~;c&-sS#_NRx;dSVLI10?yS@TV0BiNP2 z5}YQ5FezD-1L&rNG4l6ttJNNSsn!j)%VkOWv`w_4sR-2;X`rLhW!y725=!FixLmQ> zB#N8INq_o`3UmUtO@D$#Z@9f`jR4Xp-3NNEE_6)z02+qAfP@ETC+9tXdjsC)l&iWiMe7_7AP91cn1*}$x}a&D zDhNBB$E*6X@ zi)^yM&D{X3Hk)yIULQe&^UpOunn}YtGqCf@CkUOKPP-KCh~CMytm>&wBI~>jQcd|x z)QBQ9v7N9nCY&Bk*$fH>$Z1D;Q8WKe92%WT*D9M3-3nJ);4l}eMc43lNS=eaJMC$J zK@+UGs73smgo)jEZY`1Ig{eLI#AHaG&Xdf<;94UhJUWIM4k@OWs}F!do*nwUA46tb zkf6=g?RYCBj`>)uM`8XD`_9dgf1$;NPJQJ`Kc!_b!J?(SbH-iZ<==o$g##!<$B=>! zLuzyC7QXzV$ZIV=%3r?yC-loY(AQ3DA)4O+?{D~%cTTG8*PKl2$cK*@hqzf#Cbb2x zD%r82%SY4lpl!&0O~*1KPbv&v8YbjgOyMH)2ZZ7ywJ5R6k= z=9A#|9VoG65Ttv$SzCV@Jp6G4J)V4p{DmVnCp)_FQePRl`FStv*^-89SE|yJ?tJ<% zUI&FDi%_NFBd@w`0#yP&XHI*1&iVD;U`hyV(P1w|p|&8ha9x&!kYM)Yj&# zODNX1rGkLQ0B)+9Mi*zjXWR2$;GU`(z?))E*Da914|<#?t*;xYj$94FlU!*2gDaqX zQ-JEPyp2Vh0$B&WKX9mI5bH=cFS%3^eZQ@OfRVGzjWvF-`0!0o3*N^X>iM93^f&&` zm`HkjqBW$=+koqxvWP;46mELxM57)1a7OGBVz}M}bDjjk%GJW8Y`+RhEl&euziUL_ zDz&}yiV%ZK{W%YcaJssM(*y6kg64x==r%kTE+*@McF_j1NAeCBi^`II)gNe|Z-?z; zJYmbui`Wr;8@#!E@EH3e5W;D}DEE({{`H&SM&>A}^~#3V=M$)H=XX{sYYW`+@}pHl zVK&;7Wccm#^YjFVH`MiSS!<0=F8(}0T4B#p8 zF2*KmHfrC_M*qk1bmYiV@;*YDrl*W0L~9p$U0jSNr+8#nay@fjM1dCcZKW~Xe2qOb zk&0H`0Wm{POI2H++zsqP!B#(9vUV$)UkxNZFBRzJx;121{0){@BtkDVTw|V$c>_~* z-lC;wE>3j32gj`*j!?U2XXcXFOxseey1+r_F z7_#Y=V3IutO2cAcVoW*MNkzewZ}RxcXqa)dFeRoBUtz3MIs~?@!U2zI#A#^}zM0~V z!$n7#R{lHq9_~c%jW%QNcvz5@@~61>y*X8ma|Z1wYkK~{X5RX!0kCP!A`OpjKw`cZ z-7z8#vJ0<4`utk-Qqd<<5{LNeTegszV-YmL;~s{N(Wb_ca+p=ec=2|w{L{~$4qa>Q*p(QNV=Mb7*3AnRPw!7Tq+0RCCBteKhz|M~_4 z_~W1#Rku{5_w5&xth8`qE~Y_j@95G47E5q&#KareLL0Kkz>ySb4lxND^j=P@$Z7SYl-64a#Fm;TVQhr%H(denU#%rCf$ zmiBTqq%8}+3X@y2xe=8qg|{K|B=Ghe5moIA1ssPdxaD$|e!SeU%V*NS8Q z!d2+^b0wXgqDPIO96a1ciRw|vNGA1S_R?{ua zOf5_;&CIOKtS!u@o0wRdSx=j8Za&T2%)-jb#M;!##PU}y^IJDg{@3>x{8ici?WuU> zyS-6@0s_B^+OG?LJAU79Mls8ld$|1_tM$YEbPh(~SJD0(PCNJ1|BYo779G1HC@%27 z+xR~rXC$n8^*;^#>nr^0n8!W+8_3mh(Xm0{p@CeRhOb-^x-#xRQ8&TU31S@7ucH1h zNbc;$Z&8ES#f3)4hJ?ocC)mIL3RM0E<|F*OD*mI}P5u+<-^c0Sk*<~hKaouT6Y1Z_ zrRv}6w6@}RNWbZUe;+&se}lU5Td4ngSbMmMiTw|Pny}&jVu}Cv+5g-p{bPU%2&}FA i=LPP4qkf%zuKwraU&G78P4rji35avIU$6h?YyS&4o57X< literal 0 HcmV?d00001 diff --git a/processing_services/minimal/api/pipelines.py b/processing_services/minimal/api/pipelines.py index 975674b6a..5ce29e51f 100644 --- a/processing_services/minimal/api/pipelines.py +++ b/processing_services/minimal/api/pipelines.py @@ -1,4 +1,5 @@ import datetime +import hashlib import logging import math import random @@ -109,6 +110,23 @@ def make_random_detection(source_images: list[SourceImage]) -> list[Detection]: return detector_responses +# Width of the stand-in embeddings below. Matches Antenna's DetectionEmbedding column, +# which is sized for BioCLIP 2.5 (ViT-H/14). +FEATURE_DIMENSIONS = 1024 + + +def make_fake_features(label: str, dimensions: int = FEATURE_DIMENSIONS) -> list[float]: + """ + A stand-in for a backbone's output, so the embedding path can be exercised without a + real model. Seeded by the label, so crops of the same class land near each other and + nearest-neighbour search returns something meaningful. + """ + seed = int(hashlib.sha256(label.encode()).hexdigest()[:8], 16) + centroid = random.Random(seed) + jitter = random.Random() + return [centroid.gauss(0, 1) + jitter.gauss(0, 0.15) for _ in range(dimensions)] + + def make_random_prediction( algorithm: AlgorithmConfigResponse, terminal: bool = True, @@ -127,6 +145,7 @@ def make_random_prediction( labels=category_labels if len(category_labels) <= max_labels else None, scores=softmax, logits=logits, + features=make_fake_features(top_class), timestamp=datetime.datetime.now(), algorithm=AlgorithmReference(name=algorithm.name, key=algorithm.key), terminal=terminal, @@ -146,6 +165,7 @@ def make_classifications(detections: list[Detection], type: str) -> list[Detecti classification=labels[0], labels=labels, scores=[0.9], + features=make_fake_features(labels[0]), timestamp=datetime.datetime.now(), algorithm=AlgorithmReference( name=algorithms.CONSTANT_CLASSIFIER.name, key=algorithms.CONSTANT_CLASSIFIER.key diff --git a/processing_services/minimal/api/schemas.py b/processing_services/minimal/api/schemas.py index b0febba1b..6d503b988 100644 --- a/processing_services/minimal/api/schemas.py +++ b/processing_services/minimal/api/schemas.py @@ -95,6 +95,14 @@ class ClassificationResponse(pydantic.BaseModel): default_factory=list, description="The raw logits output by the model, before any calibration or normalization.", ) + features: list[float] | None = pydantic.Field( + default=None, + description=( + "The embedding the backbone produced for this crop, taken before the " + "classification head. Optional, and only useful if every value comes from " + "the same backbone." + ), + ) inference_time: float | None = None algorithm: AlgorithmReference terminal: bool = True From fcdb01ee2f0306c2707b816695da01ed0a62a7a7 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Sun, 13 Sep 2026 14:56:51 -0400 Subject: [PATCH 03/31] feat(ml): train a head from a project's taxa list and record what it learned from A retrain used to take its species list from whatever had been verified, so a project with eight verified species produced an eight-species head and the pipeline quietly went blind to everything else. The project's taxa list now decides what the head can predict, and the verified crops only decide how well it predicts each one. Species in the list with no crops yet are kept and reported rather than dropped. Every occurrence that goes into a training set is written down at the moment the set is built, and linked to the version it produced once training finishes. That is what makes a blind evaluation set possible later: without it there is no way to tell data a model learned from apart from data it has never seen, and it cannot be reconstructed after the fact. Training can outlast the request that starts it, so a service now reports its result to a callback on the job instead of holding the connection open. A processing service has no account here, so the callback is authorised with a signed token issued when the job was dispatched. A repeated result is ignored. Also closes a hole in the training-data endpoint, which checked that a caller was logged in but not that they could see the project, and moves its query parameters onto the shared validation helpers so bad input answers 400 instead of being quietly accepted. The retraining settings stored on an algorithm are now actually read: they set how the dataset is built and are passed to the service, and a job can override any of them for a single run. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/jobs/models.py | 67 ++- ami/jobs/views.py | 46 ++ .../0096_project_default_taxa_list.py | 25 ++ ami/main/models.py | 12 + .../commands/export_verified_training_data.py | 150 +++---- .../0032_training_set_membership.py | 58 +++ ami/ml/models/__init__.py | 2 + ami/ml/models/training_set.py | 112 +++++ ami/ml/tests.py | 422 +++++++++++++++++- ami/ml/training_data.py | 8 +- ami/ml/training_dataset.py | 47 +- ami/ml/training_dispatch.py | 59 ++- ami/ml/views.py | 91 ++-- 13 files changed, 946 insertions(+), 153 deletions(-) create mode 100644 ami/main/migrations/0096_project_default_taxa_list.py create mode 100644 ami/ml/migrations/0032_training_set_membership.py create mode 100644 ami/ml/models/training_set.py diff --git a/ami/jobs/models.py b/ami/jobs/models.py index 83ebb2690..fa5e5ecee 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -1089,12 +1089,19 @@ def run(cls, job: "Job"): job.progress.update_stage(cls.STAGE_PREPARE, status=JobState.STARTED, progress=0) job.save() + taxa_list = cls.target_taxa_list(job) + # The algorithm's training_config holds the defaults the service published; a job + # may override any of them for one run. + config = algorithm.training_config try: dataset = build_training_dataset( project=job.project, algorithm=algorithm, - min_per_species=params.get("min_per_species", 2), - job_id=job.pk, + min_per_species=params.get("min_per_species", config.min_per_species), + test_fraction=params.get("test_fraction", config.test_fraction), + split_salt=params.get("split_salt", config.split_salt), + taxa_list=taxa_list, + job=job, ) except NotEnoughVerifiedData as e: # A data problem, not a crash. Say so plainly rather than failing with a traceback. @@ -1110,6 +1117,22 @@ def run(cls, job: "Job"): f"Training set: {meta['rows']} verified crops over {len(meta['classes'])} species " f"({meta['train']} train / {meta['test']} held out)" ) + if meta.get("taxa_list"): + job.logger.info( + f"Species list comes from taxa list '{meta['taxa_list']['name']}' " f"({len(meta['classes'])} species)" + ) + without_data = meta.get("classes_without_verified_data") or [] + if without_data: + job.logger.info( + f"{len(without_data)} species in the list have no verified crops yet. They keep " + "whatever the current head already knows." + ) + if meta.get("dropped_species"): + job.logger.warning( + f"{len(meta['dropped_species'])} verified species are not in the taxa list and were " + f"left out: {', '.join(meta['dropped_species'][:5])}" + f"{' ...' if len(meta['dropped_species']) > 5 else ''}" + ) if meta["verified_detections_without_embedding"]: job.logger.warning( f"{meta['verified_detections_without_embedding']} verified detection(s) have no embedding " @@ -1144,6 +1167,24 @@ def run(cls, job: "Job"): job.save() cls.dispatch(job=job, service=service, algorithm=algorithm, dataset=dataset) + @classmethod + def target_taxa_list(cls, job: "Job"): + """ + Which taxa list sets the head's species. + + A job may name one; otherwise the project's default is used. Without either, the + class list falls back to whatever has been verified. + """ + from ami.main.models import TaxaList + + taxa_list_id = (job.params or {}).get("taxa_list_id") + if taxa_list_id: + taxa_list = TaxaList.objects.filter(pk=taxa_list_id).first() + if not taxa_list: + raise ValueError(f"No taxa list with id {taxa_list_id}.") + return taxa_list + return job.project.default_taxa_list + @classmethod def dispatch(cls, job: "Job", service, algorithm, dataset: dict) -> None: """Hand the service the dataset URL and leave the job running until it reports back.""" @@ -1155,9 +1196,16 @@ def dispatch(cls, job: "Job", service, algorithm, dataset: dict) -> None: job.logger.info(f"Training request accepted by {service.name}. Waiting for it to report back.") job.save() + if response is None: + # The service accepted the work and will post to the job's training-result + # endpoint when it finishes. Leaving the job STARTED is the point: a real + # training set outlasts the request that started it. + job.logger.info("Waiting for the service to report back.") + return + if response is not None: - # The service answered synchronously, which small datasets do. Record it now - # rather than leaving the job waiting for a callback that already happened. + # Answered inline, which small datasets do. Record it now rather than waiting + # for a callback that has already been overtaken. cls.record_result( job=job, payload={ @@ -1175,6 +1223,11 @@ def record_result(cls, job: "Job", payload: dict) -> None: result = payload.get("result") or {} job.result = payload + # A result can arrive through the callback on a job whose stages were never set up, + # for instance after a restart. Make sure the stage exists before reporting into it. + if not any(stage.key == cls.STAGE_TRAIN for stage in job.progress.stages): + job.progress.add_stage("Training", cls.STAGE_TRAIN) + for warning in result.get("warnings", []): job.logger.warning(warning) @@ -1187,8 +1240,14 @@ def record_result(cls, job: "Job", payload: dict) -> None: new_version = cls.register_new_version(job=job, payload=payload) if new_version: + from ami.ml.models.training_set import attach_algorithm + job.progress.add_stage_param(cls.STAGE_TRAIN, "New version", new_version.key) job.logger.info(f"Registered {new_version} as version {new_version.version}") + # The occurrences were recorded when the set was built, before this version + # existed. Point them at it now. + linked = attach_algorithm(job=job, algorithm=new_version) + job.logger.info(f"Linked {linked} occurrence(s) to {new_version.key}") job.progress.update_stage(cls.STAGE_TRAIN, status=JobState.SUCCESS, progress=1) job.finished_at = datetime.datetime.now() diff --git a/ami/jobs/views.py b/ami/jobs/views.py index 35f91ef96..749451864 100644 --- a/ami/jobs/views.py +++ b/ami/jobs/views.py @@ -7,6 +7,7 @@ from django.core.cache import cache from django.db.models import Q from django.db.models.query import QuerySet +from django.http import Http404 from django.utils import timezone from django_filters import rest_framework as filters from drf_spectacular.utils import extend_schema, extend_schema_view @@ -14,6 +15,7 @@ from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied, ValidationError from rest_framework.filters import BaseFilterBackend +from rest_framework.permissions import AllowAny from rest_framework.response import Response from ami.base.pagination import LimitOffsetPaginationWithPermissions @@ -486,6 +488,50 @@ async def get_tasks(): responses={200: MLJobResultsResponseSerializer}, parameters=[project_id_doc_param], ) + @extend_schema(exclude=True) + @action( + detail=True, + methods=["post"], + url_path="training-result", + name="training-result", + # A processing service has no Antenna account. It proves itself with the signed + # token Antenna issued when it dispatched the job, checked below. + permission_classes=[AllowAny], + authentication_classes=[], + ) + def training_result(self, request, pk=None): + """ + Receive the outcome of a retraining run from a processing service. + + Training can outlast the request that started it, so the service reports back here + instead of holding the connection open. + """ + from ami.jobs.models import TrainClassifierJob + from ami.ml.training_dispatch import verify_callback_token + + # get_object() applies project visibility, which an unauthenticated service fails. + # The token is what authorises this call, so look the job up directly. + job = Job.objects.filter(pk=pk).first() + if not job: + raise Http404("Job not found.") + + token = request.headers.get("Authorization", "").removeprefix("Token ").strip() + if not verify_callback_token(token, job): + raise PermissionDenied("Invalid or expired training callback token.") + + if job.job_type_key != TrainClassifierJob.key: + raise ValidationError(f"Job #{job.pk} is not a training job.") + + if job.status in JobState.final_states(): + # The service answered inline and the result is already recorded, or a retry + # arrived late. Either way the first answer stands. + logger.info("Ignoring a training result for job %s, which already finished", job.pk) + return Response({"status": "already recorded"}) + + TrainClassifierJob.record_result(job=job, payload=request.data) + logger.info("Recorded a training result for job %s", job.pk) + return Response({"status": "recorded"}) + @action(detail=True, methods=["post"], name="result") def result(self, request, pk=None): """ diff --git a/ami/main/migrations/0096_project_default_taxa_list.py b/ami/main/migrations/0096_project_default_taxa_list.py new file mode 100644 index 000000000..d39be1d93 --- /dev/null +++ b/ami/main/migrations/0096_project_default_taxa_list.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.10 on 2026-09-13 10:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0095_grant_sync_deployment_to_mldatamanager"), + ] + + operations = [ + migrations.AddField( + model_name="project", + name="default_taxa_list", + field=models.ForeignKey( + blank=True, + help_text="The species this project expects to see. Used as the class list when retraining a classifier head, so the head covers the region rather than only the species someone happened to verify.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="default_for_projects", + to="main.taxalist", + ), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index 3d055ff8e..33613f9ec 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -313,6 +313,18 @@ class Project(ProjectSettingsMixin, BaseModel): null=False, blank=True, ) + default_taxa_list = models.ForeignKey( + "TaxaList", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="default_for_projects", + help_text=( + "The species this project expects to see. Used as the class list when retraining " + "a classifier head, so the head covers the region rather than only the species " + "someone happened to verify." + ), + ) active = models.BooleanField(default=True) priority = models.IntegerField(default=1) diff --git a/ami/ml/management/commands/export_verified_training_data.py b/ami/ml/management/commands/export_verified_training_data.py index 9dc3b44d3..afb5903e9 100644 --- a/ami/ml/management/commands/export_verified_training_data.py +++ b/ami/ml/management/commands/export_verified_training_data.py @@ -1,31 +1,23 @@ """ -Export human-verified detections as a training set for a classifier head. +Write a classifier-head training set to a local file, for inspection. -The backbone is frozen, so retraining a head means fitting a small matrix over stored -embeddings. This command gathers the labels people have confirmed in the UI, pairs them -with the embeddings saved by the pipeline, and writes them out. +The same set a training job would send to a processing service, saved somewhere you can +open it. It builds the set through ami.ml.training_dataset so the two cannot drift apart: +a file exported here holds exactly what a retrain would have used. Usage: python manage.py export_verified_training_data --project 3 --algorithm bioclip-2-5-nf-749 - -Writes two files next to each other: - .npz embeddings, label indices, detection ids, split assignment - .json label map, counts, and the settings used to produce it - -The split is deterministic and grouped by occurrence. See training_data.split_for() for why that -matters. """ import json import pathlib -import numpy as np +from django.core.files.storage import default_storage from django.core.management.base import BaseCommand, CommandError -from ami.main.models import Project -from ami.ml import training_data +from ami.main.models import Project, TaxaList +from ami.ml import training_data, training_dataset from ami.ml.models import Algorithm -from ami.ml.models.embedding import EMBEDDING_DIMENSIONS class Command(BaseCommand): @@ -43,6 +35,12 @@ def add_arguments(self, parser): ), ) parser.add_argument("--output", type=str, default="verified_training_data", help="Output path, without suffix") + parser.add_argument( + "--taxa-list", + type=int, + default=None, + help="Taxa list to use as the species list. Defaults to the project's own default.", + ) parser.add_argument( "--test-fraction", type=float, @@ -59,7 +57,7 @@ def add_arguments(self, parser): "--min-per-species", type=int, default=2, - help="Drop species with fewer verified crops than this. A class with one example cannot be evaluated.", + help="Drop species with fewer verified crops than this. Ignored when a taxa list sets the species.", ) def handle(self, *args, **options): @@ -72,97 +70,51 @@ def handle(self, *args, **options): known = list(Algorithm.objects.values_list("key", flat=True)[:20]) raise CommandError(f"No algorithm with key '{options['algorithm']}'. Known keys: {known}") - occurrence_ids = list(training_data.verified_occurrence_ids(project)) - self.stdout.write(f"Occurrences with a standing identification: {len(occurrence_ids)}") - if not occurrence_ids: - raise CommandError("Nothing has been verified in this project yet, so there is nothing to export.") - - rows = list( - training_data.verified_training_rows(project, algorithm).values_list( - "detection_id", - "detection__occurrence_id", - "detection__occurrence__determination__name", - "vector", + taxa_list = None + if options["taxa_list"]: + taxa_list = TaxaList.objects.filter(pk=options["taxa_list"]).first() + if not taxa_list: + raise CommandError(f"No taxa list with id {options['taxa_list']}") + + try: + result = training_dataset.build_training_dataset( + project=project, + algorithm=algorithm, + min_per_species=options["min_per_species"], + split_salt=options["split_salt"], + test_fraction=options["test_fraction"], + taxa_list=taxa_list, ) - ) - self.stdout.write(f"Verified detections with an embedding from {algorithm.key}: {len(rows)}") + except training_dataset.NotEnoughVerifiedData as e: + raise CommandError(str(e)) + + meta = result["metadata"] + out = pathlib.Path(options["output"]) + with default_storage.open(result["path"], "rb") as stored: + out.with_suffix(".npz").write_bytes(stored.read()) + out.with_suffix(".json").write_text(json.dumps(meta, indent=2)) - missing = training_data.count_missing_embeddings(project, algorithm) - if missing: + if meta["taxa_list"]: + self.stdout.write(f"Species list from taxa list '{meta['taxa_list']['name']}'") + if meta["classes_without_verified_data"]: self.stdout.write( - self.style.WARNING( - f"{missing} verified detection(s) have no embedding from this algorithm and were skipped. " - "Re-run the pipeline over them with the store_classification_embeddings flag on." - ) + f"{len(meta['classes_without_verified_data'])} species in the list have no verified crops yet" ) - if not rows: - raise CommandError("No verified detection has an embedding yet, so there is nothing to train on.") - - # Drop species too rare to both train and evaluate on. - counts: dict[str, int] = {} - for _, _, name, _ in rows: - counts[name] = counts.get(name, 0) + 1 - min_per_species = options["min_per_species"] - keep = training_data.species_with_enough_examples(counts, min_per_species) - dropped = sorted(set(counts) - keep) - if dropped: + if meta["dropped_species"]: self.stdout.write( - f"Dropped {len(dropped)} species with fewer than {min_per_species} verified crops: " - f"{', '.join(dropped[:8])}{' ...' if len(dropped) > 8 else ''}" + self.style.WARNING(f"{len(meta['dropped_species'])} verified species were left out of the set") + ) + if meta["verified_detections_without_embedding"]: + self.stdout.write( + self.style.WARNING( + f"{meta['verified_detections_without_embedding']} verified detection(s) have no embedding " + "from this algorithm. Run a generate_embeddings job to include them." + ) ) - rows = [r for r in rows if r[2] in keep] - - labels = sorted(keep) - label_to_index = {name: i for i, name in enumerate(labels)} - - embeddings = np.zeros((len(rows), EMBEDDING_DIMENSIONS), dtype=np.float32) - y = np.zeros(len(rows), dtype=np.int64) - detection_ids = np.zeros(len(rows), dtype=np.int64) - splits = [] - for i, (detection_id, occurrence_id, name, vector) in enumerate(rows): - embeddings[i] = np.asarray(vector.to_list(), dtype=np.float32) - y[i] = label_to_index[name] - detection_ids[i] = detection_id - splits.append(training_data.split_for(occurrence_id, options["split_salt"], options["test_fraction"])) - split_array = np.array(splits) - - out = pathlib.Path(options["output"]) - np.savez_compressed( - out.with_suffix(".npz"), - embeddings=embeddings, - labels=y, - detection_ids=detection_ids, - split=split_array, - ) - - n_train = int((split_array == "train").sum()) - n_test = int((split_array == "test").sum()) - meta = { - "project": {"id": project.pk, "name": project.name}, - "algorithm": {"key": algorithm.key, "name": algorithm.name, "version": algorithm.version}, - "dimensions": EMBEDDING_DIMENSIONS, - "classes": labels, - "counts": {name: counts[name] for name in labels}, - "rows": len(rows), - "train": n_train, - "test": n_test, - "occurrences": len(occurrence_ids), - "verified_detections_without_embedding": missing, - "settings": { - "test_fraction": options["test_fraction"], - "split_salt": options["split_salt"], - "min_per_species": options["min_per_species"], - "split_grouped_by": "occurrence", - }, - } - out.with_suffix(".json").write_text(json.dumps(meta, indent=2)) self.stdout.write( self.style.SUCCESS( - f"Wrote {len(rows)} rows over {len(labels)} species " - f"(train {n_train} / test {n_test}) to {out.with_suffix('.npz')}" + f"Wrote {meta['rows']} rows over {len(meta['classes'])} species " + f"(train {meta['train']} / test {meta['test']}) to {out.with_suffix('.npz')}" ) ) - # Guard against a silently useless export. - if n_test == 0: - self.stdout.write(self.style.WARNING("The test split is empty. Verify more occurrences before training.")) diff --git a/ami/ml/migrations/0032_training_set_membership.py b/ami/ml/migrations/0032_training_set_membership.py new file mode 100644 index 000000000..87a0021c6 --- /dev/null +++ b/ami/ml/migrations/0032_training_set_membership.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2.10 on 2026-09-10 14:31 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0095_grant_sync_deployment_to_mldatamanager"), + ("jobs", "0025_generate_embeddings_job_type"), + ("ml", "0031_algorithm_training_fields"), + ] + + operations = [ + migrations.CreateModel( + name="TrainingSetMembership", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "algorithm", + models.ForeignKey( + blank=True, + help_text="The algorithm version this occurrence helped produce. Filled in after training finishes, because the version does not exist while the training set is built.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="training_occurrences", + to="ml.algorithm", + ), + ), + ( + "job", + models.ForeignKey( + help_text="The training job that collected this occurrence.", + on_delete=django.db.models.deletion.CASCADE, + related_name="training_set_members", + to="jobs.job", + ), + ), + ( + "occurrence", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="training_uses", to="main.occurrence" + ), + ), + ], + options={ + "indexes": [models.Index(fields=["occurrence"], name="training_use_occurrence_idx")], + }, + ), + migrations.AddConstraint( + model_name="trainingsetmembership", + constraint=models.UniqueConstraint( + fields=("occurrence", "job"), name="unique_training_use_per_occurrence_and_job" + ), + ), + ] diff --git a/ami/ml/models/__init__.py b/ami/ml/models/__init__.py index 80df44738..58ff26795 100644 --- a/ami/ml/models/__init__.py +++ b/ami/ml/models/__init__.py @@ -3,6 +3,7 @@ from ami.ml.models.pipeline import Pipeline from ami.ml.models.processing_service import ProcessingService from ami.ml.models.project_pipeline_config import ProjectPipelineConfig +from ami.ml.models.training_set import TrainingSetMembership __all__ = [ "Algorithm", @@ -11,4 +12,5 @@ "Pipeline", "ProcessingService", "ProjectPipelineConfig", + "TrainingSetMembership", ] diff --git a/ami/ml/models/training_set.py b/ami/ml/models/training_set.py new file mode 100644 index 000000000..ae07a0532 --- /dev/null +++ b/ami/ml/models/training_set.py @@ -0,0 +1,112 @@ +import logging + +from django.db import models + +from ami.base.models import BaseModel + +logger = logging.getLogger(__name__) + + +class TrainingSetMembership(BaseModel): + """ + Records that an occurrence's crops went into one training run. + + An evaluation set is only honest if it holds nothing the model learned from, and a + person verifying occurrences has no way to know which of them a past retrain already + consumed. Writing it down at the moment the training set is built is the only reliable + way to tell the two apart later. + + Kept as its own table rather than a flag on Occurrence: the same occurrence is used by + every later retrain, so this is a list, not a yes or no. + """ + + project_accessor = "occurrence__project" + + occurrence = models.ForeignKey( + "main.Occurrence", + on_delete=models.CASCADE, + related_name="training_uses", + ) + job = models.ForeignKey( + "jobs.Job", + on_delete=models.CASCADE, + related_name="training_set_members", + help_text="The training job that collected this occurrence.", + ) + algorithm = models.ForeignKey( + "ml.Algorithm", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="training_occurrences", + help_text=( + "The algorithm version this occurrence helped produce. Filled in after training " + "finishes, because the version does not exist while the training set is built." + ), + ) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["occurrence", "job"], + name="unique_training_use_per_occurrence_and_job", + ) + ] + indexes = [ + # The evaluation-set query is "occurrences never used for training", which reads + # this column and nothing else. + models.Index(fields=["occurrence"], name="training_use_occurrence_idx"), + ] + + def __str__(self) -> str: + return f"Occurrence #{self.occurrence_id} used by job #{self.job_id}" + + +def record_training_set(occurrence_ids: list[int], job) -> int: + """ + Write down which occurrences a training run used. + + Ignores rows that already exist, so re-running a job does not fail and does not + double-count. + """ + memberships = [TrainingSetMembership(occurrence_id=pk, job=job) for pk in set(occurrence_ids)] + if not memberships: + return 0 + TrainingSetMembership.objects.bulk_create(memberships, batch_size=1000, ignore_conflicts=True) + logger.info(f"Recorded {len(memberships)} occurrence(s) as used by training job #{job.pk}") + return len(memberships) + + +def attach_algorithm(job, algorithm) -> int: + """Point this job's records at the version it produced, once that version exists.""" + return TrainingSetMembership.objects.filter(job=job, algorithm__isnull=True).update(algorithm=algorithm) + + +def occurrence_ids_used_for_training(project=None) -> models.QuerySet: + """Occurrences that any training run has already consumed.""" + rows = TrainingSetMembership.objects.all() + if project: + rows = rows.filter(occurrence__project=project) + return rows.order_by().values_list("occurrence_id", flat=True).distinct() + + +def occurrences_safe_to_evaluate_on(project=None) -> models.QuerySet: + """ + Verified occurrences no training run has used. + + This is the pool an evaluation set is drawn from. Scoring a model on data it learned + from reports a number that means nothing. + """ + from ami.main.models import Occurrence + from ami.ml import training_data + + verified = training_data.verified_occurrence_ids(project) if project else None + occurrences = Occurrence.objects.filter(determination__isnull=False) + if project: + occurrences = occurrences.filter(project=project, pk__in=verified) + # Materialising the used ids follows the guidance in CLAUDE.md for anti-joins. Measured + # locally (3,193 occurrences, 219 used): 1.3 ms against 15.3 ms for + # `.exclude(training_uses__isnull=False)`. Not measured at production scale, where a + # large id list may well turn the trade-off around. + used = list(occurrence_ids_used_for_training(project)) + return occurrences.exclude(pk__in=used) diff --git a/ami/ml/tests.py b/ami/ml/tests.py index d1f0a22f5..7f8c5bf38 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -4,6 +4,7 @@ import json import pathlib import unittest +import unittest.mock import uuid import numpy as np @@ -22,6 +23,7 @@ Project, SourceImage, SourceImageCollection, + TaxaList, Taxon, TaxonRank, group_images_into_events, @@ -2431,7 +2433,7 @@ def setUp(self): self.user = User.objects.create_user(email="verifier@example.com", password="testpass123") self.images = [ SourceImage.objects.create(path=f"export{i}-2024010100{i:02d}00.jpg", project=self.project) - for i in range(6) + for i in range(20) ] self.pipeline = Pipeline.objects.create(name="Export Test Pipeline") self.algorithms = { @@ -2498,6 +2500,7 @@ def _run_export(self, **kwargs) -> dict: project=self.project.pk, algorithm=self.classifier.key, output=str(out), + test_fraction=0.5, stdout=io.StringIO(), **kwargs, ) @@ -2521,7 +2524,7 @@ def test_export_pairs_embeddings_with_human_labels(self): self._verify_all() result = self._run_export(min_per_species=1) - self.assertEqual(result["npz"]["embeddings"].shape, (len(self.images), EMBEDDING_DIMENSIONS)) + self.assertEqual(result["npz"]["features"].shape, (len(self.images), EMBEDDING_DIMENSIONS)) self.assertEqual(sorted(result["meta"]["classes"]), ["Testus duo", "Testus unus"]) self.assertEqual(result["meta"]["rows"], len(self.images)) self.assertEqual(result["meta"]["algorithm"]["key"], self.classifier.key) @@ -2724,6 +2727,9 @@ def setUp(self): } self.classifier = self.algorithms["random-species-classifier"] self.classifier.trainable = True + # Half in, half out. The held-out split is a hash of the occurrence id, so the + # default 0.2 can leave no rows on one side and make the build refuse. + self.classifier.training_config.test_fraction = 0.5 self.classifier.save() self.pipeline.algorithms.set([self.algorithms["random-detector"], self.classifier]) @@ -3111,3 +3117,415 @@ def test_defaults_exist_without_the_service_sending_any(self): algorithm = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES["random-detector"]) self.assertEqual(algorithm.training_config.min_per_species, 2) self.assertEqual(algorithm.training_info.trained_at, None) + + +class TestTrainingSetMembership(TestCase): + """ + Which occurrences a training run consumed, so an evaluation set can avoid them. + """ + + def setUp(self): + from ami.jobs.models import Job, TrainClassifierJob + + self.project = Project.objects.create(name="Training Set Membership Project") + self.user = User.objects.create_user(email="membership@example.com", password="testpass123") + self.taxon = Taxon.objects.create(name="Memberus testus", rank=TaxonRank.SPECIES.name) + self.images = [ + SourceImage.objects.create(path=f"tsm{i}-2024010100{i:02d}00.jpg", project=self.project) for i in range(4) + ] + self.occurrences = [] + for image in self.images: + detection = Detection.objects.create(source_image=image, bbox=[0, 0, 10, 10]) + occurrence = detection.associate_new_occurrence() + Identification.objects.create(occurrence=occurrence, taxon=self.taxon, user=self.user) + self.occurrences.append(occurrence) + + self.job = Job.objects.create(project=self.project, name="Retrain", job_type_key=TrainClassifierJob.key) + + def test_recording_writes_one_row_per_occurrence(self): + from ami.ml.models.training_set import TrainingSetMembership, record_training_set + + record_training_set([o.pk for o in self.occurrences], job=self.job) + self.assertEqual(TrainingSetMembership.objects.filter(job=self.job).count(), len(self.occurrences)) + + def test_recording_twice_does_not_duplicate(self): + """A retried job must not double-count what it consumed.""" + from ami.ml.models.training_set import TrainingSetMembership, record_training_set + + ids = [o.pk for o in self.occurrences] + record_training_set(ids, job=self.job) + record_training_set(ids, job=self.job) + self.assertEqual(TrainingSetMembership.objects.filter(job=self.job).count(), len(ids)) + + def test_the_version_is_attached_after_training(self): + """The version does not exist while the set is built, so it is linked afterwards.""" + from ami.ml.models.training_set import TrainingSetMembership, attach_algorithm, record_training_set + + algorithm = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES["random-species-classifier"]) + record_training_set([o.pk for o in self.occurrences], job=self.job) + self.assertEqual(TrainingSetMembership.objects.filter(algorithm__isnull=True).count(), len(self.occurrences)) + + linked = attach_algorithm(job=self.job, algorithm=algorithm) + self.assertEqual(linked, len(self.occurrences)) + self.assertEqual(TrainingSetMembership.objects.filter(algorithm=algorithm).count(), len(self.occurrences)) + + def test_used_occurrences_are_excluded_from_the_evaluation_pool(self): + """Scoring a model on what it learned from reports a number that means nothing.""" + from ami.ml.models.training_set import occurrences_safe_to_evaluate_on, record_training_set + + pool = occurrences_safe_to_evaluate_on(self.project) + self.assertEqual(pool.count(), len(self.occurrences)) + + record_training_set([self.occurrences[0].pk, self.occurrences[1].pk], job=self.job) + pool = occurrences_safe_to_evaluate_on(self.project) + self.assertEqual(pool.count(), len(self.occurrences) - 2) + self.assertNotIn(self.occurrences[0], pool) + + def test_an_unverified_occurrence_is_not_in_the_pool(self): + from ami.ml.models.training_set import occurrences_safe_to_evaluate_on + + image = SourceImage.objects.create(path="tsm-extra-20240101010000.jpg", project=self.project) + detection = Detection.objects.create(source_image=image, bbox=[0, 0, 10, 10]) + unverified = detection.associate_new_occurrence() + + self.assertNotIn(unverified, occurrences_safe_to_evaluate_on(self.project)) + + +class TestTaxaListDecidesTheClassList(TestCase): + """ + A project's taxa list sets which species a retrained head can predict. The verified + crops only decide how well it predicts each one. + """ + + def setUp(self): + self.project = Project.objects.create(name="Taxa List Training Project") + self.project.feature_flags.store_classification_embeddings = True + self.project.save() + self.user = User.objects.create_user(email="taxalist@example.com", password="testpass123") + + self.algorithms = { + key: get_or_create_algorithm_and_category_map(val) for key, val in ALGORITHM_CHOICES.items() + } + self.classifier = self.algorithms["random-species-classifier"] + self.classifier.trainable = True + self.classifier.save() + self.pipeline = Pipeline.objects.create(name="Taxa List Pipeline") + self.pipeline.algorithms.set([self.algorithms["random-detector"], self.classifier]) + + self.verified_taxa = [ + Taxon.objects.create(name=f"Verifiedus {n}", rank=TaxonRank.SPECIES.name) for n in ("alpha", "beta") + ] + self.unverified_taxa = [ + Taxon.objects.create(name=f"Unverifiedus {n}", rank=TaxonRank.SPECIES.name) for n in ("gamma", "delta") + ] + + # Twenty crops, split between the two verified species, each with an embedding. + # The held-out split is a hash of the occurrence id, so a small set can land + # entirely on one side and make the build refuse. Twenty makes that vanishingly rare. + self.images = [ + SourceImage.objects.create(path=f"tl{i}-2024010100{i:02d}00.jpg", project=self.project) for i in range(20) + ] + for i, image in enumerate(self.images): + detection = Detection.objects.create(source_image=image, bbox=[0, 0, 10, 10]) + occurrence = detection.associate_new_occurrence() + Identification.objects.create(occurrence=occurrence, taxon=self.verified_taxa[i % 2], user=self.user) + DetectionEmbedding.objects.create( + detection=detection, algorithm=self.classifier, vector=[float(i)] * EMBEDDING_DIMENSIONS + ) + + def _taxa_list(self, taxa): + taxa_list, _ = TaxaList.objects.get_or_create_for_project(name="Region list", project=self.project) + taxa_list.taxa.set(taxa) + return taxa_list + + def _build(self, **kwargs): + from ami.ml.training_dataset import build_training_dataset + + return build_training_dataset(project=self.project, algorithm=self.classifier, test_fraction=0.5, **kwargs) + + def test_without_a_taxa_list_the_classes_come_from_what_was_verified(self): + result = self._build(min_per_species=1) + self.assertEqual(sorted(result["metadata"]["classes"]), sorted(t.name for t in self.verified_taxa)) + self.assertIsNone(result["metadata"]["taxa_list"]) + default_storage.delete(result["path"]) + + def test_the_taxa_list_sets_the_classes(self): + """This is what stops a head shrinking to whatever someone happened to verify.""" + taxa_list = self._taxa_list(self.verified_taxa + self.unverified_taxa) + result = self._build(taxa_list=taxa_list, min_per_species=1) + + meta = result["metadata"] + self.assertEqual(len(meta["classes"]), 4) + self.assertEqual(meta["taxa_list"]["name"], taxa_list.name) + default_storage.delete(result["path"]) + + def test_species_in_the_list_with_no_crops_are_kept_and_reported(self): + taxa_list = self._taxa_list(self.verified_taxa + self.unverified_taxa) + result = self._build(taxa_list=taxa_list, min_per_species=1) + + meta = result["metadata"] + self.assertEqual(sorted(meta["classes_without_verified_data"]), sorted(t.name for t in self.unverified_taxa)) + for taxon in self.unverified_taxa: + self.assertEqual(meta["counts"][taxon.name], 0) + default_storage.delete(result["path"]) + + def test_verified_species_outside_the_list_are_dropped_and_reported(self): + taxa_list = self._taxa_list([self.verified_taxa[0]] + self.unverified_taxa) + result = self._build(taxa_list=taxa_list, min_per_species=1) + + meta = result["metadata"] + self.assertIn(self.verified_taxa[1].name, meta["dropped_species"]) + self.assertNotIn(self.verified_taxa[1].name, meta["classes"]) + default_storage.delete(result["path"]) + + def test_the_project_default_is_used_when_none_is_passed(self): + taxa_list = self._taxa_list(self.verified_taxa + self.unverified_taxa) + self.project.default_taxa_list = taxa_list + self.project.save() + + result = self._build(min_per_species=1) + self.assertEqual(len(result["metadata"]["classes"]), 4) + default_storage.delete(result["path"]) + + def test_an_empty_taxa_list_is_refused(self): + from ami.ml.training_dataset import NotEnoughVerifiedData + + with self.assertRaises(NotEnoughVerifiedData): + self._build(taxa_list=self._taxa_list([]), min_per_species=1) + + def test_the_job_prefers_its_own_taxa_list_over_the_project_default(self): + from ami.jobs.models import Job, TrainClassifierJob + + project_default = self._taxa_list(self.verified_taxa) + self.project.default_taxa_list = project_default + self.project.save() + + chosen, _ = TaxaList.objects.get_or_create_for_project(name="Chosen list", project=self.project) + chosen.taxa.set(self.unverified_taxa) + + job = Job.objects.create( + project=self.project, + name="Retrain", + job_type_key=TrainClassifierJob.key, + params={"algorithm_key": self.classifier.key, "taxa_list_id": chosen.pk}, + ) + self.assertEqual(TrainClassifierJob.target_taxa_list(job), chosen) + + def test_an_unknown_taxa_list_id_is_refused(self): + from ami.jobs.models import Job, TrainClassifierJob + + job = Job.objects.create( + project=self.project, + name="Retrain", + job_type_key=TrainClassifierJob.key, + params={"algorithm_key": self.classifier.key, "taxa_list_id": 999999}, + ) + with self.assertRaises(ValueError): + TrainClassifierJob.target_taxa_list(job) + + +class TestTrainingCallback(APITestCase): + """ + Training can outlast the request that started it, so the service reports back to a + callback instead of holding the connection open. + """ + + def setUp(self): + from ami.jobs.models import Job, TrainClassifierJob + + self.project = Project.objects.create(name="Callback Project") + self.algorithm = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES["random-species-classifier"]) + self.algorithm.trainable = True + self.algorithm.save() + self.job = Job.objects.create( + project=self.project, + name="Retrain", + job_type_key=TrainClassifierJob.key, + params={"algorithm_key": self.algorithm.key, "media_base_url": "http://antenna:8000"}, + ) + self.url = f"/api/v2/jobs/{self.job.pk}/training-result/" + self.payload = { + "result": { + "labels": ["Alpha one", "Beta two"], + "rows": {"total": 10, "kept": 10, "train": 8, "test": 2}, + "candidate_metrics": {"top1": 0.9, "n": 2}, + "incumbent_metrics": {"top1": 0.5, "n": 2}, + "trained_at": "2026-09-13T15:00:00", + "warnings": [], + "promote": True, + }, + "dataset": {"rows": 10}, + "dataset_url": "/media/training/x.npz", + } + + def _token(self): + from ami.ml.training_dispatch import make_callback_token + + return make_callback_token(self.job) + + def _post(self, token=None): + headers = {"HTTP_AUTHORIZATION": f"Token {token}"} if token else {} + return self.client.post(self.url, self.payload, format="json", **headers) + + def test_a_valid_token_records_the_result(self): + from ami.jobs.models import JobState + + response = self._post(self._token()) + self.assertEqual(response.status_code, 200) + self.job.refresh_from_db() + self.assertEqual(self.job.status, JobState.SUCCESS.name) + self.assertEqual(self.job.result["result"]["candidate_metrics"]["top1"], 0.9) + + def test_no_token_is_refused(self): + """The service has no Antenna account, so the token is the only thing authorising it.""" + self.assertEqual(self._post().status_code, 403) + + def test_a_forged_token_is_refused(self): + self.assertEqual(self._post("not-a-real-token").status_code, 403) + + def test_another_jobs_token_is_refused(self): + from ami.jobs.models import Job, TrainClassifierJob + from ami.ml.training_dispatch import make_callback_token + + other = Job.objects.create(project=self.project, name="Other", job_type_key=TrainClassifierJob.key) + self.assertEqual(self._post(make_callback_token(other)).status_code, 403) + + def test_a_second_result_does_not_overwrite_the_first(self): + """A retry or a late answer must not undo what already landed.""" + self._post(self._token()) + self.payload["result"]["candidate_metrics"]["top1"] = 0.1 + response = self._post(self._token()) + + self.assertEqual(response.status_code, 200) + self.job.refresh_from_db() + self.assertEqual(self.job.result["result"]["candidate_metrics"]["top1"], 0.9) + + def test_a_result_for_a_non_training_job_is_refused(self): + from ami.jobs.models import Job, MLJob + from ami.ml.training_dispatch import make_callback_token + + other = Job.objects.create(project=self.project, name="ML", job_type_key=MLJob.key) + response = self.client.post( + f"/api/v2/jobs/{other.pk}/training-result/", + self.payload, + format="json", + HTTP_AUTHORIZATION=f"Token {make_callback_token(other)}", + ) + self.assertEqual(response.status_code, 400) + + def test_the_callback_url_points_at_this_job(self): + from ami.ml.training_dispatch import callback_url_for + + self.assertEqual(callback_url_for(self.job), f"http://antenna:8000/api/v2/jobs/{self.job.pk}/training-result/") + + +class TestTrainingDataPermissions(APITestCase): + """ + Verified labels are project data, so reading them needs membership of that project, + not merely an account. + """ + + def setUp(self): + self.project = Project.objects.create(name="Permissions Project") + self.member = User.objects.create_user(email="member@example.com", password="testpass123") + self.outsider = User.objects.create_user(email="outsider@example.com", password="testpass123") + self.superuser = User.objects.create_superuser(email="super@example.com", password="testpass123") + self.project.members.add(self.member) + + self.algorithm = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES["random-species-classifier"]) + self.url = reverse_with_params("api:training-data-list") + self.params = {"project_id": self.project.pk, "algorithm": self.algorithm.key} + + def _status_for(self, user): + self.client.force_authenticate(user=user) + return self.client.get(self.url, self.params).status_code + + def test_a_member_can_read_the_training_data(self): + self.assertEqual(self._status_for(self.member), 200) + + def test_a_superuser_can_read_the_training_data(self): + self.assertEqual(self._status_for(self.superuser), 200) + + def test_anonymous_cannot(self): + self.client.force_authenticate(user=None) + self.assertIn(self.client.get(self.url, self.params).status_code, (401, 403)) + + def test_an_outsider_can_read_a_public_project(self): + """Antenna publishes non-draft projects, and these labels are that project's data.""" + self.assertEqual(self._status_for(self.outsider), 200) + + def test_an_outsider_cannot_read_a_draft_project(self): + """A draft project is private, so its verified labels are too.""" + self.project.draft = True + self.project.save() + self.assertEqual(self._status_for(self.outsider), 403) + + def test_a_member_can_still_read_their_draft_project(self): + self.project.draft = True + self.project.save() + self.assertEqual(self._status_for(self.member), 200) + + +class TestTrainingConfigIsUsed(TestCase): + """ + Settings published by the service and stored on the algorithm actually drive a run. + """ + + def setUp(self): + from ami.jobs.models import Job, TrainClassifierJob + from ami.ml.schemas import AlgorithmTrainingConfig + + self.project = Project.objects.create(name="Config Project") + self.algorithm = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES["random-species-classifier"]) + self.algorithm.trainable = True + self.algorithm.training_config = AlgorithmTrainingConfig( + min_per_species=7, test_fraction=0.4, split_salt="from-config", head_type="mlp1", epochs=42 + ) + self.algorithm.save() + self.job = Job.objects.create( + project=self.project, + name="Retrain", + job_type_key=TrainClassifierJob.key, + params={"algorithm_key": self.algorithm.key}, + ) + + def _payload(self): + from ami.ml.training_dispatch import send_training_request + + captured = {} + + class FakeResponse: + ok = True + + def json(self): + return {"labels": []} + + def fake_post(url, json=None, timeout=None): + captured.update(json) + return FakeResponse() + + with unittest.mock.patch("ami.ml.training_dispatch.create_session") as session: + session.return_value.post = fake_post + send_training_request( + job=self.job, + service=unittest.mock.Mock(endpoint_url="http://service:2000", name="svc"), + algorithm=self.algorithm, + dataset={"url": "/media/training/x.npz"}, + ) + return captured + + def test_the_fitting_settings_reach_the_service(self): + payload = self._payload() + self.assertEqual(payload["head_type"], "mlp1") + self.assertEqual(payload["epochs"], 42) + self.assertEqual(payload["min_per_species"], 7) + + def test_a_job_can_override_the_config_for_one_run(self): + self.job.params = {**self.job.params, "epochs": 5, "head_type": "linear"} + self.job.save() + + payload = self._payload() + self.assertEqual(payload["epochs"], 5) + self.assertEqual(payload["head_type"], "linear") + # Not overridden, so it still comes from the config. + self.assertEqual(payload["min_per_species"], 7) diff --git a/ami/ml/training_data.py b/ami/ml/training_data.py index 61cca7276..50d64b86b 100644 --- a/ami/ml/training_data.py +++ b/ami/ml/training_data.py @@ -17,6 +17,12 @@ DEFAULT_SPLIT_SALT = "antenna-head-v1" DEFAULT_TEST_FRACTION = 0.2 +# The two sides of the split. Named so the API, the dataset builder and the export +# cannot drift apart on spelling. +SPLIT_TRAIN = "train" +SPLIT_TEST = "test" +SPLITS = (SPLIT_TRAIN, SPLIT_TEST) + def split_for( occurrence_id: int, @@ -32,7 +38,7 @@ def split_for( data arrives — an eval set that drifts cannot compare two heads. """ digest = hashlib.sha256(f"{salt}:{occurrence_id}".encode()).hexdigest() - return "test" if (int(digest[:8], 16) / 0xFFFFFFFF) < test_fraction else "train" + return SPLIT_TEST if (int(digest[:8], 16) / 0xFFFFFFFF) < test_fraction else SPLIT_TRAIN def verified_occurrence_ids(project: Project) -> QuerySet: diff --git a/ami/ml/training_dataset.py b/ami/ml/training_dataset.py index fb4f03137..3affe7564 100644 --- a/ami/ml/training_dataset.py +++ b/ami/ml/training_dataset.py @@ -22,6 +22,7 @@ from ami.ml import training_data from ami.ml.models.algorithm import Algorithm from ami.ml.models.embedding import EMBEDDING_DIMENSIONS +from ami.ml.models.training_set import record_training_set logger = logging.getLogger(__name__) @@ -47,7 +48,8 @@ def build_training_dataset( min_per_species: int = 2, split_salt: str = training_data.DEFAULT_SPLIT_SALT, test_fraction: float = training_data.DEFAULT_TEST_FRACTION, - job_id: int | None = None, + taxa_list=None, + job=None, ) -> dict[str, typing.Any]: """ Collect verified labels and their embeddings, and save them as one npz file. @@ -57,15 +59,28 @@ def build_training_dataset( trained on. """ counts = training_data.label_counts(project, algorithm) - keep = training_data.species_with_enough_examples(counts, min_per_species) - if not keep: - raise NotEnoughVerifiedData( - f"No species in '{project.name}' has at least {min_per_species} verified crops with an " - f"embedding from {algorithm.key}. Verify more occurrences, or re-run the pipeline so the " - "verified detections get embeddings." - ) + taxa_list = taxa_list or project.default_taxa_list + + if taxa_list: + # The taxa list decides what the head can predict; the verified crops only decide + # how well it predicts each one. Without this the head shrinks to whatever happened + # to be verified, which silently narrows the pipeline. + classes = sorted(taxa_list.taxa.exclude(name="").values_list("name", flat=True)) + if not classes: + raise NotEnoughVerifiedData(f"Taxa list '{taxa_list}' is empty, so there is nothing to train.") + keep = set(classes) + outside = sorted(name for name in counts if name and name not in keep) + else: + keep = training_data.species_with_enough_examples(counts, min_per_species) + if not keep: + raise NotEnoughVerifiedData( + f"No species in '{project.name}' has at least {min_per_species} verified crops with an " + f"embedding from {algorithm.key}. Verify more occurrences, or re-run the pipeline so the " + "verified detections get embeddings." + ) + classes = sorted(keep) + outside = sorted(set(counts) - keep) - classes = sorted(keep) class_index = {name: i for i, name in enumerate(classes)} rows = training_data.verified_training_rows(project, algorithm) @@ -115,8 +130,11 @@ def build_training_dataset( "dimensions": EMBEDDING_DIMENSIONS, "dtype": np.dtype(DATASET_DTYPE).name, "classes": classes, - "counts": {name: counts[name] for name in classes}, - "dropped_species": sorted(set(counts) - keep), + # .get(): with a taxa list, a class can legitimately have no verified crops yet. + "counts": {name: counts.get(name, 0) for name in classes}, + "taxa_list": {"id": taxa_list.pk, "name": taxa_list.name} if taxa_list else None, + "classes_without_verified_data": sorted(name for name in classes if not counts.get(name)), + "dropped_species": outside, "rows": kept, "train": n_train, "test": n_test, @@ -132,7 +150,7 @@ def build_training_dataset( file_path = _save( project=project, algorithm=algorithm, - job_id=job_id, + job_id=job.pk if job else None, arrays={ "features": features, "labels": labels, @@ -144,6 +162,11 @@ def build_training_dataset( metadata=metadata, ) + if job: + # Written here rather than after training, because this is the moment the set is + # decided. A run that fails later still consumed these occurrences. + record_training_set(occurrence_ids=[int(pk) for pk in occurrence_ids[:kept]], job=job) + file_url = f"{settings.MEDIA_URL}{file_path}" logger.info(f"Wrote training dataset with {kept} rows over {len(classes)} species to {file_path}") return {"path": file_path, "url": file_url, "metadata": metadata} diff --git a/ami/ml/training_dispatch.py b/ami/ml/training_dispatch.py index 813518af4..7f9ff81e3 100644 --- a/ami/ml/training_dispatch.py +++ b/ami/ml/training_dispatch.py @@ -9,6 +9,8 @@ from urllib.parse import urljoin from django.conf import settings +from django.core import signing +from django.urls import reverse from ami.utils.requests import create_session, extract_error_message_from_response @@ -20,6 +22,42 @@ DISPATCH_TIMEOUT_SECONDS = 600 +# A processing service has no Antenna account, so the callback is authorised by a signed +# token instead. Nothing is stored: the signature carries the job id and Django's secret +# key proves Antenna issued it. +CALLBACK_SALT = "ami.ml.training.callback" +CALLBACK_MAX_AGE_SECONDS = 60 * 60 * 24 + + +def make_callback_token(job) -> str: + """A token only Antenna could have produced, tied to this one job.""" + return signing.dumps({"job_id": job.pk}, salt=CALLBACK_SALT) + + +def verify_callback_token(token: str, job) -> bool: + """True when the token is Antenna's, unexpired, and for this job.""" + if not token: + return False + try: + payload = signing.loads(token, salt=CALLBACK_SALT, max_age=CALLBACK_MAX_AGE_SECONDS) + except signing.BadSignature: + return False + return payload.get("job_id") == job.pk + + +def callback_url_for(job) -> str: + """Where the service should post its result when training finishes.""" + base = (job.params or {}).get("media_base_url") or getattr(settings, "EXTERNAL_BASE_URL", "") + if not base: + raise ValueError( + "No base URL is configured, so the processing service has no way to report back. " + "Set EXTERNAL_BASE_URL, or pass media_base_url in the job params." + ) + # reverse(), so the path follows the router rather than a copy of it here. + path = reverse("api:job-training-result", args=[job.pk]) + return urljoin(base.rstrip("/") + "/", path.lstrip("/")) + + def absolute_media_url(url: str, base_url: str | None = None) -> str: """ Turn a stored file's URL into one a processing service can fetch. @@ -48,18 +86,27 @@ def send_training_request(job, service, algorithm, dataset: dict) -> dict | None will report back later. Raises if the service refused the request. """ endpoint = urljoin(service.endpoint_url.rstrip("/") + "/", "train") + params = job.params or {} + config = algorithm.training_config payload: dict[str, typing.Any] = { - "dataset_url": absolute_media_url(dataset["url"], (job.params or {}).get("media_base_url")), + "dataset_url": absolute_media_url(dataset["url"], params.get("media_base_url")), "algorithm_key": algorithm.key, "job_id": job.pk, "name": f"{algorithm.key}-job-{job.pk}", - "min_per_species": (job.params or {}).get("min_per_species", 2), + "min_per_species": params.get("min_per_species", config.min_per_species), + # The fitting settings the service published, so an admin can tune them in Antenna + # without redeploying the service. + "min_improvement": params.get("min_improvement", config.min_improvement), + "head_type": params.get("head_type", config.head_type), + "epochs": params.get("epochs", config.epochs), + "learning_rate": params.get("learning_rate", config.learning_rate), + "weight_decay": params.get("weight_decay", config.weight_decay), } - callback = (job.params or {}).get("callback_url") - if callback: - payload["callback_url"] = callback - payload["callback_token"] = (job.params or {}).get("callback_token") + # Always sent: a service that finishes after the request times out reports back here + # instead, which is the only way a real training set can work. + payload["callback_url"] = (job.params or {}).get("callback_url") or callback_url_for(job) + payload["callback_token"] = make_callback_token(job) job.logger.info(f"Sending training request to {endpoint} for {algorithm.key}") session = create_session() diff --git a/ami/ml/views.py b/ami/ml/views.py index 2ff9fa7a5..d0ac7a229 100644 --- a/ami/ml/views.py +++ b/ami/ml/views.py @@ -3,10 +3,11 @@ from django.db import transaction from django.db.models import Prefetch from django.db.models.query import QuerySet +from django.http import Http404 from django.utils.text import slugify from drf_spectacular.utils import extend_schema from rest_framework import exceptions as api_exceptions -from rest_framework import mixins, status, viewsets +from rest_framework import mixins, serializers, status, viewsets from rest_framework.decorators import action from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request @@ -14,6 +15,7 @@ from ami.base.pagination import TrainingDataPagination from ami.base.permissions import ProjectPipelineConfigPermission +from ami.base.serializers import SingleParamSerializer from ami.base.views import ProjectMixin from ami.main.api.schemas import project_id_doc_param from ami.main.api.views import DefaultViewSet @@ -322,43 +324,70 @@ class TrainingDataViewSet(ProjectMixin, mixins.ListModelMixin, viewsets.GenericV queryset = DetectionEmbedding.objects.none() serializer_class = TrainingDataRowSerializer require_project = True + # Membership is enforced in get_queryset() via Project.objects.visible_for_user(): + # ObjectPermission maps a "list" action on a Project to check_custom_permission, which + # denies members, and IsAuthenticated alone would let any account read any project's + # verified labels. permission_classes = [IsAuthenticated] filter_backends: list = [] pagination_class = TrainingDataPagination + def _get_visible_project(self) -> Project: + """ + The requested project, if this user is allowed to see it. + + Verified labels are project data, so an account that cannot see the project must + not be able to read them. + """ + project = self.get_active_project() + if not project: + raise Http404("Project not found.") + visible = Project.objects.visible_for_user(self.request.user).filter(pk=project.pk).exists() + if not visible: + raise api_exceptions.PermissionDenied("You do not have access to this project.") + return project + def _get_algorithm(self) -> Algorithm: - key = self.request.query_params.get("algorithm") - if not key: - raise api_exceptions.ValidationError( - {"algorithm": "Required. The algorithm key whose embeddings to train on."} - ) + key = SingleParamSerializer[str].clean( + "algorithm", + serializers.CharField( + required=True, + help_text="Key of the algorithm whose embeddings to train on.", + ), + self.request.query_params, + ) algorithm = Algorithm.objects.filter(key=key).first() if not algorithm: raise api_exceptions.NotFound(f"No algorithm with key '{key}'.") return algorithm def _get_split_settings(self) -> tuple[str, float]: - params = self.request.query_params - salt = params.get("split_salt", training_data.DEFAULT_SPLIT_SALT) - raw = params.get("test_fraction", training_data.DEFAULT_TEST_FRACTION) - try: - fraction = float(raw) - except (TypeError, ValueError): - raise api_exceptions.ValidationError({"test_fraction": "Must be a number between 0 and 1."}) - if not 0 <= fraction < 1: - raise api_exceptions.ValidationError({"test_fraction": "Must be between 0 and 1."}) + salt = SingleParamSerializer[str].clean( + "split_salt", + serializers.CharField(required=False, default=training_data.DEFAULT_SPLIT_SALT), + self.request.query_params, + ) + fraction = SingleParamSerializer[float].clean( + "test_fraction", + serializers.FloatField( + required=False, + default=training_data.DEFAULT_TEST_FRACTION, + min_value=0, + max_value=0.99, + ), + self.request.query_params, + ) return salt, fraction def get_queryset(self) -> QuerySet[DetectionEmbedding]: - project = self.get_active_project() - assert project # require_project=True - self.check_object_permissions(self.request, project) + project = self._get_visible_project() qs = training_data.verified_training_rows(project, self._get_algorithm()) - split = self.request.query_params.get("split") - if split and split not in ("train", "test"): - raise api_exceptions.ValidationError({"split": "Must be 'train' or 'test'."}) - self._split_filter = split + self._split_filter = SingleParamSerializer[str].clean( + "split", + serializers.ChoiceField(choices=list(training_data.SPLITS), required=False, allow_null=True, default=None), + self.request.query_params, + ) return qs def get_serializer_context(self): @@ -366,7 +395,13 @@ def get_serializer_context(self): salt, fraction = self._get_split_settings() context["split_salt"] = salt context["test_fraction"] = fraction - context["include_features"] = self.request.query_params.get("include_features", "true").lower() != "false" + # Not url_boolean_param: it returns `value or default`, so a default of True can + # never be turned off. + context["include_features"] = SingleParamSerializer[bool].clean( + "include_features", + serializers.BooleanField(required=False, default=True), + self.request.query_params, + ) return context @extend_schema(parameters=[project_id_doc_param]) @@ -385,15 +420,13 @@ def list(self, request, *args, **kwargs): @action(detail=False, methods=["get"]) def summary(self, request, *args, **kwargs): """Counts only. Cheap enough to poll before deciding whether a retrain is worth it.""" - project = self.get_active_project() - assert project - self.check_object_permissions(request, project) + project = self._get_visible_project() algorithm = self._get_algorithm() salt, fraction = self._get_split_settings() counts = training_data.label_counts(project, algorithm) rows = training_data.verified_training_rows(project, algorithm) - splits = {"train": 0, "test": 0} + splits = {name: 0 for name in training_data.SPLITS} for occurrence_id in rows.values_list("detection__occurrence_id", flat=True): splits[training_data.split_for(occurrence_id, salt, fraction)] += 1 @@ -405,8 +438,8 @@ def summary(self, request, *args, **kwargs): "rows": sum(counts.values()), "classes": len(counts), "counts": dict(sorted(counts.items(), key=lambda kv: -kv[1])), - "train": splits["train"], - "test": splits["test"], + "train": splits[training_data.SPLIT_TRAIN], + "test": splits[training_data.SPLIT_TEST], "verified_detections_without_embedding": training_data.count_missing_embeddings(project, algorithm), "settings": { "split_salt": salt, From f60ac2d9a8ad5cb96817dcf143de5809ca983301 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Sun, 13 Sep 2026 14:57:17 -0400 Subject: [PATCH 04/31] feat(ml): keep the species a head already knew, and offer every head it retrains A retrain started from noise, so any species without freshly verified crops came out worse than before. The new head now starts from the current one's weights, and species with no new crops have those weights put back after fitting. Seeding alone is not enough: training drags them around as negatives over hundreds of rounds. With both, a retrain can only add what it learned and never lose what was already there. A retrained head was also unusable, because nothing could select it. Each head saved to disk is now offered as its own algorithm and its own pipeline, listed the moment training finishes and again whenever the service starts. The head it was trained from stays exactly where it is, so a retrain adds a choice rather than replacing one. The pipeline names were a fixed list, which meant any head trained after deployment was rejected; they are validated against the live registry instead, and an unknown name now says which ones exist. The species list Antenna sends is honoured rather than re-derived from the rows that happen to be present, which is what made a seventeen-species taxa list produce an eight-species head. Saved weights are written in a form the loader can actually read back. A request for a head shape this service cannot fit is now refused rather than quietly served as a linear one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- processing_services/bioclip/api/api.py | 36 ++- processing_services/bioclip/api/schemas.py | 12 +- .../bioclip/api/trained_heads.py | 205 ++++++++++++++++++ processing_services/bioclip/api/training.py | 129 ++++++++++- 4 files changed, 363 insertions(+), 19 deletions(-) create mode 100644 processing_services/bioclip/api/trained_heads.py diff --git a/processing_services/bioclip/api/api.py b/processing_services/bioclip/api/api.py index 9973c98da..59e0f26bf 100644 --- a/processing_services/bioclip/api/api.py +++ b/processing_services/bioclip/api/api.py @@ -31,7 +31,7 @@ ProcessingServiceInfoResponse, SourceImage, ) -from . import algorithms, training +from . import algorithms, trained_heads, training from .utils import is_base64, is_url # Configure root logger @@ -44,7 +44,7 @@ # Where retrained heads are written. Kept off the model cache so a training run # cannot overwrite the head the service is currently serving. -TRAINED_HEADS_DIR = os.environ.get("BIOCLIP_TRAINED_HEADS_DIR", "/data/bioclip-service/trained_heads") +TRAINED_HEADS_DIR = trained_heads.TRAINED_HEADS_DIR app = fastapi.FastAPI() @@ -62,6 +62,9 @@ algorithm.key: algorithm for pipeline in pipelines for algorithm in pipeline.config.algorithms } +# Heads this service has already retrained are offered alongside the ones it shipped with. +trained_heads.register(pipeline_choices, algorithm_choices) + # ----------- # API endpoints # ----------- @@ -77,7 +80,9 @@ async def info() -> ProcessingServiceInfoResponse: info = ProcessingServiceInfoResponse( name="BioCLIP ML Backend", description=("BioCLIP 2.5 with a logistic-regression classification head."), - pipelines=[pipeline.config for pipeline in pipelines], + # Built from the live registry, not the static list, so heads retrained after the + # service started are advertised too. + pipelines=[pipeline.config for pipeline in pipeline_choices.values()], # algorithms=list(algorithm_choices.values()), ) return info @@ -123,7 +128,10 @@ async def process(data: PipelineRequest) -> PipelineResultsResponse: try: Pipeline = pipeline_choices[pipeline_slug] except KeyError: - raise fastapi.HTTPException(status_code=422, detail=f"Invalid pipeline choice: {pipeline_slug}") + raise fastapi.HTTPException( + status_code=422, + detail=f"Invalid pipeline choice: {pipeline_slug}. Available: {sorted(pipeline_choices)}", + ) pipeline_request_config = PipelineRequestConfigParameters(**dict(request_config)) if request_config else {} try: @@ -265,8 +273,14 @@ class TrainRequest(pydantic.BaseModel): description="Token for the callback, so Antenna can tell a real result from a forged one.", ) name: str | None = pydantic.Field(default=None, description="Name for the produced head.") + # Defaults match this service's own training_config. Antenna sends whatever an admin + # has set there, so these are overridden in practice. min_per_species: int = 2 min_improvement: float = 0.0 + head_type: str = "linear" + epochs: int = 300 + learning_rate: float = 0.01 + weight_decay: float = 1e-4 save: bool = pydantic.Field( default=True, description="Write the head to disk. It is never loaded into the running service automatically.", @@ -278,6 +292,7 @@ class TrainResponse(pydantic.BaseModel): reason: str warnings: list[str] rows: dict + classes_restored_from_current_head: int = 0 counts: dict dropped_species: list[str] candidate_metrics: dict @@ -316,20 +331,31 @@ async def train(data: TrainRequest) -> TrainResponse: incumbent=incumbent, min_per_species=data.min_per_species, min_improvement=data.min_improvement, + epochs=data.epochs, + learning_rate=data.learning_rate, + weight_decay=data.weight_decay, + head_type=data.head_type, + # Antenna ships the species list inside the dataset; it usually comes from a + # project's taxa list and must win over whatever happens to be in the rows. + declared_classes=dataset_metadata.get("classes"), ) - except training.NotEnoughData as e: + except (training.NotEnoughData, training.UnsupportedHeadType) as e: raise fastapi.HTTPException(status_code=422, detail=str(e)) saved = None if data.save: name = data.name or f"head-{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}" saved = training.save_head(result, pathlib.Path(TRAINED_HEADS_DIR), name) + # Offer it straight away. The head it was trained from stays where it is; this + # adds a choice rather than replacing one. + trained_heads.register(pipeline_choices, algorithm_choices) response = TrainResponse( promote=result["promote"], reason=result["reason"], warnings=result["warnings"], rows=result["rows"], + classes_restored_from_current_head=result.get("classes_restored_from_current_head", 0), counts=result["counts"], dropped_species=result["dropped_species"], candidate_metrics=result["candidate_metrics"], diff --git a/processing_services/bioclip/api/schemas.py b/processing_services/bioclip/api/schemas.py index a30ea0d42..ae60e0b2e 100644 --- a/processing_services/bioclip/api/schemas.py +++ b/processing_services/bioclip/api/schemas.py @@ -262,14 +262,10 @@ class Config: extra = "ignore" -PipelineChoice = typing.Literal[ - "bioclip-2-5-logreg-pipeline", - "bioclip-2-5-panama-pipeline", - "zero-shot-hf-classifier-pipeline", - "zero-shot-object-detector-pipeline", - "zero-shot-object-detector-with-constant-classifier-pipeline", - "zero-shot-object-detector-with-random-species-classifier-pipeline", -] +# Deliberately a plain string, not a fixed list of slugs. Heads retrained after the service +# started get their own pipeline, and a Literal would reject them with an unhelpful 422. +# api.process() validates against the live registry and says which slugs exist. +PipelineChoice = str class PipelineRequestConfigParameters(pydantic.BaseModel): diff --git a/processing_services/bioclip/api/trained_heads.py b/processing_services/bioclip/api/trained_heads.py new file mode 100644 index 000000000..fdb4f42c0 --- /dev/null +++ b/processing_services/bioclip/api/trained_heads.py @@ -0,0 +1,205 @@ +""" +Serve the heads this service has retrained, alongside the one it shipped with. + +A retrained head is useless if nothing can select it. Each one saved to disk is offered as +its own algorithm and its own pipeline, so Antenna sees it in /info and a user can pick it +the same way they pick any other. The original head stays exactly where it was: a retrain +adds a choice, it never replaces one. +""" + +import json +import logging +import os +import pathlib +import typing + +logger = logging.getLogger(__name__) + +# Where the training endpoint writes heads. Kept off the model cache so a training run +# cannot overwrite the head the service is currently serving. +TRAINED_HEADS_DIR = os.environ.get("BIOCLIP_TRAINED_HEADS_DIR", "/data/bioclip-service/trained_heads") + +HEAD_SUFFIX = ".npz" +LABELS_SUFFIX = ".label_map.json" + +# Prefix for the algorithm key and pipeline slug of a retrained head. Antenna keys +# algorithms by this string, so changing it orphans everything already registered. +RETRAINED_PREFIX = "bioclip-2-5-retrained" + + +class TrainedHead(typing.NamedTuple): + """One head this service has produced, as found on disk.""" + + name: str + head_path: pathlib.Path + labels_path: pathlib.Path + metadata: dict + + @property + def algorithm_key(self) -> str: + return f"{RETRAINED_PREFIX}-{self.name}" + + @property + def pipeline_slug(self) -> str: + return f"{RETRAINED_PREFIX}-{self.name}-pipeline" + + @property + def labels(self) -> list[str]: + return [str(label) for label in self.metadata.get("labels", [])] + + +def discover(directory: str | None = None) -> list[TrainedHead]: + """ + List the heads saved on disk. + + Reads the label map rather than the weights: this runs at startup and after every + training run, and the weights are only needed once a head is actually used. + """ + path = pathlib.Path(directory or TRAINED_HEADS_DIR) + if not path.is_dir(): + return [] + + heads: list[TrainedHead] = [] + for head_path in sorted(path.glob(f"*{HEAD_SUFFIX}")): + name = head_path.name[: -len(HEAD_SUFFIX)] + labels_path = path / f"{name}{LABELS_SUFFIX}" + if not labels_path.exists(): + logger.warning(f"Skipping {head_path.name}: no label map beside it, so its classes are unknown") + continue + try: + metadata = json.loads(labels_path.read_text()) + except (OSError, ValueError) as e: + logger.warning(f"Skipping {head_path.name}: could not read its label map ({e})") + continue + if not metadata.get("labels"): + logger.warning(f"Skipping {head_path.name}: its label map lists no species") + continue + heads.append(TrainedHead(name=name, head_path=head_path, labels_path=labels_path, metadata=metadata)) + + logger.info(f"Found {len(heads)} retrained head(s) in {path}") + return heads + + +def make_classifier_class(head: TrainedHead): + """ + Build the algorithm class that serves one retrained head. + + A subclass rather than a separate implementation: the weights have the same shape as + the head it was trained from, so everything about running it is already written. Only + where the file lives and what it is called differ. + """ + from .algorithms import BioCLIP25LogRegClassifier + from .schemas import AlgorithmCategoryMapResponse, AlgorithmConfigResponse, AlgorithmTrainingInfo + + metrics = head.metadata.get("metrics") or {} + trained_at = head.metadata.get("trained_at") + labels = head.labels + + class RetrainedClassifier(BioCLIP25LogRegClassifier): + head_repo_id = None + head_local_dir = str(head.head_path.parent) + head_filename = head.head_path.name + categories_filename = head.labels_path.name + saved_models_key = f"retrained_{head.name}" + + @property + def head_uri(self): + return None + + def get_category_map(self) -> AlgorithmCategoryMapResponse: + crops = sum((head.metadata.get("counts") or {}).values()) + return AlgorithmCategoryMapResponse( + data=[{"index": i, "label": name, "taxon_rank": "SPECIES"} for i, name in enumerate(labels)], + labels=labels, + version=head.name, + description=f"Retrained on {crops} verified crops.", + ) + + def get_algorithm_config_response(self) -> AlgorithmConfigResponse: + return AlgorithmConfigResponse( + name=f"BioCLIP 2.5 + LogReg head (retrained {head.name})", + key=head.algorithm_key, + task_type="classification", + description="Retrained from species verified in Antenna.", + version=1, + version_name=head.name, + uri=None, + trainable=True, + training_config=self.training_config(), + training_info=AlgorithmTrainingInfo( + trained_at=trained_at, + metrics=metrics, + dataset_classes=len(labels), + ), + category_map=self.get_category_map(), + ) + + RetrainedClassifier.__name__ = f"RetrainedClassifier_{head.name.replace('-', '_')}" + return RetrainedClassifier + + +def make_pipeline_class(head: TrainedHead): + """ + Build the pipeline that runs one retrained head. + + Antenna runs pipelines, not algorithms, so a head with no pipeline cannot be selected. + Stage 0 is the same detector the original pipeline uses. + """ + from .algorithms import ZeroShotObjectDetector + from .pipelines import BioCLIP25LogRegPipeline + from .schemas import PipelineConfigResponse + + classifier_class = make_classifier_class(head) + + # Filled in here, not in get_stages(): /info reads the class-level config, and + # get_stages() only runs once a pipeline is actually instantiated to process images. + # A pipeline advertised with no algorithms registers in Antenna with none attached. + stage_configs = [ + ZeroShotObjectDetector().algorithm_config_response, + classifier_class().algorithm_config_response, + ] + + class RetrainedPipeline(BioCLIP25LogRegPipeline): + config = PipelineConfigResponse( + name=f"BioCLIP 2.5 Retrained Head ({head.name})", + slug=head.pipeline_slug, + description="Zero shot object detector with a head retrained from verified species.", + version=1, + algorithms=stage_configs, + ) + + def get_stages(self): + detector = ZeroShotObjectDetector() + if "candidate_labels" in self.request_config: + detector.candidate_labels = self.request_config["candidate_labels"] + return [detector, classifier_class()] + + RetrainedPipeline.__name__ = f"RetrainedPipeline_{head.name.replace('-', '_')}" + return RetrainedPipeline + + +def register(pipeline_choices: dict, algorithm_choices: dict, directory: str | None = None) -> list[str]: + """ + Add every head on disk to the service's registries. + + Called at startup and again after training, so a head becomes selectable without a + restart. Returns the pipeline slugs that were added. + """ + added: list[str] = [] + for head in discover(directory): + if head.pipeline_slug in pipeline_choices: + continue + try: + pipeline_class = make_pipeline_class(head) + classifier = make_classifier_class(head)() + config = classifier.algorithm_config_response + except Exception as e: + logger.error(f"Could not offer retrained head '{head.name}': {e}") + continue + pipeline_choices[head.pipeline_slug] = pipeline_class + algorithm_choices[config.key] = config + added.append(head.pipeline_slug) + + if added: + logger.info(f"Offering {len(added)} retrained head(s): {', '.join(added)}") + return added diff --git a/processing_services/bioclip/api/training.py b/processing_services/bioclip/api/training.py index 06db6a81a..a280ed46a 100644 --- a/processing_services/bioclip/api/training.py +++ b/processing_services/bioclip/api/training.py @@ -29,11 +29,20 @@ # reports its numbers, but says plainly that they cannot support a decision. MIN_MEANINGFUL_TEST_ROWS = 30 +# The head shapes this service can fit. Only "linear" for now: the serving path loads a +# single Linear layer, so anything else would train fine and then fail to load. An MLP-1 +# head scores far better on rare species and is worth adding, but it needs the loader too. +SUPPORTED_HEAD_TYPES = ("linear",) + class NotEnoughData(Exception): """Raised when the verified data cannot support both training and evaluation.""" +class UnsupportedHeadType(Exception): + """Raised when asked for a head shape this service cannot fit or serve.""" + + @dataclasses.dataclass class TrainingRow: detection_id: int @@ -87,6 +96,73 @@ def _matrices(rows: list[TrainingRow], labels: list[str]) -> tuple[np.ndarray, n return x, y +def warm_start( + head: torch.nn.Linear, + labels: list[str], + incumbent: dict[str, typing.Any] | None, +) -> int: + """ + Copy the current head's weights for any species they have in common. + + Without this a retrain starts from noise, so every species with no new verified crops + comes out worse than before. Seeding from the incumbent makes a retrain additive: a + species nobody verified this time keeps exactly what it already knew. + + Returns how many classes were seeded. + """ + if not incumbent: + return 0 + index = {name: i for i, name in enumerate(incumbent["labels"])} + weights = incumbent["weights"] + bias = incumbent["bias"] + + seeded = 0 + with torch.no_grad(): + for i, name in enumerate(labels): + source = index.get(name) + if source is None: + continue + head.weight[i] = torch.from_numpy(np.asarray(weights[source], dtype=np.float32)) + head.bias[i] = float(bias[source]) + seeded += 1 + logger.info(f"Warm-started {seeded} of {len(labels)} classes from the current head") + return seeded + + +def restore_untrained_classes( + weights: np.ndarray, + bias: np.ndarray, + labels: list[str], + trained_counts: dict[str, int], + incumbent: dict[str, typing.Any] | None, +) -> int: + """ + Put back the current head's weights for species that had no training rows. + + Warm-starting alone is not enough: cross-entropy still pushes those classes around as + negatives over hundreds of epochs, so they drift away from what they knew. Copying them + back afterwards is what actually makes a retrain additive. + + Returns how many classes were restored. + """ + if not incumbent: + return 0 + index = {name: i for i, name in enumerate(incumbent["labels"])} + restored = 0 + for i, name in enumerate(labels): + if trained_counts.get(name): + continue + source = index.get(name) + if source is None: + continue + weights[i] = np.asarray(incumbent["weights"][source], dtype=weights.dtype) + bias[i] = incumbent["bias"][source] + restored += 1 + if restored: + logger.info(f"Restored {restored} class(es) that had no verified crops this run") + return restored + + def train_linear_head( x: np.ndarray, y: np.ndarray, @@ -95,6 +171,8 @@ def train_linear_head( learning_rate: float = 0.01, weight_decay: float = 1e-4, device: str = "cpu", + labels: list[str] | None = None, + incumbent: dict[str, typing.Any] | None = None, ) -> torch.nn.Linear: """ Fit a single linear layer over frozen embeddings. @@ -104,6 +182,8 @@ def train_linear_head( kind of model that would need its own serving path. """ head = torch.nn.Linear(x.shape[1], num_classes).to(device) + if labels: + warm_start(head, labels, incumbent) optimizer = torch.optim.AdamW(head.parameters(), lr=learning_rate, weight_decay=weight_decay) loss_fn = torch.nn.CrossEntropyLoss() @@ -176,6 +256,11 @@ def retrain( min_per_species: int = 2, min_improvement: float = DEFAULT_MIN_IMPROVEMENT, device: str = "cpu", + declared_classes: list[str] | None = None, + epochs: int = 300, + learning_rate: float = 0.01, + weight_decay: float = 1e-4, + head_type: str = "linear", ) -> dict[str, typing.Any]: """ Fit a new head and decide whether it deserves to replace the current one. @@ -183,17 +268,31 @@ def retrain( Returns the metrics, the decision, and the new head's weights. It does not write anything: publishing is a separate, deliberate step. """ + if head_type not in SUPPORTED_HEAD_TYPES: + # Refused rather than ignored: silently fitting a different shape than the caller + # asked for would produce a head nobody could explain. + raise UnsupportedHeadType( + f"This service cannot fit a '{head_type}' head. Supported: {', '.join(SUPPORTED_HEAD_TYPES)}." + ) counts: dict[str, int] = {} for row in rows: counts[row.label] = counts.get(row.label, 0) + 1 - labels = sorted(name for name, n in counts.items() if n >= min_per_species) + + if declared_classes: + # Antenna decided the species list, usually from a project's taxa list. Honour it: + # a species with no verified crops this run still needs a column, or the head would + # quietly stop predicting it. Its weights come from the current head instead. + labels = sorted(declared_classes) + else: + labels = sorted(name for name, n in counts.items() if n >= min_per_species) if not labels: raise NotEnoughData( f"No species has at least {min_per_species} verified crops. " f"Verify more occurrences before retraining. Counts: {counts}" ) - kept = [r for r in rows if r.label in labels] + label_set = set(labels) + kept = [r for r in rows if r.label in label_set] train_rows = [r for r in kept if r.split == "train"] test_rows = [r for r in kept if r.split == "test"] if not train_rows: @@ -207,10 +306,25 @@ def retrain( x_train, y_train = _matrices(train_rows, labels) x_test, y_test = _matrices(test_rows, labels) - head = train_linear_head(x_train, y_train, num_classes=len(labels), device=device) + head = train_linear_head( + x_train, + y_train, + num_classes=len(labels), + device=device, + labels=labels, + incumbent=incumbent, + epochs=epochs, + learning_rate=learning_rate, + weight_decay=weight_decay, + ) weights = head.weight.detach().cpu().numpy() bias = head.bias.detach().cpu().numpy() + trained_counts: dict[str, int] = {} + for row in train_rows: + trained_counts[row.label] = trained_counts.get(row.label, 0) + 1 + restored = restore_untrained_classes(weights, bias, labels, trained_counts, incumbent) + candidate = evaluate(weights, bias, x_test, y_test) incumbent_metrics = None if incumbent: @@ -256,9 +370,10 @@ def retrain( "labels": labels, "weights": weights, "bias": bias, - "counts": {name: counts[name] for name in labels}, - "dropped_species": sorted(set(counts) - set(labels)), + "counts": {name: counts.get(name, 0) for name in labels}, + "dropped_species": sorted(set(counts) - label_set), "rows": {"total": len(rows), "kept": len(kept), "train": len(train_rows), "test": len(test_rows)}, + "classes_restored_from_current_head": restored, "candidate_metrics": candidate, "incumbent_metrics": incumbent_metrics, "promote": promote, @@ -284,7 +399,9 @@ def save_head(result: dict[str, typing.Any], directory: pathlib.Path, name: str) head_path, W=result["weights"], b=result["bias"], - classes=np.array(result["labels"], dtype=object), + # A plain string array, not dtype=object. An object array can only be read back + # with allow_pickle=True, and the loader that serves these heads does not set it. + classes=np.array([str(label) for label in result["labels"]]), ) labels_path.write_text( json.dumps( From d0cb601c9bc2542b856df86442ebcfde4b48a1ed Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Sun, 13 Sep 2026 14:57:18 -0400 Subject: [PATCH 05/31] fix(compose): stop nginx rejecting real trap captures in local development Captures run to several megabytes and nginx defaults to a one megabyte body limit, so importing real images into the local MinIO failed with HTTP 413. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- compose/local/minio/nginx.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compose/local/minio/nginx.conf b/compose/local/minio/nginx.conf index 8f1c97387..1c90ecc9e 100644 --- a/compose/local/minio/nginx.conf +++ b/compose/local/minio/nginx.conf @@ -9,6 +9,10 @@ http { listen 9000 default; server_name _; + # Real trap captures run to several MB. nginx defaults to a 1 MB body limit, so + # without this it rejects them with 413 before they reach MinIO. + client_max_body_size 0; + location / { # Set Host header to mitigate MinIO's signature discrepancy issue # e.g. "SignatureDoesNotMatch: The request signature we calculated From 101a83d8b42a0d9c0db0365b16c7830e50621b6c Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Mon, 14 Sep 2026 12:15:56 -0400 Subject: [PATCH 06/31] fix(ml): stop a retrain from registering two versions of the same head A processing service posts its result to the job's callback before it returns from the training request, so a run that finishes quickly reported twice: once through the callback and once through the code that dispatched it. Both paths registered a version, so every retrain left two identical algorithms behind and the job it came from sat at sixty-seven per cent with a finished result. The dispatching code now re-reads the job before writing anything and stops if a result has already landed, and the recording path ignores a second report for a job that is already finished. Progress from the dispatch is written field by field rather than as a whole row, so a stale copy held from before the callback can no longer put a finished job back into "started". Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/jobs/models.py | 24 ++++++++++++++++- ami/ml/tests.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/ami/jobs/models.py b/ami/jobs/models.py index fa5e5ecee..1d8cf63f5 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -1191,10 +1191,19 @@ def dispatch(cls, job: "Job", service, algorithm, dataset: dict) -> None: from ami.ml.training_dispatch import send_training_request response = send_training_request(job=job, service=service, algorithm=algorithm, dataset=dataset) + + # The service posts its callback before returning from /train, so a fast run is + # already finished by the time this line is reached. Writing this copy's progress + # would put the training stage back to "started" on a job that is done. + job.refresh_from_db() + if job.status in JobState.final_states(): + job.logger.info(f"{service.name} already reported its result.") + return + job.progress.update_stage(cls.STAGE_DISPATCH, status=JobState.SUCCESS, progress=1) job.progress.update_stage(cls.STAGE_TRAIN, status=JobState.STARTED, progress=0) job.logger.info(f"Training request accepted by {service.name}. Waiting for it to report back.") - job.save() + job.save(update_fields=["progress", "updated_at"]) if response is None: # The service accepted the work and will post to the job's training-result @@ -1220,6 +1229,15 @@ def dispatch(cls, job: "Job", service, algorithm, dataset: dict) -> None: @classmethod def record_result(cls, job: "Job", payload: dict) -> None: """Store what the service reported, register the new version, and finish the job.""" + # A service posts its callback before returning from /train, so a fast run reports + # twice: once through the callback and once inline. Without this guard each retrain + # registered two algorithm versions. Re-read first, because the inline caller holds + # a copy from before the callback landed. + job.refresh_from_db(fields=["status", "result"]) + if job.status in JobState.final_states(): + job.logger.info("A training result is already recorded for this job; ignoring a duplicate.") + return + result = payload.get("result") or {} job.result = payload @@ -1227,6 +1245,10 @@ def record_result(cls, job: "Job", payload: dict) -> None: # for instance after a restart. Make sure the stage exists before reporting into it. if not any(stage.key == cls.STAGE_TRAIN for stage in job.progress.stages): job.progress.add_stage("Training", cls.STAGE_TRAIN) + # A service that is reporting back plainly received the request, and the callback + # can arrive before the dispatching code closes that stage. + if any(stage.key == cls.STAGE_DISPATCH for stage in job.progress.stages): + job.progress.update_stage(cls.STAGE_DISPATCH, status=JobState.SUCCESS, progress=1) for warning in result.get("warnings", []): job.logger.warning(warning) diff --git a/ami/ml/tests.py b/ami/ml/tests.py index 7f8c5bf38..d5d572319 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -3529,3 +3529,67 @@ def test_a_job_can_override_the_config_for_one_run(self): self.assertEqual(payload["head_type"], "linear") # Not overridden, so it still comes from the config. self.assertEqual(payload["min_per_species"], 7) + + +class TestTrainingResultIsRecordedOnce(TestCase): + """ + A service posts its callback before returning from /train, so a fast run reports twice. + Only the first report may count, or each retrain registers two algorithm versions. + """ + + def setUp(self): + from ami.jobs.models import Job, TrainClassifierJob + + self.project = Project.objects.create(name="Duplicate Result Project") + self.parent = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES["random-species-classifier"]) + self.parent.trainable = True + self.parent.save() + self.job = Job.objects.create( + project=self.project, + name="Retrain", + job_type_key=TrainClassifierJob.key, + params={"algorithm_key": self.parent.key}, + ) + self.payload = { + "result": { + "labels": ["Alpha one", "Beta two"], + "rows": {"total": 10, "kept": 10, "train": 8, "test": 2}, + "candidate_metrics": {"top1": 0.9, "n": 2}, + "incumbent_metrics": {"top1": 0.5, "n": 2}, + "trained_at": "2026-09-13T19:00:00", + "warnings": [], + "promote": True, + }, + "dataset": {"rows": 10}, + "dataset_url": "/media/training/x.npz", + } + + def test_reporting_twice_registers_one_version(self): + from ami.jobs.models import TrainClassifierJob + + before = Algorithm.objects.filter(name=self.parent.name).count() + TrainClassifierJob.record_result(job=self.job, payload=self.payload) + TrainClassifierJob.record_result(job=self.job, payload=self.payload) + + self.assertEqual(Algorithm.objects.filter(name=self.parent.name).count(), before + 1) + + def test_a_stale_copy_of_the_job_cannot_report_again(self): + """The inline caller holds a copy from before the callback landed.""" + from ami.jobs.models import Job, TrainClassifierJob + + stale = Job.objects.get(pk=self.job.pk) + TrainClassifierJob.record_result(job=self.job, payload=self.payload) + + before = Algorithm.objects.filter(name=self.parent.name).count() + TrainClassifierJob.record_result(job=stale, payload=self.payload) + self.assertEqual(Algorithm.objects.filter(name=self.parent.name).count(), before) + + def test_the_first_result_is_the_one_kept(self): + from ami.jobs.models import TrainClassifierJob + + TrainClassifierJob.record_result(job=self.job, payload=self.payload) + second = {**self.payload, "result": {**self.payload["result"], "candidate_metrics": {"top1": 0.1}}} + TrainClassifierJob.record_result(job=self.job, payload=second) + + self.job.refresh_from_db() + self.assertEqual(self.job.result["result"]["candidate_metrics"]["top1"], 0.9) From bf5e050d78209a1ab267eca946782fe1c5e31d66 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Mon, 14 Sep 2026 12:16:26 -0400 Subject: [PATCH 07/31] feat(ml): score a model against a fixed set of occurrences people have verified There was no way to say whether a retrained head was better than the one it came from. Comparing two models means asking them the same questions, so this adds an occurrence set: a fixed, stored list of verified occurrences that does not move when new data arrives. A set that belongs to no project is available everywhere, which is how one set compares models across the platform, the same way a taxa list with no project already works. Scoring reads what is already in the database. The human identification and the model's classification are both stored, so an algorithm that has processed the set is scored in a single pass with no images opened and no GPU. A new job type runs it and writes down both the overall share correct and the average over species, because trap data is long-tailed and a model that only handles the common species otherwise looks excellent. The per-species breakdown is kept as rows so it can be read the other way round: every model's score for one species. An algorithm is only asked about species it can actually predict. Its category map is the list of answers available to it, so an occurrence of anything else is left out and counted separately rather than marked wrong; otherwise a regional head looks bad for not knowing a species nobody trained it on. An algorithm that has never run on the set is reported as a missing step rather than an accuracy of zero. Scoring the same pair again replaces the earlier result, since a second run over the same occurrences is a correction and not a new fact. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/jobs/migrations/0026_evaluation.py | 33 +++++ ami/jobs/models.py | 74 ++++++++++ ami/ml/evaluation.py | 180 +++++++++++++++++++++++++ ami/ml/migrations/0033_evaluation.py | 134 ++++++++++++++++++ ami/ml/models/__init__.py | 4 + ami/ml/models/evaluation.py | 113 ++++++++++++++++ ami/ml/tests.py | 141 ++++++++++++++++++- 7 files changed, 678 insertions(+), 1 deletion(-) create mode 100644 ami/jobs/migrations/0026_evaluation.py create mode 100644 ami/ml/evaluation.py create mode 100644 ami/ml/migrations/0033_evaluation.py create mode 100644 ami/ml/models/evaluation.py diff --git a/ami/jobs/migrations/0026_evaluation.py b/ami/jobs/migrations/0026_evaluation.py new file mode 100644 index 000000000..93a4b7066 --- /dev/null +++ b/ami/jobs/migrations/0026_evaluation.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.10 on 2026-09-13 15:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("jobs", "0025_generate_embeddings_job_type"), + ] + + operations = [ + migrations.AlterField( + model_name="job", + name="job_type_key", + field=models.CharField( + choices=[ + ("ml", "ML pipeline"), + ("populate_captures_collection", "Populate capture set"), + ("data_storage_sync", "Data storage sync"), + ("regroup_events", "Regroup sessions"), + ("unknown", "Unknown"), + ("data_export", "Data Export"), + ("post_processing", "Post Processing"), + ("train_classifier", "Train classifier"), + ("generate_embeddings", "Generate embeddings"), + ("evaluate_algorithm", "Evaluate algorithm"), + ], + default="unknown", + max_length=255, + verbose_name="Job Type", + ), + ), + ] diff --git a/ami/jobs/models.py b/ami/jobs/models.py index 1d8cf63f5..ce65f1c28 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -1039,6 +1039,79 @@ def embedding_count(cls, algorithm) -> int: return DetectionEmbedding.objects.filter(algorithm=algorithm).count() +class EvaluateAlgorithmJob(JobType): + """ + Score an algorithm against a fixed set of occurrences people have verified. + + Reads predictions that already exist rather than running anything, so an algorithm that + has processed the set is scored in one pass with no GPU. Comparing two models means + scoring both on the same set, which is why the set's membership is fixed. + """ + + name = "Evaluate algorithm" + key = "evaluate_algorithm" + + STAGE_SCORE = "score" + + @classmethod + def run(cls, job: "Job"): + from ami.ml import evaluation + from ami.ml.models import Algorithm, OccurrenceSet + + params = job.params or {} + algorithm_key = params.get("algorithm_key") + occurrence_set_id = params.get("occurrence_set_id") + if not algorithm_key or not occurrence_set_id: + raise ValueError("An evaluate_algorithm job needs 'algorithm_key' and 'occurrence_set_id' in its params.") + + algorithm = Algorithm.objects.filter(key=algorithm_key).first() + if not algorithm: + raise ValueError(f"No algorithm with key '{algorithm_key}'.") + occurrence_set = OccurrenceSet.objects.filter(pk=occurrence_set_id).first() + if not occurrence_set: + raise ValueError(f"No occurrence set with id {occurrence_set_id}.") + + job.progress.add_stage("Scoring", cls.STAGE_SCORE) + job.update_status(JobState.STARTED) + job.started_at = datetime.datetime.now() + job.finished_at = None + job.progress.update_stage(cls.STAGE_SCORE, status=JobState.STARTED, progress=0) + job.save() + + try: + result = evaluation.score(occurrence_set, algorithm) + except evaluation.NothingToScore as e: + # A missing step, not a bad model. Saying so beats recording an accuracy of zero. + job.logger.error(str(e)) + job.progress.update_stage(cls.STAGE_SCORE, status=JobState.FAILURE, progress=0) + job.finished_at = datetime.datetime.now() + job.result = {"error": str(e)} + job.update_status(JobState.FAILURE, save=True) + return + + stored = evaluation.save_evaluation(occurrence_set, algorithm, result, job=job) + + job.logger.info( + f"{algorithm.key} on '{occurrence_set.name}': " + f"{result['micro_accuracy']:.3f} overall, {result['macro_accuracy']:.3f} averaged over " + f"{result['species_scored']} species" + ) + if result["occurrences_skipped"]: + job.logger.info( + f"{result['occurrences_skipped']} occurrence(s) were left out: their species is not one " + "this algorithm can predict, or it never classified them." + ) + job.progress.add_stage_param(cls.STAGE_SCORE, "Accuracy", round(result["micro_accuracy"], 3)) + job.progress.add_stage_param(cls.STAGE_SCORE, "Averaged over species", round(result["macro_accuracy"], 3)) + job.progress.add_stage_param(cls.STAGE_SCORE, "Occurrences scored", result["occurrences_scored"]) + job.progress.add_stage_param(cls.STAGE_SCORE, "Species", result["species_scored"]) + job.progress.update_stage(cls.STAGE_SCORE, status=JobState.SUCCESS, progress=1) + job.result = {"evaluation_id": stored.pk, "micro_accuracy": result["micro_accuracy"]} + job.finished_at = datetime.datetime.now() + job.update_status(JobState.SUCCESS, save=True) + job.save() + + class TrainClassifierJob(JobType): """ Retrain a classifier head from the species people have verified in this project. @@ -1409,6 +1482,7 @@ def run(cls, job: "Job"): PostProcessingJob, TrainClassifierJob, GenerateEmbeddingsJob, + EvaluateAlgorithmJob, ] diff --git a/ami/ml/evaluation.py b/ami/ml/evaluation.py new file mode 100644 index 000000000..70568cb16 --- /dev/null +++ b/ami/ml/evaluation.py @@ -0,0 +1,180 @@ +""" +Score an algorithm against a fixed set of occurrences people have verified. + +The comparison is between what a person said and what the algorithm predicted, both of +which are already in the database. No images are opened and no model runs: an algorithm +that has processed the set can be scored in a single pass over its classifications. + +An algorithm is only asked about species it can actually predict. Its category map is the +list of answers available to it, so an occurrence of anything else is left out rather than +counted wrong — otherwise a regional head looks bad for not knowing a species nobody +trained it on. +""" + +import collections +import logging +import typing + +from django.db.models import QuerySet + +from ami.main.models import Classification, Occurrence +from ami.ml.models.algorithm import Algorithm +from ami.ml.models.evaluation import OccurrenceSet + +logger = logging.getLogger(__name__) + + +class NothingToScore(Exception): + """The set holds no occurrence this algorithm was asked about.""" + + +def predictable_taxa(algorithm: Algorithm) -> set[str]: + """The species names this algorithm can answer with.""" + if not algorithm.category_map: + return set() + return {str(label) for label in (algorithm.category_map.labels or [])} + + +def occurrences_to_score(occurrence_set: OccurrenceSet, algorithm: Algorithm) -> QuerySet[Occurrence]: + """Occurrences in the set that carry a human determination.""" + return ( + occurrence_set.occurrences.filter(determination__isnull=False).select_related("determination").order_by("pk") + ) + + +def predictions_for(occurrence_ids: list[int], algorithm: Algorithm) -> dict[int, int]: + """ + This algorithm's latest prediction per occurrence, as occurrence id to taxon id. + + One query rather than one per occurrence: an evaluation set runs to thousands of rows. + Ordered by score so the strongest classification on a detection wins. + """ + rows = ( + Classification.objects.filter( + algorithm=algorithm, + detection__occurrence_id__in=occurrence_ids, + taxon__isnull=False, + ) + .order_by("detection__occurrence_id", "-score") + .values_list("detection__occurrence_id", "taxon_id") + ) + best: dict[int, int] = {} + for occurrence_id, taxon_id in rows: + best.setdefault(occurrence_id, taxon_id) + return best + + +def score(occurrence_set: OccurrenceSet, algorithm: Algorithm) -> dict[str, typing.Any]: + """ + Compare an algorithm's predictions against what people verified. + + Returns overall accuracy, a per-species breakdown, and how much of the set was left + out. Raises NothingToScore when the algorithm has never run on the set, so that reads + as a missing step rather than an accuracy of zero. + """ + answerable = predictable_taxa(algorithm) + if not answerable: + raise NothingToScore( + f"Algorithm '{algorithm.key}' has no category map, so there is no way to know " + "which species it was asked about." + ) + + occurrences = list(occurrences_to_score(occurrence_set, algorithm)) + if not occurrences: + raise NothingToScore(f"'{occurrence_set.name}' holds no verified occurrence to score.") + + predictions = predictions_for([o.pk for o in occurrences], algorithm) + if not predictions: + raise NothingToScore( + f"Algorithm '{algorithm.key}' has not classified anything in '{occurrence_set.name}'. " + "Run it over the set first, then evaluate." + ) + + per_taxon: dict[int, dict[str, typing.Any]] = collections.defaultdict( + lambda: {"scored": 0, "correct": 0, "taxon": None} + ) + scored = 0 + correct = 0 + skipped = 0 + + for occurrence in occurrences: + truth = occurrence.determination + if truth.name not in answerable: + skipped += 1 + continue + predicted = predictions.get(occurrence.pk) + if predicted is None: + skipped += 1 + continue + + bucket = per_taxon[truth.pk] + bucket["taxon"] = truth + bucket["scored"] += 1 + scored += 1 + if predicted == truth.pk: + bucket["correct"] += 1 + correct += 1 + + if not scored: + raise NothingToScore(f"None of the species in '{occurrence_set.name}' are ones '{algorithm.key}' can predict.") + + accuracies = [b["correct"] / b["scored"] for b in per_taxon.values() if b["scored"]] + return { + "micro_accuracy": correct / scored, + "macro_accuracy": sum(accuracies) / len(accuracies), + "occurrences_scored": scored, + "occurrences_skipped": skipped, + "species_scored": len(per_taxon), + "per_taxon": [ + { + "taxon": bucket["taxon"], + "accuracy": bucket["correct"] / bucket["scored"], + "occurrences_scored": bucket["scored"], + "correct": bucket["correct"], + } + for bucket in per_taxon.values() + if bucket["scored"] + ], + } + + +def save_evaluation(occurrence_set: OccurrenceSet, algorithm: Algorithm, result: dict, job=None): + """ + Store a score, replacing any earlier one for the same algorithm and set. + + Replaced rather than appended: the pair is unique, and a second run over the same set + is a correction, not a new fact. + """ + from ami.ml.models.evaluation import AlgorithmEvaluation, TaxonEvaluation + + evaluation, _ = AlgorithmEvaluation.objects.update_or_create( + algorithm=algorithm, + occurrence_set=occurrence_set, + defaults={ + "job": job, + "micro_accuracy": result["micro_accuracy"], + "macro_accuracy": result["macro_accuracy"], + "occurrences_scored": result["occurrences_scored"], + "occurrences_skipped": result["occurrences_skipped"], + "species_scored": result["species_scored"], + }, + ) + evaluation.taxa.all().delete() + TaxonEvaluation.objects.bulk_create( + [ + TaxonEvaluation( + evaluation=evaluation, + taxon=row["taxon"], + accuracy=row["accuracy"], + occurrences_scored=row["occurrences_scored"], + correct=row["correct"], + ) + for row in result["per_taxon"] + ], + batch_size=500, + ) + logger.info( + f"Scored {algorithm.key} on '{occurrence_set.name}': " + f"{result['micro_accuracy']:.3f} over {result['occurrences_scored']} occurrences" + ) + return evaluation diff --git a/ami/ml/migrations/0033_evaluation.py b/ami/ml/migrations/0033_evaluation.py new file mode 100644 index 000000000..5164132a3 --- /dev/null +++ b/ami/ml/migrations/0033_evaluation.py @@ -0,0 +1,134 @@ +# Generated by Django 4.2.10 on 2026-09-13 15:49 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0096_project_default_taxa_list"), + ("jobs", "0026_evaluation"), + ("ml", "0032_training_set_membership"), + ] + + operations = [ + migrations.CreateModel( + name="AlgorithmEvaluation", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "micro_accuracy", + models.FloatField(help_text="Share of occurrences the algorithm named correctly.", null=True), + ), + ( + "macro_accuracy", + models.FloatField( + help_text="Mean of the per-species accuracies. Trap data is long-tailed, so this says far more than the plain share about whether rare species are handled.", + null=True, + ), + ), + ("occurrences_scored", models.PositiveIntegerField(default=0)), + ( + "occurrences_skipped", + models.PositiveIntegerField( + default=0, + help_text="In the set but not scored, because the species is outside what this algorithm can predict. Scoring those would punish it for a question it was never asked.", + ), + ), + ("species_scored", models.PositiveIntegerField(default=0)), + ( + "algorithm", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="evaluations", to="ml.algorithm" + ), + ), + ( + "job", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="evaluations", + to="jobs.job", + ), + ), + ], + options={ + "ordering": ["-created_at"], + }, + ), + migrations.CreateModel( + name="TaxonEvaluation", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "accuracy", + models.FloatField(help_text="Share of this species' occurrences the algorithm named correctly."), + ), + ("occurrences_scored", models.PositiveIntegerField(default=0)), + ("correct", models.PositiveIntegerField(default=0)), + ( + "evaluation", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="taxa", to="ml.algorithmevaluation" + ), + ), + ( + "taxon", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="evaluations", to="main.taxon" + ), + ), + ], + options={ + "ordering": ["-occurrences_scored", "taxon__name"], + }, + ), + migrations.CreateModel( + name="OccurrenceSet", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("name", models.CharField(max_length=255)), + ("description", models.TextField(blank=True)), + ( + "occurrences", + models.ManyToManyField(blank=True, related_name="evaluation_sets", to="main.occurrence"), + ), + ( + "projects", + models.ManyToManyField( + blank=True, + help_text="Projects this set belongs to. A set with none is available everywhere.", + related_name="occurrence_sets", + to="main.project", + ), + ), + ], + options={ + "ordering": ["name"], + }, + ), + migrations.AddField( + model_name="algorithmevaluation", + name="occurrence_set", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="evaluations", to="ml.occurrenceset" + ), + ), + migrations.AddConstraint( + model_name="taxonevaluation", + constraint=models.UniqueConstraint(fields=("evaluation", "taxon"), name="unique_taxon_per_evaluation"), + ), + migrations.AddConstraint( + model_name="algorithmevaluation", + constraint=models.UniqueConstraint( + fields=("algorithm", "occurrence_set"), name="unique_evaluation_per_algorithm_and_set" + ), + ), + ] diff --git a/ami/ml/models/__init__.py b/ami/ml/models/__init__.py index 58ff26795..c5a94e23b 100644 --- a/ami/ml/models/__init__.py +++ b/ami/ml/models/__init__.py @@ -1,5 +1,6 @@ from ami.ml.models.algorithm import Algorithm, AlgorithmCategoryMap from ami.ml.models.embedding import DetectionEmbedding +from ami.ml.models.evaluation import AlgorithmEvaluation, OccurrenceSet, TaxonEvaluation from ami.ml.models.pipeline import Pipeline from ami.ml.models.processing_service import ProcessingService from ami.ml.models.project_pipeline_config import ProjectPipelineConfig @@ -8,7 +9,10 @@ __all__ = [ "Algorithm", "AlgorithmCategoryMap", + "AlgorithmEvaluation", "DetectionEmbedding", + "OccurrenceSet", + "TaxonEvaluation", "Pipeline", "ProcessingService", "ProjectPipelineConfig", diff --git a/ami/ml/models/evaluation.py b/ami/ml/models/evaluation.py new file mode 100644 index 000000000..10f51c161 --- /dev/null +++ b/ami/ml/models/evaluation.py @@ -0,0 +1,113 @@ +import logging + +from django.db import models + +from ami.base.models import BaseModel, BaseQuerySet + +logger = logging.getLogger(__name__) + + +class OccurrenceSetQuerySet(BaseQuerySet): + def for_project(self, project) -> models.QuerySet: + """Sets this project can use: its own, plus any that belong to no project.""" + return self.filter(models.Q(projects=project) | models.Q(projects__isnull=True)).distinct() + + +class OccurrenceSet(BaseModel): + """ + A fixed list of occurrences to score models against. + + Two models can only be compared if they were scored on the same occurrences, so the + membership is stored rather than re-sampled. A set with no projects is global, which is + how one set compares models across the platform; that follows how TaxaList already + treats a list with no project. + """ + + name = models.CharField(max_length=255) + description = models.TextField(blank=True) + occurrences = models.ManyToManyField("main.Occurrence", related_name="evaluation_sets", blank=True) + projects = models.ManyToManyField( + "main.Project", + related_name="occurrence_sets", + blank=True, + help_text="Projects this set belongs to. A set with none is available everywhere.", + ) + + objects = OccurrenceSetQuerySet.as_manager() + + class Meta: + ordering = ["name"] + + def __str__(self) -> str: + return f"{self.name} ({self.occurrences.count()} occurrences)" + + @property + def is_global(self) -> bool: + return not self.projects.exists() + + +class AlgorithmEvaluation(BaseModel): + """ + How one algorithm scored against one occurrence set. + + Scores are stored rather than computed on demand because every view that shows them — + the best model for a taxa list, a species' performance across algorithms — would + otherwise recompute the same thing for every row. + """ + + algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="evaluations") + occurrence_set = models.ForeignKey(OccurrenceSet, on_delete=models.CASCADE, related_name="evaluations") + job = models.ForeignKey("jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="evaluations") + + micro_accuracy = models.FloatField(null=True, help_text="Share of occurrences the algorithm named correctly.") + macro_accuracy = models.FloatField( + null=True, + help_text=( + "Mean of the per-species accuracies. Trap data is long-tailed, so this says " + "far more than the plain share about whether rare species are handled." + ), + ) + occurrences_scored = models.PositiveIntegerField(default=0) + occurrences_skipped = models.PositiveIntegerField( + default=0, + help_text=( + "In the set but not scored, because the species is outside what this algorithm " + "can predict. Scoring those would punish it for a question it was never asked." + ), + ) + species_scored = models.PositiveIntegerField(default=0) + + class Meta: + ordering = ["-created_at"] + constraints = [ + models.UniqueConstraint( + fields=["algorithm", "occurrence_set"], + name="unique_evaluation_per_algorithm_and_set", + ) + ] + + def __str__(self) -> str: + return f"{self.algorithm} on {self.occurrence_set}: {self.micro_accuracy}" + + +class TaxonEvaluation(BaseModel): + """ + How one algorithm did on one species, within one evaluation. + + The per-species breakdown the taxon page shows. Kept as rows rather than a blob so it + can be queried the other way round: every algorithm's score for a given species. + """ + + evaluation = models.ForeignKey(AlgorithmEvaluation, on_delete=models.CASCADE, related_name="taxa") + taxon = models.ForeignKey("main.Taxon", on_delete=models.CASCADE, related_name="evaluations") + + accuracy = models.FloatField(help_text="Share of this species' occurrences the algorithm named correctly.") + occurrences_scored = models.PositiveIntegerField(default=0) + correct = models.PositiveIntegerField(default=0) + + class Meta: + ordering = ["-occurrences_scored", "taxon__name"] + constraints = [models.UniqueConstraint(fields=["evaluation", "taxon"], name="unique_taxon_per_evaluation")] + + def __str__(self) -> str: + return f"{self.taxon}: {self.accuracy}" diff --git a/ami/ml/tests.py b/ami/ml/tests.py index d5d572319..94db9bb1c 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -29,7 +29,15 @@ group_images_into_events, ) from ami.ml import training_data -from ami.ml.models import Algorithm, DetectionEmbedding, Pipeline, ProcessingService +from ami.ml.models import ( + Algorithm, + AlgorithmCategoryMap, + AlgorithmEvaluation, + DetectionEmbedding, + OccurrenceSet, + Pipeline, + ProcessingService, +) from ami.ml.models.embedding import EMBEDDING_DIMENSIONS from ami.ml.models.pipeline import collect_images, get_or_create_algorithm_and_category_map, save_results from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask @@ -3593,3 +3601,134 @@ def test_the_first_result_is_the_one_kept(self): self.job.refresh_from_db() self.assertEqual(self.job.result["result"]["candidate_metrics"]["top1"], 0.9) + + +class TestAlgorithmEvaluation(TestCase): + """ + Scoring an algorithm against a fixed set of verified occurrences, so two models can be + compared on identical data. + """ + + def setUp(self): + self.project = Project.objects.create(name="Evaluation Project") + self.user = User.objects.create_user(email="evaluator@example.com", password="testpass123") + self.taxa = [Taxon.objects.create(name=f"Evaluus {n}", rank=TaxonRank.SPECIES.name) for n in ("alpha", "beta")] + self.outsider_taxon = Taxon.objects.create(name="Outsideus ignotus", rank=TaxonRank.SPECIES.name) + + self.algorithm = get_or_create_algorithm_and_category_map(ALGORITHM_CHOICES["random-species-classifier"]) + self.category_map = AlgorithmCategoryMap.objects.create( + labels=[t.name for t in self.taxa], + data=[{"index": i, "label": t.name} for i, t in enumerate(self.taxa)], + version="eval-test", + ) + self.algorithm.category_map = self.category_map + self.algorithm.save() + + self.occurrence_set = OccurrenceSet.objects.create(name="Blind set") + self.occurrence_set.projects.add(self.project) + + def _occurrence(self, truth, predicted=None, index=0): + """One verified occurrence, optionally with a prediction from the algorithm.""" + image = SourceImage.objects.create( + path=f"ev{index}-{truth.pk}-2024010100{index:02d}00.jpg", project=self.project + ) + detection = Detection.objects.create(source_image=image, bbox=[0, 0, 10, 10]) + occurrence = detection.associate_new_occurrence() + Identification.objects.create(occurrence=occurrence, taxon=truth, user=self.user) + if predicted: + Classification.objects.create( + detection=detection, + taxon=predicted, + algorithm=self.algorithm, + score=0.9, + timestamp=datetime.datetime.now(), + category_map=self.category_map, + ) + self.occurrence_set.occurrences.add(occurrence) + return occurrence + + def test_a_perfect_run_scores_one(self): + from ami.ml import evaluation + + for i, taxon in enumerate(self.taxa): + self._occurrence(truth=taxon, predicted=taxon, index=i) + + result = evaluation.score(self.occurrence_set, self.algorithm) + self.assertEqual(result["micro_accuracy"], 1.0) + self.assertEqual(result["occurrences_scored"], 2) + + def test_a_wrong_prediction_lowers_the_score(self): + from ami.ml import evaluation + + self._occurrence(truth=self.taxa[0], predicted=self.taxa[0], index=0) + self._occurrence(truth=self.taxa[1], predicted=self.taxa[0], index=1) + + result = evaluation.score(self.occurrence_set, self.algorithm) + self.assertEqual(result["micro_accuracy"], 0.5) + + def test_species_the_algorithm_cannot_predict_are_left_out(self): + """Counting those wrong would punish a regional head for a question nobody asked it.""" + from ami.ml import evaluation + + self._occurrence(truth=self.taxa[0], predicted=self.taxa[0], index=0) + self._occurrence(truth=self.outsider_taxon, predicted=self.taxa[0], index=1) + + result = evaluation.score(self.occurrence_set, self.algorithm) + self.assertEqual(result["occurrences_scored"], 1) + self.assertEqual(result["occurrences_skipped"], 1) + self.assertEqual(result["micro_accuracy"], 1.0) + + def test_the_per_species_average_differs_from_the_plain_share(self): + """Long-tailed data: one common species must not hide failure on a rare one.""" + from ami.ml import evaluation + + for i in range(4): + self._occurrence(truth=self.taxa[0], predicted=self.taxa[0], index=i) + self._occurrence(truth=self.taxa[1], predicted=self.taxa[0], index=9) + + result = evaluation.score(self.occurrence_set, self.algorithm) + self.assertEqual(result["micro_accuracy"], 0.8) + self.assertEqual(result["macro_accuracy"], 0.5) + + def test_an_algorithm_that_never_ran_says_so(self): + """A missing step should read as a missing step, not an accuracy of zero.""" + from ami.ml import evaluation + + self._occurrence(truth=self.taxa[0], predicted=None, index=0) + with self.assertRaises(evaluation.NothingToScore): + evaluation.score(self.occurrence_set, self.algorithm) + + def test_results_are_stored_per_species(self): + from ami.ml import evaluation + + self._occurrence(truth=self.taxa[0], predicted=self.taxa[0], index=0) + self._occurrence(truth=self.taxa[1], predicted=self.taxa[0], index=1) + + result = evaluation.score(self.occurrence_set, self.algorithm) + stored = evaluation.save_evaluation(self.occurrence_set, self.algorithm, result) + + self.assertEqual(stored.micro_accuracy, 0.5) + self.assertEqual(stored.taxa.count(), 2) + self.assertEqual(stored.taxa.get(taxon=self.taxa[0]).accuracy, 1.0) + self.assertEqual(stored.taxa.get(taxon=self.taxa[1]).accuracy, 0.0) + + def test_scoring_again_replaces_the_earlier_result(self): + """A second run over the same set is a correction, not a new fact.""" + from ami.ml import evaluation + + self._occurrence(truth=self.taxa[0], predicted=self.taxa[0], index=0) + result = evaluation.score(self.occurrence_set, self.algorithm) + evaluation.save_evaluation(self.occurrence_set, self.algorithm, result) + evaluation.save_evaluation(self.occurrence_set, self.algorithm, result) + + self.assertEqual( + AlgorithmEvaluation.objects.filter(algorithm=self.algorithm, occurrence_set=self.occurrence_set).count(), + 1, + ) + + def test_a_set_with_no_project_is_global(self): + """One set can compare models across projects.""" + shared = OccurrenceSet.objects.create(name="Shared across projects") + self.assertTrue(shared.is_global) + self.assertFalse(self.occurrence_set.is_global) + self.assertIn(shared, OccurrenceSet.objects.for_project(self.project)) From f0be00133d5b82cd849727bbaedf8495c6c68d25 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Mon, 14 Sep 2026 12:16:58 -0400 Subject: [PATCH 08/31] feat(ml): show how each model performs, per species and per taxa list The scores had nowhere to be read. Four places now show them, following the mockups from the September 3 meeting. The species table gains a "training images ready" count: the verified crops that species has, which is what a head can be fit on. It counts crops rather than occurrences, because one insect photographed over three frames is three images to train on, and it does not roll up to genus or family, because a head is fit on the label itself and a crop verified as a species is not training data for its genus. The count is annotated once per page the same way the verification counts already are, so it costs one query whether the page holds one row or a thousand. A species page lists every model that has been scored on it, with the accuracy for that species and the set it was scored on, each linking to the model. A taxa list shows its best model, ranked on the per-species average rather than the plain share so that a model which only handles the common species cannot win. An algorithm's details panel shows what it has scored, on which sets. Each of these reads "n/a" until something has actually been evaluated. The evaluations behind the algorithms list are prefetched and the best model is found in one query, so neither list issues a query per row. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/main/api/serializers.py | 37 ++++++++++ ami/main/api/views.py | 13 +++- ami/main/models.py | 40 +++++++++++ ami/main/tests.py | 46 ++++++++++++ ami/ml/reporting.py | 89 ++++++++++++++++++++++++ ami/ml/serializers.py | 8 +++ ami/ml/tests.py | 135 ++++++++++++++++++++++++++++++++++++ ami/ml/views.py | 4 ++ 8 files changed, 370 insertions(+), 2 deletions(-) create mode 100644 ami/ml/reporting.py diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 5e46a790d..95b694a7c 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -621,6 +621,7 @@ def get_taxa(self, obj): class TaxonListSerializer(DefaultSerializer): # latest_detection = DetectionNestedSerializer(read_only=True) occurrences = serializers.SerializerMethodField() + training_crops_ready = serializers.SerializerMethodField() parents = TaxonParentSerializer(many=True, read_only=True, source="parents_json") parent_id = serializers.PrimaryKeyRelatedField(queryset=Taxon.objects.all(), source="parent") tags = serializers.SerializerMethodField() @@ -640,6 +641,7 @@ class Meta: "details", "occurrences_count", "verified_count", + "training_crops_ready", "occurrences", "tags", "last_detected", @@ -649,6 +651,10 @@ class Meta: "updated_at", ] + def get_training_crops_ready(self, obj): + """Verified crops of this species, which is what a head can be trained on.""" + return getattr(obj, "training_crops_count", None) or 0 + def get_occurrences(self, obj): """ Return URL to the occurrences endpoint filtered by this taxon. @@ -669,6 +675,7 @@ class TaxaListSerializer(DefaultSerializer): taxa = serializers.SerializerMethodField() taxa_count = serializers.SerializerMethodField() projects = serializers.SerializerMethodField() + best_model = serializers.SerializerMethodField() class Meta: model = TaxaList @@ -679,10 +686,26 @@ class Meta: "taxa", "taxa_count", "projects", + "best_model", "created_at", "updated_at", ] + def get_best_model(self, obj): + """The algorithm scoring highest on this list's species, or null if none has been scored.""" + from ami.ml import reporting + + evaluation = reporting.best_evaluation_for_taxa_list(obj) + if not evaluation: + return None + return { + "id": evaluation.algorithm_id, + "name": evaluation.algorithm.name, + "accuracy": evaluation.micro_accuracy, + "accuracy_by_species": evaluation.macro_accuracy, + "occurrence_set": evaluation.occurrence_set.name, + } + def get_taxa(self, obj): """ Return URL to the taxa endpoint filtered by this taxalist. @@ -963,6 +986,8 @@ class Meta: class TaxonSerializer(DefaultSerializer): + training_crops_ready = serializers.SerializerMethodField() + algorithm_performance = serializers.SerializerMethodField() # latest_detection = DetectionNestedSerializer(read_only=True) occurrences = TaxonOccurrenceNestedSerializer(many=True, read_only=True, source="example_occurrences") parent = TaxonNoParentNestedSerializer(read_only=True) @@ -1004,8 +1029,20 @@ class Meta: "cover_image_credit", "summary_data", "common_name_en", + "training_crops_ready", + "algorithm_performance", ] + def get_training_crops_ready(self, obj): + """Verified crops of this species, which is what a head can be trained on.""" + return getattr(obj, "training_crops_count", None) or 0 + + def get_algorithm_performance(self, obj): + """How each scored algorithm has done on this species. Empty until one is evaluated.""" + from ami.ml import reporting + + return reporting.performance_for_taxon(obj) + class CaptureOccurrenceSerializer(DefaultSerializer): determination = TaxonNoParentNestedSerializer(read_only=True) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 7a17d4a09..a5e3c2667 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -1949,7 +1949,8 @@ def get_taxa_observed( The sparse verification rollup (``verified_count`` / ``agreed_*``) is the same on either path — a Python pass over the verified subset applied as ``CASE`` - annotations, see :meth:`TaxonQuerySet.with_verification_counts`. + annotations, see :meth:`TaxonQuerySet.with_verification_counts`. ``training_crops_count`` + is annotated the same way, see :meth:`TaxonQuerySet.with_training_crop_counts`. """ request = self.request use_aggregation = "collection" in request.query_params @@ -1986,7 +1987,7 @@ def get_taxa_observed( if self.action == "list" and "verified" in request.query_params: verified_param = BooleanField(required=False).clean(request.query_params.get("verified")) - return qs.with_verification_counts( + qs = qs.with_verification_counts( project, request, occurrence_filters=direct_filters, @@ -1995,6 +1996,14 @@ def get_taxa_observed( verified=verified_param, ) + return qs.with_training_crop_counts( + project, + request, + occurrence_filters=direct_filters, + apply_default_score_filter=apply_default_score_filter, + apply_default_taxa_filter=apply_default_taxa_filter, + ) + def attach_tags_by_project(self, qs: QuerySet, project: Project) -> QuerySet: """ Prefetch and override the `.tags` attribute on each Taxon diff --git a/ami/main/models.py b/ami/main/models.py index 33613f9ec..d82d743f7 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -4117,6 +4117,46 @@ def with_verification_counts( return qs + def with_training_crop_counts( + self, + project: Project, + request: Request | None, + *, + occurrence_filters: models.Q, + apply_default_score_filter: bool = True, + apply_default_taxa_filter: bool = True, + ): + """Annotate ``training_crops_count``: verified crops a classifier head can be fit on. + + Counted against the exact determination, without the hierarchical rollup + :meth:`with_verification_counts` does — a head is fit on the label itself, so a crop + verified as a species is not training data for its genus. Crops still need an + embedding from the chosen feature extractor before a job can use them; this is the + upper bound, not the row count of the next training set. + """ + default_q = build_occurrence_default_filters_q( + project, + request, + occurrence_accessor="", + apply_default_score_filter=apply_default_score_filter, + apply_default_taxa_filter=apply_default_taxa_filter, + ) + verified_occurrences = ( + Occurrence.objects.filter(occurrence_filters) + .filter(default_q) + .filter(determination_id__isnull=False) + .filter(Exists(Identification.objects.filter(occurrence=OuterRef("pk"), withdrawn=False))) + ) + crop_counts = { + row["occurrence__determination_id"]: row["crops"] + for row in ( + Detection.objects.filter(occurrence__in=verified_occurrences) + .values("occurrence__determination_id") + .annotate(crops=models.Count("pk")) + ) + } + return self.annotate(training_crops_count=_case_from_map(crop_counts, 0, models.IntegerField())) + def filter_by_project_default_taxa(self, project: Project | None = None, request: Request | None = None): """ Filter taxa according to a project's default include and exclude settings, diff --git a/ami/main/tests.py b/ami/main/tests.py index 0a2ce6b7f..ccdba586a 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -6349,6 +6349,52 @@ def test_list_field_values(self): self.assertEqual(rows["Vanessa atalanta"]["verified_count"], 1) self.assertEqual(rows["Vanessa itea"]["verified_count"], 0) + # --- training_crops_ready (verified crops, no rollup) --- + + def test_training_crops_ready_counts_crops_not_occurrences(self): + """A second frame of the same insect is a second crop a head can be fit on.""" + first_detection = self.occ_pred.detections.first() + assert first_detection is not None + Detection.objects.create( + source_image=first_detection.source_image, + occurrence=self.occ_pred, + timestamp=first_detection.timestamp, + bbox=[0.3, 0.3, 0.4, 0.4], + path="detections/second_frame.jpg", + ) + + detail = self._detail(self.cardui) + self.assertEqual(detail["verified_count"], 2) + self.assertEqual(detail["training_crops_ready"], 3) + + def test_training_crops_ready_does_not_roll_up_to_ancestors(self): + """A head is fit on the label itself, so species crops are not genus training data.""" + for ancestor in (self.genus, self.family, self.order): + self.assertEqual(self._detail(ancestor)["training_crops_ready"], 0, ancestor.name) + + def test_training_crops_ready_in_list(self): + rows = self._list_by_name() + self.assertEqual(rows["Vanessa cardui"]["training_crops_ready"], 2) + self.assertEqual(rows["Vanessa atalanta"]["training_crops_ready"], 1) + self.assertEqual(rows["Vanessa itea"]["training_crops_ready"], 0) + + def test_training_crops_ready_does_not_scale_with_page_size(self): + """One aggregate for the page, not one query per row.""" + + from django.core.cache import caches + + def query_count(limit: int) -> int: + # Cold cache on both runs, or a warm one hides the scaling. + caches["default"].clear() + with CaptureQueriesContext(connection) as ctx: + res = self.client.get(f"/api/v2/taxa/?project_id={self.project.pk}&limit={limit}") + self.assertEqual(res.status_code, status.HTTP_200_OK) + return len(ctx.captured_queries) + + small = query_count(limit=1) + large = query_count(limit=1000) + self.assertLessEqual(large, small, f"Query count grew with page size: {small} -> {large}") + # --- verified=true|false filter --- def test_verified_filter_true_false_complement(self): diff --git a/ami/ml/reporting.py b/ami/ml/reporting.py new file mode 100644 index 000000000..761b8015b --- /dev/null +++ b/ami/ml/reporting.py @@ -0,0 +1,89 @@ +""" +The numbers the model-performance views read. + +Each of these answers one question a screen asks, in a single query. They are kept apart +from the models so a view can ask for them without pulling in the training machinery, and +so the same number means the same thing everywhere it is shown. +""" + +import typing + +from ami.main.models import TaxaList, Taxon +from ami.ml.models.algorithm import Algorithm +from ami.ml.models.evaluation import AlgorithmEvaluation, TaxonEvaluation + +DEFAULT_EVALUATION_LIMIT = 5 + + +def best_evaluation_for_taxa_list(taxa_list: TaxaList) -> AlgorithmEvaluation | None: + """ + The algorithm that scores highest on the species in this list. + + Ranked on the per-species average rather than the plain share: trap data is long-tailed, + so a model that only handles the common species would otherwise look like the best one. + """ + # No .distinct(): the join repeats an evaluation once per species it scored in the list, + # which cannot change which row sorts first. + return ( + AlgorithmEvaluation.objects.filter(taxa__taxon__lists=taxa_list) + .select_related("algorithm", "occurrence_set") + .order_by("-macro_accuracy", "-micro_accuracy") + .first() + ) + + +def performance_for_taxon(taxon: Taxon) -> list[dict[str, typing.Any]]: + """ + How each algorithm has done on one species. + + The breakdown the taxon page shows: one row per algorithm that has been scored on a set + containing this species. + """ + rows = ( + TaxonEvaluation.objects.filter(taxon=taxon) + .select_related("evaluation__algorithm", "evaluation__occurrence_set") + .order_by("-accuracy") + ) + return [ + { + "algorithm": { + "id": row.evaluation.algorithm_id, + "name": row.evaluation.algorithm.name, + "key": row.evaluation.algorithm.key, + "version": row.evaluation.algorithm.version, + }, + "occurrence_set": { + "id": row.evaluation.occurrence_set_id, + "name": row.evaluation.occurrence_set.name, + }, + "accuracy": row.accuracy, + "occurrences_scored": row.occurrences_scored, + "correct": row.correct, + "overall_accuracy": row.evaluation.micro_accuracy, + "overall_accuracy_by_species": row.evaluation.macro_accuracy, + } + for row in rows + ] + + +def latest_evaluations(algorithm: Algorithm, limit: int = DEFAULT_EVALUATION_LIMIT) -> list[dict[str, typing.Any]]: + """ + What this algorithm has scored, most recent first. Shown on its details panel. + + Read through the related manager without filtering or reordering, so a list view that + prefetched the evaluations is served from that cache instead of one query per row. + """ + rows = sorted(algorithm.evaluations.all(), key=lambda row: row.created_at, reverse=True)[:limit] + return [ + { + "id": row.pk, + "occurrence_set": {"id": row.occurrence_set_id, "name": row.occurrence_set.name}, + "accuracy": row.micro_accuracy, + "accuracy_by_species": row.macro_accuracy, + "occurrences_scored": row.occurrences_scored, + "occurrences_skipped": row.occurrences_skipped, + "species_scored": row.species_scored, + "created_at": row.created_at, + } + for row in rows + ] diff --git a/ami/ml/serializers.py b/ami/ml/serializers.py index 3d50f5461..ddceec4ab 100644 --- a/ami/ml/serializers.py +++ b/ami/ml/serializers.py @@ -35,6 +35,7 @@ class AlgorithmSerializer(DefaultSerializer): # A bare name in Meta.fields raises ImproperlyConfigured at import time. training_config = SchemaField(schema=AlgorithmTrainingConfig, required=False) training_info = SchemaField(schema=AlgorithmTrainingInfo, read_only=True) + evaluations = serializers.SerializerMethodField() class Meta: model = Algorithm @@ -51,12 +52,19 @@ class Meta: "trainable", "training_config", "training_info", + "evaluations", "category_map", "category_count", "created_at", "updated_at", ] + def get_evaluations(self, obj): + """How this algorithm has scored on each evaluation set. Empty until one is scored.""" + from ami.ml import reporting + + return reporting.latest_evaluations(obj) + class AlgorithmNestedSerializer(DefaultSerializer): class Meta: diff --git a/ami/ml/tests.py b/ami/ml/tests.py index 94db9bb1c..971556eb0 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -37,6 +37,7 @@ OccurrenceSet, Pipeline, ProcessingService, + TaxonEvaluation, ) from ami.ml.models.embedding import EMBEDDING_DIMENSIONS from ami.ml.models.pipeline import collect_images, get_or_create_algorithm_and_category_map, save_results @@ -3732,3 +3733,137 @@ def test_a_set_with_no_project_is_global(self): self.assertTrue(shared.is_global) self.assertFalse(self.occurrence_set.is_global) self.assertIn(shared, OccurrenceSet.objects.for_project(self.project)) + + +class TestPerformanceReporting(TestCase): + """ + The numbers the model-performance screens read: the best model for a taxa list, and how + each algorithm has done on one species. + """ + + def setUp(self): + self.project = Project.objects.create(name="Reporting Project") + self.user = User.objects.create_user(email="reporter@example.com", password="testpass123") + self.common = Taxon.objects.create(name="Reportus communis", rank=TaxonRank.SPECIES.name) + self.rare = Taxon.objects.create(name="Reportus rarus", rank=TaxonRank.SPECIES.name) + self.taxa_list = TaxaList.objects.create(name="Reporting list") + self.taxa_list.taxa.set([self.common, self.rare]) + self.taxa_list.projects.add(self.project) + self.occurrence_set = OccurrenceSet.objects.create(name="Reporting set") + + def _algorithm(self, key: str) -> Algorithm: + return Algorithm.objects.create(name=key, key=key) + + def _evaluation(self, algorithm: Algorithm, micro: float, macro: float, per_taxon: dict) -> AlgorithmEvaluation: + evaluation = AlgorithmEvaluation.objects.create( + algorithm=algorithm, + occurrence_set=self.occurrence_set, + micro_accuracy=micro, + macro_accuracy=macro, + occurrences_scored=sum(scored for _, scored in per_taxon.values()), + species_scored=len(per_taxon), + ) + for taxon, (correct, scored) in per_taxon.items(): + TaxonEvaluation.objects.create( + evaluation=evaluation, + taxon=taxon, + accuracy=correct / scored, + occurrences_scored=scored, + correct=correct, + ) + return evaluation + + def test_the_best_model_is_the_one_that_handles_the_rare_species(self): + """Ranked on the per-species average, or a model that only knows the common one wins.""" + from ami.ml import reporting + + self._evaluation(self._algorithm("common-only"), micro=0.9, macro=0.5, per_taxon={self.common: (9, 10)}) + even = self._evaluation( + self._algorithm("handles-both"), + micro=0.8, + macro=0.8, + per_taxon={self.common: (8, 10), self.rare: (8, 10)}, + ) + + self.assertEqual(reporting.best_evaluation_for_taxa_list(self.taxa_list), even) + + def test_a_list_nothing_has_been_scored_on_has_no_best_model(self): + from ami.ml import reporting + + self.assertIsNone(reporting.best_evaluation_for_taxa_list(self.taxa_list)) + + def test_a_species_lists_every_algorithm_scored_on_it(self): + from ami.ml import reporting + + self._evaluation(self._algorithm("first"), micro=1.0, macro=1.0, per_taxon={self.rare: (2, 2)}) + self._evaluation(self._algorithm("second"), micro=0.5, macro=0.5, per_taxon={self.rare: (1, 2)}) + + rows = reporting.performance_for_taxon(self.rare) + self.assertEqual([row["algorithm"]["key"] for row in rows], ["first", "second"]) + self.assertEqual([row["accuracy"] for row in rows], [1.0, 0.5]) + self.assertEqual(rows[0]["occurrence_set"]["name"], self.occurrence_set.name) + + def test_a_species_nothing_has_been_scored_on_lists_nothing(self): + from ami.ml import reporting + + self.assertEqual(reporting.performance_for_taxon(self.common), []) + + def test_an_algorithm_lists_its_own_scores(self): + from ami.ml import reporting + + algorithm = self._algorithm("scored-twice") + self._evaluation(algorithm, micro=1.0, macro=1.0, per_taxon={self.common: (2, 2)}) + other_set = OccurrenceSet.objects.create(name="Second set") + AlgorithmEvaluation.objects.create( + algorithm=algorithm, + occurrence_set=other_set, + micro_accuracy=0.5, + macro_accuracy=0.5, + occurrences_scored=2, + species_scored=1, + ) + + rows = reporting.latest_evaluations(algorithm) + self.assertEqual({row["occurrence_set"]["name"] for row in rows}, {"Reporting set", "Second set"}) + self.assertEqual(len(reporting.latest_evaluations(algorithm, limit=1)), 1) + + def test_the_algorithms_list_does_not_query_per_row(self): + """Evaluations are prefetched: adding scored algorithms must not add queries.""" + from django.core.cache import caches + from django.db import connection + from django.test.utils import CaptureQueriesContext + + from ami.ml.models import Pipeline, ProcessingService, ProjectPipelineConfig + + service = ProcessingService.objects.create(name="Reporting service", endpoint_url="http://example.com") + service.projects.add(self.project) + pipeline = Pipeline.objects.create(name="Reporting pipeline", slug="reporting-pipeline") + pipeline.projects.add(self.project) + ProjectPipelineConfig.objects.update_or_create( + project=self.project, pipeline=pipeline, defaults={"enabled": True} + ) + + self.client.force_login(self.user) + url = f"/api/v2/ml/algorithms/?project_id={self.project.pk}" + + def query_count() -> int: + # Cold cache on both runs, or a warm one hides the scaling. + caches["default"].clear() + with CaptureQueriesContext(connection) as ctx: + res = self.client.get(url) + self.assertEqual(res.status_code, 200) + return len(ctx.captured_queries) + + for index in range(2): + algorithm = self._algorithm(f"listed-{index}") + pipeline.algorithms.add(algorithm) + self._evaluation(algorithm, micro=1.0, macro=1.0, per_taxon={self.common: (1, 1)}) + two_algorithms = query_count() + + for index in range(2, 6): + algorithm = self._algorithm(f"listed-{index}") + pipeline.algorithms.add(algorithm) + self._evaluation(algorithm, micro=1.0, macro=1.0, per_taxon={self.common: (1, 1)}) + six_algorithms = query_count() + + self.assertLessEqual(six_algorithms, two_algorithms) diff --git a/ami/ml/views.py b/ami/ml/views.py index d0ac7a229..fee75db8d 100644 --- a/ami/ml/views.py +++ b/ami/ml/views.py @@ -25,6 +25,7 @@ from .models.algorithm import Algorithm, AlgorithmCategoryMap from .models.embedding import EMBEDDING_DIMENSIONS, DetectionEmbedding +from .models.evaluation import AlgorithmEvaluation from .models.pipeline import Pipeline from .models.processing_service import ProcessingService from .models.project_pipeline_config import ProjectPipelineConfig @@ -63,6 +64,9 @@ class AlgorithmViewSet(DefaultViewSet, ProjectMixin): def get_queryset(self) -> QuerySet["Algorithm"]: qs: QuerySet["Algorithm"] = super().get_queryset() qs = qs.with_category_count() # type: ignore[union-attr] # Custom queryset method + qs = qs.prefetch_related( + Prefetch("evaluations", queryset=AlgorithmEvaluation.objects.select_related("occurrence_set")) + ) # Only scope the list by project. Detail stays unscoped so links from historical # classifications whose pipeline is no longer enabled still resolve. if getattr(self, "action", None) == "list": From 9c8bdd76d6fa2b585934ce309678c30d1eebfd7f Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Tue, 15 Sep 2026 12:17:39 -0400 Subject: [PATCH 09/31] feat(jobs): let people start training and evaluation jobs without the API Retraining a head, computing embeddings and scoring a model could only be started by posting to the API by hand. The job form now asks which kind of job to make and then asks for what that kind needs: a pipeline for an ML or embeddings run, a head to retrain, a head and an evaluation set to score. Nothing else is shown, so the form no longer demands captures for a job that never looks at one. Each job type now declares what it cannot run without, and the API refuses a job that is missing any of it. A gap is reported while the person still has the form open rather than surfacing minutes later as a job that failed for want of an algorithm, and a job type nobody recognises is rejected outright instead of being stored as something that can never run. Evaluation sets are readable over the API for the first place that needs to offer them as a choice. They stay read-only: membership is built deliberately, because two models can only be compared if they were scored on exactly the same occurrences. Algorithms can be filtered to the trainable ones, so the form offers only heads a processing service will actually accept. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/jobs/models.py | 19 +++++++++++++ ami/jobs/serializers.py | 49 ++++++++++++++++++++++++++++++-- ami/jobs/tests/test_jobs.py | 56 +++++++++++++++++++++++++++++++++++++ ami/jobs/views.py | 2 -- ami/ml/serializers.py | 22 +++++++++++++++ ami/ml/views.py | 32 +++++++++++++++++++-- config/api_router.py | 1 + 7 files changed, 173 insertions(+), 8 deletions(-) diff --git a/ami/jobs/models.py b/ami/jobs/models.py index ce65f1c28..4e5426161 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -444,6 +444,17 @@ class JobType: name: str key: str + # What a job of this type cannot run without. The API refuses to create one that is + # missing any of these, so a gap is a 400 when the job is made rather than a failure + # minutes later when it runs. ``required_params`` are keys inside ``Job.params``; + # ``required_fields`` are fields on the job itself. + required_fields: tuple[str, ...] = () + required_params: tuple[str, ...] = () + + # Whether a person can start one from the UI. The rest are created by the platform: + # a data sync, an export, a regroup. Nothing should offer those as a choice. + user_creatable: bool = False + # @TODO Consider adding custom vocabulary for job types to be used in the UI # verb: str = "Sync" # present_participle: str = "syncing" @@ -460,6 +471,8 @@ def run(cls, job: "Job"): class MLJob(JobType): name = "ML pipeline" key = "ml" + required_fields = ("pipeline",) + user_creatable = True @classmethod def run(cls, job: "Job"): @@ -941,6 +954,8 @@ class GenerateEmbeddingsJob(JobType): name = "Generate embeddings" key = "generate_embeddings" + required_fields = ("pipeline",) + user_creatable = True @classmethod def run(cls, job: "Job"): @@ -1050,6 +1065,8 @@ class EvaluateAlgorithmJob(JobType): name = "Evaluate algorithm" key = "evaluate_algorithm" + required_params = ("algorithm_key", "occurrence_set_id") + user_creatable = True STAGE_SCORE = "score" @@ -1128,6 +1145,8 @@ class TrainClassifierJob(JobType): name = "Train classifier" key = "train_classifier" + required_params = ("algorithm_key",) + user_creatable = True STAGE_PREPARE = "prepare" STAGE_DISPATCH = "dispatch" diff --git a/ami/jobs/serializers.py b/ami/jobs/serializers.py index b739797cb..ba069c62f 100644 --- a/ami/jobs/serializers.py +++ b/ami/jobs/serializers.py @@ -14,7 +14,16 @@ from ami.ml.schemas import PipelineProcessingTask, PipelineTaskResult, ProcessingServiceClientInfo from ami.ml.serializers import PipelineNestedSerializer -from .models import JOB_LOGS_DEFAULT_LIMIT, Job, JobProgress, MLJob, _legacy_logs_shape, serialize_job_logs +from .models import ( + JOB_LOGS_DEFAULT_LIMIT, + VALID_JOB_TYPES, + Job, + JobProgress, + MLJob, + _legacy_logs_shape, + get_job_type_by_key, + serialize_job_logs, +) from .schemas import QueuedTaskAcknowledgment @@ -52,8 +61,8 @@ class JobListSerializer(DefaultSerializer): progress = SchemaField(schema=JobProgress, read_only=True) logs = serializers.SerializerMethodField() job_type = JobTypeSerializer(read_only=True) - # All jobs created from the Jobs UI are ML jobs (datasync, etc. are created for the user) - # @TODO Remove this when the UI is updated pass a job type. This should be a required field. + # Defaulted rather than required: an API client written before job types were + # selectable still means an ML job. job_type_key = serializers.SlugField(write_only=True, default=MLJob.key) # Free-form, and read by the job type that consumes it. A train_classifier job # carries its algorithm_key here; an ML job ignores it entirely. @@ -152,6 +161,40 @@ class Meta: "dispatch_mode", ] + def validate_job_type_key(self, value): + job_type = get_job_type_by_key(value) + if not job_type: + known = sorted(t.key for t in VALID_JOB_TYPES) + raise serializers.ValidationError(f"Unknown job type '{value}'. Known types: {known}") + return value + + def validate(self, attrs): + """ + Refuse a job that could never run. + + Each job type says what it cannot run without. Checking here means a missing + pipeline or algorithm is reported while the person is still looking at the form, + rather than as a failed job minutes later. + """ + attrs = super().validate(attrs) + job_type = get_job_type_by_key(attrs.get("job_type_key", MLJob.key)) + if not job_type: + return attrs + + missing_fields = [name for name in job_type.required_fields if not attrs.get(name)] + if missing_fields: + raise serializers.ValidationError( + {f"{name}_id": f"A {job_type.name} job needs a {name}." for name in missing_fields} + ) + + params = attrs.get("params") or {} + missing_params = [name for name in job_type.required_params if not params.get(name)] + if missing_params: + raise serializers.ValidationError( + {"params": f"A {job_type.name} job needs {', '.join(missing_params)} in its params."} + ) + return attrs + @extend_schema_field( { "type": "object", diff --git a/ami/jobs/tests/test_jobs.py b/ami/jobs/tests/test_jobs.py index 4aab76437..945c36928 100644 --- a/ami/jobs/tests/test_jobs.py +++ b/ami/jobs/tests/test_jobs.py @@ -10,6 +10,7 @@ from ami.base.serializers import reverse_with_params from ami.jobs.models import ( DataStorageSyncJob, + EvaluateAlgorithmJob, Job, JobDispatchMode, JobLog, @@ -328,6 +329,61 @@ def test_create_job(self): # progress = JobProgress(**data["progress"]) # self.assertEqual(progress.summary.status, JobState.CREATED) + def test_creating_a_job_of_an_unknown_type_is_refused(self): + jobs_create_url = reverse_with_params("api:job-list", params={"project_id": self.project.pk}) + self.client.force_authenticate(user=self.user) + + resp = self.client.post( + jobs_create_url, + {"project_id": self.project.pk, "name": "Nonsense", "delay": 0, "job_type_key": "not-a-job-type"}, + ) + + self.assertEqual(resp.status_code, 400) + self.assertIn("job_type_key", resp.json()) + + def test_a_job_missing_what_its_type_needs_is_refused(self): + """ + The gap is reported while the form is still open, rather than as a job that fails + minutes later for want of an algorithm. + """ + jobs_create_url = reverse_with_params("api:job-list", params={"project_id": self.project.pk}) + self.client.force_authenticate(user=self.user) + + resp = self.client.post( + jobs_create_url, + { + "project_id": self.project.pk, + "name": "Evaluate nothing in particular", + "delay": 0, + "job_type_key": EvaluateAlgorithmJob.key, + }, + format="json", + ) + + self.assertEqual(resp.status_code, 400) + self.assertIn("params", resp.json()) + + def test_a_job_carrying_what_its_type_needs_is_created(self): + jobs_create_url = reverse_with_params("api:job-list", params={"project_id": self.project.pk}) + self.client.force_authenticate(user=self.user) + + resp = self.client.post( + jobs_create_url, + { + "project_id": self.project.pk, + "name": "Evaluate a head", + "delay": 0, + "job_type_key": EvaluateAlgorithmJob.key, + "params": {"algorithm_key": "some-head", "occurrence_set_id": 1}, + }, + format="json", + ) + + self.assertEqual(resp.status_code, 201) + job = Job.objects.get(pk=resp.json()["id"]) + self.assertEqual(job.job_type_key, EvaluateAlgorithmJob.key) + self.assertEqual(job.params["algorithm_key"], "some-head") + def test_run_job(self): data = self._create_job("Test run job", start_now=False) diff --git a/ami/jobs/views.py b/ami/jobs/views.py index 749451864..ba80f803c 100644 --- a/ami/jobs/views.py +++ b/ami/jobs/views.py @@ -299,8 +299,6 @@ def perform_create(self, serializer): """ If the ``start_now`` parameter is passed, enqueue the job immediately. """ - # All jobs created from the Jobs UI are ML jobs. - # @TODO Remove this when the UI is updated pass a job type # Get an instance for the model without saving obj = serializer.Meta.model(**serializer.validated_data) # Check permissions before saving diff --git a/ami/ml/serializers.py b/ami/ml/serializers.py index ddceec4ab..3af09e25a 100644 --- a/ami/ml/serializers.py +++ b/ami/ml/serializers.py @@ -5,6 +5,7 @@ from ami.ml import training_data from .models.algorithm import Algorithm, AlgorithmCategoryMap +from .models.evaluation import OccurrenceSet from .models.pipeline import Pipeline, PipelineStage from .models.processing_service import ProcessingService from .models.project_pipeline_config import ProjectPipelineConfig @@ -66,6 +67,27 @@ def get_evaluations(self, obj): return reporting.latest_evaluations(obj) +class OccurrenceSetSerializer(DefaultSerializer): + """The evaluation sets a project can score a model against.""" + + occurrences_count = serializers.SerializerMethodField() + + class Meta: + model = OccurrenceSet + fields = [ + "id", + "details", + "name", + "description", + "occurrences_count", + "created_at", + "updated_at", + ] + + def get_occurrences_count(self, obj) -> int: + return getattr(obj, "annotated_occurrences_count", None) or obj.occurrences.count() + + class AlgorithmNestedSerializer(DefaultSerializer): class Meta: model = Algorithm diff --git a/ami/ml/views.py b/ami/ml/views.py index fee75db8d..c917d824a 100644 --- a/ami/ml/views.py +++ b/ami/ml/views.py @@ -1,6 +1,6 @@ import logging -from django.db import transaction +from django.db import models, transaction from django.db.models import Prefetch from django.db.models.query import QuerySet from django.http import Http404 @@ -25,13 +25,14 @@ from .models.algorithm import Algorithm, AlgorithmCategoryMap from .models.embedding import EMBEDDING_DIMENSIONS, DetectionEmbedding -from .models.evaluation import AlgorithmEvaluation +from .models.evaluation import AlgorithmEvaluation, OccurrenceSet from .models.pipeline import Pipeline from .models.processing_service import ProcessingService from .models.project_pipeline_config import ProjectPipelineConfig from .serializers import ( AlgorithmCategoryMapSerializer, AlgorithmSerializer, + OccurrenceSetSerializer, PipelineRegistrationSerializer, PipelineSerializer, ProcessingServiceSerializer, @@ -48,7 +49,9 @@ class AlgorithmViewSet(DefaultViewSet, ProjectMixin): queryset = Algorithm.objects.all() serializer_class = AlgorithmSerializer - filterset_fields = ["name", "version"] + # ``trainable`` is filterable so a form that starts a retrain can offer only the heads + # a service will actually accept. + filterset_fields = ["name", "version", "trainable"] ordering_fields = [ "id", "created_at", @@ -103,6 +106,29 @@ class AlgorithmCategoryMapViewSet(DefaultViewSet): ] +class OccurrenceSetViewSet(DefaultViewSet, ProjectMixin): + """ + API endpoint listing the fixed occurrence sets a model can be scored against. + + Read-only: membership is built deliberately, not edited in passing, because two models + can only be compared if they were scored on exactly the same occurrences. + """ + + queryset = OccurrenceSet.objects.all() + serializer_class = OccurrenceSetSerializer + http_method_names = ["get", "head", "options"] + ordering_fields = ["name", "created_at", "updated_at"] + search_fields = ["name"] + + def get_queryset(self) -> QuerySet["OccurrenceSet"]: + qs = super().get_queryset().annotate(annotated_occurrences_count=models.Count("occurrences")) + project = self.get_active_project() + if project: + # A set with no project is global, so it is offered everywhere. + return qs.for_project(project) + return qs + + class PipelineViewSet(DefaultViewSet, ProjectMixin): """ API endpoint that allows pipelines to be viewed or edited. diff --git a/config/api_router.py b/config/api_router.py index 93af2afa0..6972ac5e8 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -54,6 +54,7 @@ router.register(r"tags", views.TagViewSet) router.register(r"ml/algorithms", ml_views.AlgorithmViewSet) router.register(r"ml/labels", ml_views.AlgorithmCategoryMapViewSet) +router.register(r"ml/occurrence-sets", ml_views.OccurrenceSetViewSet) router.register(r"ml/pipelines", ml_views.PipelineViewSet) router.register(r"ml/processing_services", ml_views.ProcessingServiceViewSet) router.register(r"ml/training-data", ml_views.TrainingDataViewSet, basename="training-data") From cdd09bc029233644de9f65ebd4a57ff559134c3e Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 17 Sep 2026 12:53:39 -0400 Subject: [PATCH 10/31] fix(ml): stop the BioCLIP service losing the requests module inside a loop The loop over grouped detection requests bound its value to `requests`, which is also the name of the HTTP module imported at the top of the file. Every later reference in that function resolved to the loop's list rather than the module. The loop variable is renamed, two genuinely unused imports go, and the files are brought up to what the repo's pre-commit hooks produce, including the modern Django test-client `headers=` form. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/main/api/serializers.py | 4 ++-- ami/ml/serializers.py | 2 +- ami/ml/tests.py | 2 +- processing_services/bioclip/api/algorithms.py | 5 ++--- processing_services/bioclip/api/api.py | 11 ++++------- processing_services/bioclip/api/schemas.py | 1 - processing_services/bioclip/api/training.py | 8 ++++---- .../weights_test/bioclip25_logreg_categories.json | 2 +- 8 files changed, 15 insertions(+), 20 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 057fb9699..f5799a3e2 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -735,7 +735,7 @@ class Meta: "updated_at", ] - def get_best_model(self, obj): + def get_best_model(self, obj) -> dict | None: """The algorithm scoring highest on this list's species, or null if none has been scored.""" from ami.ml import reporting @@ -1086,7 +1086,7 @@ def get_training_crops_ready(self, obj) -> int | None: """ return getattr(obj, "training_crops_count", None) - def get_algorithm_performance(self, obj): + def get_algorithm_performance(self, obj) -> list[dict]: """How each scored algorithm has done on this species. Empty until one is evaluated.""" from ami.ml import reporting diff --git a/ami/ml/serializers.py b/ami/ml/serializers.py index 3af09e25a..a0f104f8b 100644 --- a/ami/ml/serializers.py +++ b/ami/ml/serializers.py @@ -60,7 +60,7 @@ class Meta: "updated_at", ] - def get_evaluations(self, obj): + def get_evaluations(self, obj) -> list[dict]: """How this algorithm has scored on each evaluation set. Empty until one is scored.""" from ami.ml import reporting diff --git a/ami/ml/tests.py b/ami/ml/tests.py index 971556eb0..138195282 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -3418,7 +3418,7 @@ def test_a_result_for_a_non_training_job_is_refused(self): f"/api/v2/jobs/{other.pk}/training-result/", self.payload, format="json", - HTTP_AUTHORIZATION=f"Token {make_callback_token(other)}", + headers={"authorization": f"Token {make_callback_token(other)}"}, ) self.assertEqual(response.status_code, 400) diff --git a/processing_services/bioclip/api/algorithms.py b/processing_services/bioclip/api/algorithms.py index 76846743b..6d9d1e5bd 100644 --- a/processing_services/bioclip/api/algorithms.py +++ b/processing_services/bioclip/api/algorithms.py @@ -10,8 +10,8 @@ from .schemas import ( AlgorithmCategoryMapResponse, AlgorithmConfigResponse, - AlgorithmTrainingConfig, AlgorithmReference, + AlgorithmTrainingConfig, BoundingBox, ClassificationResponse, Detection, @@ -631,8 +631,7 @@ def get_category_map(self) -> AlgorithmCategoryMapResponse: # Build description description_text = ( - f"Logistic-regression head over frozen {self.model_name} embeddings. " - f"Contains {len(labels)} classes." + f"Logistic-regression head over frozen {self.model_name} embeddings. " f"Contains {len(labels)} classes." ) return AlgorithmCategoryMapResponse( diff --git a/processing_services/bioclip/api/api.py b/processing_services/bioclip/api/api.py index 59e0f26bf..e692a4974 100644 --- a/processing_services/bioclip/api/api.py +++ b/processing_services/bioclip/api/api.py @@ -5,13 +5,13 @@ import datetime import inspect import logging -import os import pathlib import fastapi import pydantic import requests +from . import algorithms, trained_heads, training from .pipelines import ( BioCLIP25LogRegPipeline, BioCLIPPanamaPipeline, @@ -31,7 +31,6 @@ ProcessingServiceInfoResponse, SourceImage, ) -from . import algorithms, trained_heads, training from .utils import is_base64, is_url # Configure root logger @@ -176,7 +175,7 @@ def create_detections( # Process each source image and its detection requests detections = [] - for source_image_id, requests in grouped_detection_requests.items(): + for source_image_id, detection_requests in grouped_detection_requests.items(): if source_image_id not in source_image_map: raise ValueError( f"A detection request for source image {source_image_id} was received, " @@ -185,7 +184,7 @@ def create_detections( logger.info(f"Processing existing detections for source image {source_image_id}.") - for request in requests: + for request in detection_requests: source_image = source_image_map[source_image_id] cropped_image_id = ( f"{source_image.id}-crop-{request.bbox.x1}-{request.bbox.y1}-{request.bbox.x2}-{request.bbox.y2}" @@ -254,9 +253,7 @@ def create_detections( class TrainRequest(pydantic.BaseModel): """Retrain a classifier head from a dataset Antenna has already prepared.""" - dataset_url: str = pydantic.Field( - description="URL of the npz training set Antenna wrote to storage." - ) + dataset_url: str = pydantic.Field(description="URL of the npz training set Antenna wrote to storage.") algorithm_key: str = pydantic.Field( description="Which head to retrain. Its current weights are the baseline the new head must beat." ) diff --git a/processing_services/bioclip/api/schemas.py b/processing_services/bioclip/api/schemas.py index ae60e0b2e..c198384d0 100644 --- a/processing_services/bioclip/api/schemas.py +++ b/processing_services/bioclip/api/schemas.py @@ -2,7 +2,6 @@ import datetime import logging import pathlib -import typing import PIL.Image import pydantic diff --git a/processing_services/bioclip/api/training.py b/processing_services/bioclip/api/training.py index a280ed46a..bf97650f8 100644 --- a/processing_services/bioclip/api/training.py +++ b/processing_services/bioclip/api/training.py @@ -51,7 +51,9 @@ class TrainingRow: features: np.ndarray -def fetch_dataset(url: str, timeout: int = 300, session: requests.Session | None = None) -> tuple[list[TrainingRow], dict]: +def fetch_dataset( + url: str, timeout: int = 300, session: requests.Session | None = None +) -> tuple[list[TrainingRow], dict]: """ Download the training set Antenna prepared and unpack it. @@ -328,9 +330,7 @@ def retrain( candidate = evaluate(weights, bias, x_test, y_test) incumbent_metrics = None if incumbent: - incumbent_metrics = score_incumbent( - incumbent["weights"], incumbent["bias"], incumbent["labels"], test_rows - ) + incumbent_metrics = score_incumbent(incumbent["weights"], incumbent["bias"], incumbent["labels"], test_rows) if incumbent_metrics is None: promote = False diff --git a/processing_services/bioclip/weights_test/bioclip25_logreg_categories.json b/processing_services/bioclip/weights_test/bioclip25_logreg_categories.json index fe5eb3800..54548c0cb 100644 --- a/processing_services/bioclip/weights_test/bioclip25_logreg_categories.json +++ b/processing_services/bioclip/weights_test/bioclip25_logreg_categories.json @@ -59,4 +59,4 @@ "taxon_rank": "SPECIES", "source_class": "9" } -] \ No newline at end of file +] From be1abd10d6aa4bcc353ec25e9e3a61f3ea5e9eb7 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 17 Sep 2026 13:16:05 -0400 Subject: [PATCH 11/31] fix(jobs): stop requiring a pipeline on ML jobs that never needed one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Declaring ``pipeline`` required for the ML job type changed behaviour for a job shape that has always been accepted: the role-permission tests post a job with a name and a delay and nothing else, and expect it created or refused on permission grounds. Requiring a pipeline turned both outcomes into a validation error, so an unauthorised caller was told their payload was wrong rather than that they may not do this. The job types added here keep their requirements — nothing has ever created one of those without them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/jobs/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ami/jobs/models.py b/ami/jobs/models.py index 4e5426161..abf39f367 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -471,7 +471,6 @@ def run(cls, job: "Job"): class MLJob(JobType): name = "ML pipeline" key = "ml" - required_fields = ("pipeline",) user_creatable = True @classmethod From 8ec3704f5a0d4c9effa1da4e40d9fe984ec32857 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 17 Sep 2026 14:10:05 -0400 Subject: [PATCH 12/31] refactor(ml): move the BioCLIP service to the companion repo where the models live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The service was added under processing_services/, which the README describes as somewhere to keep a local demo backend copied from `example` — and that is exactly what it was, a fork of `example` with BioCLIP bolted on. Two thirds of it was that scaffolding: a second Algorithm base class, a second copy of fifteen schema classes already defined in the companion repo, and four demo classifiers that had nothing to do with BioCLIP. It now lives in ami-data-companion, which is where the team's real inference code, model loading and weight handling already are, and where the trainable flag it depends on was added. The classifier is written against that repo's InferenceBaseClass rather than a parallel one, so it is a model alongside the others instead of a service beside the service. See RolnickLab/ami-data-companion#167. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- processing_services/bioclip/Dockerfile | 7 - processing_services/bioclip/api/__init__.py | 0 processing_services/bioclip/api/algorithms.py | 747 ------------------ processing_services/bioclip/api/api.py | 428 ---------- processing_services/bioclip/api/pipelines.py | 438 ---------- processing_services/bioclip/api/schemas.py | 380 --------- processing_services/bioclip/api/test.py | 64 -- .../bioclip/api/trained_heads.py | 205 ----- processing_services/bioclip/api/training.py | 417 ---------- processing_services/bioclip/api/utils.py | 149 ---- .../bioclip/docker-compose.yml | 25 - processing_services/bioclip/main.py | 4 - processing_services/bioclip/requirements.txt | 12 - .../bioclip/scripts/export_logreg_head.py | 144 ---- .../bioclip/scripts/run_on_remote_gpu.sh | 49 -- .../bioclip25_logreg_categories.json | 62 -- .../weights_test/bioclip25_logreg_head.pt | Bin 43093 -> 0 bytes 17 files changed, 3131 deletions(-) delete mode 100644 processing_services/bioclip/Dockerfile delete mode 100644 processing_services/bioclip/api/__init__.py delete mode 100644 processing_services/bioclip/api/algorithms.py delete mode 100644 processing_services/bioclip/api/api.py delete mode 100644 processing_services/bioclip/api/pipelines.py delete mode 100644 processing_services/bioclip/api/schemas.py delete mode 100644 processing_services/bioclip/api/test.py delete mode 100644 processing_services/bioclip/api/trained_heads.py delete mode 100644 processing_services/bioclip/api/training.py delete mode 100644 processing_services/bioclip/api/utils.py delete mode 100644 processing_services/bioclip/docker-compose.yml delete mode 100644 processing_services/bioclip/main.py delete mode 100644 processing_services/bioclip/requirements.txt delete mode 100644 processing_services/bioclip/scripts/export_logreg_head.py delete mode 100755 processing_services/bioclip/scripts/run_on_remote_gpu.sh delete mode 100644 processing_services/bioclip/weights_test/bioclip25_logreg_categories.json delete mode 100644 processing_services/bioclip/weights_test/bioclip25_logreg_head.pt diff --git a/processing_services/bioclip/Dockerfile b/processing_services/bioclip/Dockerfile deleted file mode 100644 index 3e0781f92..000000000 --- a/processing_services/bioclip/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM python:3.11-slim - -# Set up ml backend FastAPI -WORKDIR /app -COPY . /app -RUN pip install -r ./requirements.txt -CMD ["python", "/app/main.py"] diff --git a/processing_services/bioclip/api/__init__.py b/processing_services/bioclip/api/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/processing_services/bioclip/api/algorithms.py b/processing_services/bioclip/api/algorithms.py deleted file mode 100644 index 6d9d1e5bd..000000000 --- a/processing_services/bioclip/api/algorithms.py +++ /dev/null @@ -1,747 +0,0 @@ -import datetime -import json -import logging -import math -import os -import random - -import torch - -from .schemas import ( - AlgorithmCategoryMapResponse, - AlgorithmConfigResponse, - AlgorithmReference, - AlgorithmTrainingConfig, - BoundingBox, - ClassificationResponse, - Detection, - SourceImage, -) - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -SAVED_MODELS = {} - - -def get_best_device() -> str: - """ - Returns the best available device for running the model. - - MPS is not supported by the current algoritms. - """ - if torch.cuda.is_available(): - return f"cuda:{torch.cuda.current_device()}" - else: - return "cpu" - - -class Algorithm: - algorithm_config_response: AlgorithmConfigResponse - - def compile(self): - raise NotImplementedError("Subclasses must implement the compile method") - - def run(self, inputs: list[SourceImage] | list[Detection]) -> list[Detection]: - raise NotImplementedError("Subclasses must implement the run method") - - def _build_categories(self, classes, label_map) -> list[dict]: - """ - Map each head row to an Antenna category. - - The label map is keyed by the class value, which is the iNat taxon id for the - Newfoundland head. - """ - return [ - { - "index": index, - "label": label_map[str(source_class)]["species_name"], - "taxon_rank": "SPECIES", - "source_class": str(source_class), - "inat_taxon_id": label_map[str(source_class)]["inat_taxon_id"], - } - for index, source_class in enumerate(classes) - ] - - def get_category_map(self) -> AlgorithmCategoryMapResponse: - return AlgorithmCategoryMapResponse( - data=[], - labels=[], - version="v1", - description="A model without labels.", - uri=None, - ) - - def get_algorithm_config_response(self) -> AlgorithmConfigResponse: - return AlgorithmConfigResponse( - name="Base Algorithm", - key="base", - task_type="base", - description="A base class for all algorithms.", - version=1, - version_name="v1", - category_map=self.get_category_map(), - ) - - def __init__(self): - self.algorithm_config_response = self.get_algorithm_config_response() - - -class ZeroShotObjectDetector(Algorithm): - """ - Huggingface Zero-Shot Object Detection model. - Produces both a bounding box and a classification for each detection. - The classification is based on the candidate labels. - """ - - candidate_labels: list[str] = ["insect"] - - def compile(self, device: str | None = None): - saved_models_key = "zero_shot_object_detector" # generate a key for each uniquely compiled algorithm - - if saved_models_key not in SAVED_MODELS: - from transformers import pipeline - - device_choice = device or get_best_device() - device_index = int(device_choice.split(":")[-1]) if ":" in device_choice else -1 - logger.info(f"Compiling {self.algorithm_config_response.name} on device {device_choice}...") - checkpoint = "google/owlv2-base-patch16-ensemble" - self.model = pipeline( - model=checkpoint, - task="zero-shot-object-detection", - use_fast=True, - device=device_index, - ) - SAVED_MODELS[saved_models_key] = self.model - else: - logger.info(f"Using saved model for {self.algorithm_config_response.name}...") - self.model = SAVED_MODELS[saved_models_key] - - def run(self, source_images: list[SourceImage], intermediate=False) -> list[Detection]: - detector_responses: list[Detection] = [] - for source_image in source_images: - if source_image.width and source_image.height and source_image._pil: - start_time = datetime.datetime.now() - logger.info("Predicting...") - if not self.candidate_labels: - raise ValueError("No candidate labels are provided during inference.") - logger.info(f"Predicting with candidate labels: {self.candidate_labels}") - predictions = self.model(source_image._pil, candidate_labels=self.candidate_labels) - end_time = datetime.datetime.now() - elapsed_time = (end_time - start_time).total_seconds() - - for prediction in predictions: - logger.info("Prediction: %s", prediction) - bbox = BoundingBox( - x1=prediction["box"]["xmin"], - x2=prediction["box"]["xmax"], - y1=prediction["box"]["ymin"], - y2=prediction["box"]["ymax"], - ) - cropped_image_pil = source_image._pil.crop((bbox.x1, bbox.y1, bbox.x2, bbox.y2)) - detection = Detection( - id=f"{source_image.id}-crop-{bbox.x1}-{bbox.y1}-{bbox.x2}-{bbox.y2}", - url=source_image.url, # @TODO: ideally, should save cropped image at separate url - width=cropped_image_pil.width, - height=cropped_image_pil.height, - timestamp=datetime.datetime.now(), - source_image=source_image, - bbox=bbox, - inference_time=elapsed_time, - algorithm=AlgorithmReference( - name=self.algorithm_config_response.name, - key=self.algorithm_config_response.key, - ), - classifications=[ - ClassificationResponse( - classification=prediction["label"], - labels=[prediction["label"]], - scores=[prediction["score"]], - logits=[prediction["score"]], - inference_time=elapsed_time, - timestamp=datetime.datetime.now(), - algorithm=AlgorithmReference( - name=self.algorithm_config_response.name, - key=self.algorithm_config_response.key, - ), - terminal=not intermediate, - ) - ], - ) - detection._pil = cropped_image_pil - detector_responses.append(detection) - else: - raise ValueError(f"Source image {source_image.id} does not have width and height attributes.") - - return detector_responses - - def get_category_map(self) -> AlgorithmCategoryMapResponse: - return AlgorithmCategoryMapResponse( - data=[{"index": i, "label": label} for i, label in enumerate(self.candidate_labels)], - labels=self.candidate_labels, - version="v1", - description="Candidate labels used for zero-shot object detection.", - uri=None, - ) - - def get_algorithm_config_response(self) -> AlgorithmConfigResponse: - return AlgorithmConfigResponse( - name="Zero Shot Object Detector", - key="zero-shot-object-detector", - task_type="detection", - description=( - "Huggingface Zero Shot Object Detection model." - "Produces both a bounding box and a candidate label classification for each detection." - ), - version=1, - version_name="v1", - category_map=self.get_category_map(), - ) - - -class HFImageClassifier(Algorithm): - """ - A local classifier that uses the Hugging Face pipeline to classify images. - """ - - model_name: str = "google/vit-base-patch16-224" # Vision Transformer model trained on ImageNet-1k - - def compile(self): - saved_models_key = "hf_image_classifier" # generate a key for each uniquely compiled algorithm - - if saved_models_key not in SAVED_MODELS: - from transformers import pipeline - - logger.info(f"Compiling {self.algorithm_config_response.name} from scratch...") - self.model = pipeline("image-classification", model=self.model_name, device=get_best_device()) - SAVED_MODELS[saved_models_key] = self.model - else: - logger.info(f"Using saved model for {self.algorithm_config_response.name}...") - self.model = SAVED_MODELS[saved_models_key] - - def run(self, detections: list[Detection]) -> list[Detection]: - detections_to_return: list[Detection] = [] - start_time = datetime.datetime.now() - - opened_cropped_images = [detection._pil for detection in detections] # type: ignore - - # Process the entire batch of cropped images at once - results = self.model(images=opened_cropped_images) - - end_time = datetime.datetime.now() - elapsed_time = (end_time - start_time).total_seconds() - - for detection, preds in zip(detections, results): - labels = [pred["label"] for pred in preds] - scores = [pred["score"] for pred in preds] - max_score_index = scores.index(max(scores)) - classification = labels[max_score_index] - logger.info(f"Classification: {classification}") - logger.info(f"labels: {labels}") - logger.info(f"scores: {scores}") - - existing_classifications = detection.classifications - - detection_with_classification = detection.copy(deep=True) - detection_with_classification.classifications = existing_classifications + [ - ClassificationResponse( - classification=classification, - labels=labels, - scores=scores, - logits=scores, - inference_time=elapsed_time, - timestamp=datetime.datetime.now(), - algorithm=AlgorithmReference( - name=self.algorithm_config_response.name, key=self.algorithm_config_response.key - ), - terminal=True, - ) - ] - - detections_to_return.append(detection_with_classification) - - return detections_to_return - - def get_category_map(self) -> AlgorithmCategoryMapResponse: - """ - Extract the category map from the model. - Returns an AlgorithmCategoryMapResponse with labels, data, and model information. - """ - from transformers.models.auto.configuration_auto import AutoConfig - - logger.info(f"Loading configuration for {self.model_name}") - config = AutoConfig.from_pretrained(self.model_name) - - # Extract label information - if not hasattr(config, "id2label") or not config.id2label: - raise ValueError( - f"Cannot create category map for model {self.model_name}, no id2label mapping found in config" - ) - else: - # Sort labels by index - # Ensure keys are strings for consistent access - id2label: dict[str, str] = {str(k): v for k, v in config.id2label.items()} - indices = sorted([int(k) for k in id2label.keys()]) - - # Create labels and data - labels = [id2label[str(i)] for i in indices] - data = [{"label": label, "index": idx} for idx, label in zip(indices, labels)] - - # Build description - description_text = ( - f"Vision Transformer model trained on ImageNet-1k. " - f"Contains {len(labels)} object classes. Model: {self.model_name}" - ) - - return AlgorithmCategoryMapResponse( - data=data, - labels=labels, - version="ImageNet-1k", - description=description_text, - uri=f"https://huggingface.co/{self.model_name}", - ) - - def get_algorithm_config_response(self) -> AlgorithmConfigResponse: - return AlgorithmConfigResponse( - name="HF Image Classifier", - key="hf-image-classifier", - task_type="classification", - description="HF ViT for image classification.", - version=1, - version_name="v1", - category_map=self.get_category_map(), - ) - - -class RandomSpeciesClassifier(Algorithm): - """ - A local classifier that produces random butterfly species classifications. - """ - - def compile(self): - pass - - def _make_random_prediction( - self, - terminal: bool = True, - max_labels: int = 2, - ) -> ClassificationResponse: - assert self.algorithm_config_response.category_map is not None - category_labels = self.algorithm_config_response.category_map.labels - logits = [random.random() for _ in category_labels] - softmax = [math.exp(logit) / sum([math.exp(logit) for logit in logits]) for logit in logits] - top_class = category_labels[softmax.index(max(softmax))] - return ClassificationResponse( - classification=top_class, - labels=category_labels if len(category_labels) <= max_labels else None, - scores=softmax, - logits=logits, - timestamp=datetime.datetime.now(), - algorithm=AlgorithmReference( - name=self.algorithm_config_response.name, - key=self.algorithm_config_response.key, - ), - terminal=terminal, - ) - - def run(self, detections: list[Detection]) -> list[Detection]: - detections_to_return: list[Detection] = [] - for detection in detections: - detection_with_classification = detection.copy(deep=True) - detection_with_classification.classifications = [self._make_random_prediction(terminal=True)] - detections_to_return.append(detection_with_classification) - return detections_to_return - - algorithm_config_response = AlgorithmConfigResponse( - name="Random species classifier", - key="random-species-classifier", - task_type="classification", - description="A random species classifier", - version=1, - version_name="v1", - uri="https://huggingface.co/RolnickLab/random-species-classifier", - category_map=AlgorithmCategoryMapResponse( - data=[ - { - "index": 0, - "gbif_key": "1234", - "label": "Vanessa atalanta", - "source": "manual", - "taxon_rank": "SPECIES", - }, - { - "index": 1, - "gbif_key": "4543", - "label": "Vanessa cardui", - "source": "manual", - "taxon_rank": "SPECIES", - }, - { - "index": 2, - "gbif_key": "7890", - "label": "Vanessa itea", - "source": "manual", - "taxon_rank": "SPECIES", - }, - ], - labels=["Vanessa atalanta", "Vanessa cardui", "Vanessa itea"], - version="v1", - description="A simple species classifier", - uri="https://huggingface.co/RolnickLab/random-species-classifier", - ), - ) - - -class ConstantClassifier(Algorithm): - """ - A local classifier that always returns a constant species classification. - """ - - def compile(self): - pass - - def _make_constant_prediction( - self, - terminal: bool = True, - ) -> ClassificationResponse: - assert self.algorithm_config_response.category_map is not None - labels = self.algorithm_config_response.category_map.labels - return ClassificationResponse( - classification=labels[0], - labels=labels, - scores=[0.9], # Constant score for each detection - timestamp=datetime.datetime.now(), - algorithm=AlgorithmReference( - name=self.algorithm_config_response.name, - key=self.algorithm_config_response.key, - ), - terminal=terminal, - ) - - def run(self, detections: list[Detection]) -> list[Detection]: - detections_to_return: list[Detection] = [] - for detection in detections: - detection_with_classification = detection.copy(deep=True) - detection_with_classification.classifications = [self._make_constant_prediction(terminal=True)] - detections_to_return.append(detection_with_classification) - return detections_to_return - - algorithm_config_response = AlgorithmConfigResponse( - name="Constant classifier", - key="constant-classifier", - task_type="classification", - description="Always return a classification of 'Moth'", - version=1, - version_name="v1", - uri="https://huggingface.co/RolnickLab/constant-classifier", - category_map=AlgorithmCategoryMapResponse( - data=[ - { - "index": 0, - "gbif_key": "1234", - "label": "Moth", - "source": "manual", - "taxon_rank": "SUPERFAMILY", - } - ], - labels=["Moth"], - version="v1", - description="A classifier that always returns 'Moth'", - uri="https://huggingface.co/RolnickLab/constant-classifier", - ), - ) - - -class BioCLIPWithLinearHead(torch.nn.Module): - """ - A frozen BioCLIP image encoder with a linear classification head on top. - - Keeps the encoder, the head and the image transform in a single module so that the - whole classifier can be cached in SAVED_MODELS as one object. - """ - - def __init__(self, encoder, head, preprocess): - super().__init__() - self.encoder = encoder - self.head = head - self.preprocess = preprocess - - def forward(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - """Returns (logits, features). The features are what the head consumes, so they - are the right thing to store for retraining a head later.""" - features = self.encoder.encode_image(images).float() - # The head was fit on L2-normalised embeddings, so normalise here too. - features = features / features.norm(dim=-1, keepdim=True) - return self.head(features), features - - -class BioCLIP25LogRegClassifier(Algorithm): - """ - A local classifier that uses a frozen BioCLIP 2.5 image encoder with a linear - logistic-regression head. - - The head is a single Linear layer whose weights come from an sklearn - LogisticRegression fit on L2-normalised BioCLIP embeddings, so softmax over the - linear output reproduces sklearn's multinomial predict_proba exactly. See - scripts/export_logreg_head.py for the conversion. - """ - - model_name: str = "hf-hub:imageomics/bioclip-2.5-vith14" # BioCLIP 2.5 ViT-H/14, loaded by open_clip - # The head and its label map are read from the Hugging Face repo that already serves - # them to the Newfoundland trap classifier demo, so there is a single source of truth - # for the weights rather than a second copy that can drift. - head_repo_id: str = "mohammedelabbas/newfoundland-leps-trap-classifier" - head_repo_type: str = "space" - head_filename: str = "logreg_head_antenna.npz" - categories_filename: str = "label_map.json" - # Read the head from this directory instead of the hub, for offline or local testing. - # The directory must contain the same two filenames. - head_local_dir: str | None = os.environ.get("BIOCLIP_HEAD_DIR") or None - # One cache slot per head. Two heads sharing a slot would silently serve the wrong - # weights, because the encoder is identical and only the Linear layer differs. - saved_models_key: str = "bioclip_25_logreg_nf749" - # Antenna's schema allows omitting the label list per classification when the class - # list is large. The full list always travels in the category map instead. - max_labels_in_response: int = 100 - - @property - def head_uri(self) -> str: - prefix = "spaces/" if self.head_repo_type == "space" else "" - return f"https://huggingface.co/{prefix}{self.head_repo_id}" - - def _head_file(self, filename: str) -> str: - if self.head_local_dir: - logger.info(f"Loading {filename} from local directory {self.head_local_dir}") - return f"{self.head_local_dir}/{filename}" - - from huggingface_hub import hf_hub_download - - logger.info(f"Loading {filename} from {self.head_repo_id} ({self.head_repo_type})") - return hf_hub_download(repo_id=self.head_repo_id, filename=filename, repo_type=self.head_repo_type) - - def _load_head_arrays(self): - """ - Load the exported sklearn LogisticRegression as (weight, bias, classes). - - `classes` holds the label-map key for each row of the weight matrix, so the row - order of the head and the order of the category map stay tied together. - """ - import numpy as np - - checkpoint = np.load(self._head_file(self.head_filename)) - return checkpoint["W"], checkpoint["b"], checkpoint["classes"] - - def _load_head(self, embed_dim: int) -> torch.nn.Module: - weight, bias, _classes = self._load_head_arrays() - - if weight.shape[1] != embed_dim: - raise ValueError( - f"Head was fit on {weight.shape[1]}-dim embeddings but {self.model_name} " - f"produces {embed_dim}-dim embeddings." - ) - - head = torch.nn.Linear(embed_dim, weight.shape[0]) - head.weight.data = torch.from_numpy(weight).float() - head.bias.data = torch.from_numpy(bias).float() - return head - - def compile(self): - saved_models_key = self.saved_models_key - - self.device = get_best_device() - if saved_models_key not in SAVED_MODELS: - import open_clip - - logger.info(f"Compiling {self.algorithm_config_response.name} from scratch...") - encoder, _, preprocess = open_clip.create_model_and_transforms(self.model_name) - head = self._load_head(encoder.visual.output_dim) - self.model = BioCLIPWithLinearHead(encoder, head, preprocess).eval().to(self.device) - SAVED_MODELS[saved_models_key] = self.model - else: - logger.info(f"Using saved model for {self.algorithm_config_response.name}...") - self.model = SAVED_MODELS[saved_models_key] - - def run(self, detections: list[Detection]) -> list[Detection]: - detections_to_return: list[Detection] = [] - start_time = datetime.datetime.now() - - opened_cropped_images = [detection._pil for detection in detections] # type: ignore - - # Process the entire batch of cropped images at once - crops = torch.stack([self.model.preprocess(image.convert("RGB")) for image in opened_cropped_images]) - with torch.inference_mode(): - batch_logits, batch_features = self.model(crops.to(self.device)) - batch_scores = torch.nn.functional.softmax(batch_logits, dim=-1) - - end_time = datetime.datetime.now() - elapsed_time = (end_time - start_time).total_seconds() - - assert self.algorithm_config_response.category_map is not None - category_labels = self.algorithm_config_response.category_map.labels - - for detection, logits, scores, features in zip( - detections, batch_logits.cpu(), batch_scores.cpu(), batch_features.cpu() - ): - classification = category_labels[int(scores.argmax())] - logger.info(f"Classification: {classification}") - - existing_classifications = detection.classifications - - detection_with_classification = detection.copy(deep=True) - detection_with_classification.classifications = existing_classifications + [ - ClassificationResponse( - classification=classification, - labels=category_labels if len(category_labels) <= self.max_labels_in_response else None, - scores=scores.tolist(), - logits=logits.tolist(), - features=features.tolist(), - inference_time=elapsed_time, - timestamp=datetime.datetime.now(), - algorithm=AlgorithmReference( - name=self.algorithm_config_response.name, key=self.algorithm_config_response.key - ), - terminal=True, - ) - ] - - detections_to_return.append(detection_with_classification) - - return detections_to_return - - def get_category_map(self) -> AlgorithmCategoryMapResponse: - """ - Load the category map exported alongside the head. - Returns an AlgorithmCategoryMapResponse with labels, data, and model information. - """ - categories_path = self._head_file(self.categories_filename) - with open(categories_path) as f: - label_map = json.load(f) - - if not label_map: - raise ValueError( - f"Cannot create category map for model {self.model_name}, " - f"no categories found in {self.categories_filename}" - ) - - # Row i of the head predicts classes[i], so walk the classes in row order to keep - # the head and the labels aligned. - _weight, _bias, classes = self._load_head_arrays() - categories = self._build_categories(classes, label_map) - labels = [category["label"] for category in categories] - - # Build description - description_text = ( - f"Logistic-regression head over frozen {self.model_name} embeddings. " f"Contains {len(labels)} classes." - ) - - return AlgorithmCategoryMapResponse( - data=categories, - labels=labels, - version="v1", - description=description_text, - uri=self.head_uri, - ) - - def training_config(self) -> AlgorithmTrainingConfig: - """ - Defaults for retraining this head. Antenna seeds its own copy once, so changing - these afterwards only affects algorithms registered from here on. - """ - return AlgorithmTrainingConfig( - # A linear head matches what is deployed, so a retrained one is a drop-in. - # An MLP-1 head scores far better on rare species and is the likely next default. - head_type="linear", - epochs=300, - learning_rate=0.01, - min_per_species=2, - ) - - def get_algorithm_config_response(self) -> AlgorithmConfigResponse: - return AlgorithmConfigResponse( - name="BioCLIP 2.5 + LogReg head (Newfoundland, 749 species)", - key="bioclip-2-5-vith14-logreg-nf749", - task_type="classification", - description="Frozen BioCLIP 2.5 (ViT-H/14) encoder with a linear logistic-regression head.", - # A different head means a different key, not a new version. Antenna looks - # algorithms up with get_or_create(key=..., version=...) but has a unique - # constraint on key alone, so bumping the version of an existing key raises - # an IntegrityError instead of registering the new category map. - version=1, - version_name="v1", - uri=self.head_uri, - # The encoder is frozen, so only this linear head is retrained. The zero-shot - # detector in stage 0 is not trainable and stays False. - trainable=True, - training_config=self.training_config(), - category_map=self.get_category_map(), - ) - - -class BioCLIPPanamaLogRegClassifier(BioCLIP25LogRegClassifier): - """ - The Panama (BCI + Mount Totumas) head over the same frozen BioCLIP 2.5 encoder. - - Same architecture as the Newfoundland head, but the export carries its label - vocabulary inside the npz instead of a separate label map, and the vocabulary is - larger than the number of trained rows: 1,095 names, 900 output classes. The 195 - species with no training data can never be predicted, so they are not categories. - """ - - head_repo_id: str | None = None # not published; read from a local directory - head_filename: str = "head_combined.npz" - categories_filename: str | None = None # labels travel inside the npz - head_local_dir: str | None = os.environ.get("BIOCLIP_PANAMA_HEAD_DIR") or None - saved_models_key: str = "bioclip_25_logreg_panama900" - - @property - def head_uri(self) -> str | None: - return None - - def _load_label_vocabulary(self): - import numpy as np - - return np.load(self._head_file(self.head_filename), allow_pickle=True)["labels"] - - def _build_categories(self, classes, label_map) -> list[dict]: - # `label_map` is the vocabulary array here, indexed by class value rather than - # keyed by it. - return [ - { - "index": index, - "label": str(label_map[int(source_class)]), - "taxon_rank": "SPECIES", - "source_class": str(int(source_class)), - } - for index, source_class in enumerate(classes) - ] - - def get_category_map(self) -> AlgorithmCategoryMapResponse: - _weight, _bias, classes = self._load_head_arrays() - vocabulary = self._load_label_vocabulary() - categories = self._build_categories(classes, vocabulary) - labels = [category["label"] for category in categories] - - return AlgorithmCategoryMapResponse( - data=categories, - labels=labels, - version="v1", - description=( - f"Logistic-regression head over frozen {self.model_name} embeddings. " - f"Contains {len(labels)} classes from a {len(vocabulary)}-species vocabulary." - ), - uri=self.head_uri, - ) - - def get_algorithm_config_response(self) -> AlgorithmConfigResponse: - return AlgorithmConfigResponse( - name="BioCLIP 2.5 + LogReg head (Panama, 900 species)", - key="bioclip-2-5-vith14-logreg-panama900", - task_type="classification", - description="Frozen BioCLIP 2.5 (ViT-H/14) encoder with a linear logistic-regression head.", - version=1, - version_name="v1", - uri=self.head_uri, - trainable=True, - training_config=self.training_config(), - category_map=self.get_category_map(), - ) diff --git a/processing_services/bioclip/api/api.py b/processing_services/bioclip/api/api.py deleted file mode 100644 index e692a4974..000000000 --- a/processing_services/bioclip/api/api.py +++ /dev/null @@ -1,428 +0,0 @@ -""" -Fast API interface for processing images through the localization and classification pipelines. -""" - -import datetime -import inspect -import logging -import pathlib - -import fastapi -import pydantic -import requests - -from . import algorithms, trained_heads, training -from .pipelines import ( - BioCLIP25LogRegPipeline, - BioCLIPPanamaPipeline, - Pipeline, - ZeroShotHFClassifierPipeline, - ZeroShotObjectDetectorPipeline, - ZeroShotObjectDetectorWithConstantClassifierPipeline, - ZeroShotObjectDetectorWithRandomSpeciesClassifierPipeline, -) -from .schemas import ( - AlgorithmConfigResponse, - Detection, - DetectionRequest, - PipelineRequest, - PipelineRequestConfigParameters, - PipelineResultsResponse, - ProcessingServiceInfoResponse, - SourceImage, -) -from .utils import is_base64, is_url - -# Configure root logger -logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S" -) - -# Get the root logger -logger = logging.getLogger(__name__) - -# Where retrained heads are written. Kept off the model cache so a training run -# cannot overwrite the head the service is currently serving. -TRAINED_HEADS_DIR = trained_heads.TRAINED_HEADS_DIR - -app = fastapi.FastAPI() - - -pipelines: list[type[Pipeline]] = [ - BioCLIP25LogRegPipeline, - BioCLIPPanamaPipeline, - ZeroShotHFClassifierPipeline, - ZeroShotObjectDetectorPipeline, - ZeroShotObjectDetectorWithConstantClassifierPipeline, - ZeroShotObjectDetectorWithRandomSpeciesClassifierPipeline, -] -pipeline_choices: dict[str, type[Pipeline]] = {pipeline.config.slug: pipeline for pipeline in pipelines} -algorithm_choices: dict[str, AlgorithmConfigResponse] = { - algorithm.key: algorithm for pipeline in pipelines for algorithm in pipeline.config.algorithms -} - -# Heads this service has already retrained are offered alongside the ones it shipped with. -trained_heads.register(pipeline_choices, algorithm_choices) - -# ----------- -# API endpoints -# ----------- - - -@app.get("/") -async def root(): - return fastapi.responses.RedirectResponse("/docs") - - -@app.get("/info", tags=["services"]) -async def info() -> ProcessingServiceInfoResponse: - info = ProcessingServiceInfoResponse( - name="BioCLIP ML Backend", - description=("BioCLIP 2.5 with a logistic-regression classification head."), - # Built from the live registry, not the static list, so heads retrained after the - # service started are advertised too. - pipelines=[pipeline.config for pipeline in pipeline_choices.values()], - # algorithms=list(algorithm_choices.values()), - ) - return info - - -# Check if the server is online -@app.get("/livez", tags=["health checks"]) -async def livez(): - return fastapi.responses.JSONResponse(status_code=200, content={"status": True}) - - -# Check if the pipelines are ready to process data -@app.get("/readyz", tags=["health checks"]) -async def readyz(): - """ - Check if the server is ready to process data. - - Returns a list of pipeline slugs that are online and ready to process data. - @TODO may need to simplify this to just return True/False. Pipeline algorithms will likely be loaded into memory - on-demand when the pipeline is selected. - """ - if pipeline_choices: - return fastapi.responses.JSONResponse(status_code=200, content={"status": list(pipeline_choices.keys())}) - else: - return fastapi.responses.JSONResponse(status_code=503, content={"status": []}) - - -@app.post("/process", tags=["services"]) -async def process(data: PipelineRequest) -> PipelineResultsResponse: - pipeline_slug = data.pipeline - request_config = data.config - - source_images = [SourceImage(**img.model_dump()) for img in data.source_images] - # Open source images once before processing - for img in source_images: - img.open(raise_exception=True) - - detections = create_detections( - source_images=source_images, - detection_requests=data.detections, - ) - - try: - Pipeline = pipeline_choices[pipeline_slug] - except KeyError: - raise fastapi.HTTPException( - status_code=422, - detail=f"Invalid pipeline choice: {pipeline_slug}. Available: {sorted(pipeline_choices)}", - ) - - pipeline_request_config = PipelineRequestConfigParameters(**dict(request_config)) if request_config else {} - try: - pipeline = Pipeline( - source_images=source_images, - request_config=pipeline_request_config, - existing_detections=detections, - ) - pipeline.compile() - except Exception as e: - logger.error(f"Error compiling pipeline: {e}") - raise fastapi.HTTPException(status_code=422, detail=f"{e}") - - try: - response = pipeline.run() - except Exception as e: - logger.error(f"Error running pipeline: {e}") - raise fastapi.HTTPException(status_code=422, detail=f"{e}") - - return response - - -# ----------- -# Helper functions -# ----------- - - -def create_detections( - source_images: list[SourceImage], - detection_requests: list[DetectionRequest] | None, -): - if not detection_requests: - return [] - - # Group detection requests by source image id - source_image_map = {img.id: img for img in source_images} - grouped_detection_requests = {} - for request in detection_requests: - if request.source_image.id not in grouped_detection_requests: - grouped_detection_requests[request.source_image.id] = [] - grouped_detection_requests[request.source_image.id].append(request) - - # Process each source image and its detection requests - detections = [] - for source_image_id, detection_requests in grouped_detection_requests.items(): - if source_image_id not in source_image_map: - raise ValueError( - f"A detection request for source image {source_image_id} was received, " - "but no source image with that ID was provided." - ) - - logger.info(f"Processing existing detections for source image {source_image_id}.") - - for request in detection_requests: - source_image = source_image_map[source_image_id] - cropped_image_id = ( - f"{source_image.id}-crop-{request.bbox.x1}-{request.bbox.y1}-{request.bbox.x2}-{request.bbox.y2}" - ) - if not request.crop_image_url: - logger.info("Detection request does not have a crop_image_url, crop the original source image.") - assert source_image._pil is not None, "Source image must be opened before cropping." - cropped_image_pil = source_image._pil.crop( - (request.bbox.x1, request.bbox.y1, request.bbox.x2, request.bbox.y2) - ) - else: - try: - logger.info(f"Opening existing cropped image from {request.crop_image_url}.") - if is_url(request.crop_image_url): - cropped_image = SourceImage( - id=cropped_image_id, - url=request.crop_image_url, - ) - elif is_base64(request.crop_image_url): - logger.info("Decoding base64 cropped image.") - cropped_image = SourceImage( - id=cropped_image_id, - b64=request.crop_image_url, - ) - else: - # Must be a filepath - cropped_image = SourceImage( - id=cropped_image_id, - filepath=request.crop_image_url, - ) - cropped_image.open(raise_exception=True) - cropped_image_pil = cropped_image._pil - except Exception as e: - logger.warning(f"Error opening cropped image: {e}") - logger.info(f"Falling back to cropping the original source image {source_image_id}.") - assert source_image._pil is not None, "Source image must be opened before cropping." - cropped_image_pil = source_image._pil.crop( - (request.bbox.x1, request.bbox.y1, request.bbox.x2, request.bbox.y2) - ) - - # Create a Detection object - det = Detection( - source_image=SourceImage( - id=source_image.id, - url=source_image.url, - ), - bbox=request.bbox, - id=cropped_image_id, - url=request.crop_image_url or source_image.url, - algorithm=request.algorithm, - ) - # Set the _pil attribute to the cropped image - det._pil = cropped_image_pil - detections.append(det) - logger.info(f"Created detection {det.id} for source image {source_image_id}.") - - return detections - - -if __name__ == "__main__": - import uvicorn - - uvicorn.run(app, host="0.0.0.0", port=2000) - - -class TrainRequest(pydantic.BaseModel): - """Retrain a classifier head from a dataset Antenna has already prepared.""" - - dataset_url: str = pydantic.Field(description="URL of the npz training set Antenna wrote to storage.") - algorithm_key: str = pydantic.Field( - description="Which head to retrain. Its current weights are the baseline the new head must beat." - ) - job_id: int | None = pydantic.Field( - default=None, - description="Antenna job to report back to. Without it the result is only returned in this response.", - ) - callback_url: str | None = pydantic.Field( - default=None, - description="Where to post the result when training finishes.", - ) - callback_token: str | None = pydantic.Field( - default=None, - description="Token for the callback, so Antenna can tell a real result from a forged one.", - ) - name: str | None = pydantic.Field(default=None, description="Name for the produced head.") - # Defaults match this service's own training_config. Antenna sends whatever an admin - # has set there, so these are overridden in practice. - min_per_species: int = 2 - min_improvement: float = 0.0 - head_type: str = "linear" - epochs: int = 300 - learning_rate: float = 0.01 - weight_decay: float = 1e-4 - save: bool = pydantic.Field( - default=True, - description="Write the head to disk. It is never loaded into the running service automatically.", - ) - - -class TrainResponse(pydantic.BaseModel): - promote: bool - reason: str - warnings: list[str] - rows: dict - classes_restored_from_current_head: int = 0 - counts: dict - dropped_species: list[str] - candidate_metrics: dict - incumbent_metrics: dict | None - labels: list[str] - saved: dict[str, str] | None - trained_at: str - reported_to_antenna: bool = False - - -@app.post("/train", tags=["training"]) -async def train(data: TrainRequest) -> TrainResponse: - """ - Retrain a classifier head from human-verified labels. - - Antenna prepares the dataset and hands over a URL; this downloads it, fits a new head, - and scores it against the head currently in service on the same held-out rows. It never - swaps the running head: promoting is a separate, deliberate step, because an automatic - swap would let one bad training run quietly degrade every later classification. - """ - try: - rows, dataset_metadata = training.fetch_dataset(data.dataset_url) - except requests.HTTPError as e: - raise fastapi.HTTPException(status_code=502, detail=f"Could not download the training set: {e}") - except Exception as e: - raise fastapi.HTTPException(status_code=422, detail=f"Could not read the training set: {e}") - - if not rows: - raise fastapi.HTTPException(status_code=422, detail="The training set is empty.") - - incumbent = _incumbent_head(data.algorithm_key) - - try: - result = training.retrain( - rows=rows, - incumbent=incumbent, - min_per_species=data.min_per_species, - min_improvement=data.min_improvement, - epochs=data.epochs, - learning_rate=data.learning_rate, - weight_decay=data.weight_decay, - head_type=data.head_type, - # Antenna ships the species list inside the dataset; it usually comes from a - # project's taxa list and must win over whatever happens to be in the rows. - declared_classes=dataset_metadata.get("classes"), - ) - except (training.NotEnoughData, training.UnsupportedHeadType) as e: - raise fastapi.HTTPException(status_code=422, detail=str(e)) - - saved = None - if data.save: - name = data.name or f"head-{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}" - saved = training.save_head(result, pathlib.Path(TRAINED_HEADS_DIR), name) - # Offer it straight away. The head it was trained from stays where it is; this - # adds a choice rather than replacing one. - trained_heads.register(pipeline_choices, algorithm_choices) - - response = TrainResponse( - promote=result["promote"], - reason=result["reason"], - warnings=result["warnings"], - rows=result["rows"], - classes_restored_from_current_head=result.get("classes_restored_from_current_head", 0), - counts=result["counts"], - dropped_species=result["dropped_species"], - candidate_metrics=result["candidate_metrics"], - incumbent_metrics=result["incumbent_metrics"], - labels=result["labels"], - saved=saved, - trained_at=result["trained_at"], - ) - - if data.callback_url: - response.reported_to_antenna = _report_to_antenna(data, response, dataset_metadata) - - return response - - -def _report_to_antenna(data: TrainRequest, response: "TrainResponse", dataset_metadata: dict) -> bool: - """ - Tell the Antenna job how the run went. - - Reported rather than raised on failure: the training itself succeeded, and losing the - callback should not make the caller think it did not. - """ - payload = { - "job_id": data.job_id, - "algorithm_key": data.algorithm_key, - "dataset": dataset_metadata, - "result": response.model_dump(), - } - headers = {"Content-Type": "application/json"} - if data.callback_token: - headers["Authorization"] = f"Token {data.callback_token}" - try: - reply = requests.post(data.callback_url, json=payload, headers=headers, timeout=60) - reply.raise_for_status() - return True - except Exception as e: - logger.error(f"Trained successfully but could not report to Antenna at {data.callback_url}: {e}") - return False - - -def _incumbent_head(algorithm_key: str) -> dict | None: - """ - Load the weights of the head currently in service, so a new head can be compared to it. - - Scans the algorithm classes rather than a pipeline's `stages`, because stages only - exist once a pipeline is instantiated. Returns None when the key names no local head; - the caller then refuses to promote rather than promoting something it could not compare. - """ - for candidate in vars(algorithms).values(): - if not inspect.isclass(candidate) or not issubclass(candidate, algorithms.Algorithm): - continue - if not hasattr(candidate, "_load_head_arrays"): - continue - try: - instance = candidate() - if instance.algorithm_config_response.key != algorithm_key: - continue - weights, bias, classes = instance._load_head_arrays() - # The npz's `classes` holds column indices, not names. The species names come - # from the category map, so use that; otherwise every label comparison against - # Antenna's taxon names silently finds nothing in common. - labels = [str(label) for label in instance.get_category_map().labels] - if len(labels) != weights.shape[0]: - logger.warning( - f"{candidate.__name__}: category map has {len(labels)} labels but the head has " - f"{weights.shape[0]} outputs. Falling back to the names stored in the npz." - ) - labels = [str(c) for c in classes] - except Exception as e: - logger.warning(f"Could not load the current head for {candidate.__name__}: {e}") - continue - return {"weights": weights, "bias": bias, "labels": labels} - return None diff --git a/processing_services/bioclip/api/pipelines.py b/processing_services/bioclip/api/pipelines.py deleted file mode 100644 index f15e21d14..000000000 --- a/processing_services/bioclip/api/pipelines.py +++ /dev/null @@ -1,438 +0,0 @@ -import datetime -import logging -from typing import final - -from .algorithms import ( - Algorithm, - BioCLIP25LogRegClassifier, - BioCLIPPanamaLogRegClassifier, - ConstantClassifier, - HFImageClassifier, - RandomSpeciesClassifier, - ZeroShotObjectDetector, -) -from .schemas import ( - Detection, - DetectionResponse, - PipelineConfigResponse, - PipelineRequestConfigParameters, - PipelineResultsResponse, - SourceImage, - SourceImageResponse, -) - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -class Pipeline: - """ - A base class for defining and running a pipeline consisting of multiple stages. - Each stage is represented by an algorithm that processes inputs and produces - outputs. The pipeline is designed to handle batch processing using custom batch - sizes for each stage. - - Attributes: - stages (list[Algorithm]): A list of algorithms representing the stages of - the pipeline in order of execution. Typically [Detector(), Classifier()]. - batch_sizes (list[int]): A list of integers specifying the batch size for - each stage. For example, [1, 1] means that the detector can process 1 - source image a time and the classifier can process 1 detection at a time. - config (PipelineConfigResponse): Pipeline metadata. - """ - - stages: list[Algorithm] - batch_sizes: list[int] - request_config: dict - config: PipelineConfigResponse - - stages = [] - batch_sizes = [] - config = PipelineConfigResponse( - name="Base Pipeline", - slug="base", - description="A base class for all pipelines.", - version=1, - algorithms=[], - ) - - def __init__( - self, - source_images: list[SourceImage], - request_config: PipelineRequestConfigParameters | dict = {}, - existing_detections: list[Detection] = [], - custom_batch_sizes: list[int] = [], - ): - self.source_images = source_images - self.request_config = request_config if isinstance(request_config, dict) else request_config.model_dump() - self.existing_detections = existing_detections - - logger.info("Initializing algorithms....") - self.stages = self.stages or self.get_stages() - self.batch_sizes = custom_batch_sizes or self.batch_sizes or [1] * len(self.stages) - assert len(self.batch_sizes) == len(self.stages), "Number of batch sizes must match the number of stages." - - def get_stages(self) -> list[Algorithm]: - """ - An optional function to initialize and return a list of algorithms/stages. - Any pipeline config values relevant to a particular algorithm should be passed or set here. - """ - return [] - - @final - def compile(self): - logger.info("Compiling algorithms....") - for stage_idx, stage in enumerate(self.stages): - logger.info(f"[{stage_idx+1}/{len(self.stages)}] Compiling {stage.algorithm_config_response.name}...") - stage.compile() - - def run(self) -> PipelineResultsResponse: - """ - This function must always return a PipelineResultsResponse object. - """ - raise NotImplementedError("Subclasses must implement") - - @final - def _batchify_inputs(self, inputs: list, batch_size: int) -> list[list]: - """ - Helper function to split the inputs into batches of the specified size. - """ - batched_inputs = [] - for i in range(0, len(inputs), batch_size): - start_id = i - end_id = i + batch_size - batched_inputs.append(inputs[start_id:end_id]) - return batched_inputs - - @final - def _get_detections( - self, algorithm: Algorithm, inputs: list[SourceImage] | list[Detection], batch_size: int, **kwargs - ) -> list[Detection]: - """A single stage, step, or algorithm in a pipeline. Batchifies inputs and produces Detections as outputs.""" - outputs: list[Detection] = [] - batched_inputs = self._batchify_inputs(inputs, batch_size) - for batch in batched_inputs: - outputs.extend(algorithm.run(batch, **kwargs)) - return outputs - - @final - def _get_pipeline_response(self, detections: list[Detection], elapsed_time: float) -> PipelineResultsResponse: - """ - Final stage of the pipeline to format the detections. - """ - detection_responses = [ - DetectionResponse( - source_image_id=detection.source_image.id, - bbox=detection.bbox, - inference_time=detection.inference_time, - algorithm=detection.algorithm, - timestamp=datetime.datetime.now(), - classifications=detection.classifications, - ) - for detection in detections - ] - source_image_responses = [SourceImageResponse(**image.model_dump()) for image in self.source_images] - - return PipelineResultsResponse( - pipeline=self.config.slug, # type: ignore - # algorithms={algorithm.key: algorithm for algorithm in self.config.algorithms}, - total_time=elapsed_time, - source_images=source_image_responses, - detections=detection_responses, - ) - - -class ZeroShotHFClassifierPipeline(Pipeline): - """ - A pipeline that uses the Zero Shot Object Detector to produce bounding boxes - and then applies the HuggingFace image classifier. - """ - - batch_sizes = [1, 1] - config = PipelineConfigResponse( - name="Zero Shot HF Classifier Pipeline", - slug="zero-shot-hf-classifier-pipeline", - description=("Zero Shot Object Detector with HF image classifier."), - version=1, - algorithms=[ - ZeroShotObjectDetector().algorithm_config_response, - HFImageClassifier().algorithm_config_response, - ], - ) - - def get_stages(self) -> list[Algorithm]: - zero_shot_object_detector = ZeroShotObjectDetector() - if "candidate_labels" in self.request_config: - logger.info( - "Setting candidate labels for zero shot object detector to %s", self.request_config["candidate_labels"] - ) - zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] - self.config.algorithms = [ - zero_shot_object_detector.algorithm_config_response, - HFImageClassifier().algorithm_config_response, - ] - - return [zero_shot_object_detector, HFImageClassifier()] - - def run(self) -> PipelineResultsResponse: - start_time = datetime.datetime.now() - detections_with_candidate_labels: list[Detection] = [] - if self.existing_detections: - logger.info("[1/2] Skipping the localizer, use existing detections...") - detections_with_candidate_labels = self.existing_detections - else: - logger.info("[1/2] No existing detections, generating detections...") - detections_with_candidate_labels: list[Detection] = self._get_detections( - self.stages[0], self.source_images, self.batch_sizes[0], intermediate=True - ) - - logger.info("[2/2] Running the classifier...") - detections_with_classifications: list[Detection] = self._get_detections( - self.stages[1], detections_with_candidate_labels, self.batch_sizes[1] - ) - end_time = datetime.datetime.now() - elapsed_time = (end_time - start_time).total_seconds() - - pipeline_response: PipelineResultsResponse = self._get_pipeline_response( - detections_with_classifications, elapsed_time - ) - logger.info(f"Successfully processed {len(detections_with_classifications)} detections.") - - return pipeline_response - - -class ZeroShotObjectDetectorPipeline(Pipeline): - """ - A pipeline that uses the HuggingFace zero shot object detector. - Produces both a bounding box and a classification for each detection. - The classification is based on the candidate labels provided in the request. - """ - - batch_sizes = [1] - config = PipelineConfigResponse( - name="Zero Shot Object Detector Pipeline", - slug="zero-shot-object-detector-pipeline", - description=("Zero shot object detector (bbox and classification)."), - version=1, - algorithms=[ZeroShotObjectDetector().algorithm_config_response], - ) - - def get_stages(self) -> list[Algorithm]: - zero_shot_object_detector = ZeroShotObjectDetector() - if "candidate_labels" in self.request_config: - logger.info( - "Setting candidate labels for zero shot object detector to %s", self.request_config["candidate_labels"] - ) - zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] - self.config.algorithms = [zero_shot_object_detector.algorithm_config_response] - - return [zero_shot_object_detector] - - def run(self) -> PipelineResultsResponse: - start_time = datetime.datetime.now() - logger.info("[1/1] Running the zero shot object detector...") - detections_with_classifications: list[Detection] = self._get_detections( - self.stages[0], self.source_images, self.batch_sizes[0] - ) - end_time = datetime.datetime.now() - elapsed_time = (end_time - start_time).total_seconds() - pipeline_response: PipelineResultsResponse = self._get_pipeline_response( - detections_with_classifications, elapsed_time - ) - logger.info(f"Successfully processed {len(detections_with_classifications)} detections.") - - return pipeline_response - - -class ZeroShotObjectDetectorWithRandomSpeciesClassifierPipeline(Pipeline): - """ - A pipeline that uses the HuggingFace zero shot object detector and a random species classifier. - """ - - batch_sizes = [1, 1] - config = PipelineConfigResponse( - name="Zero Shot Object Detector With Random Species Classifier Pipeline", - slug="zero-shot-object-detector-with-random-species-classifier-pipeline", - description=("HF zero shot object detector with random species classifier."), - version=1, - algorithms=[ - ZeroShotObjectDetector().algorithm_config_response, - RandomSpeciesClassifier().algorithm_config_response, - ], - ) - - def get_stages(self) -> list[Algorithm]: - zero_shot_object_detector = ZeroShotObjectDetector() - if "candidate_labels" in self.request_config: - zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] - - self.config.algorithms = [ - zero_shot_object_detector.algorithm_config_response, - RandomSpeciesClassifier().algorithm_config_response, - ] - - return [zero_shot_object_detector, RandomSpeciesClassifier()] - - def run(self) -> PipelineResultsResponse: - start_time = datetime.datetime.now() - detections: list[Detection] = [] - if self.existing_detections: - logger.info("[1/2] Skipping the localizer, use existing detections...") - detections = self.existing_detections - else: - logger.info("[1/2] No existing detections, generating detections...") - detections = self._get_detections(self.stages[0], self.source_images, self.batch_sizes[0]) - - logger.info("[2/2] Running the classifier...") - detections_with_classifications: list[Detection] = self._get_detections( - self.stages[1], detections, self.batch_sizes[1] - ) - end_time = datetime.datetime.now() - elapsed_time = (end_time - start_time).total_seconds() - pipeline_response: PipelineResultsResponse = self._get_pipeline_response( - detections_with_classifications, elapsed_time - ) - logger.info(f"Successfully processed {len(detections_with_classifications)} detections.") - - return pipeline_response - - -class ZeroShotObjectDetectorWithConstantClassifierPipeline(Pipeline): - """ - A pipeline that uses the HuggingFace zero shot object detector and a constant classifier. - """ - - batch_sizes = [1, 1] - config = PipelineConfigResponse( - name="Zero Shot Object Detector With Constant Classifier Pipeline", - slug="zero-shot-object-detector-with-constant-classifier-pipeline", - description=("HF zero shot object detector with constant classifier."), - version=1, - algorithms=[ - ZeroShotObjectDetector().algorithm_config_response, - ConstantClassifier().algorithm_config_response, - ], - ) - - def get_stages(self) -> list[Algorithm]: - zero_shot_object_detector = ZeroShotObjectDetector() - if "candidate_labels" in self.request_config: - zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] - - self.config.algorithms = [ - zero_shot_object_detector.algorithm_config_response, - ConstantClassifier().algorithm_config_response, - ] - - return [zero_shot_object_detector, ConstantClassifier()] - - def run(self) -> PipelineResultsResponse: - start_time = datetime.datetime.now() - detections: list[Detection] = [] - if self.existing_detections: - logger.info("[1/2] Skipping the localizer, use existing detections...") - detections = self.existing_detections - else: - logger.info("[1/2] No existing detections, generating detections...") - detections = self._get_detections(self.stages[0], self.source_images, self.batch_sizes[0]) - - logger.info("[2/2] Running the classifier...") - detections_with_classifications: list[Detection] = self._get_detections( - self.stages[1], detections, self.batch_sizes[1] - ) - end_time = datetime.datetime.now() - elapsed_time = (end_time - start_time).total_seconds() - pipeline_response: PipelineResultsResponse = self._get_pipeline_response( - detections_with_classifications, elapsed_time - ) - logger.info(f"Successfully processed {len(detections_with_classifications)} detections.") - - return pipeline_response - - -class BioCLIP25LogRegPipeline(Pipeline): - """ - Zero-shot object detector for bounding boxes, then BioCLIP 2.5 + logistic-regression - head for the terminal species classification. - - The detector is a placeholder for local testing. Swap stage 0 for the production - moth detector before using this on real deployments. - """ - - batch_sizes = [1, 1] - config = PipelineConfigResponse( - name="BioCLIP 2.5 + LogReg Head Pipeline", - slug="bioclip-2-5-logreg-pipeline", - description="Zero shot object detector with a BioCLIP 2.5 + logistic-regression species classifier.", - version=1, - algorithms=[ - ZeroShotObjectDetector().algorithm_config_response, - BioCLIP25LogRegClassifier().algorithm_config_response, - ], - ) - - def get_stages(self) -> list[Algorithm]: - zero_shot_object_detector = ZeroShotObjectDetector() - if "candidate_labels" in self.request_config: - zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] - - classifier = BioCLIP25LogRegClassifier() - self.config.algorithms = [ - zero_shot_object_detector.algorithm_config_response, - classifier.algorithm_config_response, - ] - return [zero_shot_object_detector, classifier] - - def run(self) -> PipelineResultsResponse: - start_time = datetime.datetime.now() - - if self.existing_detections: - logger.info("[1/2] Skipping the localizer, use existing detections...") - detections = self.existing_detections - else: - logger.info("[1/2] No existing detections, generating detections...") - detections = self._get_detections( - self.stages[0], self.source_images, self.batch_sizes[0], intermediate=True - ) - - logger.info("[2/2] Running the BioCLIP 2.5 classifier...") - detections_with_classifications = self._get_detections(self.stages[1], detections, self.batch_sizes[1]) - - elapsed_time = (datetime.datetime.now() - start_time).total_seconds() - pipeline_response = self._get_pipeline_response(detections_with_classifications, elapsed_time) - logger.info(f"Successfully processed {len(detections_with_classifications)} detections.") - - return pipeline_response - - -@final -class BioCLIPPanamaPipeline(BioCLIP25LogRegPipeline): - """ - Same two stages as the Newfoundland pipeline, with the Panama head as the terminal - classifier. Both pipelines share the frozen encoder through SAVED_MODELS, so running - both on one GPU costs one copy of the backbone plus two small Linear heads. - """ - - batch_sizes = [1, 1] - config = PipelineConfigResponse( - name="BioCLIP 2.5 + Panama LogReg Head Pipeline", - slug="bioclip-2-5-panama-pipeline", - description="Zero shot object detector with a BioCLIP 2.5 + Panama logistic-regression species classifier.", - version=1, - algorithms=[ - ZeroShotObjectDetector().algorithm_config_response, - BioCLIPPanamaLogRegClassifier().algorithm_config_response, - ], - ) - - def get_stages(self) -> list[Algorithm]: - zero_shot_object_detector = ZeroShotObjectDetector() - if "candidate_labels" in self.request_config: - zero_shot_object_detector.candidate_labels = self.request_config["candidate_labels"] - - classifier = BioCLIPPanamaLogRegClassifier() - self.config.algorithms = [ - zero_shot_object_detector.algorithm_config_response, - classifier.algorithm_config_response, - ] - return [zero_shot_object_detector, classifier] diff --git a/processing_services/bioclip/api/schemas.py b/processing_services/bioclip/api/schemas.py deleted file mode 100644 index c198384d0..000000000 --- a/processing_services/bioclip/api/schemas.py +++ /dev/null @@ -1,380 +0,0 @@ -# Can these be imported from the OpenAPI spec yaml? -import datetime -import logging -import pathlib - -import PIL.Image -import pydantic - -from .utils import get_image - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -class BoundingBox(pydantic.BaseModel): - x1: float - y1: float - x2: float - y2: float - - @classmethod - def from_coords(cls, coords: list[float]): - return cls(x1=coords[0], y1=coords[1], x2=coords[2], y2=coords[3]) - - def to_string(self): - return f"{self.x1},{self.y1},{self.x2},{self.y2}" - - def to_path(self): - return "-".join([str(int(x)) for x in [self.x1, self.y1, self.x2, self.y2]]) - - def to_tuple(self): - return (self.x1, self.y1, self.x2, self.y2) - - -class BaseImage(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="ignore", arbitrary_types_allowed=True) - - id: str - url: str | None = None - b64: str | None = None - filepath: str | pathlib.Path | None = None - _pil: PIL.Image.Image | None = None - width: int | None = None - height: int | None = None - timestamp: datetime.datetime | None = None - - # Validate that there is at least one of the following fields - @pydantic.model_validator(mode="after") - def validate_source(self): - if not any([self.url, self.b64, self.filepath, self._pil]): - raise ValueError("At least one of the following fields must be provided: url, b64, filepath, pil") - return self - - def open(self, raise_exception=False) -> PIL.Image.Image | None: - if not self._pil: - logger.warn(f"Opening image {self.id} for the first time") - self._pil = get_image( - url=self.url, - b64=self.b64, - filepath=self.filepath, - raise_exception=raise_exception, - ) - else: - logger.info(f"Using already loaded image {self.id}") - if self._pil: - self.width, self.height = self._pil.size - return self._pil - - -class SourceImage(BaseImage): - pass - - -class AlgorithmReference(pydantic.BaseModel): - name: str - key: str - - -class ClassificationResponse(pydantic.BaseModel): - classification: str - labels: list[str] | None = pydantic.Field( - default=None, - description=( - "A list of all possible labels for the model, in the correct order. " - "Omitted if the model has too many labels to include for each classification in the response. " - "Use the category map from the algorithm to get the full list of labels and metadata." - ), - ) - scores: list[float] = pydantic.Field( - default_factory=list, - description="The calibrated probabilities for each class label, most commonly the softmax output.", - ) - logits: list[float] = pydantic.Field( - default_factory=list, - description="The raw logits output by the model, before any calibration or normalization.", - ) - features: list[float] | None = pydantic.Field( - default=None, - description=( - "The embedding the backbone produced for this crop, taken before the " - "classification head. Optional, and only useful if every value comes from " - "the same backbone." - ), - ) - inference_time: float | None = None - algorithm: AlgorithmReference - terminal: bool = True - timestamp: datetime.datetime - - -class SourceImageRequest(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="ignore") - - id: str - url: str - # b64: str | None = None - # @TODO bring over new SourceImage & b64 validation from the lepsAI repo - - -class SourceImageResponse(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="ignore") - - id: str - url: str - - -class DetectionRequest(pydantic.BaseModel): - source_image: SourceImageRequest # the 'original' image - bbox: BoundingBox - crop_image_url: str | None = None - algorithm: AlgorithmReference - - -class DetectionResponse(pydantic.BaseModel): - # these fields are populated with values from a Detection, excluding source_image details - source_image_id: str - bbox: BoundingBox - inference_time: float | None = None - algorithm: AlgorithmReference - timestamp: datetime.datetime - crop_image_url: str | None = None - classifications: list[ClassificationResponse] = [] - - -class Detection(BaseImage): - """ - An internal representation of a detection with reference to a source image instance. - """ - - source_image: SourceImage # the 'original' uncropped image - bbox: BoundingBox - inference_time: float | None = None - algorithm: AlgorithmReference - classifications: list[ClassificationResponse] = [] - - -class AlgorithmCategoryMapResponse(pydantic.BaseModel): - data: list[dict] = pydantic.Field( - default_factory=dict, - description="Complete data for each label, such as id, gbif_key, explicit index, source, etc.", - examples=[ - [ - {"label": "Moth", "index": 0, "gbif_key": 1234}, - {"label": "Not a moth", "index": 1, "gbif_key": 5678}, - ] - ], - ) - labels: list[str] = pydantic.Field( - default_factory=list, - description="A simple list of string labels, in the correct index order used by the model.", - examples=[["Moth", "Not a moth"]], - ) - version: str | None = pydantic.Field( - default=None, - description="The version of the category map. Can be a descriptive string or a version number.", - examples=["LepNet2021-with-2023-mods"], - ) - description: str | None = pydantic.Field( - default=None, - description="A description of the category map used to train. e.g. source, purpose and modifications.", - examples=["LepNet2021 with Schmidt 2023 corrections. Limited to species with > 1000 observations."], - ) - uri: str | None = pydantic.Field( - default=None, - description="A URI to the category map file, could be a public web URL or object store path.", - ) - - -class AlgorithmTrainingConfig(pydantic.BaseModel): - """ - Default settings for retraining this algorithm. Antenna seeds its own copy from this - and an admin can then override it, so these are defaults, not rules. - """ - - # Used by Antenna when it builds the training set. - min_per_species: int = 2 - test_fraction: float = 0.2 - split_salt: str = "antenna-head-v1" - - # Used here when fitting. - head_type: str = "linear" - epochs: int = 300 - learning_rate: float = 0.01 - weight_decay: float = 1e-4 - min_improvement: float = 0.0 - - model_config = pydantic.ConfigDict(extra="allow") - - -class AlgorithmTrainingInfo(pydantic.BaseModel): - """Where a retrained version's weights came from.""" - - trained_at: datetime.datetime | None = None - dataset_url: str | None = None - dataset_rows: int | None = None - dataset_classes: int | None = None - metrics: dict = pydantic.Field(default_factory=dict) - previous_metrics: dict = pydantic.Field(default_factory=dict) - parent_algorithm_key: str | None = None - job_id: int | None = None - warnings: list[str] = pydantic.Field(default_factory=list) - - model_config = pydantic.ConfigDict(extra="allow") - - -class AlgorithmConfigResponse(pydantic.BaseModel): - name: str - key: str = pydantic.Field( - description=("A unique key for an algorithm to lookup the category map (class list) and other metadata."), - ) - description: str | None = None - task_type: str | None = pydantic.Field( - default=None, - description="The type of task the model is trained for. e.g. 'detection', 'classification', 'embedding', etc.", - examples=["detection", "classification", "segmentation", "embedding"], - ) - version: int = pydantic.Field( - default=1, - description="A sortable version number for the model. Increment this number when the model is updated.", - ) - version_name: str | None = pydantic.Field( - default=None, - description="A complete version name e.g. '2021-01-01', 'LepNet2021'.", - ) - uri: str | None = pydantic.Field( - default=None, - description="A URI to the weights or model details, could be a public web URL or object store path.", - ) - trainable: bool = pydantic.Field( - default=False, - description=( - "Whether this algorithm can be retrained from labelled data. Only the classifier head " - "over the frozen backbone is cheap enough to retrain; the backbone and the detector are not." - ), - ) - training_config: AlgorithmTrainingConfig | None = None - training_info: AlgorithmTrainingInfo | None = None - category_map: AlgorithmCategoryMapResponse | None = None - - class Config: - extra = "ignore" - - -# Deliberately a plain string, not a fixed list of slugs. Heads retrained after the service -# started get their own pipeline, and a Literal would reject them with an unhelpful 422. -# api.process() validates against the live registry and says which slugs exist. -PipelineChoice = str - - -class PipelineRequestConfigParameters(pydantic.BaseModel): - """Parameters used to configure a pipeline request. - - Accepts any serializable key-value pair. - Example: {"force_reprocess": True, "auth_token": "abc123"} - - Supported parameters are defined by the pipeline in the processing service - and should be published in the Pipeline's info response. - """ - - force_reprocess: bool = pydantic.Field( - default=False, - description="Force reprocessing of the image, even if it has already been processed.", - ) - auth_token: str | None = pydantic.Field( - default=None, - description="An optional authentication token to use for the pipeline.", - ) - candidate_labels: list[str] | None = pydantic.Field( - default=None, - description="A list of candidate labels to use for the zero-shot object detector.", - ) - - -class PipelineRequest(pydantic.BaseModel): - pipeline: PipelineChoice - source_images: list[SourceImageRequest] - detections: list[DetectionRequest] | None = None - config: PipelineRequestConfigParameters | dict | None = None - - # Example for API docs: - class Config: - json_schema_extra = { - "example": { - "pipeline": "random", - "source_images": [ - { - "id": "123", - "url": "https://archive.org/download/mma_various_moths_and_butterflies_54143/54143.jpg", - } - ], - "config": {"force_reprocess": True, "auth_token": "abc123"}, - } - } - - -class PipelineResultsResponse(pydantic.BaseModel): - pipeline: PipelineChoice - total_time: float - algorithms: dict[str, AlgorithmConfigResponse] = pydantic.Field( - default_factory=dict, - description=( - "A dictionary of all algorithms used in the pipeline, including their class list and other " - "metadata, keyed by the algorithm key. " - "DEPRECATED: Algorithms should only be provided in the ProcessingServiceInfoResponse." - ), - depreciated=True, - ) - source_images: list[SourceImageResponse] - detections: list[DetectionResponse] - errors: list | str | None = None - - -class PipelineStageParam(pydantic.BaseModel): - """A configurable parameter of a stage of a pipeline.""" - - name: str - key: str - category: str = "default" - - -class PipelineStage(pydantic.BaseModel): - """A configurable stage of a pipeline.""" - - key: str - name: str - params: list[PipelineStageParam] = [] - description: str | None = None - - -class PipelineConfigResponse(pydantic.BaseModel): - """Details about a pipeline, its algorithms and category maps.""" - - name: str - slug: str - version: int - description: str | None = None - algorithms: list[AlgorithmConfigResponse] = [] - stages: list[PipelineStage] = [] - - -class ProcessingServiceInfoResponse(pydantic.BaseModel): - """Information about the processing service.""" - - name: str = pydantic.Field(example="Mila Research Lab - Moth AI Services") - description: str | None = pydantic.Field( - default=None, - examples=["Algorithms developed by the Mila Research Lab for analysis of moth images."], - ) - pipelines: list[PipelineConfigResponse] = pydantic.Field( - default=list, - examples=[ - [ - PipelineConfigResponse(name="Random Pipeline", slug="random", version=1, algorithms=[]), - ] - ], - ) - # algorithms: list[AlgorithmConfigResponse] = pydantic.Field( - # default=list, - # examples=[RANDOM_BINARY_CLASSIFIER], - # ) diff --git a/processing_services/bioclip/api/test.py b/processing_services/bioclip/api/test.py deleted file mode 100644 index b5b1b5f7c..000000000 --- a/processing_services/bioclip/api/test.py +++ /dev/null @@ -1,64 +0,0 @@ -import unittest - -from fastapi.testclient import TestClient - -from .api import app -from .pipelines import CustomPipeline -from .schemas import PipelineRequest, SourceImage, SourceImageRequest - - -class TestPipeline(unittest.TestCase): - def test_custom_pipeline(self): - # @TODO: Load actual antenna images? - pipeline = CustomPipeline( - source_images=[ - SourceImage( - id="1001", - url=( - "https://huggingface.co/datasets/huggingface/" - "documentation-images/resolve/main/pipeline-cat-chonk.jpeg" - ), - ), - SourceImage(id="1002", url="https://cdn.britannica.com/79/191679-050-C7114D2B/Adult-capybara.jpg"), - ], - detector_batch_size=2, - classifier_batch_size=2, - ) - detections = pipeline.run() - - self.assertEqual(len(detections), 20) - expected_labels = ["lynx, catamount", "beaver"] - for detection_id, detection in enumerate(detections): - self.assertEqual(detection.source_image_id, pipeline.source_images[detection_id].id) - self.assertIsNotNone(detection.bbox) - self.assertEqual(len(detection.classifications), 1) - classification = detection.classifications[0] - self.assertEqual(classification.classification, expected_labels[detection_id]) - self.assertGreaterEqual(classification.scores[0], 0.0) - self.assertLessEqual(classification.scores[0], 1.0) - - -class TestAPI(unittest.TestCase): - def setUp(self): - self.client = TestClient(app) - - def test_root(self): - response = self.client.get("/") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.url, "http://testserver/docs") - - def test_process(self): - source_images = [ - SourceImage(id="1", url="https://example.com/image1.jpg"), - SourceImage(id="2", url="https://example.com/image2.jpg"), - ] - source_image_requests = [SourceImageRequest(**image.dict()) for image in source_images] - request = PipelineRequest(pipeline="local-pipeline", source_images=source_image_requests, config={}) - response = self.client.post("/process", json=request.dict()) - - self.assertEqual(response.status_code, 200) - data = response.json() - self.assertEqual(data["pipeline"], "local-pipeline") - self.assertEqual(len(data["source_images"]), 2) - self.assertEqual(len(data["detections"]), 2) - self.assertGreater(data["total_time"], 0.0) diff --git a/processing_services/bioclip/api/trained_heads.py b/processing_services/bioclip/api/trained_heads.py deleted file mode 100644 index fdb4f42c0..000000000 --- a/processing_services/bioclip/api/trained_heads.py +++ /dev/null @@ -1,205 +0,0 @@ -""" -Serve the heads this service has retrained, alongside the one it shipped with. - -A retrained head is useless if nothing can select it. Each one saved to disk is offered as -its own algorithm and its own pipeline, so Antenna sees it in /info and a user can pick it -the same way they pick any other. The original head stays exactly where it was: a retrain -adds a choice, it never replaces one. -""" - -import json -import logging -import os -import pathlib -import typing - -logger = logging.getLogger(__name__) - -# Where the training endpoint writes heads. Kept off the model cache so a training run -# cannot overwrite the head the service is currently serving. -TRAINED_HEADS_DIR = os.environ.get("BIOCLIP_TRAINED_HEADS_DIR", "/data/bioclip-service/trained_heads") - -HEAD_SUFFIX = ".npz" -LABELS_SUFFIX = ".label_map.json" - -# Prefix for the algorithm key and pipeline slug of a retrained head. Antenna keys -# algorithms by this string, so changing it orphans everything already registered. -RETRAINED_PREFIX = "bioclip-2-5-retrained" - - -class TrainedHead(typing.NamedTuple): - """One head this service has produced, as found on disk.""" - - name: str - head_path: pathlib.Path - labels_path: pathlib.Path - metadata: dict - - @property - def algorithm_key(self) -> str: - return f"{RETRAINED_PREFIX}-{self.name}" - - @property - def pipeline_slug(self) -> str: - return f"{RETRAINED_PREFIX}-{self.name}-pipeline" - - @property - def labels(self) -> list[str]: - return [str(label) for label in self.metadata.get("labels", [])] - - -def discover(directory: str | None = None) -> list[TrainedHead]: - """ - List the heads saved on disk. - - Reads the label map rather than the weights: this runs at startup and after every - training run, and the weights are only needed once a head is actually used. - """ - path = pathlib.Path(directory or TRAINED_HEADS_DIR) - if not path.is_dir(): - return [] - - heads: list[TrainedHead] = [] - for head_path in sorted(path.glob(f"*{HEAD_SUFFIX}")): - name = head_path.name[: -len(HEAD_SUFFIX)] - labels_path = path / f"{name}{LABELS_SUFFIX}" - if not labels_path.exists(): - logger.warning(f"Skipping {head_path.name}: no label map beside it, so its classes are unknown") - continue - try: - metadata = json.loads(labels_path.read_text()) - except (OSError, ValueError) as e: - logger.warning(f"Skipping {head_path.name}: could not read its label map ({e})") - continue - if not metadata.get("labels"): - logger.warning(f"Skipping {head_path.name}: its label map lists no species") - continue - heads.append(TrainedHead(name=name, head_path=head_path, labels_path=labels_path, metadata=metadata)) - - logger.info(f"Found {len(heads)} retrained head(s) in {path}") - return heads - - -def make_classifier_class(head: TrainedHead): - """ - Build the algorithm class that serves one retrained head. - - A subclass rather than a separate implementation: the weights have the same shape as - the head it was trained from, so everything about running it is already written. Only - where the file lives and what it is called differ. - """ - from .algorithms import BioCLIP25LogRegClassifier - from .schemas import AlgorithmCategoryMapResponse, AlgorithmConfigResponse, AlgorithmTrainingInfo - - metrics = head.metadata.get("metrics") or {} - trained_at = head.metadata.get("trained_at") - labels = head.labels - - class RetrainedClassifier(BioCLIP25LogRegClassifier): - head_repo_id = None - head_local_dir = str(head.head_path.parent) - head_filename = head.head_path.name - categories_filename = head.labels_path.name - saved_models_key = f"retrained_{head.name}" - - @property - def head_uri(self): - return None - - def get_category_map(self) -> AlgorithmCategoryMapResponse: - crops = sum((head.metadata.get("counts") or {}).values()) - return AlgorithmCategoryMapResponse( - data=[{"index": i, "label": name, "taxon_rank": "SPECIES"} for i, name in enumerate(labels)], - labels=labels, - version=head.name, - description=f"Retrained on {crops} verified crops.", - ) - - def get_algorithm_config_response(self) -> AlgorithmConfigResponse: - return AlgorithmConfigResponse( - name=f"BioCLIP 2.5 + LogReg head (retrained {head.name})", - key=head.algorithm_key, - task_type="classification", - description="Retrained from species verified in Antenna.", - version=1, - version_name=head.name, - uri=None, - trainable=True, - training_config=self.training_config(), - training_info=AlgorithmTrainingInfo( - trained_at=trained_at, - metrics=metrics, - dataset_classes=len(labels), - ), - category_map=self.get_category_map(), - ) - - RetrainedClassifier.__name__ = f"RetrainedClassifier_{head.name.replace('-', '_')}" - return RetrainedClassifier - - -def make_pipeline_class(head: TrainedHead): - """ - Build the pipeline that runs one retrained head. - - Antenna runs pipelines, not algorithms, so a head with no pipeline cannot be selected. - Stage 0 is the same detector the original pipeline uses. - """ - from .algorithms import ZeroShotObjectDetector - from .pipelines import BioCLIP25LogRegPipeline - from .schemas import PipelineConfigResponse - - classifier_class = make_classifier_class(head) - - # Filled in here, not in get_stages(): /info reads the class-level config, and - # get_stages() only runs once a pipeline is actually instantiated to process images. - # A pipeline advertised with no algorithms registers in Antenna with none attached. - stage_configs = [ - ZeroShotObjectDetector().algorithm_config_response, - classifier_class().algorithm_config_response, - ] - - class RetrainedPipeline(BioCLIP25LogRegPipeline): - config = PipelineConfigResponse( - name=f"BioCLIP 2.5 Retrained Head ({head.name})", - slug=head.pipeline_slug, - description="Zero shot object detector with a head retrained from verified species.", - version=1, - algorithms=stage_configs, - ) - - def get_stages(self): - detector = ZeroShotObjectDetector() - if "candidate_labels" in self.request_config: - detector.candidate_labels = self.request_config["candidate_labels"] - return [detector, classifier_class()] - - RetrainedPipeline.__name__ = f"RetrainedPipeline_{head.name.replace('-', '_')}" - return RetrainedPipeline - - -def register(pipeline_choices: dict, algorithm_choices: dict, directory: str | None = None) -> list[str]: - """ - Add every head on disk to the service's registries. - - Called at startup and again after training, so a head becomes selectable without a - restart. Returns the pipeline slugs that were added. - """ - added: list[str] = [] - for head in discover(directory): - if head.pipeline_slug in pipeline_choices: - continue - try: - pipeline_class = make_pipeline_class(head) - classifier = make_classifier_class(head)() - config = classifier.algorithm_config_response - except Exception as e: - logger.error(f"Could not offer retrained head '{head.name}': {e}") - continue - pipeline_choices[head.pipeline_slug] = pipeline_class - algorithm_choices[config.key] = config - added.append(head.pipeline_slug) - - if added: - logger.info(f"Offering {len(added)} retrained head(s): {', '.join(added)}") - return added diff --git a/processing_services/bioclip/api/training.py b/processing_services/bioclip/api/training.py deleted file mode 100644 index bf97650f8..000000000 --- a/processing_services/bioclip/api/training.py +++ /dev/null @@ -1,417 +0,0 @@ -""" -Retrain the classifier head from labels people verified in Antenna. - -The backbone is frozen, so a head is just a linear layer over stored embeddings. Antenna -already holds those embeddings, so nothing here touches an image: pull rows, fit a head, -score it against the one in service, and keep the new one only if it wins. -""" - -import dataclasses -import datetime -import json -import logging -import pathlib -import tempfile -import typing - -import numpy as np -import requests -import torch - -logger = logging.getLogger(__name__) - -# A new head must beat the incumbent STRICTLY, by more than this margin. A tie must not -# promote: swapping heads is not free (it invalidates comparisons against past results), so -# an equal score is not a reason to change. -DEFAULT_MIN_IMPROVEMENT = 0.0 - -# Below this many held-out rows the comparison is noise, not evidence. The run still -# reports its numbers, but says plainly that they cannot support a decision. -MIN_MEANINGFUL_TEST_ROWS = 30 - -# The head shapes this service can fit. Only "linear" for now: the serving path loads a -# single Linear layer, so anything else would train fine and then fail to load. An MLP-1 -# head scores far better on rare species and is worth adding, but it needs the loader too. -SUPPORTED_HEAD_TYPES = ("linear",) - - -class NotEnoughData(Exception): - """Raised when the verified data cannot support both training and evaluation.""" - - -class UnsupportedHeadType(Exception): - """Raised when asked for a head shape this service cannot fit or serve.""" - - -@dataclasses.dataclass -class TrainingRow: - detection_id: int - label: str - split: str - features: np.ndarray - - -def fetch_dataset( - url: str, timeout: int = 300, session: requests.Session | None = None -) -> tuple[list[TrainingRow], dict]: - """ - Download the training set Antenna prepared and unpack it. - - An npz of float16 vectors rather than JSON rows: the values are stored as two bytes in - Postgres, so float16 loses nothing, and it is roughly ten times smaller and far faster - to parse than the equivalent JSON. - """ - session = session or requests.Session() - response = session.get(url, timeout=timeout, stream=True) - response.raise_for_status() - - with tempfile.NamedTemporaryFile(suffix=".npz") as handle: - for chunk in response.iter_content(chunk_size=1 << 20): - handle.write(chunk) - handle.flush() - archive = np.load(handle.name, allow_pickle=True) - - metadata = json.loads(str(archive["metadata"])) - classes = [str(c) for c in archive["classes"]] - features = archive["features"] - labels = archive["labels"] - detection_ids = archive["detection_ids"] - splits = archive["split"] - - rows = [ - TrainingRow( - detection_id=int(detection_id), - label=classes[int(label)], - split=str(split), - features=vector, - ) - for detection_id, label, split, vector in zip(detection_ids, labels, splits, features) - ] - logger.info(f"Loaded {len(rows)} training rows over {len(classes)} species from {url}") - return rows, metadata - - -def _matrices(rows: list[TrainingRow], labels: list[str]) -> tuple[np.ndarray, np.ndarray]: - index = {name: i for i, name in enumerate(labels)} - x = np.asarray([r.features for r in rows], dtype=np.float32) - y = np.asarray([index[r.label] for r in rows], dtype=np.int64) - return x, y - - -def warm_start( - head: torch.nn.Linear, - labels: list[str], - incumbent: dict[str, typing.Any] | None, -) -> int: - """ - Copy the current head's weights for any species they have in common. - - Without this a retrain starts from noise, so every species with no new verified crops - comes out worse than before. Seeding from the incumbent makes a retrain additive: a - species nobody verified this time keeps exactly what it already knew. - - Returns how many classes were seeded. - """ - if not incumbent: - return 0 - index = {name: i for i, name in enumerate(incumbent["labels"])} - weights = incumbent["weights"] - bias = incumbent["bias"] - - seeded = 0 - with torch.no_grad(): - for i, name in enumerate(labels): - source = index.get(name) - if source is None: - continue - head.weight[i] = torch.from_numpy(np.asarray(weights[source], dtype=np.float32)) - head.bias[i] = float(bias[source]) - seeded += 1 - logger.info(f"Warm-started {seeded} of {len(labels)} classes from the current head") - return seeded - - -def restore_untrained_classes( - weights: np.ndarray, - bias: np.ndarray, - labels: list[str], - trained_counts: dict[str, int], - incumbent: dict[str, typing.Any] | None, -) -> int: - """ - Put back the current head's weights for species that had no training rows. - - Warm-starting alone is not enough: cross-entropy still pushes those classes around as - negatives over hundreds of epochs, so they drift away from what they knew. Copying them - back afterwards is what actually makes a retrain additive. - - Returns how many classes were restored. - """ - if not incumbent: - return 0 - index = {name: i for i, name in enumerate(incumbent["labels"])} - restored = 0 - for i, name in enumerate(labels): - if trained_counts.get(name): - continue - source = index.get(name) - if source is None: - continue - weights[i] = np.asarray(incumbent["weights"][source], dtype=weights.dtype) - bias[i] = incumbent["bias"][source] - restored += 1 - if restored: - logger.info(f"Restored {restored} class(es) that had no verified crops this run") - return restored - - -def train_linear_head( - x: np.ndarray, - y: np.ndarray, - num_classes: int, - epochs: int = 300, - learning_rate: float = 0.01, - weight_decay: float = 1e-4, - device: str = "cpu", - labels: list[str] | None = None, - incumbent: dict[str, typing.Any] | None = None, -) -> torch.nn.Linear: - """ - Fit a single linear layer over frozen embeddings. - - Deliberately the same shape as the head already in service (a Linear over L2-normalised - BioCLIP features), so a retrained head is a drop-in replacement rather than a different - kind of model that would need its own serving path. - """ - head = torch.nn.Linear(x.shape[1], num_classes).to(device) - if labels: - warm_start(head, labels, incumbent) - optimizer = torch.optim.AdamW(head.parameters(), lr=learning_rate, weight_decay=weight_decay) - loss_fn = torch.nn.CrossEntropyLoss() - - inputs = torch.from_numpy(x).to(device) - targets = torch.from_numpy(y).to(device) - - head.train() - for epoch in range(epochs): - optimizer.zero_grad() - loss = loss_fn(head(inputs), targets) - loss.backward() - optimizer.step() - if epoch % 50 == 0: - logger.debug(f"epoch {epoch} loss {loss.item():.4f}") - head.eval() - return head - - -def evaluate( - weights: np.ndarray, - bias: np.ndarray, - x: np.ndarray, - y: np.ndarray, -) -> dict[str, float]: - """Top-1 and macro recall. Macro matters because trap data is heavily long-tailed.""" - logits = x @ weights.T + bias - predicted = logits.argmax(axis=1) - correct = predicted == y - - per_class = [] - for cls in np.unique(y): - mask = y == cls - per_class.append(float(correct[mask].mean())) - - return { - "top1": float(correct.mean()), - "macro_recall": float(np.mean(per_class)) if per_class else 0.0, - "n": int(len(y)), - "classes": int(len(np.unique(y))), - } - - -def score_incumbent( - incumbent_weights: np.ndarray, - incumbent_bias: np.ndarray, - incumbent_labels: list[str], - rows: list[TrainingRow], -) -> dict[str, float] | None: - """ - Score the head currently in service on the same held-out rows. - - Only rows whose species the incumbent can actually predict are counted. Scoring it on - species it was never trained to output would understate it and make any new head look - better than it is. - """ - index = {name: i for i, name in enumerate(incumbent_labels)} - usable = [r for r in rows if r.label in index] - if not usable: - return None - x = np.asarray([r.features for r in usable], dtype=np.float32) - y = np.asarray([index[r.label] for r in usable], dtype=np.int64) - result = evaluate(incumbent_weights, incumbent_bias, x, y) - result["skipped_unknown_species"] = len(rows) - len(usable) - return result - - -def retrain( - rows: list[TrainingRow], - incumbent: dict[str, typing.Any] | None = None, - min_per_species: int = 2, - min_improvement: float = DEFAULT_MIN_IMPROVEMENT, - device: str = "cpu", - declared_classes: list[str] | None = None, - epochs: int = 300, - learning_rate: float = 0.01, - weight_decay: float = 1e-4, - head_type: str = "linear", -) -> dict[str, typing.Any]: - """ - Fit a new head and decide whether it deserves to replace the current one. - - Returns the metrics, the decision, and the new head's weights. It does not write - anything: publishing is a separate, deliberate step. - """ - if head_type not in SUPPORTED_HEAD_TYPES: - # Refused rather than ignored: silently fitting a different shape than the caller - # asked for would produce a head nobody could explain. - raise UnsupportedHeadType( - f"This service cannot fit a '{head_type}' head. Supported: {', '.join(SUPPORTED_HEAD_TYPES)}." - ) - counts: dict[str, int] = {} - for row in rows: - counts[row.label] = counts.get(row.label, 0) + 1 - - if declared_classes: - # Antenna decided the species list, usually from a project's taxa list. Honour it: - # a species with no verified crops this run still needs a column, or the head would - # quietly stop predicting it. Its weights come from the current head instead. - labels = sorted(declared_classes) - else: - labels = sorted(name for name, n in counts.items() if n >= min_per_species) - if not labels: - raise NotEnoughData( - f"No species has at least {min_per_species} verified crops. " - f"Verify more occurrences before retraining. Counts: {counts}" - ) - - label_set = set(labels) - kept = [r for r in rows if r.label in label_set] - train_rows = [r for r in kept if r.split == "train"] - test_rows = [r for r in kept if r.split == "test"] - if not train_rows: - raise NotEnoughData("The train split is empty.") - if not test_rows: - raise NotEnoughData( - "The test split is empty, so a new head cannot be compared against the current one. " - "Verify more occurrences." - ) - - x_train, y_train = _matrices(train_rows, labels) - x_test, y_test = _matrices(test_rows, labels) - - head = train_linear_head( - x_train, - y_train, - num_classes=len(labels), - device=device, - labels=labels, - incumbent=incumbent, - epochs=epochs, - learning_rate=learning_rate, - weight_decay=weight_decay, - ) - weights = head.weight.detach().cpu().numpy() - bias = head.bias.detach().cpu().numpy() - - trained_counts: dict[str, int] = {} - for row in train_rows: - trained_counts[row.label] = trained_counts.get(row.label, 0) + 1 - restored = restore_untrained_classes(weights, bias, labels, trained_counts, incumbent) - - candidate = evaluate(weights, bias, x_test, y_test) - incumbent_metrics = None - if incumbent: - incumbent_metrics = score_incumbent(incumbent["weights"], incumbent["bias"], incumbent["labels"], test_rows) - - if incumbent_metrics is None: - promote = False - reason = "No incumbent head was scored, so the new head is not promoted automatically." - elif candidate["top1"] > incumbent_metrics["top1"] + min_improvement: - promote = True - reason = ( - f"New head top-1 {candidate['top1']:.3f} beats current {incumbent_metrics['top1']:.3f} " - f"by more than {min_improvement:.3f}." - ) - else: - promote = False - reason = ( - f"New head top-1 {candidate['top1']:.3f} does not beat current " - f"{incumbent_metrics['top1']:.3f} by more than {min_improvement:.3f}." - ) - - warnings: list[str] = [] - if len(test_rows) < MIN_MEANINGFUL_TEST_ROWS: - warnings.append( - f"Only {len(test_rows)} held-out row(s). Anything under {MIN_MEANINGFUL_TEST_ROWS} is too " - "few to tell two heads apart, so treat this result as a smoke test, not evidence." - ) - if incumbent_metrics and incumbent_metrics.get("skipped_unknown_species"): - warnings.append( - f"{incumbent_metrics['skipped_unknown_species']} held-out row(s) name species the current " - "head cannot predict, so they were left out of its score. The two heads were not scored on " - "identical rows." - ) - if len(labels) < 5: - warnings.append( - f"The new head covers only {len(labels)} species. The head in service covers far more, so " - "promoting this one would narrow what the pipeline can predict." - ) - - return { - "labels": labels, - "weights": weights, - "bias": bias, - "counts": {name: counts.get(name, 0) for name in labels}, - "dropped_species": sorted(set(counts) - label_set), - "rows": {"total": len(rows), "kept": len(kept), "train": len(train_rows), "test": len(test_rows)}, - "classes_restored_from_current_head": restored, - "candidate_metrics": candidate, - "incumbent_metrics": incumbent_metrics, - "promote": promote, - "reason": reason, - "warnings": warnings, - "trained_at": datetime.datetime.now().isoformat(), - } - - -def save_head(result: dict[str, typing.Any], directory: pathlib.Path, name: str) -> dict[str, str]: - """ - Write the head in the same format the service already loads (npz + label_map.json). - - Written under its own name rather than overwriting the current head: Antenna keys - algorithms by key, and a head that is swapped in place makes past classifications - impossible to attribute. - """ - directory.mkdir(parents=True, exist_ok=True) - head_path = directory / f"{name}.npz" - labels_path = directory / f"{name}.label_map.json" - - np.savez( - head_path, - W=result["weights"], - b=result["bias"], - # A plain string array, not dtype=object. An object array can only be read back - # with allow_pickle=True, and the loader that serves these heads does not set it. - classes=np.array([str(label) for label in result["labels"]]), - ) - labels_path.write_text( - json.dumps( - { - "labels": result["labels"], - "counts": result["counts"], - "metrics": result["candidate_metrics"], - "trained_at": result["trained_at"], - }, - indent=2, - ) - ) - return {"head": str(head_path), "labels": str(labels_path)} diff --git a/processing_services/bioclip/api/utils.py b/processing_services/bioclip/api/utils.py deleted file mode 100644 index a7fcb6a75..000000000 --- a/processing_services/bioclip/api/utils.py +++ /dev/null @@ -1,149 +0,0 @@ -import base64 -import binascii -import io -import logging -import pathlib -import re -import tempfile -from urllib.parse import urlparse - -import PIL.Image -import PIL.ImageFile -import requests - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -PIL.ImageFile.LOAD_TRUNCATED_IMAGES = True - -# This is polite and required by some hosts -# see: https://foundation.wikimedia.org/wiki/Policy:User-Agent_policy -USER_AGENT = "AntennaInsectDataPlatform/1.0 (https://insectai.org)" - -# ----------- -# File handling functions -# ----------- - - -def is_url(path: str) -> bool: - return path.startswith("http://") or path.startswith("https://") - - -def is_base64(s: str) -> bool: - try: - # Check if string can be decoded from base64 - return base64.b64encode(base64.b64decode(s)).decode() == s - except Exception: - return False - - -def get_or_download_file(path_or_url, tempdir_prefix="antenna") -> pathlib.Path: - """ - Fetch a file from a URL or local path. If the path is a URL, download the file. - If the URL has already been downloaded, return the existing local path. - If the path is a local path, return the path. - - >>> filepath = get_or_download_file("https://example.uk/images/31-20230919033000-snapshot.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=451d406b7eb1113e1bb05c083ce51481%2F20240429%2F") # noqa: E501 - >>> filepath.name - '31-20230919033000-snapshot.jpg' - >>> filepath = get_or_download_file("/home/user/images/31-20230919033000-snapshot.jpg") - >>> filepath.name - '31-20230919033000-snapshot.jpg' - """ - if not path_or_url: - raise Exception("Specify a URL or path to fetch file from.") - - # If path is a local path instead of a URL then urlretrieve will just return that path - - destination_dir = pathlib.Path(tempfile.mkdtemp(prefix=tempdir_prefix)) - fname = pathlib.Path(urlparse(path_or_url).path).name - if not destination_dir.exists(): - destination_dir.mkdir(parents=True, exist_ok=True) - local_filepath = pathlib.Path(destination_dir) / fname - - if local_filepath and local_filepath.exists(): - logger.info(f"Using existing {local_filepath}") - return local_filepath - - else: - logger.info(f"Downloading {path_or_url} to {local_filepath}") - headers = {"User-Agent": USER_AGENT} - response = requests.get(path_or_url, stream=True, headers=headers) - response.raise_for_status() # Raise an exception for HTTP errors - - with open(local_filepath, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - - resulting_filepath = pathlib.Path(local_filepath).resolve() - logger.info(f"Downloaded to {resulting_filepath}") - return resulting_filepath - - -def open_image(fp: str | bytes | pathlib.Path | io.BytesIO, raise_exception: bool = True) -> PIL.Image.Image | None: - """ - Wrapper from PIL.Image.open that handles errors and converts to RGB. - """ - img = None - try: - img = PIL.Image.open(fp) - except PIL.UnidentifiedImageError: - logger.warn(f"Unidentified image: {str(fp)[:100]}...") - if raise_exception: - raise - except OSError: - logger.warn(f"Could not open image: {str(fp)[:100]}...") - if raise_exception: - raise - else: - # Convert to RGB if necessary - if img.mode != "RGB": - img = img.convert("RGB") - - return img - - -def decode_base64_string(string) -> io.BytesIO: - image_data = re.sub("^data:image/.+;base64,", "", string) - decoded = base64.b64decode(image_data) - buffer = io.BytesIO(decoded) - buffer.seek(0) - return buffer - - -def get_image( - url: str | None = None, - filepath: str | pathlib.Path | None = None, - b64: str | None = None, - raise_exception: bool = True, -) -> PIL.Image.Image | None: - """ - Given a URL, local file path or base64 image, return a PIL image. - """ - - if url: - logger.info(f"Fetching image from URL: {url}") - tempdir = tempfile.TemporaryDirectory(prefix="ami_images") - img_path = get_or_download_file(url, tempdir_prefix=tempdir.name) - return open_image(img_path, raise_exception=raise_exception) - - elif filepath: - logger.info(f"Loading image from local filesystem: {filepath}") - return open_image(filepath, raise_exception=raise_exception) - - elif b64: - logger.info(f"Loading image from base64 string: {b64[:30]}...") - try: - buffer = decode_base64_string(b64) - except binascii.Error as e: - logger.warn(f"Could not decode base64 image: {e}") - if raise_exception: - raise - else: - return None - else: - return open_image(buffer, raise_exception=raise_exception) - - else: - raise Exception("Specify a URL, path or base64 image.") diff --git a/processing_services/bioclip/docker-compose.yml b/processing_services/bioclip/docker-compose.yml deleted file mode 100644 index 90e6897aa..000000000 --- a/processing_services/bioclip/docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -services: - ml_backend_bioclip: - build: - context: . - volumes: - - ./:/app:z - - ./huggingface_cache:/root/.cache/huggingface - - ./pytorch_cache:/root/.cache/torch - ports: - - "2004:2000" - extra_hosts: - - minio:host-gateway - networks: - - antenna_network - # deploy: - # resources: - # reservations: - # devices: - # - driver: nvidia - # count: 1 - # capabilities: [ gpu ] - -networks: - antenna_network: - name: antenna_network diff --git a/processing_services/bioclip/main.py b/processing_services/bioclip/main.py deleted file mode 100644 index 2ed50004d..000000000 --- a/processing_services/bioclip/main.py +++ /dev/null @@ -1,4 +0,0 @@ -if __name__ == "__main__": - import uvicorn - - uvicorn.run("api.api:app", host="0.0.0.0", port=2000, reload=True) diff --git a/processing_services/bioclip/requirements.txt b/processing_services/bioclip/requirements.txt deleted file mode 100644 index 5f0a86dd0..000000000 --- a/processing_services/bioclip/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -fastapi==0.116.0 -uvicorn==0.35.0 -pydantic==2.11.7 -Pillow==11.3.0 -requests==2.32.4 -transformers==4.50.3 -torch==2.6.0 -torchvision==0.21.0 -scipy==1.16.0 -open_clip_torch==2.32.0 -huggingface_hub==0.34.4 -numpy==2.2.6 diff --git a/processing_services/bioclip/scripts/export_logreg_head.py b/processing_services/bioclip/scripts/export_logreg_head.py deleted file mode 100644 index ca3d5713a..000000000 --- a/processing_services/bioclip/scripts/export_logreg_head.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Convert a trained sklearn LogisticRegression probe into the two files this service needs: - - 1. an npz holding the linear head, as W, b and classes - 2. a label map JSON, keyed by class value, with species_name and inat_taxon_id - -Upload both to the Hugging Face repo named by BioCLIP25LogRegClassifier.head_repo_id. -The service downloads them from there with huggingface_hub.hf_hub_download. - -This is the same layout the Newfoundland trap classifier demo publishes, so a head -exported here can be dropped in next to the existing one without a second format. - -The head is fit on L2-normalised BioCLIP image embeddings, so softmax(x @ W.T + b) -reproduces sklearn's multinomial predict_proba exactly. This script verifies that -numerically before writing anything. - -Example: - - python scripts/export_logreg_head.py \ - --classifier probe.joblib \ - --labels species_labels.csv \ - --model-name hf-hub:imageomics/bioclip-2.5-vith14 \ - --out-head logreg_head.npz \ - --out-labels label_map.json - ---labels is a CSV with a `class` column matching the classifier's own class values -(clf.classes_, e.g. iNat taxon ids) and a `label` column with the name Antenna should -show. An optional `inat_taxon_id` column is passed through; it defaults to the class -value, which is already the iNat taxon id in the existing heads. -""" - -import argparse -import csv -import json -from pathlib import Path - -import numpy as np - - -def load_classifier(path: Path): - if path.suffix == ".joblib": - import joblib - - return joblib.load(path) - - import pickle - - with open(path, "rb") as f: - return pickle.load(f) - - -def load_label_rows(path: Path) -> dict[str, dict]: - """Map the classifier's class value (as a string) to its category metadata.""" - if path.suffix == ".json": - rows = json.loads(path.read_text()) - else: - with open(path, newline="") as f: - rows = list(csv.DictReader(f)) - - label_rows = {} - for row in rows: - if "class" not in row or "label" not in row: - raise ValueError(f"Each row of {path} needs at least a 'class' and a 'label' column, got: {row}") - label_rows[str(row["class"])] = row - return label_rows - - -def verify_softmax_equivalence(clf, embed_dim: int, tolerance: float = 1e-6) -> None: - """ - sklearn's one-vs-rest mode produces per-class sigmoids that are then renormalised, - which a single softmax cannot reproduce. Catch that here rather than in production. - """ - rng = np.random.default_rng(0) - probe = rng.normal(size=(8, embed_dim)) - probe = probe / np.linalg.norm(probe, axis=1, keepdims=True) - - sklearn_proba = clf.predict_proba(probe) - - scores = probe @ clf.coef_.T + clf.intercept_ - scores = scores - scores.max(axis=1, keepdims=True) - softmax_proba = np.exp(scores) / np.exp(scores).sum(axis=1, keepdims=True) - - max_difference = float(np.abs(sklearn_proba - softmax_proba).max()) - if max_difference > tolerance: - raise ValueError( - f"softmax(x @ W.T + b) differs from predict_proba by {max_difference:.2e}. " - "The head is probably one-vs-rest, not multinomial. Refit with a multinomial " - "solver (lbfgs or saga) so a single Linear layer can reproduce it." - ) - print(f"Verified softmax equivalence with sklearn (max diff {max_difference:.2e})") - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--classifier", type=Path, required=True, help="Pickled/joblib LogisticRegression") - parser.add_argument("--labels", type=Path, required=True, help="CSV or JSON mapping class value -> label") - parser.add_argument("--model-name", default="hf-hub:imageomics/bioclip-2.5-vith14") - parser.add_argument("--out-head", type=Path, default=Path("logreg_head.npz")) - parser.add_argument("--out-labels", type=Path, default=Path("label_map.json")) - args = parser.parse_args() - - clf = load_classifier(args.classifier) - num_classes, embed_dim = clf.coef_.shape - print(f"Loaded classifier with {num_classes} classes over {embed_dim}-dim embeddings") - - if num_classes != len(clf.classes_): - raise ValueError( - f"Classifier has {len(clf.classes_)} classes but {num_classes} coefficient rows. " - "Binary logistic regression is not supported; this service expects a multinomial head." - ) - - verify_softmax_equivalence(clf, embed_dim) - - label_rows = load_label_rows(args.labels) - missing = [str(c) for c in clf.classes_ if str(c) not in label_rows] - if missing: - raise ValueError(f"{len(missing)} classes have no label, e.g. {missing[:5]}") - - # The label map is keyed by class value, so the service can walk `classes` in head - # row order and look each one up. - label_map = {} - for class_value in clf.classes_: - row = label_rows[str(class_value)] - label_map[str(class_value)] = { - "species_name": row["label"], - "inat_taxon_id": int(row.get("inat_taxon_id") or class_value), - } - - args.out_head.parent.mkdir(parents=True, exist_ok=True) - np.savez( - args.out_head, - W=clf.coef_.astype("float32"), - b=clf.intercept_.astype("float32"), - classes=clf.classes_, - ) - print(f"Wrote head to {args.out_head} (W {clf.coef_.shape}, {num_classes} classes)") - - args.out_labels.parent.mkdir(parents=True, exist_ok=True) - args.out_labels.write_text(json.dumps(label_map, indent=2)) - print(f"Wrote {len(label_map)} labels to {args.out_labels}") - - -if __name__ == "__main__": - main() diff --git a/processing_services/bioclip/scripts/run_on_remote_gpu.sh b/processing_services/bioclip/scripts/run_on_remote_gpu.sh deleted file mode 100755 index 8d6001af3..000000000 --- a/processing_services/bioclip/scripts/run_on_remote_gpu.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# -# Tunnel the bioclip processing service running on a remote GPU host into a local -# Antenna stack. -# -# Usage: ./run_on_remote_gpu.sh [local-port] [remote-port] -# -# Opens two forwards over one SSH connection: -# -L local:2004 -> remote:2000 Antenna calls the service (sync/push mode) -# -R remote:9000 -> local:9000 the service downloads captures from minio -# -R remote:8000 -> local:8000 the ADC worker calls Antenna (async/pull mode) -# -# The reverse forward is required because Antenna signs capture URLs as -# http://minio:9000/... and the host name is covered by the S3 signature, so it -# cannot be rewritten. The remote host needs "127.0.0.1 minio" in /etc/hosts. -# -# Register the service in Antenna as http://host.docker.internal:2004 -# -# The connection is reopened automatically if it drops, which it will over a long -# job or an idle period. Ctrl-C to stop for good. -set -uo pipefail - -SSH_HOST="${1:?usage: $0 [local-port] [remote-port]}" -LOCAL_PORT="${2:-2004}" -REMOTE_PORT="${3:-2000}" -MINIO_PORT=9000 -API_PORT=8000 -RETRY_DELAY=5 - -echo "Antenna -> http://host.docker.internal:${LOCAL_PORT} -> ${SSH_HOST}:${REMOTE_PORT}" -echo "service -> http://minio:${MINIO_PORT} -> local minio-proxy" -echo "worker -> http://127.0.0.1:${API_PORT}/api/v2 -> local Antenna API" -echo "Ctrl-C to close the tunnel." - -trap 'echo; echo "tunnel closed."; exit 0' INT TERM - -while true; do - ssh -N -T \ - -o ExitOnForwardFailure=yes \ - -o ServerAliveInterval=15 \ - -o ServerAliveCountMax=3 \ - -o TCPKeepAlive=yes \ - -L "127.0.0.1:${LOCAL_PORT}:127.0.0.1:${REMOTE_PORT}" \ - -R "127.0.0.1:${MINIO_PORT}:127.0.0.1:${MINIO_PORT}" \ - -R "127.0.0.1:${API_PORT}:127.0.0.1:${API_PORT}" \ - "$SSH_HOST" - echo "$(date '+%H:%M:%S') tunnel dropped, reconnecting in ${RETRY_DELAY}s..." >&2 - sleep "$RETRY_DELAY" -done diff --git a/processing_services/bioclip/weights_test/bioclip25_logreg_categories.json b/processing_services/bioclip/weights_test/bioclip25_logreg_categories.json deleted file mode 100644 index 54548c0cb..000000000 --- a/processing_services/bioclip/weights_test/bioclip25_logreg_categories.json +++ /dev/null @@ -1,62 +0,0 @@ -[ - { - "index": 0, - "label": "Actias luna", - "taxon_rank": "SPECIES", - "source_class": "0" - }, - { - "index": 1, - "label": "Automeris io", - "taxon_rank": "SPECIES", - "source_class": "1" - }, - { - "index": 2, - "label": "Hyalophora cecropia", - "taxon_rank": "SPECIES", - "source_class": "2" - }, - { - "index": 3, - "label": "Malacosoma disstria", - "taxon_rank": "SPECIES", - "source_class": "3" - }, - { - "index": 4, - "label": "Orgyia leucostigma", - "taxon_rank": "SPECIES", - "source_class": "4" - }, - { - "index": 5, - "label": "Lymantria dispar", - "taxon_rank": "SPECIES", - "source_class": "5" - }, - { - "index": 6, - "label": "Spilosoma virginica", - "taxon_rank": "SPECIES", - "source_class": "6" - }, - { - "index": 7, - "label": "Halysidota tessellaris", - "taxon_rank": "SPECIES", - "source_class": "7" - }, - { - "index": 8, - "label": "Datana ministra", - "taxon_rank": "SPECIES", - "source_class": "8" - }, - { - "index": 9, - "label": "Euchaetes egle", - "taxon_rank": "SPECIES", - "source_class": "9" - } -] diff --git a/processing_services/bioclip/weights_test/bioclip25_logreg_head.pt b/processing_services/bioclip/weights_test/bioclip25_logreg_head.pt deleted file mode 100644 index 8abbe4eb3d65753d1b50cfb3d2de6c2a2ddbf56f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43093 zcmbrlc~p+k_cz=;sZb;pNrNV-DC)lUiBJ?0N=S*K(xAzZB+Zkelu`(VGFOJWuf1gm zMF`0pNlGF^=6LS!^LyUE-t}GUdDnZbb?$quYhCA@&)MgkefID<^PMH7#Kh#}#QtB; zXfY|Vl>wovf&$iAT6qVBt_k;BKpt5 z*7!$;iLZ(b4PWJN<{cFo5EP-{9qzX>Dj>+$JJK&CA~f84y`@5!grT8_tf)qW=u4k9 zeqoaTQ$@icDAXr%;s3l;2$S*<7ky(MChZ|9`d_s!Dokdsm|3X zM1(2(i!k_0`u|TK7GZ<^WtkI*TWTsigV;r><&b*suj*!#(6hb^U@@ z`uTeM1_XzxiF)=JBzhGR73{q#$R{GgFCuJ&vx0|$=v{EAuV0XNh)=Lzn7W6a=!O64 z@%~XOZ3BWuqYVuXSQRnh|44ScrJ2?E^#PIo7L&q8dJOwdd!gaMK0yJ|e%}APGv9!a zH4$MNsZnP0on__LYdm=zD3Z#5%KZO2bNXV_|I5sot^QBzk>2b5!XreIa;K;&v!3)H0CpR~x+O%N z|EK(S0NP>}{{_IxO_6?~|5-E+_T&HW`u%^T78DQ}8RYjL3`y{Mqt8s^Y9-2 za39R1en``#o$74Z-7Gfp>`pkJ-HaOohoRibuP~`4+GbVhGS1~k3?I40Sh)T53YaNe z%--04$6NV?UfPZnw4(_B%>Ba6U-%GI_xPiVxDi{E6$3i2H-f%=IJ#)falXFttpL8m!pF4zs%GmpdMdF@b{@sZE3*}`TY%?1B~`*@vFK+Px?lbe<4 zrNd3gzkPkF|nE&G$`nr{(+dDHhti>76?he7n3twLyn6{54qFY=Ap8^VrBkj7@2>5xmZ^-I}L+iL6{ zmItqYrGsm_4V7ls;01?x7HmHoJzNy<*?b|qco@gcQiZNb%#1Pog&EoFbzo}sZxmGPQROLpE<^1X z)VSs_^FtXJ&E}J-Mi~}Vo6}CscD|q`gvJQ3!?9PAEPB;-*pwH9xwSp``(Fbzk9u7@ z!(EdlzwhUL)q>d6B~5(BlW3aix|EYu`^~o-B=DNYZsSR*JKQ0sHX$56%31Auj)y-l zWlneYgY-8II_jp)+|);M?#IKphY`x8d-*PZw9fsya)_iQ7eRUQTj;19O_Q(eg7xREnB=`Ij2q!bMlaX0dXtq@6Pv^fzP{qi zoVP*OzdyXwCpjjUy936VHQ~$~%^=aML*M(C(u!%?5PmWppM6wg>gBcg&G9f?^;9Ht z=^mk4$uQiA55Y7t1O;ygvzcm|Y_*jxB^bU3<*>78|8YEPQcr>BFH^vEj5J9HiqZN* zS0E`SfK^QCgNx3ow!6=dqsAq(s9~xnJ!-4OhI704{sn!o{P_r|XOnQS<8fSUd<71i zFr#~x`S@e;J>in&BbnIvXPDG7oa}#?bMX&5xk-sVc#n9{xSm3P=bhw&uRjI<^3(7$ zu^&$D{Rak8Q{c_=?XbbPO)y)umbJF4(e_$v{C3Qcoz9rRc0Ca1N9>qR)8RFrSUr~A zRzC%w_w9g2r>$80)tR{jZNu}}0^{`7hf zlf4N8-?p*^dpjZXSq3NRT?E!yn<-+A0ZIHg%Mwt&Ra+DR@=zd2APn-+cnTRua*1N!gJT( zWzgYA2OurzCDiv?L*;_iY<)#LUv55y9OneF-&`)P7tUhYPd>t)@u5uNz9T(jT2$Pe z0E&O|@a(v8Z2qzgxY%)#{V8i;dKE%!^p{7i$OPz+oXC#rb>Z}8J8Vk02#Pawsn?B1 z;rzkS)8fRoitCV>v@wNgHt~u4SuRd(33G`0$IV_@54uhR=$bb(yE5KGiTzTgb>!>X+Sw6EJ=(& zscP%-w)R=>HLcmnlEdEf8yu$FI^CYlzQ=qLmdf0LPP=SM z$kJfDHdn(ks}P)et_s2~Ps4?i6sTdY7pI%(ReOuG<#{{$7SEJkkX=J&%M zmu7JF3t?}-k>u~s#J5i?F}`yGds|S2ex2cvI%gTizL3Y#Ti%%6G>tl}ZOPxy7lrc_ zm~w;dk=1kY@aFH)mUQCSjr0v!sWa2NpkoA%Bo$5bG|whs-DDZ!e#Ke z^&-~u`5acxRv>|$2J1O&4@$q&*aF82{-ODOf#tt_TxmoNKdM+c~g=P}gyrV-b;l5S`2=8`)-u=2||Y&2iQp33h74KX#|HQ*t@lTqyV$3k|)(GQ%S zYOraOGuh0-gZ%RF@k}$|4_+OwWLuH83(g-~PL1BT@!hK{IL4X7#qXwraJLL?m{f+| zK3(9aSPh9|18KNU3=Zqj#*Nv2tY5z!29!5-<6@1H*l@WBcHZ+3S5t78zcJz+PX9Q7*GhG0(VJ|1 z*#Mw*E%d;)nhBP-T4SWjIG2Nq%_cB<)tuT>AfzfY1bO?{9n{RW*Ers~ePlg6Q z*5IyNEwEeO1Fy&zv9RM(?BV=hyn63^x_UDUrZqnS8|64wm%kqxW{qS|49BtuR)c8m zULjq5`V2!~9Kzq>Zurg33~mpIahsby@O~)~?CGr-Fkc@*lONPzcFtCqIP)Ej8d@)O zKUG2(|CF$hVa<3=yAJi^7jRF@ck!__1{GiDu<;)?$Xsp*1)Ms`suql=EL@7-FBRZN zKnfHaNwT5S!kOyTdVw%-0&~<~!6dKz!{}LMe6c_thmKI73VSn}vq+JaIF-_D*_AA( ztdB1fmn2o!c&cm_kVV7X+LR|UxIBCs#DwTm-{MH>dl|>)Y*nIbjs;jhs+FIe`W1t$ zZgW=71>}>lmpiaQi#hu9oY*oM_Cp^Xx+RN&SGfZ)Dn(5YBWu2*Nl$Mt7nl2{e@)%GT)efY+YahV0X^Asq`T9$u! z{S7y0vjTOcw&Cpa%lMnwVobg}1|JXGi^({gtIOu`bes0iQ*$OTy zVJ!5?cUV2y7INqQ#jM;!<{fHACaKxDq$Qcw31i{>R$W*Srprb+81UiubMe%u#W45b zBsL}QG>kqx1!mmM$7;bBo96DQ&2ij#2fr6HvF z{((Wgqv?L1CHu477BgSXQ&<`mfzX~Oci$xJ714DG#{!;jKQAg7>cXt>Y{A(n;A zrpJcOzmmZ@9N9@bYc8U+W7r9UcH|TPHl(k z^8DI6Mw(37M@tZ#@dhd+-{3N~44R+*m3vr za38rD_uj)@1APjgC`LykGGX^hJ4h~Di?$~??#sz@utC)k29F(uM>@~ryue0$R`C?~ zyG+ zxZnbw*qFqurDP!f;BQp$o&XjP`-O)Zq%eMWJuG`H3IA>@u!EUS)S;1vY^^tyUww%k zwCQobcMu(1782C93&$+!q2h}rh@wx{$I7wBI(T>@+Rr%%*S=<=Oqu)`SBXh6??Da1y`TC~Zp0e6?y!$f$QwuX4bq&$ z;2YT6a*NyZITC6Py~b|Cu~gx045kkXSlsntY~jYEn64H>$)dCTcKSWOpO)H2RV@+8 z)|$lfJ9vJ+1*>p84qAGi__N>zw&-M|e{&B%{a`+3PD&w*+;oYApe=ic-6C{F`)=|#Lotgt7q8h zGwabTw-p>ePhoX-oAJbGQ||E|X>dz^jGL;nvCHcdDrzY*_t$lF;Z!qDZNGzul^Vdw z$&_|?S~8v7(_FA^6S*7QfXdAseDTp8n6qDW_CELoS#9gs?klRab#nrm_~mhL&C}T& z5MqkS9FWx4Vs8&j#ks5RqeoC|EgfISCMf0#mswxMUB@n9^qhdIP-2Ym6hi5`$J=Jdzl_p zD~uFqR~Jo4H8Ud^nXqgYjoWsj)?gcH5l8k!g2vZOtZ{g3c)M;s~}}wE=#X zs>qKTPG1#x^e8Ra$M37NaWYb@e?bCN1r>sl;WvEpyMm3ongnOvCQ@B`J6;v$;Dg;#Snsow z`5pcWGaO6NXSpkN9>~FujdHa1$!EyfnFQB7E3t1@FIU~D!{m3)fS=!=;}5|^x)F0l z5c68JKsr~mo~t#+2bMrc4vG4cW${9o&5<0cDI(Vipm~?8(c2;JY*sKc+UohxNa4(spCY zHj$&Va~|=xj771?zM%{=2eZ=RTew889j7cj%T<*tvO1OH%&;nlMU5MP6~^P3X8Uda ziPd08an``ImfiSwPC4HZWz23TE#T7+ucl2jhB*6GsH3Mr-%4L%i`ss`_iy0)4|muS zB~KM2oDt(c!m&wOwD{9sNKd>a(7zMMo2XBwv6YH!_piHr-yavI{$L01_duE@p8g^H zdh;gNUmFKjmlo4$D8#H)@t_s{hOa#=#<;tWICJqXEP1+>CN-Xen)M+NmF-N0d-l+x zbEdfMLLZ#&S;#F2DZr>@z;61E#uZDCQdE62gcp3}8`@vt*1SlF$S=ZMMWvM9Rt{gc z7~+AskKuPgDs;44(+yeTFD6|B@d1}w*@As+&YKW+WScrO+GW74Z@r2!X_HaT{WZF8 zJV7ZEVq_q1iG8o)F?Q`);V0*MeD$#m{?SIAuLy9H;VI3gMpt;>;Lslnbo26kzf z$Zo%Q15Z5`C-+Os=w-xensKWR&YkGO^xCN`c>EL4%KFS-NHoH=M;=37Z~>pX_9s8a zb`eJ9&Eq|DQn9woo6ndi&wjkgrCO^Vm{VoSUVS;t{q~K4{Rdyc7G)iplYANuj8Vs{ zAc4aN{DhmnSkdA=`qXxG9EJW&#HO#E+$P6IFv#1KDXNU8wXHG`kYkB$dFlk$vT*|Y z1PLGV;j+$NGP&3x*konNYi2&dm*?U@p>rMjy*dw%{w*P6sRTT$H3K#;O^4;}P1qYb zh#a0(!Ths4^kxpF4Jxt1&3qz^94U`0EOy|Q8cQBr<%L3n!&pD@Xl-xzTexy)Itw-` z<2FtdXDjRy(R0u_s`*(?d+!9%j)<#Zw7wB9ZTN=q!6T^4@+Vh)avQysoek}i2Gh1_ zbGhtOx@zFRdWtx*sjI{CPLie;!De?$+6_5GITe13`4~) z^6@JHmjxBzt>Y&6Bcz^FtCE3Xv+9^nbsualzl}58{h)K`PaNO!92GlWgC!)B+;%af z$D+OXxe~kzM#Dw74hWoC&q>5x2Fv1c6zwvTYG>_WzY?@pp2I^DUH02pB((y)Su$qj$eDJadZv@YMkNK)(v782cDq4h82x# z3KLB0Qzo(R&u}v(kf91kYy4{1R%KrpIH1iQKiSRdq=InKb2nx$+>O1{;yH)(W9-4J zDlpocj&e!GoQ*PLrL`m|YaY!!N3}zfpEENM=tE3r0vuLZ58wKi(*l=ZDEH3-hX<-S z^T;SBcOjE5Epn;l>VM)ix(lBltFSp=MswU0245Fe@h_Z*flBobTokM;kX7gLOKUhr ze<{QZeX6wfUas)1m=v{*O~*U4BMI$eDEePNPPPc^+y=l0P3*7tQ_MqQ|ML8*%I}9rjUcHU`HRvDOiGEaprHoNYB@*HgxGFFyZ*-KjnBwaJkc zHN59vnUx45rJul}7n^AE!(}Y{+W>^WISmb)w=!uHT?$^j63;$Ev^Kg1RmW4IX6JMM z^P~ZsU^aqMHm<^hcKacE{u^kz5C8!p|IFDcoqQ98psKMN>{?W*dHYSid-E6Wr%MT6 zq4AINGqUBZ&<;M#FsD#48}7S*EQRhLN(HkkIms0lIJvV^Nv&UL`sRg$W`-a+gOj*Cve3%~bg}-flmG?cYMrRra(Ki%I0&)ybsbB@{SBiqp(5Vf5tHe&{sL2b<$__>?1m@$SFbG~)bD?q#A8m^IqbzHtxv z^EtzqeBf|0cdP{6i@TY%!w#l7XFTeOe7W3Pcksh|Z|KXlVPocZ;;Xpv=veUsqQ+X% zHXmg&vde~;-Uax{MFn3*X0k`NQk3bv034fhh&h$Q38mp|X{H301X{!VE7M`4bu|Vx zPbdA4s+7l@aI@zPf<6AaRO~XJyPG!Pi`f#{Wq#w><`^RzYN<9_)pd3}U$YZ1+SL`FiSc&%f$jNNhl()8SF~02np;DB0Z{QXR74hL| zP4-1}E-Pv7K;`CE{BC;@j*J1!>oTA_{qKc$l2=md&tY`n;1G(5TnU@Bcff`%bx;pJ z{EQpAC^#U?yS`b*DBut8EAqvM8l~{Lsc~SD>qRynRoL#iQ`odY1>m^PfL&bm7ZTNS zQ8|sLTl!z1eZ^b8q*#vmbY`&4haaKa@-m!V^9Qg6wAqpC+gatbFs3~z z4XUo?GqE;Va=tEy?;F;mv{40~xsZ+5hCD`3zhe60d<2dBifxY{zXK90PSom4?|})+w~W zd&5DH-EoUIDG?Kx&&p=YOU^RsYLV`2XyX(k&%w5SIY^Bhi(b}|n5d^k`A0s$5t)me z<(q5@ifctN%Lg#){3^J0daS@Ny$Lg9o$!k0N4QBz+{z6~@Mha#>Z%ckT6=fEf;5s& zsNgm^uLYZ}RrqpoHQyrCC$V4KY46wP{JxQWs8e;2YDddapy_HFeefaoOfEp6erFY< z21C>}3}l0oa``nUQ?an@Dfjl2Ib?^VV~>c(^~iDuMM|pDu}78o%S4iGEYZW$w*pac z;}awqP9dl9(X_+00PjC4V*QK6NI6S_h2363zmEL@&4ZFObbAKo*?BO_MWu8(_5~{4 zH~~XL>udMVK7tmVDHz+@hEsnZ#oJ~N!K!y6NEAu2XP=LttzI%4?VJYAuC8!+unAo- zH>B@Liso4@OAHG{@(53c(+)L*4~{+mbbfXJAPc_H=fF5gOp{s>8+7)+c|Eez}1$>}DIr@0}iTtZ8XC~g6TjCE*uXf#u5V({hbQErdxI@q{Kz#3CmQC^xYb>TSnA!RjL zzPDkTdU0f>aYrz=XHu=%>CLQe7?0;;<7?HAbi$gfhoB{`MSA0ovA?_gnfX~CI&{sC z@g08!@!swvJNP~hHG2dhTDDyKjze&K$!ySBrVcwoBFSn^rifP-!rV76Wzs$qN#UIY zyWi4+a@K>H!4OT9ODqA)s29AMUlzt}^`l~SIj-%=evp+eg-uM03vdnLLR0U6*s5dv z&ztTPStP^E=H7-qtNx%vR|0pfR+mCks-WfaNDzLV%z0%0#=&9V;9~DPRD7t$Tv~rJ zK6EB7T(8bvZ+neKh8kGXSb%e1UPPbdLR=SHg!iw$#)rKcOv?26iNw;!1!YbtCqbAdD4f`!R0bsR(T*4CRpJ) z?@1K7Dg*GsQEtP)9Tc08%PXGU1n0{V*|n8>P)JprNZK^V`3LbF;`~ z{1s_DGx-zuKuI3v)-}N6zauDQe>s2i@HD)7J_QdRZR6}F?Sm`7qba_(l~>-}gA-EJ zD0an3+t*)`7~YCz#g(`Cx!+5`Vr2*BUQ%U!5CvXo$>i`&h5D~m5VBY2doh1qF?1gc5!r_0S^L=>&@-Hj5BSM= z)Vq;C=r#tbkk@5i4rufe$mvnk}gAv{=?AlUj_oo(2sOk0%p z(voG#`22|#{q(qoQ*$I(o0$NQ>ZC!&tm(W`ZZWp&-4VQ<;>)%hbEK;(#*%8b^XDu5 zP$FUoboDy3x??R!;Gerhzi!jID2MZ>Rr)n4>n76nOU+2tX4iZhg z)5>7FWM&JtXL@1gXeZRk@8YKX9z{=G-lNcr-6I*4IY#VBj;U39x7 z&hCVN!`sqB$>XxL?dHu>$ev$>Qp=~)!ki<_N%k==j9AIEzb>SuJM1xGj5gWqN@dn* z2{7jBcF-|Xg_-R+;B$Hu3p~-x*?v|bJ7&j9M7bax&s=QCwV@}oIkxWk5IP|iK>g#Q zuzmXiI^QAzy1B3U&9^7f&fNpt?_yVaQya%rd$Y++YZBG$Jp|6*^XTTSm;7B(zLi>4 zz@%as?d{#kHVqYWmJKtQ#`8lsqqh4%unPxTILd@NFJ@Q!|95CSMX-Z;)c`mFd*6#TJxvCo&ro zNs7C#g*`*f$ibu(Qw$eVdxkyT3!aYOALo)ah{pIeoR(K6uzkm6NoDaK&PsDPZaK~i z-OLME(Y;gf>{=eAEZKAEN`OU zl2lyW`~U_zv&jF;7)p07M2B_R{3P>OHf5rKmFho4C-bFLGw&83Vl!TJZcoO6MYlCoLM-LwKiO!x{f~FZQ;VY7c63{GSBjNwCA(N$>sdff&*;Y*AhH_HHXD@ zPvtz<@fh8+54GRjgPN5_WD%swz7#ow_VZ%!^1lM7D{`=+Z$I32{fWwU$-Hfx8X8g{ zWPdy^viZ({X-*x4eo-VlQEc>MQyKTv+n9X`9|w*;jx6+3Jvzm{<|HYf{lz4v_?9jT9_zS z|LZEZUAju3w{0;q9qR&7Du)HbZS+7lbSfT`ZlIO(Rrwp4q;+81YV`|EsYEf~WZA8%ue zrcY*lH?~o=@)@o)eU z%U7ZKvn}ZJ!S%w0@#!#zbLZo3eCHM!uA}iA#i=9Af<Dx@P_Qyb+!fm(h?@%x@vq5m;Y<~?jEw_s*6DUQs*(Qp>*Bx`vY zS6PU_O(@@)N}ok_uiq+xb4ezwL?IshcebFa-VSKmYs>%f-^YtLsv*8;13MirmgI2( zMsCX^t5xN^#NPc})3Ru$HTo*LjwqqW%L7Q$J(+gu-p4UtNAtf!!D>Jo0|vAxH1GRt{I_ad??v-3)z12xu6Zcp3vm>E?Njv=cfx=#|ClVvnN91g%{{Hzm_?- zM6*M!*RW|wA@|~U1Dr^iN)}hW$=2Xl0% zqv!8J)?p`yy1I&}mE+FhvSxDcJYCqAYh`?zFaXwwwc!AFlHG5f$*<^_gf|!WvTa3s z_+trGXudp=HbgH4*T-@&f6p)oIH*pZ8{R%i z4Px4r7*{35zu0St6Q{OdL#!36*v_L%mB^3ET}#U~hO*JuTkzahO_86I0MjZS!k1zT z`m6d2H@(YXj@{>?G7Irm#DzBqB8lK;T@9GLo4 zo+*6wWc9JPFf&vE_FNu;f3K9l-{G?CY||ZHp;evrJksD#iR;peIeDD+l$|U-*cac7 z72RjJv|Pk*(qX2hL+C)uDoFf@FkrluX0H*z*uke!A^Sc*Vv{_Kf0vAj(R=9K%urgT z_zu=fPiH@w4R!>$uu<8|FjRaZtdY)vvJIa+yC>N2#0C@Q z)K(?-%d=>g{xuH0K5^AS zEkeV~Dr{eyH0>Pmo~wFSB~Y300w!j!V9OyL=Zo&2)W>WCl|eNmbIyu%-|WNpTIrCp z{y6G8CBT4!Cat;V#5!AL+2@LLu&U02=C*f3eZL%S?^I3=2DH5QMYdm6zo2R z(JK_NNGSvCK8)f+#2Zn5x-n_1Jw&zQ*|catj9R|df_bb4e$|#j=V308I?J7O1sOR7&kqyaT-e=l zdf!^ab$uNOsB1 zo<+oOK+_kqnfY4@d>~>yAMdZA_|nr*-G2+0CoY3(lRPL8#a)JPF=J+W7PP^{i0xEM zq8hV*HmxTY(#?5KVNInrt+^it8z&8i?_-=u`O|dTQazU@7EWe<_948y)*SXLWEo62 z){8>jiR|FDTo@Ltf%mmUHoamDd)I6W$3pi(!u}1Y(DR$e<~dx>rhd5m>nWt%God-J zm00Vc*`O?`N3NR_`KB&Sc6)jR`5m5)4ue**K-CrW;c@5Q7-hr-fP^32GylFRBU1Yu7GXuLLMM=i&q@!lM6!dyN0dWXlAB70#(<`ic7 z{Uxk!(q<*k8@OsuH8v&OisDShGednzn)*9~x~5h#X}4u8_NzX9T_nn#w^nl3o_dMy z!5C4d^bn@5Q3}DH%D7}#7yf!t%1V!jbnC1j{HZ;Oj^tTVd6OG;s%-|=AH&XM*}-7# zK{V|9YhDojmG_*w1RGthqqK`5OCG(L&aZQ0zMcp8@e>!)#bC3kfP)h+i<0~A|>6Q%YaFd z{uNOiWb0aD!xX6b$$30FOpPy%_hMqr>sW#8N`e2m5=dCB!4`bh2cr|G;i=9)?n!tF zNq!##y8>(RkyH;Dgzshh!Zx5!y($BVDm*F50rXY9<&OTcp-lok_!4P_eKLPw)H5SU z&edSUpY-FnkQ%UQyvne80~qWWN7f5Nva3+q#eNZnb~lqYaS-vyv-kpwSrwTJL%g& zYwp75A#84;B>YK{h7G1_bouRf{>0lC@X&S_X}8G&>Wu>5-^R?`+Ke{oi1OS!HnF$A zHAzO-nPvB?(uk09I50RHj(ca|vxVc~L8Usmx?ICaxh`B=j01fO@Wqscr7&qg6$ZyV z<>Wm~Sysy=Hujw>2Bp6MjWKrgXTJ^GK6?|kybGbQH^nGzmB!k4xzhgh4Vbs$2o6~8 zWzL#w=zR4;GL7Ae^MsFZ@X$)UB~^ho&mHNJvkkc9sxkk6v!JFr84ITQ!Q|KLS#7@~ zGtcv7-RH)z4C6qiWjBg-WEb*VBpWfosTHy&7m-wT6`MCy7h;R1QKa8B=szX~<~iTF zWamZP(XO4fuVy^u&#aE)PG%=l`KFVwfXZ3!=e791C^`>+EZ;AVD?1YGvt;utr zBZQ)DR)n-jX(&obiO5cUHH?H%R3zlN&lOE2Dh(Pc4W*%_LF;$_{sOOOxS#7fpZDi| z*3aQZTO}s59B*Ifo0Q16|2&8Wc!En>5sERnmiSFW4R1@tf&HX+;BKCQUCEb0yRMd$ z7R{x?zj|mgJ&@Tv7IK302VCOdU;Lv`M|MUsfhJ#Tpjmb=srp%>`1jl|Abl*JJV=gd z2Q8*klHIWRj?i<_MI7YaLj6j^@W0e>SR&B}3xeYCnRY5=q$}cE;mq9PI}5jpK9lv6 zUh-=^MY4&3bn>4DojKP_+qLgN(koYZS-1nD8f4kA@D|?U>QksJZKGYkB(YHOAAjaT zE$rQVfc;>Pp=j7lHl}B%c#e+)Ho8UOf@kmWM&U=gS*XTLjpgm!5~j0}=O>_E)IZwg z><0d>Pswo6A1WWE4FhVAa1{nR?Bvy3ut#SzI5s^Jr|b&CDRvB#zAEC6E(NHa<-k=> zd<9vK!65$J+OR#5oJwt z$Dw2X@XO3bv6A!pto@^_Fqyy8`!S#wt8)@Rs>(p%c;$LtJGsRzr!iB<+%LCJ9recitX+>OJ0*c!L@=R?CWT0e3iCQ zJod{Tbp7Qd{=8x&JEyP9-_;$$M%dh?$yW&+Hw$dXq5*VUMF~zWIzu{DDI|YoEynxB zvT-Smu%Z1fIexHVbJ9A6Ipzkj+sMV76sGY51Gn&MZ(SgK=T1&7Y&$=uEJ_@6XBDef zn~e?C5kRkwi!RBiz>o+T>`*Uc_73g*i-)hNEXx@?BxA66$xx7NUI#x0?q*T9J?PuU zbapdGOh<;4Qul3XywtN5zP%pECMU|^y1f=C9W5vBd|yMeWcP?}w65l!E}6>xQYZ$u z33K=()t#`?Bc0MF=5t^F1XI%cHMlg^0t@%{LuStraas0mo9icZ!0nAQlO8vY9Y|~9 zn;m`W%hxM3y6y-c@$mwcE*(o9rXw*T=of7~`H{1Ww4)ARfr~n{A1*$%fjhVV@rxec z1&YzZdvh!izsIm850dHAGzk`H(g((`-_g&ztNDL-Y;m!gD_OZ5fz59ls6ShS8YVk) zDfh!@m4h$4JWq@Dmi2M_+@JFHcKNtEodeB#TzG^TYeHTN9pMAk;LA=ndZsIij9r*1gFUWZ24bEfy z8pz1#pv(>9g^bvQ-Tx^DL)yUx6*O~60Sz>5aU0y9HW6|Yl6jG$2j6&jDz6(qj)m`- zPeIXFC^IFF<-7O56~XFRg|v5uFrQS3!5$OqgP~H!WW-@X*H%;Y-~ucwFy;QDwTQ zeZre6mOiD6n8a&t6S{EyY8Xp;8asrU zm&6#lZTblw!ZkP<{)_%L`*UVvIl6OrF&GBs!1w12I(;VbPR^HTn^hkCY|?-MyAM*~ z>vHthssR1)<-C9ESZ>>q7u=D&p;Q#4fm#)bY)H5nG!zZRw~lX!n_mK^8{Bc!m#?C! z0b{VY(pO~PHUXZDT}f;Bv;4`*JUE(h+;+8L5xlaQ#}^srko$>78e7=Li<5#7|9G%M zm%aR2&9nT|Ic8j5Vm^AwS!2ubgY53a43zL3k2R^e@Lv5qH0Qpd_LFtcptl)f=uYORV~gwEyI_dX3!X2%FM65wkbmNFm>e`zkZ<0A{htt- zco|iO#fq}j_OU7c_1voca;$e$2ELHjf?B&T;O%OK4_pNRZec$p`u(DO|8sm_ge-d- ze1_k!J`^%uU*Igi)yH5_o+2FutZY`u0y^5$XSQUqN1+x_; ztGI=OELnJZjL^kaV3rOE@NT~Z20Txrom1SQO|KjB!`pe$*)Euxnoa?iyQwboDaQ}~ z406eetmJScE(+d7Rm%qAedQAF&gyKe2(w|LCv{A+pcJga)!EvY3otKo3(hpxLF-9( zY2!fyIvVmF7HQ0fd*SOiWgA8O;-$?_+Ue4upN(+$b*8BG=tYR^Ea$ov&cNw&{V;rT z7N4>DBs7j2hy9D1>6mf=os@Nh#6UkbZe0N7yTplVwGZ(pP1Z40IxU_ zdytAV68Nmr$83mu8Mm;!fS2BR5;Uf~5$&+mW0Ik+n4R28YcdBjt6?^*FLEYMlQP1W zV{Gx#J9#j5y~&l?88f|^vh2V$chIC9(i5G>XGsIlHGB$t>pzeiDr5y0wtuFkLArE6 z{2!|b6ga4g0c_UmdTMhs;6yRE!0w9*K6|C4htnF`ONW4U>aB+OotNukA_)V6IW;|1k>dc-+CYGnznK9tY)Ep=x5*68A~ z8ZG8}=>q;fH4=SGglECi9zMsRRk(|klJa;;3^6Ly%>#o`!t?o}tbrGBPwfwNHO z-6~pleFC+N4F$uq9LrvHQMC7UFjhbLLTanJxwi8u7(RRqhID_TO&?}5`=5opeo7lX zU+M^-4y>iGlBY3H?E$y1X%F-*RNxzW>*1Qv8FlC$!ta05$RclKijofl*3BXDs<+*o z>$;(MNui|f&z}S2AjNaXZe-)}KcOgj^e8m*(GX~}iSowoVXXnl{Hikn6ssG;I@4;Q z@V+n8ixE2aK05fY>j2C)6y}bJ=6FO|mwa97L2a7>=)1Miz{r7QIn140zkBeHc8^B= z*H^*%=V%NxXyUS_)I!49Y_R*@52lxHQ_J}{)cTqS*87k1Q{0m2PD3^YOg+Me-gpmZ zW@%Er@pYPWYBzp4>cg%b3c`VN9WZISH=F*qk9va2`SpW5Ao;5BjG4HO{a)10J~*C-Md9W~^9k8H;SGgL1Xiq&&9_(|x1pYn=t7V6Mu z_!p7&P`ff5t$QbO?PdqacAz4=rBcpy1`Cd*mVPK7D8f+hMGckclNH4VYkHIi)gmkeH7WI&u_EDLz6!zVC*nyJPYD;lp&W@37cQ=85>o z^1s|3w*<(w$R}E7&H}u(*qcay{50MYrrP*2eU+=A{Zs>shHR$kCXe`CLRWZfV+B>o zOR})djr{&M!r7>MF(p>V;RTI;>WO*@lFv&Z-aVQ5XzRi;J1GkLW=I;RQ#k!(W#$#1 z$Sw~^r)Sz(U^gaCeC&nbKRCG(E>G&IGuot0ZLRGzd}SVcTPee=2UPG+&t}1iGZR^t z)-fp29fJM15*=>@V~}Ypl>9biCp*(f(Qh9q>*!$2)MAQTrAh6=UA|@gAxKm@4wsFt z@Y~io!M!u)SRi30^o5T-kfwB!p^+(j7j=yS^5zIWo8fG&(_R=^If1oV z=%b|WR+9hnn`|wYvM7rHyf^tIdR5=!>Z&8?^(%ik8*G6k#kXm$w-?QRr^2?SD6knG z>J+P`iY3wJ?5gW7-2OQmy1a$##@dj*^>${_l0#s(<`$M!Cc{21(?he8?R5N}4i-e0 zL(9ZjELXb@io3jtc{dOwB9K3wLmNHCx z!V^;c?T94l2kI6LBB$6(qBZ?9Xzh`0Fq;aYd-()5b$1c7FF22Rc9&>i=s4bU+G}!q zJPIG#I#K*Pj^)M;;D1bzzz@uhO3dY9P`o2oByozXkS`O>U$qB3%-mS&>+Mu5^^fbg zxst`mIIz>5+E{p~hI@cRF}r047=%t^?srFs58jNT>gLP*^d=uDv=Urd6aXa|@A+Q_ zH=wK4nGjCE8BQKwhNkc=wL#n>>5Jyi}LlO?U{noVK_GI2&`F_Nxi-<5T<#G z-8Emzj?Z&M|Bxu!y?Y-86)OlVX*3M{e26{0Fj%;Qq=3_CdAJl+NpDrHITv4l?t?>N z-I%op$>gjF4p_gG)>z!Ou`Ni)dmc6{t8FNkVLbrmuC%5jLN92#whih1v|{DsQ(;qx zIm8Niy|_pnoANW^#vOZB_hJvTDb1p@HlIY$DK$Y()fw(md;#Ie!~8PAVNlyTlr8pG zKyQJ6u({(1*Kg-?T@Jn!M2QvPWg85eya-&o{M4G1sRsSxIWdn`$PO7ABoaf+k7 zSj4sx7Vo%x^88|ZF#WU-Of9a{{$;EA_NW{D++pvi)c*sQX)qEmjG2M5X4BcT#}T;k zO%MyY5yuby`;B5po8qypRs3N4*~~`MjV-X$B)^$@`0Vr|7ILy5)`5_1TBy>K`2}Pq zWQ=<&UAQraSMtMOh*+8Q1Ul>Q5Bh%(vaEL};K9T?QDVv%w6LwDd14d1=)E0g69cjv2AocbP9j4UJVsS?Pn#$wVa<(E7v_o6*c1=c%P6pkWwa3`yWrnLydx;Qoaoix%hy0 zk7Tin28_-uM$Mj5u-hhiU3sKy|M(>t|XIhKN?3N0@S>S4BCYhs& z=O}jJ=6ssC^c6(}3ioW8kzCyc9cJWr6Rb=Xpu<(@7z=kkhwH~k^h_VdhGlb?m51W9 zsYdL<)k0`WdrHl5KPge^8vi$Y8>gFW4@b{=lE!CuY%p7cQ#Snvvdj12w9}?wP&pYj zh6uc_1i>^7dFHcX1{?U*lbv}b%%w}8vdsQnXe9i;>M7KK=Wt2(kjyD~pE0wUO#IuQ zX7Jf>1beeFf}w zg&Q&JymOqs&ULQgTCeS2BjA;TD)^j=cKSIki>2-q7bKfHKhAH=^tL241Ir2X2I zlmGsSTYV;mTP8IQ$689^ej$q~*mZy;^EKFk5h1jpw?-5=e=ttXJHqLG(MI_#&$$!5 z6F6hXli>8FoC?+AVU~smehKR2=A6_;o2swGd#`6!VRIn)%YDds_M6U3SxB{q^I2?M zG~1%R3R9nW(f*H@MW@tevbJAB#+%{`9~KT{bEoK_Qn3oYuk4`}ztq{B-d4)?y+^k- zlvwF$4|rKD&!!peXE938cnPOs60DzKeC;u(TUJ01)rVl9qL7C>3y!4la+?ODonq@9 zbEsqaR+9GgVs=y4V84eMUe8UVq^DW@q?6ld{ly@#sgb6h+x7gc(21CNOaqleeW34F zAg=qn58ns7;Mid!SigTJ{m8$_o%pev4kz4(YcsyV%y-ivY}88x4+<-|( zF2KluKCX7@Q~qVA8$W2>ZirfUlXtD#B|f0{9{T1buw#u?(A5d+sBKl%Pov`Ktv2(tX%Di7@(Ow2vwe)>3Tl z7ix4S$i2S}5|77{j9CIx{u#p}dxoIH_GPHwo(Of8Q?Z-hM^+i>Y{t}hx-eOW3toMd zTNp5sE)A1|ia*;#|0y++fx8y|`>2P147Kp|t1aZSIF#?dtVee~yoS(+jufsge6Pko zg#{LK&}%0!zps0_Gn%?M%jOdp%+dto3DfY&JtL-8JeUSJ*@MFS`B2(607uM~qY*!j z!x(3ICR_5IB7FAK(w!UF4YgdDKGO;7j4aq+GodG{B8Ml1*Ui&!I=*EYdcIu5ma2I$ zrPCAeuf|?J%w;O`$=-#LMe=NAz9ZAfn8am?q}kuh?+|fN3Y+=fx@|^r_)XPV)N1zv zblOMK%I#O^=Pm35G zxJH3B(bm^w_AfdhdP^)coNnQ~lgeQF<(+iQ`626RR}@&(>-6u(Q4Eq=#Fm={vZq;t z;E<&j{HZMg_t8=0C*eUa_pip=+neb0dU;;8tOZ6tosYc%Kk3!@H?TcO=zN_wM91l4 z_^Vq02j-7N6N7(rJKLHmj~zw_XbG#?H5Aoc5d6$-ahYa4t>&6p`Q2$yIQs~UOK*Ud zz45eP`yZ^bp2DK~{z1INPOj~uFb7bPrE61WkkOKHX#7eUV~uoJ8quDyV=!q!6OHnlh$lV@Gb`76(mmnNhi)}sXGFH*tX>`VV6q$gD82+U zO4@A4q{Tv4@jj?u62>f_-G!O@0W9~jHEesNjHfRhhL&`Wsz?>5DsI853m2ir{Q-ES zda}=$#Ikes*`AhEOcQkUST;CgNv*aT~XU70OSMPub- zFk#yucE+e5lpOOh;_yP6Ju!lHsFcD0-Qn1nv{^J%?mzr2`vms+-G_RUOuDo*jLLf=&@)%J71>7xz$f+D|9MJA|#tv z4By3`=d0o+6KT|1`k9`Ws?oOqcQ~&63NE%8!w;VgY(|0v3-(MQ^;ZTk;YJjU+~&yS zmhNQZOT1b9S|gU2r-ZKtv~hXuxh$@F6228Oy9Q+)P*%Go^eTxTem)k**9)!zjYY85 zP>fs0^g&qUX%K%?!`aynESr%z;Ka3~e z-hgnCA?uEK#82`tX4jgIL)^@@(0s`lmZxgLvYelEZ1`@>yzmdMAKS$ex3^Q?TH(E` zy$4Fd#55%532c>OQH7X!D1WHD%K!71?ODQ zg)@qySY*4ZFdJ>+#zd^+mL+N61*!8~iF*u_8|I9@$rs`I0X;6~D-W@Yp3OUpeHTWWL&|=0OrYl2Hvj1RXVg(7B@qmUT@< zm+PYiuXi?VI9AK2W#^z}2cTS-Dht{>l*X+cPnXT)s2kpbPSIsrd*Bwx2=kZCHw^(>X&w?wq*^hH!K>1XD zWzHgIHERmf-_Z+h?N01YYc!^>U4(O8r&DiH8PlD{bA}NW6t&k9o1VYqCG1~NdDsEi z*c}W(H)F9Y&>E|&=HeCmpXA=N5^S@65!bnet-o}KU-H#%8FnIJJ_4Gi;{&z3p}&()PHxB07~_Qx>uGOM3fH6d5VFF%DWGNq`2T&xpDkIA?{!pI!c;G)STUIMb@1h6 zUdnJDC4&EMdLHccY2l;V`)T5?{VXnZ88esDf)CD3R36hwX0eyJdt)!bAuU-}cI+

tDds3_n|57s>lGzr;@bml@*%5e#vR7Ea8{-6~cqS+w|{66Fm(zr5{0u+2@

s+MvxTOif1h@L+sf9mB-V2l?id zRcPBX9+&*5!n{NY7#{SMTAmfKXU*2^VUxgLE^8o9#f`jlUMJtxX3anD34o0z@^pA^ zAhtfzV7B}AgWuJqWTSD96X8StTE@e=<+8^iJ9h(Tb*Kc&w1%=MaTNPJ+k-yHXQ6#! zDSfW6pxHa;gPY1xrYZOUBR?S4mTtlKGfRL)M!^QXe(JWgW})|Gnc*#hze@6OH9^dq zms;bBg!7E82A+~eDAsMa=#UDgrq`9%c~dG;~9mL4sfhyRLmy5zB9xj8O~ z(jn8yJA`a`F{eLf108x`PZxBox%S0d@Q3LV$S7jW#C0;;(OO4YjisP?M4OH;wHGpJ z3)=H4JTzbfcX>QVF>iQF4geZ&#OG#ttB}Kk0B8^F#D&w`( z)zH$fMbc~=y*1~Ub7nc!{0U`>`)1LBc1hGt$ReW$A7IG6lN6OW7Wkn@`A-kUqOVVg zcKJ%;X{Ttm`}{!Wn|%{ZJDk{1#ahz(Zy&ANB<8$U48tvgb8J_tHg~W7hsb@@0qB0^ z#CmRihB3>&LUwXEWjxIloJcdo0}a=cdgW;t{qQCHmc9e#$)CB`Yn3>&{YCJ7+g11$ z-3pfiCX@4oEjXqd-?m^ zA4ps_jg4^FjC1V;*SC5MbTmDr_dmyqa=uK(Y$ttwX+;eeAra1wpH>8?#n~`@Oc7M; z6~c+{L8!SUlZD)l6Mf0p0WS_@(6(3MWOl@q76z!Yxbzb7`!q*T_87@z)E)Wh%Gbfs zDp6!K{|!KvE50^Z1WwikjI*?7*R+P>iu_mj=uQh4H)0suwoRQWtXnVcYmUS>?xmc2 zL=z{YpJw|^V>W(o41tYjWoWU6t@za0uXJL!4hwlxkA?~-XwI^+IOEqd?%Re+Zcb4K zOgvTtSzdb~clt=wx<3_G$D}~P*E(>x_?UVYIT zPV*ZQNOEf)ck#6*s>se}4vBXlSvQqLJ@TY;#ezkq4g>8C=Cok>8D7Kj0F3;#9qo)f z1?E&0i;q8}Cudf$qMu$kF$ehj^HyP;hv4+v)IiU7UZSkGmmsPwi>e200<0g6Dl>;r zNP#o0SMK6M<6PON%!lwI_9U1c`U+nMKc|-ZO`O`7y~I!72vyEs;E2gqwoS{6t{yL= z9ox5&wjNTj-)Bl!e#JSm^{i#4FAl3U#uNiR7Lk_?!%7oq*~yhmrv3zeJvxqUy;?2S zI60k}8+MAEx%I5nsFjMg__6lccWI)GB(@5Ux~eRW$@}i7t(LPuX3RLeSO1EC`+Gd= zT4=~xT@G^-7x>dw$8=cn!)G=lkUS^^=f?VUWDbb=eaQ>XVcIYo}|O(V{EUGS8~qWfs%R{_+F2VIRA*8j@9Ci zf|>YlUnu)5Ig{pAdC;A0C&543mbC`QvKf!mQTl2JEnjm67MIN>Kluvqy4nGW{{**` zMKwIPHe>Wem8DgGg}(N8ppjc6`mH!0<<+a8<|e_SAZNx6PljhIGgwr*9jVL6;rCZA zRB3XV^D!F1tkip9{uD2~78-{yS8QTA+8bHl@)69$(2C6sz0K$6AaCuPg8aJ}v4p;%l3`=46sZ;c4;z*_ux0~f<3JkogCr!~$Mc=03BwhTT zen%UFe@Hb*UGia5kG_M@m>49#`K(d(Al^4IVq|cQe9I;Y`JMvm7`M`j4{}U*ST_ew zlGrT%#hsmTk)si4U)#G%EQ4H@6prVrTi7;N;n;D%F1RNviT0yIq-pN&dFvl#bTeR+J z8ijZ7W+iDq_@|n$;V54za?0!hv*W5ECg-=c-o0pFg&^%V z1?=rIFkD)S220yuXk8inQ-2O~A4IZu+2e84AAx_dxFS*?smS*JxWn7ZZQ^eioDrwb z9K;?34QEAbHsa(a34A2I8*7S2(EV*u?E2{cxYu3j6noi7U~J=2w#grIi;|dKK@6-Z z$OAX!X7P(`ceHK~sH-WO%@jPxu#G}ilQ3a4Gp)`>tp+&^8WF-S?M-GeKh-gM-UzPN zOOg{UHN#;`K5=Cq<7w9Rd{Q3yjEqh?v1NV81|6Hk-`F;p8Ll6W-^~2jPK$h)uC9+_ zgCz1Pj)#eHM@j3v(CL|70}K3I87OKaGZ>7wj6Rc9yc8bGI>4K2?c_fHdgP@rSRhXDb7R*SN-;jeM!eMujA?xfcC!XkjWu*sWszM&%rDR4!5} z7P59dHKax-l%5$yIo$`Kxl;)ptdNB+JHV|RyBrIYq+z&(1Y0ThT=*PDvz+%+xH0pM zz*S-wg+6kC?hn)9#LG&&we%XD37iB8l7-wgh2L;ZYbMg1YA8+|#PW^L;|Pm~)Uqjn z_{YI)NVWysx_FcK`Sp+A_F)LaFDcL>IhVCfYM>wAqQwS21b1F)5execG7EBz zj^9s%}fXBd2;K9BlOw3Hw9!Fcmgi= zC4tnmtN6jiPc+SL2Smiku*8i5$D?2iJIyb`gF`Eshw@H1A@P{5?eW2uu;q--nZfUq zUj;w5D=Tr)XAczSutx@NFy~+e#YOkh5@&(WbMnDI8%D8OA>WWRX5uBfE9maT9rQ9I z4XjrWVQuH_Q0dSZc2_b{9QC7%o~F;iM~mF}BT^gi+x{!yW~joZ9Qwld?cKnnMl@2a zZ#^j$yy3ToFepz(R7rmgtA%Y7la79eTSslsD(@M6>XxR0jS2MpbT;SxB$TC%_hS4V zFKD+p$Nx896)QVs;752H6>j`QwE~+ke924_3axC9_hN7r4Pv?iul%a*FLlUIrhW}Z zl|#e0M^_!$g-PYu%Aer}jh5l7jhwJ!-C^VteBE{+g9-zR9)QDX)|AR( z1b*UO=6>W?shF|H??!MdUuKYZVtVl^?U~Hm`A&YL}!lMan{3=Et-^k$-iPbPUqz_)^TcergIWoN@ z&rFSC=uE+Je$;>wP&q9YVYo3)N?44^Bhx{~H46i+-@>QQTflYsM6$nN4rSN0nQFa1 zJJ{gD8I=jo_aFXj{PhWNXhREZ3|)&~b6UagPc|uPAp6gu7%untU=R0^KYUJ~k7?52 zYFAp4w7>@hD7~PEt5=HZdTKdO8pO)D zrPs0MHTK*tpWD>Dz!xSgj3s>VTU4XBfVG?MhTg3c*~%3^NaE=z`Yi1K_$spo77R<~ z&AQt8?CJBlYEOHp?K~ycFH6ERzf-aC))^4>jKxiP)2OL`4{O_yD`WsCguBQx?(Lkr zkn3&0S@M@b$+H&}tQ)vub#vIaDhQv;RKwWpHo8CSE=esu11?1qV2!Cf+r1-{X~2JlP?|5Vl9?+&Szlf=QOepqu^^ zu1ubW#WPGIb+)|S7wviPfeGUn5N}a(T6LJ&{5D% zN$X^p1O5=NRteyS-OmC2_EOs8vKwu=>pW&3q;r}FaCBNESaAe?X6|gCrvb6%b98^k zEpE$+9$SB5C!La3E)2OMcmPjdhl_iKZuI?yXn*MqNY|g>sw-E3#Ix(<-Q7Z6^G30> zYu1pr`R%bhmBB zfYGE@(*sK*_JL-qun+KeFv7A_Hu{s`fa!QbQ4+)1b8Z6$>Y1bFV?WjsUC#SR?Z@yv zmuS2<$D;Ed(0fZamXq|6bH932{4LZLOoOs$!2vJmu}cMwqH0iiG=bE;c#t-9#TQ?d znDvzwE;f50#+rG+_w9wG`}r4*zimcP8UaK4j*x$O7&;lehEK0dnL}m~J?=dPzao}U zp;8+a%k6`*W+OVNGZ4>o-=}xE_Q+;b&D{ec&h`Dd*saGDwW%~$w=AMhL zUs1zl-N%{RuUA}};J^!G(ik;vB)-c6E^D+r-O?J#JD1IXhRlon^7X5D!`Q1(FL00A zseib2wzl|6>kh>x{{a`s!W~X>yrYy0GgvqShh(R79?|>Qjg|XI-TN!oX>^F6`7w_h zv2YI`w`&Bmk8Oi1d+q4TTnqN)N(1M4y#zM?InMX3NQX6Hgr!Dv@S2kd)8~|d&hsn$ zgNtfdJaZ|0aok4l9$a8Demt%H5JnawO5s|=A2Q9;Mekyz^Uk$V?E6h`7Vq`0qDxnvVji#y$PpkKuh4MtDE12ygpcgz(ZxigU6 z>^en_l`c$sq6{-AdngL-nTaa7%i-DmbWqt6!wUD7(Yz(KpnvrTd}@(oCmjrM>Wn(J ztSXjiu06mNS!~92?oxDbdosVm^D2Z-I>+To>r{G>^cp~*2op21{{$$IuuEmp3-KYoR!cD>9?`W=e z#suh_Fo_*Y%&hw{Ruxs-{@Lt|Oa+m}Dn7EjOjQ2(6j=u3QJbVP4vcvS*L(Ma>n#~} zI^+w1 z+ze}f==-O@#<_cPTWozyPr5J09{_+-8gHr(c=7_H$d{nZ2GxekXdeA*p3RP)cmT6Y=AlUc0RJzvmSpBO^CQi#LC#oNHflvM z3|hCH&3`w5C5~GUOnA;}C

q;kyNRzvrE1%GGnATE6ABoM9rP0g=tus-52{4qWt zy0Ux*>mO!~x58r4ah4ss^xG205M0~|tX1GrTk=M*$sW?I>0Tymi5dzkPH{N+d^&vouZ1qk1(DHJZPBpxclfH8 zgW=BRmvGw5oAl-UxsZonIJMi;Aw>BUD%%g|M)wtv)kHPcf8iHB=pa-dIvJ8*o`>6i zHQ4j@ZzxMwf^8_?!?u;>!0kX6AqyQ3Ph;vSdcUw+E-VKJ-4j!}%G7hp&J(yst}JBx-6=tbwi?eyMT3vLO{g2&~8y~?DGF7Hc#j9;4Uv-mb?)r#2MRY{zY1yIN3 z9Za?K6IVC>Eky7?Y4y{5?s54n(6GqD(8XmmMz4e)Gg_5B59#0@Wyi1rU0wVuHGoaH zwTMI?)!C7`5-1-skGXH|B5&C+JijLi4PS{kpM%0K&QFK2{NZNyG5;1fD|Qx&HudsD zoSXRME8pQ;Gt$n)fAldk2X49sQlW;6l@HT1P9()uAK*x>_2q4eq3sBLzO>{68A(cL!4U z4aQ%rnp_1ZV9(4$e1caoH(-D<{9bkl~@BO6mok{R0Ujcmbo`_CXXtS&H(xBEQfJJvMF4PcW_+~9L>3v+-|Pf0f4hFu$U zhL&#ZqpdfNL&pa@h$<5tjVuo9M?40rO*)tr8g*{wcRQBzEfH1w&2XBo7n|L`8GkJv zjVnFX@yPtaO!4fw}-*0GRUN3q1KBr?=n`rqCo`Txr(6TiR z#~rSO(H4R)W1=g`e-9$L`Mcofu)|cBHJZ(FD1(CyR!pa_lm9RICeWn}7^XUgZ!oiG z`$s&4m7W%`>ho0o>_0PZa`reDxio-3xIBnO)SiIJX0bHYNt$gMxQZ*&+)45a&U2+v z_DokXhW&WGg6m9?W_w1)(Eas#+})v7Vt?CPqQSFt+0spK#nR0qgc-CnmoIP1uDgbj zRAK{dQmLlNGSQa!}>Q2YT@89NddI3*JkzH@D6?bk zIZ$&x03ke)w$vD4%<$`6Fmq+&#^=zLpF>a=64}%{o_J4DIA8YiRMMZuMy|_;A@T!> zkNZcrCT4-j;5qmyUmt6y4Z*N+izvr15z`8DpmCxS{inH(JG8~0r92seb4L}z2KQkY zx8fZ63i~_D1O1uzr^n(kA3t%sJ#{EfUCccTj^JmD=5Y6PbeMebE)G`eL2cCw`gb~$ z>|RM>ai0?tu9J$)wf*4nA%C)<~EgL=@&V0Jg&xSXm zhZpuSSX{}qtbWP0Yg|J!fkSQ76&%ak7I0sD-;hqQCY_wN8n?L1Fwc-T+;HPX&{yN3$QVGCqN)rgPy+k!)57mYHO!Fi=)*|oM@I=ApBd=7TNPxG@V z|GX{vE!fWm45M5`w2Ao4h$kSpPDeUex~LHT@Zv zGq(u64Mwt)H}uJ8_h33=FaiC0ZJC1ZEO@)Kk(WGaL|XH8z+2(~HS#(5^K&e=oJoKn zH4z)7;)%nrN3l`aK=-T)VemUSobK_B_dlk>u9>}on@%n`y0075WpdcsnZwv|lc9L| zq#61gyC)j@_c`4>^Ask}R~Mgj(#F0!*0h~7=i(z~!U^X><{;)FxV(Z7Ur|D7-d1$s zK@K%8E#UM1o&v?i%h^T8{UBL2hIuS$r)BR9z)CuR2B1QqCL|TChF_LRwl~bCF=^wE(7ex9{6#XIET<-sS9~Sac{cJ})*PXz)~kGo zaG$t1>o}|$*h$}24~U8_25}t*GuX$@8L;8rZvOejJa{19$Umse;?)B^#daD0tL#g_ zsqEUe&5>k?NQQ(m&m#N2RtOnNA|gp-ZXk(5Whz5x5K$^+ra>~qzOU7!o>FNJ&8nw) zP$?ykzPtDRzVCgX_xSpcHLN$Bv$Dd%= zWC))P-=M0b6B)mvgq)wT2sjD>dHDKc&ZpQuPD)6P~ zC-XSH3cmOp!mHYc7>LgY4V@pjq+^GT&=hYvbleoCdA2YU#u%}F&J&op$6Mj#D|?QN ze*hhVtWfj(W-au~=H!}B8E!qqS zf#`Z~a1J@b+)VN&rOxFvVg6}s(-Z;o9R_5k#3K|N??eZbjakL^n_NF?v4gL5e|dzE zBPoS&-h3(k<$IW}IPeBn?)#2G8As6!|I z5PV%a!s>DPU5ni_*k!q^u&BbCb)0UB+SlaiPI*b{mhJ$wSdGf;+emGM+~~NM0^Axg ziM`+K2WmI=!jp5$XwfZ)UsX27n{fR^^MTz}zWP}~3 zREyGY#Ng!mlhkLmKJ_X;&MV`0T+nW`}>M zfbAcPK=qC?eRn~Np3HyD>Swf}+&&rR!?<)@nk+^h`XZxQXGu;6?xvHc)KaphmS)Zj zBO;dXVf!u{Qk!N8onxn%({&+tZA+aX}Ue0tPu)@bsN$c70cU9w9XZ5vB+Xvpc~b^PGeqNB&!gw zMe3g;m=A71Ez7;MWmuY48uFO)rIX(M}RPj7pdj1En1$%eG<@r+5gTU(8cHmMzd2s-^ZPVJpD zd^^`Mup?HSxHfyEsfGcp=u9Jm??TDL##^BF@Gaoh?ee?OnUSVsMg0CGW*)PP1m=OZa482~AZ)5C@8+RB8ZH#3bmf#0cUsz{1owqlr;mIqB-x zAs$>#=UCe`dgSU%nj;-egk((d&9^*e+L9bc0x#M{b~x({nL$sW92t zC<{vqhH;6B2s3f;I^-BnU_;l5Ge7mW(TNv-z$DQGQZd~Eu3u~i7iFO5xR~0H`)YVU z*Pb3ZaTUMMNJPnuT+ZM58Xn0MBT~CEiNlU1v~Q&y8%s;bK@TsKcPwQSUYmkIyAo)h z6Q`mgf<(GF6jaS3`Jze_KxC5@FJgBGGwwt@4QWuPQ3bD9s~HS&tS#kD5H`f8^b(Wv zR~sG^x1kvWZ<&jK+EdePZ%EjyKp#9h10^zT%)o*SG*(64=EI+1pkXbipL7&X9*n0C zM;nm9*9OFOjwjsnwBmd`Z0MY2t>~eg&E>XiqVMb&dQz^0+}_m0Jku$Jm0YgMdUaqWwFF%hnritIc znJmSR+`41)!DP}WE==lk2T=EM4Q0;CL+q&xl4v5$3_f!qT@#PPt&P)3LW421W_poU z6BmZ9)TG}BCF#`UEqqIEK34YJy-irF4K&qvD%Rxpq3QgWjw_ggcIjh6Qk9}(C%wWzl`z(izAm&|I$IW zA^1G9eTrnhzC4xOo&r;jjipmPO|WKGCi~~$bXq4}3kNHApzM}r42{cTo?IxRy$b~K zTW=QTsO=<8HhcNgdZv=LZ#{Iex(pd=G9Wi6Hj|hneYWUfB&2Q2V#~NSZJm$PV#as@ zDtCB0DU44fJNI%P8U0Ie=e%&dyGk5;1qG>TMjsyYcA|5F9^>I)3;bN^46Sb~Agi{P z4PKB$p0#cv3Rp#S&Yj@7H}~P9?Wb_;#3FodqDwBB3(`-)VZ@QsQ!bI~=h1D~Al_FM zML%5u(d<0T3Dl$`-+ZX%6bbUwaz6y*AE3rY{kXzfo*4Fu;Gy9sc$rBAi?Oj#RBph{ z$rdrKcn3Ug?uB-oL8n+RZT69SuEtNjPhoIF0e6n0@#;%KnTdmp+#B)n_&0t7==yu0F#y9lC~#!*0TC z;YjAaQ6ZBRC5Zl85z_DNVuPADlJk=eQKgW*q@L4_JF?phJgax(-M0N0x%DFWg;Gp06$;UBPa5^z&k=1qQ-AvlsVqX*wi2@m7)MUyOnXF8zH@aIHQK7 z5S40aWb7wRq6cnmCx&PCfdt=&`t0H2rr80|GHO5lzNZ`5G&{EWu?F6?`oiSx4nv*y zMl|?sKZd5Qps@je;dm~`I^nYsYsdk&-JL*P^KQe98w9RJt23RAC1kP5ItUs%$4Xsm zhK#|>ti9JDZ?a1^m!q#reI$y}Z|f*hyI=x-e|`Y_?>kUib|L-HGlQu?HJU8c#YD}h zqg(8JNz9u@{A1F&*3x(;az$CuL&`t=y0!XLYs z+mEauMPuYhjCCz+f4qe3@8II@mVC|^!->;_Y((v+dr(39CA%$iKlXHWvgy`FL|n;) z$UK!G1(H|D=MQ0I-i3M4rWw!Hx*L=%cIKpUWY$bL{=^VdGoa`_9$a||O zPxpc$eGZ%_N!ivV z1t$(HW_ZVYS+$rxUR@`l&d1~k=9kjU$P@^zk!E(D8G!-i&CK|WcU?DbS86)fVKz?;iC)9cxaRuefx;h z+ZD|K!Sr1y`9g`T)RLi{$|bn>hZ?2Qc!9f$Mc??M~rnVu%S+t?H!{+ z?)pw39ZCmydtwIhQh^=0?Ux3_0{7WVpOmSCTM#+6M~h0FosT}3twFfVg%Lbh4HrZw zlk355%>2E{Fiq_zJ6U8CWR-7&NOvv{dhR1&wm(aq#@t_Jl@)xde`oyQ%#;SinjSUTgQ|Qu;(0R}6=L^O zPFuSbW_&DRcDcU7#LjLUU0lG<-p(+?tsC*$r3bjKTZe9I5+Jwr=g@0oa^OShHe&ao z0T1lUM4QttG`VIUS!Wpl=OXQ>Xatx0r+EnUXAPrQ#!d8kbhb^Yb3IBWy2IFy$MCd| zC_@aj$%E@YkYgGMcVDzaZQ%_(8k5Urb$ntHZe9Vwu2*RJV>CIVJ()>wID@G*kq~`; zHEPVRAnsaA_^!o^$i^-EVeQ@lNE8hx2OF2MqO)Xapn?&(n)@e4f8UBkNdu1-o6V+q%->xC*~h)<)$WrJu*(*7LR{#()&wf_&K~OztMJ4hErN|u z0|)(lK;m{9Tzn8nFNSOa#}rTM1Zx34dQW6NLK z%;)yE%a1BiF7XsZmaU@`^2F&MVLqVzd?$6E9x*+$v=D740#Z`?vPR4v$_ZIbNzy^CpKTQ9FldLpj&Pr-{MP3^*#|t75ZdA5(}`2 z&0!{gm8L^l6_6b817h5|SaUL&Jj)+Tysb6KzAP4|RlmWcN}Kl9m4i^#E=6sgzJ(>% zequ>~Wc!@4>Qtj8hsuS_r)?wwchpGZbcfT-X{)<%IwX@iWv4Qs?+=6S&`r?IUxD@P zPjF`9iPOb%u$|NM$q@2^zNnda@3JhJSl$6fS57eQ;>);wjYlL&GYEQQz7jp*3$QNP zg0-)`)GnhOMt5sv^4F&B2it``Ov=gx>O5N$Wf_kDb8<7iR341A_M7=TE*8qhmS#@r;#Pp#1eET0NgbXGfcpnCNW~WR-%|qfAKU z-Cii_+l^vV(_xFbCp~0fO*9|o!=SwnGxwk|6_oqQx^L~kTfSAyxMB-xGVDZ81v#+l zcg#V-^#K1+s4}sBtxwfr-hg{zKJ1^R#H+~ZLJRw;r2oA<5fv__)k~8|&Id)Xvq+_f zZ-0cIpjuSk;00mp<;mCLk8Hr3cX()E3yk|R31t5gz+BI{m@$nq4kuh_#1mDzFc8@f zWvwVFt_qSZGIYi0L|pp59=xekPS;>9JmZ+J^5a)cpg7yY~{Gv$hz!5s0_dCN5Wj)AG%uDH1MP%i zjPJ@}SF|*+*BjJGv78H7TsLCt>LjUdk^)^Er%v4+3flNwJm&bT4KVJOIC>_fkTnaDIdoD1ufv4xCpDKG(xAo9}(Z6)&5Ro2`Sbyq(b%gaIU2z3D^CGwyWc@ zgxfPp4-lc}r&p8A$O*JLupM$7HECV`UK`WJ)?~xwsCOiKQ@&gOkwOq=miMVn}6=Nz%U# z>*5_Tefs7gr%82Hm0K@Nr(ptvysA1iDySv}y{~o>i`5C_M`Hx8Gy81g&3PUgz852Q z><@NJTL9E)=}>)1R}gN_q<`($f$J9ulB25^lFrgSWWG=VzR;DWLhK7j-!PG~J|!6S zZ8TluokVW=8q!evGGd(74h`eF!7Ok##}!~;c&-sS#_NRx;dSVLI10?yS@TV0BiNP2 z5}YQ5FezD-1L&rNG4l6ttJNNSsn!j)%VkOWv`w_4sR-2;X`rLhW!y725=!FixLmQ> zB#N8INq_o`3UmUtO@D$#Z@9f`jR4Xp-3NNEE_6)z02+qAfP@ETC+9tXdjsC)l&iWiMe7_7AP91cn1*}$x}a&D zDhNBB$E*6X@ zi)^yM&D{X3Hk)yIULQe&^UpOunn}YtGqCf@CkUOKPP-KCh~CMytm>&wBI~>jQcd|x z)QBQ9v7N9nCY&Bk*$fH>$Z1D;Q8WKe92%WT*D9M3-3nJ);4l}eMc43lNS=eaJMC$J zK@+UGs73smgo)jEZY`1Ig{eLI#AHaG&Xdf<;94UhJUWIM4k@OWs}F!do*nwUA46tb zkf6=g?RYCBj`>)uM`8XD`_9dgf1$;NPJQJ`Kc!_b!J?(SbH-iZ<==o$g##!<$B=>! zLuzyC7QXzV$ZIV=%3r?yC-loY(AQ3DA)4O+?{D~%cTTG8*PKl2$cK*@hqzf#Cbb2x zD%r82%SY4lpl!&0O~*1KPbv&v8YbjgOyMH)2ZZ7ywJ5R6k= z=9A#|9VoG65Ttv$SzCV@Jp6G4J)V4p{DmVnCp)_FQePRl`FStv*^-89SE|yJ?tJ<% zUI&FDi%_NFBd@w`0#yP&XHI*1&iVD;U`hyV(P1w|p|&8ha9x&!kYM)Yj&# zODNX1rGkLQ0B)+9Mi*zjXWR2$;GU`(z?))E*Da914|<#?t*;xYj$94FlU!*2gDaqX zQ-JEPyp2Vh0$B&WKX9mI5bH=cFS%3^eZQ@OfRVGzjWvF-`0!0o3*N^X>iM93^f&&` zm`HkjqBW$=+koqxvWP;46mELxM57)1a7OGBVz}M}bDjjk%GJW8Y`+RhEl&euziUL_ zDz&}yiV%ZK{W%YcaJssM(*y6kg64x==r%kTE+*@McF_j1NAeCBi^`II)gNe|Z-?z; zJYmbui`Wr;8@#!E@EH3e5W;D}DEE({{`H&SM&>A}^~#3V=M$)H=XX{sYYW`+@}pHl zVK&;7Wccm#^YjFVH`MiSS!<0=F8(}0T4B#p8 zF2*KmHfrC_M*qk1bmYiV@;*YDrl*W0L~9p$U0jSNr+8#nay@fjM1dCcZKW~Xe2qOb zk&0H`0Wm{POI2H++zsqP!B#(9vUV$)UkxNZFBRzJx;121{0){@BtkDVTw|V$c>_~* z-lC;wE>3j32gj`*j!?U2XXcXFOxseey1+r_F z7_#Y=V3IutO2cAcVoW*MNkzewZ}RxcXqa)dFeRoBUtz3MIs~?@!U2zI#A#^}zM0~V z!$n7#R{lHq9_~c%jW%QNcvz5@@~61>y*X8ma|Z1wYkK~{X5RX!0kCP!A`OpjKw`cZ z-7z8#vJ0<4`utk-Qqd<<5{LNeTegszV-YmL;~s{N(Wb_ca+p=ec=2|w{L{~$4qa>Q*p(QNV=Mb7*3AnRPw!7Tq+0RCCBteKhz|M~_4 z_~W1#Rku{5_w5&xth8`qE~Y_j@95G47E5q&#KareLL0Kkz>ySb4lxND^j=P@$Z7SYl-64a#Fm;TVQhr%H(denU#%rCf$ zmiBTqq%8}+3X@y2xe=8qg|{K|B=Ghe5moIA1ssPdxaD$|e!SeU%V*NS8Q z!d2+^b0wXgqDPIO96a1ciRw|vNGA1S_R?{ua zOf5_;&CIOKtS!u@o0wRdSx=j8Za&T2%)-jb#M;!##PU}y^IJDg{@3>x{8ici?WuU> zyS-6@0s_B^+OG?LJAU79Mls8ld$|1_tM$YEbPh(~SJD0(PCNJ1|BYo779G1HC@%27 z+xR~rXC$n8^*;^#>nr^0n8!W+8_3mh(Xm0{p@CeRhOb-^x-#xRQ8&TU31S@7ucH1h zNbc;$Z&8ES#f3)4hJ?ocC)mIL3RM0E<|F*OD*mI}P5u+<-^c0Sk*<~hKaouT6Y1Z_ zrRv}6w6@}RNWbZUe;+&se}lU5Td4ngSbMmMiTw|Pny}&jVu}Cv+5g-p{bPU%2&}FA i=LPP4qkf%zuKwraU&G78P4rji35avIU$6h?YyS&4o57X< From 93acc47687c8a50f5b08dcce7242d0761bf91623 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 17 Sep 2026 16:27:37 -0400 Subject: [PATCH 13/31] refactor(main): keep occurrence sets with the other named collections of occurrences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An occurrence set is a named, project-scoped list of occurrences, which is the same shape as a capture set or a taxa list — and both of those live in the main app and are routed under the domain noun they collect. This one was in the ml app under `ml/occurrence-sets`, so the one place someone would look for sets of occurrences was the one place they were not. The model and its viewset move to main; the route becomes `occurrences/sets`, registered before `occurrences` so the detail route does not swallow it. The evaluations themselves stay in ml and reference it across apps, the way they already reference main.Taxon — they are about how a model performed, not about which occurrences were chosen. Nothing is deployed yet, so the migrations are regenerated rather than moved with state operations: main/0097 creates the set, ml/0033 creates the two evaluation tables that depend on it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/jobs/models.py | 3 +- ami/main/api/serializers.py | 22 ++++++++++++ ami/main/api/views.py | 25 ++++++++++++++ ami/main/migrations/0097_evaluation.py | 38 +++++++++++++++++++++ ami/main/models.py | 39 +++++++++++++++++++++ ami/ml/evaluation.py | 3 +- ami/ml/migrations/0033_evaluation.py | 47 ++++++-------------------- ami/ml/models/__init__.py | 5 ++- ami/ml/models/evaluation.py | 43 ++--------------------- ami/ml/serializers.py | 22 ------------ ami/ml/tests.py | 2 +- ami/ml/views.py | 28 ++------------- config/api_router.py | 2 +- 13 files changed, 146 insertions(+), 133 deletions(-) create mode 100644 ami/main/migrations/0097_evaluation.py diff --git a/ami/jobs/models.py b/ami/jobs/models.py index abf39f367..000541dce 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -1071,8 +1071,9 @@ class EvaluateAlgorithmJob(JobType): @classmethod def run(cls, job: "Job"): + from ami.main.models import OccurrenceSet from ami.ml import evaluation - from ami.ml.models import Algorithm, OccurrenceSet + from ami.ml.models import Algorithm params = job.params or {} algorithm_key = params.get("algorithm_key") diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index f5799a3e2..95fe6fe30 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -26,6 +26,7 @@ Event, Identification, Occurrence, + OccurrenceSet, Page, Project, ProjectSettingsMixin, @@ -780,6 +781,27 @@ def get_projects(self, obj): return list(obj.projects.values_list("id", flat=True)) +class OccurrenceSetSerializer(DefaultSerializer): + """The evaluation sets a project can score a model against.""" + + occurrences_count = serializers.SerializerMethodField() + + class Meta: + model = OccurrenceSet + fields = [ + "id", + "details", + "name", + "description", + "occurrences_count", + "created_at", + "updated_at", + ] + + def get_occurrences_count(self, obj) -> int: + return getattr(obj, "annotated_occurrences_count", None) or obj.occurrences.count() + + class TaxaListTaxonInputSerializer(serializers.Serializer): """Serializer for adding a taxon to a taxa list.""" diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 14dfbf3a4..7acddca64 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -49,6 +49,7 @@ Event, Identification, Occurrence, + OccurrenceSet, Page, Project, ProjectQuerySet, @@ -84,6 +85,7 @@ ModelAgreementSerializer, OccurrenceListSerializer, OccurrenceSerializer, + OccurrenceSetSerializer, PageListSerializer, PageSerializer, ProjectListSerializer, @@ -953,6 +955,29 @@ class CaptureSetChoicesPagination(LimitOffsetPaginationWithPermissions): max_limit = 100 +class OccurrenceSetViewSet(DefaultViewSet, ProjectMixin): + """ + API endpoint listing the fixed occurrence sets a model can be scored against. + + Read-only: membership is built deliberately, not edited in passing, because two models + can only be compared if they were scored on exactly the same occurrences. + """ + + queryset = OccurrenceSet.objects.all() + serializer_class = OccurrenceSetSerializer + http_method_names = ["get", "head", "options"] + ordering_fields = ["name", "created_at", "updated_at"] + search_fields = ["name"] + + def get_queryset(self) -> QuerySet["OccurrenceSet"]: + qs = super().get_queryset().annotate(annotated_occurrences_count=models.Count("occurrences")) + project = self.get_active_project() + if project: + # A set with no project is global, so it is offered everywhere. + return qs.for_project(project) + return qs + + class SourceImageCollectionViewSet(DefaultViewSet, ProjectMixin): """ Endpoint for viewing capture sets or samples of captures. diff --git a/ami/main/migrations/0097_evaluation.py b/ami/main/migrations/0097_evaluation.py new file mode 100644 index 000000000..d7425f47d --- /dev/null +++ b/ami/main/migrations/0097_evaluation.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.10 on 2026-09-17 16:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0096_project_default_taxa_list"), + ] + + operations = [ + migrations.CreateModel( + name="OccurrenceSet", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("name", models.CharField(max_length=255)), + ("description", models.TextField(blank=True)), + ( + "occurrences", + models.ManyToManyField(blank=True, related_name="evaluation_sets", to="main.occurrence"), + ), + ( + "projects", + models.ManyToManyField( + blank=True, + help_text="Projects this set belongs to. A set with none is available everywhere.", + related_name="occurrence_sets", + to="main.project", + ), + ), + ], + options={ + "ordering": ["name"], + }, + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index e5ff8b13c..3d582bb8a 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -4928,6 +4928,45 @@ def html(self) -> str: ] +class OccurrenceSetQuerySet(BaseQuerySet): + def for_project(self, project) -> models.QuerySet: + """Sets this project can use: its own, plus any that belong to no project.""" + return self.filter(models.Q(projects=project) | models.Q(projects__isnull=True)).distinct() + + +class OccurrenceSet(BaseModel): + """ + A fixed list of occurrences to score models against. + + Two models can only be compared if they were scored on the same occurrences, so the + membership is stored rather than re-sampled. A set with no projects is global, which is + how one set compares models across the platform; that follows how TaxaList already + treats a list with no project. + """ + + name = models.CharField(max_length=255) + description = models.TextField(blank=True) + occurrences = models.ManyToManyField("main.Occurrence", related_name="evaluation_sets", blank=True) + projects = models.ManyToManyField( + "main.Project", + related_name="occurrence_sets", + blank=True, + help_text="Projects this set belongs to. A set with none is available everywhere.", + ) + + objects = OccurrenceSetQuerySet.as_manager() + + class Meta: + ordering = ["name"] + + def __str__(self) -> str: + return f"{self.name} ({self.occurrences.count()} occurrences)" + + @property + def is_global(self) -> bool: + return not self.projects.exists() + + class SourceImageCollectionQuerySet(BaseQuerySet): def with_source_images_count(self): return self.annotate( diff --git a/ami/ml/evaluation.py b/ami/ml/evaluation.py index 70568cb16..5ca1409da 100644 --- a/ami/ml/evaluation.py +++ b/ami/ml/evaluation.py @@ -17,9 +17,8 @@ from django.db.models import QuerySet -from ami.main.models import Classification, Occurrence +from ami.main.models import Classification, Occurrence, OccurrenceSet from ami.ml.models.algorithm import Algorithm -from ami.ml.models.evaluation import OccurrenceSet logger = logging.getLogger(__name__) diff --git a/ami/ml/migrations/0033_evaluation.py b/ami/ml/migrations/0033_evaluation.py index 5164132a3..0bc2ed475 100644 --- a/ami/ml/migrations/0033_evaluation.py +++ b/ami/ml/migrations/0033_evaluation.py @@ -1,13 +1,13 @@ -# Generated by Django 4.2.10 on 2026-09-13 15:49 +# Generated by Django 4.2.10 on 2026-09-17 16:20 -import django.db.models.deletion from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ("main", "0096_project_default_taxa_list"), ("jobs", "0026_evaluation"), + ("main", "0097_evaluation"), ("ml", "0032_training_set_membership"), ] @@ -54,6 +54,14 @@ class Migration(migrations.Migration): to="jobs.job", ), ), + ( + "occurrence_set", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="evaluations", + to="main.occurrenceset", + ), + ), ], options={ "ordering": ["-created_at"], @@ -88,39 +96,6 @@ class Migration(migrations.Migration): "ordering": ["-occurrences_scored", "taxon__name"], }, ), - migrations.CreateModel( - name="OccurrenceSet", - fields=[ - ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ("created_at", models.DateTimeField(auto_now_add=True)), - ("updated_at", models.DateTimeField(auto_now=True)), - ("name", models.CharField(max_length=255)), - ("description", models.TextField(blank=True)), - ( - "occurrences", - models.ManyToManyField(blank=True, related_name="evaluation_sets", to="main.occurrence"), - ), - ( - "projects", - models.ManyToManyField( - blank=True, - help_text="Projects this set belongs to. A set with none is available everywhere.", - related_name="occurrence_sets", - to="main.project", - ), - ), - ], - options={ - "ordering": ["name"], - }, - ), - migrations.AddField( - model_name="algorithmevaluation", - name="occurrence_set", - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, related_name="evaluations", to="ml.occurrenceset" - ), - ), migrations.AddConstraint( model_name="taxonevaluation", constraint=models.UniqueConstraint(fields=("evaluation", "taxon"), name="unique_taxon_per_evaluation"), diff --git a/ami/ml/models/__init__.py b/ami/ml/models/__init__.py index c5a94e23b..43a6bce6b 100644 --- a/ami/ml/models/__init__.py +++ b/ami/ml/models/__init__.py @@ -1,6 +1,6 @@ from ami.ml.models.algorithm import Algorithm, AlgorithmCategoryMap from ami.ml.models.embedding import DetectionEmbedding -from ami.ml.models.evaluation import AlgorithmEvaluation, OccurrenceSet, TaxonEvaluation +from ami.ml.models.evaluation import AlgorithmEvaluation, TaxonEvaluation from ami.ml.models.pipeline import Pipeline from ami.ml.models.processing_service import ProcessingService from ami.ml.models.project_pipeline_config import ProjectPipelineConfig @@ -11,10 +11,9 @@ "AlgorithmCategoryMap", "AlgorithmEvaluation", "DetectionEmbedding", - "OccurrenceSet", - "TaxonEvaluation", "Pipeline", "ProcessingService", "ProjectPipelineConfig", + "TaxonEvaluation", "TrainingSetMembership", ] diff --git a/ami/ml/models/evaluation.py b/ami/ml/models/evaluation.py index 10f51c161..4cb85a4f4 100644 --- a/ami/ml/models/evaluation.py +++ b/ami/ml/models/evaluation.py @@ -2,50 +2,11 @@ from django.db import models -from ami.base.models import BaseModel, BaseQuerySet +from ami.base.models import BaseModel logger = logging.getLogger(__name__) -class OccurrenceSetQuerySet(BaseQuerySet): - def for_project(self, project) -> models.QuerySet: - """Sets this project can use: its own, plus any that belong to no project.""" - return self.filter(models.Q(projects=project) | models.Q(projects__isnull=True)).distinct() - - -class OccurrenceSet(BaseModel): - """ - A fixed list of occurrences to score models against. - - Two models can only be compared if they were scored on the same occurrences, so the - membership is stored rather than re-sampled. A set with no projects is global, which is - how one set compares models across the platform; that follows how TaxaList already - treats a list with no project. - """ - - name = models.CharField(max_length=255) - description = models.TextField(blank=True) - occurrences = models.ManyToManyField("main.Occurrence", related_name="evaluation_sets", blank=True) - projects = models.ManyToManyField( - "main.Project", - related_name="occurrence_sets", - blank=True, - help_text="Projects this set belongs to. A set with none is available everywhere.", - ) - - objects = OccurrenceSetQuerySet.as_manager() - - class Meta: - ordering = ["name"] - - def __str__(self) -> str: - return f"{self.name} ({self.occurrences.count()} occurrences)" - - @property - def is_global(self) -> bool: - return not self.projects.exists() - - class AlgorithmEvaluation(BaseModel): """ How one algorithm scored against one occurrence set. @@ -56,7 +17,7 @@ class AlgorithmEvaluation(BaseModel): """ algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="evaluations") - occurrence_set = models.ForeignKey(OccurrenceSet, on_delete=models.CASCADE, related_name="evaluations") + occurrence_set = models.ForeignKey("main.OccurrenceSet", on_delete=models.CASCADE, related_name="evaluations") job = models.ForeignKey("jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="evaluations") micro_accuracy = models.FloatField(null=True, help_text="Share of occurrences the algorithm named correctly.") diff --git a/ami/ml/serializers.py b/ami/ml/serializers.py index a0f104f8b..38534c662 100644 --- a/ami/ml/serializers.py +++ b/ami/ml/serializers.py @@ -5,7 +5,6 @@ from ami.ml import training_data from .models.algorithm import Algorithm, AlgorithmCategoryMap -from .models.evaluation import OccurrenceSet from .models.pipeline import Pipeline, PipelineStage from .models.processing_service import ProcessingService from .models.project_pipeline_config import ProjectPipelineConfig @@ -67,27 +66,6 @@ def get_evaluations(self, obj) -> list[dict]: return reporting.latest_evaluations(obj) -class OccurrenceSetSerializer(DefaultSerializer): - """The evaluation sets a project can score a model against.""" - - occurrences_count = serializers.SerializerMethodField() - - class Meta: - model = OccurrenceSet - fields = [ - "id", - "details", - "name", - "description", - "occurrences_count", - "created_at", - "updated_at", - ] - - def get_occurrences_count(self, obj) -> int: - return getattr(obj, "annotated_occurrences_count", None) or obj.occurrences.count() - - class AlgorithmNestedSerializer(DefaultSerializer): class Meta: model = Algorithm diff --git a/ami/ml/tests.py b/ami/ml/tests.py index 138195282..e10dbc93c 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -20,6 +20,7 @@ Event, Identification, Occurrence, + OccurrenceSet, Project, SourceImage, SourceImageCollection, @@ -34,7 +35,6 @@ AlgorithmCategoryMap, AlgorithmEvaluation, DetectionEmbedding, - OccurrenceSet, Pipeline, ProcessingService, TaxonEvaluation, diff --git a/ami/ml/views.py b/ami/ml/views.py index c917d824a..4399676b2 100644 --- a/ami/ml/views.py +++ b/ami/ml/views.py @@ -1,6 +1,6 @@ import logging -from django.db import models, transaction +from django.db import transaction from django.db.models import Prefetch from django.db.models.query import QuerySet from django.http import Http404 @@ -25,14 +25,13 @@ from .models.algorithm import Algorithm, AlgorithmCategoryMap from .models.embedding import EMBEDDING_DIMENSIONS, DetectionEmbedding -from .models.evaluation import AlgorithmEvaluation, OccurrenceSet +from .models.evaluation import AlgorithmEvaluation from .models.pipeline import Pipeline from .models.processing_service import ProcessingService from .models.project_pipeline_config import ProjectPipelineConfig from .serializers import ( AlgorithmCategoryMapSerializer, AlgorithmSerializer, - OccurrenceSetSerializer, PipelineRegistrationSerializer, PipelineSerializer, ProcessingServiceSerializer, @@ -106,29 +105,6 @@ class AlgorithmCategoryMapViewSet(DefaultViewSet): ] -class OccurrenceSetViewSet(DefaultViewSet, ProjectMixin): - """ - API endpoint listing the fixed occurrence sets a model can be scored against. - - Read-only: membership is built deliberately, not edited in passing, because two models - can only be compared if they were scored on exactly the same occurrences. - """ - - queryset = OccurrenceSet.objects.all() - serializer_class = OccurrenceSetSerializer - http_method_names = ["get", "head", "options"] - ordering_fields = ["name", "created_at", "updated_at"] - search_fields = ["name"] - - def get_queryset(self) -> QuerySet["OccurrenceSet"]: - qs = super().get_queryset().annotate(annotated_occurrences_count=models.Count("occurrences")) - project = self.get_active_project() - if project: - # A set with no project is global, so it is offered everywhere. - return qs.for_project(project) - return qs - - class PipelineViewSet(DefaultViewSet, ProjectMixin): """ API endpoint that allows pipelines to be viewed or edited. diff --git a/config/api_router.py b/config/api_router.py index 6972ac5e8..1c075cd36 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -41,6 +41,7 @@ router.register(r"detections", views.DetectionViewSet) # Register before r"occurrences" — see docs/claude/reference/api-stats-pattern.md (guarded by regression test). router.register(r"occurrences/stats", views.OccurrenceStatsViewSet, basename="occurrence-stats") +router.register(r"occurrences/sets", views.OccurrenceSetViewSet) router.register(r"occurrences", views.OccurrenceViewSet) router.register(r"taxa/lists", views.TaxaListViewSet) # NESTED: /taxa/lists/{taxalist_id}/taxa/ @@ -54,7 +55,6 @@ router.register(r"tags", views.TagViewSet) router.register(r"ml/algorithms", ml_views.AlgorithmViewSet) router.register(r"ml/labels", ml_views.AlgorithmCategoryMapViewSet) -router.register(r"ml/occurrence-sets", ml_views.OccurrenceSetViewSet) router.register(r"ml/pipelines", ml_views.PipelineViewSet) router.register(r"ml/processing_services", ml_views.ProcessingServiceViewSet) router.register(r"ml/training-data", ml_views.TrainingDataViewSet, basename="training-data") From e971071adff0c651bc6516b416b3fee68e9087e0 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Thu, 17 Sep 2026 16:43:38 -0400 Subject: [PATCH 14/31] fix(ml): count the crops a retrain will actually use, not the ones passing display filters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "training images ready" count went through the project's default filters, which the training set itself does not. The score threshold hides predictions the model was unsure about — but these rows are human answers, and someone correcting a low-confidence prediction is the most useful crop there is. So the column dropped exactly the data most worth training on, and disagreed with the set a retrain builds: on a verified occurrence scored below the threshold, the column said 6 where the training set used 7. The count now mirrors `ami.ml.training_data` and applies no display filters, with a test that pins the two together. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/main/api/views.py | 18 ++---------------- ami/main/models.py | 17 ++++++----------- ami/main/tests.py | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 7acddca64..bd3aa98ad 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -2105,13 +2105,7 @@ def get_taxa_observed( verified_counts=verified_counts, ) - qs = self.annotate_training_crop_counts( - qs, - project, - occurrence_filters=direct_filters, - apply_default_score_filter=apply_default_score_filter, - apply_default_taxa_filter=apply_default_taxa_filter, - ) + qs = self.annotate_training_crop_counts(qs, project, occurrence_filters=direct_filters) return self.annotate_example_occurrences( qs, @@ -2128,8 +2122,6 @@ def annotate_training_crop_counts( project: Project, *, occurrence_filters: models.Q, - apply_default_score_filter=True, - apply_default_taxa_filter=True, ) -> QuerySet: """Add the ``training_crops_count`` annotation behind the ``with_training_crop_counts`` opt-in param. @@ -2146,13 +2138,7 @@ def annotate_training_crop_counts( ) if not include_counts: return qs.annotate(training_crops_count=models.Value(None, output_field=models.IntegerField())) - return qs.with_training_crop_counts( - project, - self.request, - occurrence_filters=occurrence_filters, - apply_default_score_filter=apply_default_score_filter, - apply_default_taxa_filter=apply_default_taxa_filter, - ) + return qs.with_training_crop_counts(project, occurrence_filters=occurrence_filters) def annotate_example_occurrences( self, diff --git a/ami/main/models.py b/ami/main/models.py index 3d582bb8a..242317d1a 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -4147,11 +4147,8 @@ def with_verification_counts( def with_training_crop_counts( self, project: Project, - request: Request | None, *, occurrence_filters: models.Q, - apply_default_score_filter: bool = True, - apply_default_taxa_filter: bool = True, ): """Annotate ``training_crops_count``: verified crops a classifier head can be fit on. @@ -4160,17 +4157,15 @@ def with_training_crop_counts( verified as a species is not training data for its genus. Crops still need an embedding from the chosen feature extractor before a job can use them; this is the upper bound, not the row count of the next training set. + + The project's default filters are deliberately not applied, so that this matches what + a retrain would actually use (``ami.ml.training_data``). The score threshold hides + predictions the model was unsure about, but these rows are human answers — and a + person correcting a low-confidence prediction is the most useful crop there is, so + filtering on the model's confidence would hide exactly the data worth training on. """ - default_q = build_occurrence_default_filters_q( - project, - request, - occurrence_accessor="", - apply_default_score_filter=apply_default_score_filter, - apply_default_taxa_filter=apply_default_taxa_filter, - ) verified_occurrences = ( Occurrence.objects.filter(occurrence_filters) - .filter(default_q) .filter(determination_id__isnull=False) .filter(Exists(Identification.objects.filter(occurrence=OuterRef("pk"), withdrawn=False))) ) diff --git a/ami/main/tests.py b/ami/main/tests.py index 05845d74f..10101347d 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -6511,6 +6511,23 @@ def test_training_crops_ready_counts_crops_not_occurrences(self): self.assertEqual(detail["verified_count"], 2) self.assertEqual(detail["training_crops_ready"], 3) + def test_training_crops_ready_ignores_the_project_score_threshold(self): + """ + The count must match what a retrain would actually use. + + The score threshold hides predictions the model was unsure about, but these rows + are human answers. Someone correcting a low-confidence prediction is the most + useful crop there is, so filtering on the model's confidence would hide exactly + the data worth training on — and make the column disagree with the training set. + """ + self.project.default_filters_score_threshold = 0.5 + self.project.save() + Occurrence.objects.filter(pk=self.occ_pred.pk).update(determination_score=0.1) + + detail = self._detail(self.cardui, "&with_training_crop_counts=true") + + self.assertEqual(detail["training_crops_ready"], 2) + def test_training_crops_ready_does_not_roll_up_to_ancestors(self): """A head is fit on the label itself, so species crops are not genus training data.""" for ancestor in (self.genus, self.family, self.order): From d0827dffd81131761bf8c4069ca8ca5a97ded310 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Fri, 18 Sep 2026 16:24:23 -0400 Subject: [PATCH 15/31] fix(ml): record which training set each retrained version was fitted on Every retrain creates a new algorithm version, and training_info.dataset_url is meant to name the exact set it learned from. It was always null, so no retrained version could be traced back to its data. The dataset's own metadata is echoed back by the processing service and is where that field is read from, but the url was computed after the file was written and so never went in. The path is deterministic, so it is worked out first and the file now records where it was written, which also makes the archive self-describing as its comment already claimed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/ml/tests.py | 22 ++++++++++++++++++++++ ami/ml/training_dataset.py | 34 +++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/ami/ml/tests.py b/ami/ml/tests.py index e10dbc93c..81c21ee46 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -2813,6 +2813,28 @@ def test_vectors_are_stored_as_float16(self): self.assertEqual(np.load(f, allow_pickle=True)["features"].dtype, np.float16) default_storage.delete(result["path"]) + def test_the_dataset_names_the_url_it_was_written_to(self): + """ + The version a retrain produces has to name the exact set it was fitted on. + + The service echoes this metadata back in its result, and that is where + ``Algorithm.training_info.dataset_url`` comes from, so a missing url leaves every + retrained version with no record of its training data. + """ + from ami.ml.training_dataset import build_training_dataset + + self._verify_all() + result = build_training_dataset( + project=self.project, algorithm=self.classifier, min_per_species=1, test_fraction=0.5 + ) + + with default_storage.open(result["path"], "rb") as f: + metadata = json.loads(str(np.load(f, allow_pickle=True)["metadata"])) + + self.assertEqual(metadata["url"], result["url"]) + self.assertIn(result["path"], metadata["url"]) + default_storage.delete(result["path"]) + def test_nothing_verified_means_no_file_is_written(self): from ami.ml.training_dataset import NotEnoughVerifiedData, build_training_dataset diff --git a/ami/ml/training_dataset.py b/ami/ml/training_dataset.py index 3affe7564..779f8baff 100644 --- a/ami/ml/training_dataset.py +++ b/ami/ml/training_dataset.py @@ -124,7 +124,12 @@ def build_training_dataset( "before a new head can be compared against the current one." ) + file_path = dataset_path(project, algorithm, job.pk if job else None) metadata = { + # The file records where it was written. A retrain is only auditable if the + # version it produced can name the exact set it was fitted on, and the service + # echoes this metadata back in its result. + "url": f"{settings.MEDIA_URL}{file_path}", "project": {"id": project.pk, "name": project.name}, "algorithm": {"key": algorithm.key, "name": algorithm.name, "version": algorithm.version}, "dimensions": EMBEDDING_DIMENSIONS, @@ -148,9 +153,7 @@ def build_training_dataset( } file_path = _save( - project=project, - algorithm=algorithm, - job_id=job.pk if job else None, + file_path=file_path, arrays={ "features": features, "labels": labels, @@ -167,15 +170,20 @@ def build_training_dataset( # decided. A run that fails later still consumed these occurrences. record_training_set(occurrence_ids=[int(pk) for pk in occurrence_ids[:kept]], job=job) - file_url = f"{settings.MEDIA_URL}{file_path}" + file_url = metadata["url"] logger.info(f"Wrote training dataset with {kept} rows over {len(classes)} species to {file_path}") return {"path": file_path, "url": file_url, "metadata": metadata} +def dataset_path(project: Project, algorithm: Algorithm, job_id: int | None) -> str: + """Where this project's training set for this algorithm is written.""" + stem = f"{slugify(project.name)}-{slugify(algorithm.key)}" + suffix = f"job-{job_id}" if job_id else "manual" + return f"{DATASET_DIRECTORY}/{stem}-{suffix}.npz" + + def _save( - project: Project, - algorithm: Algorithm, - job_id: int | None, + file_path: str, arrays: dict[str, np.ndarray], metadata: dict[str, typing.Any], ) -> str: @@ -185,10 +193,6 @@ def _save( default_storage is the local filesystem in development and the project's S3 bucket in production, so this follows wherever captures already live without a special case. """ - stem = f"{slugify(project.name)}-{slugify(algorithm.key)}" - suffix = f"job-{job_id}" if job_id else "manual" - file_path = f"{DATASET_DIRECTORY}/{stem}-{suffix}.npz" - with tempfile.TemporaryDirectory() as tmp: local = pathlib.Path(tmp) / "dataset.npz" # Uncompressed: embeddings are close to random, so compression buys about 10 per @@ -198,6 +202,10 @@ def _save( # A re-run of the same job replaces its dataset instead of piling up copies. default_storage.delete(file_path) with open(local, "rb") as f: - file_path = default_storage.save(file_path, f) + saved_path = default_storage.save(file_path, f) - return file_path + if saved_path != file_path: + # The metadata inside the archive records the URL it was written to, so a rename + # by the storage backend would leave the file describing somewhere it is not. + logger.warning(f"Storage wrote the training set to {saved_path}, not {file_path}.") + return saved_path From 721449140d811ad4efc2715ad43506ced2c4c06c Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Sun, 20 Sep 2026 21:05:08 -0400 Subject: [PATCH 16/31] fix(ml): show a project the evaluation sets it is allowed to score against An occurrence set belonging to no project is global: it is how one set compares models across the whole platform. The endpoint never returned one. The inherited visibility filter keeps a row only if it reaches a non-draft project, and a set with no project never does, so global sets were hidden from everyone but a superuser -- the opposite of what global means. The endpoint also had no test, which is why this went unnoticed. It now has one, and the viewset requires a project like the taxa-lists endpoint it mirrors: without one the queryset has nothing to scope to and would list every set on the platform. Only a set's name and size are exposed, never the occurrences inside it, so a global set carries nothing from the projects that contributed to it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/main/api/views.py | 10 +++--- ami/main/models.py | 12 +++++++ ami/ml/tests.py | 76 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 5 deletions(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index bd3aa98ad..94991a8e4 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -968,14 +968,14 @@ class OccurrenceSetViewSet(DefaultViewSet, ProjectMixin): http_method_names = ["get", "head", "options"] ordering_fields = ["name", "created_at", "updated_at"] search_fields = ["name"] + # Scoping is the whole point of the queryset below, so without a project there is + # nothing sensible to return: it would list every set on the platform. + require_project = True def get_queryset(self) -> QuerySet["OccurrenceSet"]: qs = super().get_queryset().annotate(annotated_occurrences_count=models.Count("occurrences")) - project = self.get_active_project() - if project: - # A set with no project is global, so it is offered everywhere. - return qs.for_project(project) - return qs + # A set with no project is global, so it is offered everywhere. + return qs.for_project(self.get_active_project()) class SourceImageCollectionViewSet(DefaultViewSet, ProjectMixin): diff --git a/ami/main/models.py b/ami/main/models.py index 242317d1a..f332bfc07 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -4928,6 +4928,18 @@ def for_project(self, project) -> models.QuerySet: """Sets this project can use: its own, plus any that belong to no project.""" return self.filter(models.Q(projects=project) | models.Q(projects__isnull=True)).distinct() + def visible_for_user(self, user) -> models.QuerySet: + """ + Global sets stay visible; project sets follow their project. + + The inherited filter keeps a row only if it reaches a non-draft project, which a + set belonging to no project never does. Without this a global set is hidden from + everyone but a superuser, which is the opposite of what global means. Only a set's + name and size are exposed, never the occurrences inside it. + """ + visible = super().visible_for_user(user) + return self.filter(models.Q(pk__in=visible.values("pk")) | models.Q(projects__isnull=True)).distinct() + class OccurrenceSet(BaseModel): """ diff --git a/ami/ml/tests.py b/ami/ml/tests.py index 81c21ee46..3a8ecc3ef 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -3626,6 +3626,82 @@ def test_the_first_result_is_the_one_kept(self): self.assertEqual(self.job.result["result"]["candidate_metrics"]["top1"], 0.9) +class TestOccurrenceSetAPI(APITestCase): + """ + The endpoint the evaluation-set picker reads. + + A set is either one project's or global, so what this returns decides which sets a + person can score a model against. + """ + + def setUp(self): + self.project = Project.objects.create(name="Set Project") + self.other_project = Project.objects.create(name="Someone Else's Project") + self.user = User.objects.create_user(email="sets@example.com", password="testpass123") + self.project.members.add(self.user) + + self.own = OccurrenceSet.objects.create(name="This project's blind set") + self.own.projects.add(self.project) + self.global_set = OccurrenceSet.objects.create(name="Platform-wide blind set") + self.theirs = OccurrenceSet.objects.create(name="Another project's blind set") + self.theirs.projects.add(self.other_project) + + self.url = reverse_with_params("api:occurrenceset-list") + + def _names(self, response): + return sorted(item["name"] for item in response.json()["results"]) + + def test_a_project_sees_its_own_sets_and_the_global_ones(self): + self.client.force_authenticate(user=self.user) + response = self.client.get(self.url, {"project_id": self.project.pk}) + + self.assertEqual(response.status_code, 200) + self.assertEqual(self._names(response), sorted([self.own.name, self.global_set.name])) + + def test_another_project_s_set_is_not_offered(self): + """Scoring against a set from another project would compare models on data + this project cannot see.""" + self.client.force_authenticate(user=self.user) + response = self.client.get(self.url, {"project_id": self.project.pk}) + + self.assertNotIn(self.theirs.name, self._names(response)) + + def test_listing_without_a_project_is_refused(self): + """Without a project the queryset has nothing to scope to, so it would return + every set on the platform.""" + self.client.force_authenticate(user=self.user) + + self.assertEqual(self.client.get(self.url).status_code, 400) + + def test_the_sets_are_read_only(self): + """Membership is built deliberately: two models can only be compared if they were + scored on exactly the same occurrences.""" + self.client.force_authenticate(user=self.user) + response = self.client.post(self.url, {"name": "Made up in passing"}, format="json") + + # 405 if the method is refused first, 403 if the permission check gets there + # first. Either way the endpoint will not create one. + self.assertIn(response.status_code, (403, 405)) + + def test_a_draft_project_s_sets_are_hidden_from_outsiders(self): + """A draft project is not published, so neither is what it scores models on.""" + draft = Project.objects.create(name="Unpublished Project", draft=True) + secret = OccurrenceSet.objects.create(name="Draft project's set") + secret.projects.add(draft) + + self.client.force_authenticate(user=None) + response = self.client.get(self.url, {"project_id": draft.pk}) + + self.assertNotIn(secret.name, self._names(response)) + + def test_a_global_set_is_offered_to_anonymous_readers_too(self): + """Global means every project, and Antenna publishes non-draft projects.""" + self.client.force_authenticate(user=None) + response = self.client.get(self.url, {"project_id": self.project.pk}) + + self.assertIn(self.global_set.name, self._names(response)) + + class TestAlgorithmEvaluation(TestCase): """ Scoring an algorithm against a fixed set of verified occurrences, so two models can be From cc7154786a8e2a685834847ad514e4a31eb998ff Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Sun, 20 Sep 2026 21:05:17 -0400 Subject: [PATCH 17/31] perf(ml): look up the best model once per page, not once per taxa list Every list on the taxa-lists page shows the model that handles its species best, and each one was fetched with its own query: a page of six lists cost 28 queries against 10 for a single list. The lookup is now five correlated subqueries annotated onto the page, following how collection counts are already done here, so the cost no longer grows with the number of lists. The ranking rule moves into one constant. The page and the single-list lookup have to agree on what "best" means, and they were two copies of the same order_by. A multi-row fixture is the only way to see this: with one list an N+1 and a flat query are indistinguishable. The guard counts only evaluation queries, so it reports this endpoint's own scaling rather than the per-row project and taxa lookups that already exist on the serializer. A second test reads best_model back from the endpoint, because a missing annotation would otherwise turn the field null with the query count still looking healthy. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/main/api/serializers.py | 21 +++++----- ami/main/api/views.py | 2 + ami/ml/reporting.py | 34 ++++++++++++---- ami/ml/tests.py | 79 +++++++++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 16 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 95fe6fe30..7470e551b 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -737,18 +737,21 @@ class Meta: ] def get_best_model(self, obj) -> dict | None: - """The algorithm scoring highest on this list's species, or null if none has been scored.""" - from ami.ml import reporting + """ + The algorithm scoring highest on this list's species, or null if none has been scored. - evaluation = reporting.best_evaluation_for_taxa_list(obj) - if not evaluation: + Read from the annotations the viewset attaches, so a page of lists costs one query + rather than one per list. + """ + algorithm_id = getattr(obj, "best_algorithm_id", None) + if algorithm_id is None: return None return { - "id": evaluation.algorithm_id, - "name": evaluation.algorithm.name, - "accuracy": evaluation.micro_accuracy, - "accuracy_by_species": evaluation.macro_accuracy, - "occurrence_set": evaluation.occurrence_set.name, + "id": algorithm_id, + "name": obj.best_algorithm_name, + "accuracy": obj.best_micro_accuracy, + "accuracy_by_species": obj.best_macro_accuracy, + "occurrence_set": obj.best_occurrence_set_name, } def get_taxa(self, obj): diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 94991a8e4..434e644d6 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -36,6 +36,7 @@ from ami.main.api.serializers import TagSerializer from ami.main.models_future.identifications import create_identifications_batch, resolve_occurrences from ami.main.models_future.occurrence import model_agreement_for_project, top_identifiers_for_project +from ami.ml import reporting from ami.ml.models.algorithm import Algorithm from ami.ml.serializers import AlgorithmSerializer from ami.utils.requests import get_default_classification_threshold @@ -2268,6 +2269,7 @@ def get_queryset(self): qs = super().get_queryset() # Annotate with taxa count for better performance qs = qs.annotate(annotated_taxa_count=models.Count("taxa")) + qs = reporting.annotate_best_model(qs) project = self.get_active_project() if project: return qs.filter(projects=project) diff --git a/ami/ml/reporting.py b/ami/ml/reporting.py index 761b8015b..03b3f3e78 100644 --- a/ami/ml/reporting.py +++ b/ami/ml/reporting.py @@ -8,6 +8,8 @@ import typing +from django.db import models + from ami.main.models import TaxaList, Taxon from ami.ml.models.algorithm import Algorithm from ami.ml.models.evaluation import AlgorithmEvaluation, TaxonEvaluation @@ -15,23 +17,41 @@ DEFAULT_EVALUATION_LIMIT = 5 -def best_evaluation_for_taxa_list(taxa_list: TaxaList) -> AlgorithmEvaluation | None: - """ - The algorithm that scores highest on the species in this list. +# How "best" is decided, in one place so the single-list lookup and the list page cannot +# drift. Ranked on the per-species average rather than the plain share: trap data is +# long-tailed, so a model that only handles the common species would otherwise win. +BEST_MODEL_ORDERING = ("-macro_accuracy", "-micro_accuracy") - Ranked on the per-species average rather than the plain share: trap data is long-tailed, - so a model that only handles the common species would otherwise look like the best one. - """ + +def best_evaluation_for_taxa_list(taxa_list: TaxaList) -> AlgorithmEvaluation | None: + """The algorithm that scores highest on the species in this list.""" # No .distinct(): the join repeats an evaluation once per species it scored in the list, # which cannot change which row sorts first. return ( AlgorithmEvaluation.objects.filter(taxa__taxon__lists=taxa_list) .select_related("algorithm", "occurrence_set") - .order_by("-macro_accuracy", "-micro_accuracy") + .order_by(*BEST_MODEL_ORDERING) .first() ) +def annotate_best_model(taxa_lists: models.QuerySet) -> models.QuerySet: + """ + Attach each list's best-scoring evaluation, for a page of lists. + + Correlated subqueries rather than a lookup per row: every list on the page renders its + best model, so a per-row lookup costs one query each. + """ + best = AlgorithmEvaluation.objects.filter(taxa__taxon__lists=models.OuterRef("pk")).order_by(*BEST_MODEL_ORDERING) + return taxa_lists.annotate( + best_algorithm_id=models.Subquery(best.values("algorithm_id")[:1]), + best_algorithm_name=models.Subquery(best.values("algorithm__name")[:1]), + best_micro_accuracy=models.Subquery(best.values("micro_accuracy")[:1]), + best_macro_accuracy=models.Subquery(best.values("macro_accuracy")[:1]), + best_occurrence_set_name=models.Subquery(best.values("occurrence_set__name")[:1]), + ) + + def performance_for_taxon(taxon: Taxon) -> list[dict[str, typing.Any]]: """ How each algorithm has done on one species. diff --git a/ami/ml/tests.py b/ami/ml/tests.py index 3a8ecc3ef..a871e86ee 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -3833,6 +3833,85 @@ def test_a_set_with_no_project_is_global(self): self.assertIn(shared, OccurrenceSet.objects.for_project(self.project)) +class TestTaxaListQueryCount(APITestCase): + """ + Guard against an N+1 on the taxa-lists page. + + best_model is a per-row lookup, which is the shape that silently turns a page of lists + into one query each. A multi-row fixture is the only way to see it: with a single list + an N+1 and a flat query look identical. + """ + + def setUp(self): + self.project = Project.objects.create(name="Query Count Project") + self.user = User.objects.create_user(email="qcount-lists@example.com", password="testpass123") + self.project.members.add(self.user) + self.client.force_authenticate(user=self.user) + + self.taxon = Taxon.objects.create(name="Countus communis", rank=TaxonRank.SPECIES.name) + algorithm = Algorithm.objects.create(name="Scored model", key="scored-model") + occurrence_set = OccurrenceSet.objects.create(name="Query count set") + evaluation = AlgorithmEvaluation.objects.create( + algorithm=algorithm, + occurrence_set=occurrence_set, + micro_accuracy=0.9, + macro_accuracy=0.9, + occurrences_scored=10, + species_scored=1, + ) + TaxonEvaluation.objects.create( + evaluation=evaluation, taxon=self.taxon, accuracy=0.9, occurrences_scored=10, correct=9 + ) + self._add_list("First list") + + def _add_list(self, name: str) -> None: + taxa_list = TaxaList.objects.create(name=name) + taxa_list.taxa.set([self.taxon]) + taxa_list.projects.add(self.project) + + def _evaluation_queries(self) -> int: + """How many times a page of lists asks the evaluation tables anything.""" + from django.core.cache import caches + from django.db import connection + from django.test.utils import CaptureQueriesContext + + url = f"/api/v2/taxa/lists/?project_id={self.project.pk}&limit=100" + # A warm cachalot cache hides query-scaling regressions, so start cold. + caches["default"].clear() + with CaptureQueriesContext(connection) as ctx: + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + return len([q for q in ctx.captured_queries if "algorithmevaluation" in q["sql"].lower()]) + + def test_the_endpoint_reports_the_best_model(self): + """ + Guards the other half of the annotation: read from a field the viewset forgot to + attach, best_model would quietly go null with the query count still flat. + """ + response = self.client.get(f"/api/v2/taxa/lists/?project_id={self.project.pk}") + best = response.json()["results"][0]["best_model"] + + self.assertIsNotNone(best, "best_model is null; the viewset is not annotating it") + self.assertEqual(best["name"], "Scored model") + self.assertAlmostEqual(best["accuracy_by_species"], 0.9) + self.assertEqual(best["occurrence_set"], "Query count set") + + def test_best_model_does_not_query_once_per_taxa_list(self): + one_list = self._evaluation_queries() + + for n in range(5): + self._add_list(f"List {n}") + + six_lists = self._evaluation_queries() + + self.assertEqual( + six_lists, + one_list, + f"Evaluation queries grew with the number of taxa lists: {one_list} -> {six_lists} " + "(best_model is being looked up per row)", + ) + + class TestPerformanceReporting(TestCase): """ The numbers the model-performance screens read: the best model for a taxa list, and how From 8b30351abb1270d95310a1d8abed5eec84419f32 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Sun, 20 Sep 2026 22:40:00 -0400 Subject: [PATCH 18/31] fix(ml): show a project only the evaluations it is allowed to see A taxon and an algorithm are shared across the whole platform, but an evaluation set belongs to one project. The three screens that report model performance read through the shared side and never scoped back: a species page listed every evaluation that had scored that taxon anywhere, a taxa list ranked its best model against them, and an algorithm's panel showed every set it had ever run on. A draft project's set name and numbers were visible from another project. Scope each read to the sets a project may see -- its own, plus the global ones -- the same shape DetectionViewSet uses for the detections list (#1390). The three call sites resolve the project the way get_summary_data already does, from the request in the serializer context, and the taxa-lists viewset passes the project it has already resolved into the annotation. Give both evaluation models a project_accessor. Without one, visible_for_user() treats "no project relation" as public and returns them unfiltered, which is the mechanism behind all three leaks. Add four tests: one per screen, plus one pinning that a global set is still offered to every project, so the fix cannot quietly become "hide everything". All four fail against the previous behaviour. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/main/api/serializers.py | 10 +++++-- ami/main/api/views.py | 2 +- ami/ml/models/evaluation.py | 6 ++++ ami/ml/reporting.py | 59 +++++++++++++++++++++++++----------- ami/ml/serializers.py | 11 +++++-- ami/ml/tests.py | 60 +++++++++++++++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 23 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 7470e551b..2b2eb7677 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -1112,10 +1112,16 @@ def get_training_crops_ready(self, obj) -> int | None: return getattr(obj, "training_crops_count", None) def get_algorithm_performance(self, obj) -> list[dict]: - """How each scored algorithm has done on this species. Empty until one is evaluated.""" + """ + How each scored algorithm has done on this species. Empty until one is evaluated. + + Scoped to the project being viewed: a taxon is shared across the platform but an + evaluation set is not, so without this the page reports another project's numbers. + """ from ami.ml import reporting - return reporting.performance_for_taxon(obj) + project = get_active_project(request=self.context["request"], required=False) + return reporting.performance_for_taxon(obj, project=project) class CaptureOccurrenceSerializer(DefaultSerializer): diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 434e644d6..32cdd872f 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -2269,8 +2269,8 @@ def get_queryset(self): qs = super().get_queryset() # Annotate with taxa count for better performance qs = qs.annotate(annotated_taxa_count=models.Count("taxa")) - qs = reporting.annotate_best_model(qs) project = self.get_active_project() + qs = reporting.annotate_best_model(qs, project=project) if project: return qs.filter(projects=project) return qs diff --git a/ami/ml/models/evaluation.py b/ami/ml/models/evaluation.py index 4cb85a4f4..1466e4f3e 100644 --- a/ami/ml/models/evaluation.py +++ b/ami/ml/models/evaluation.py @@ -16,6 +16,10 @@ class AlgorithmEvaluation(BaseModel): otherwise recompute the same thing for every row. """ + # Reached through the set it scored. Without this, visible_for_user() leaves these + # rows unfiltered, because the base class treats "no project relation" as public. + project_accessor = "occurrence_set__projects" + algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="evaluations") occurrence_set = models.ForeignKey("main.OccurrenceSet", on_delete=models.CASCADE, related_name="evaluations") job = models.ForeignKey("jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="evaluations") @@ -59,6 +63,8 @@ class TaxonEvaluation(BaseModel): can be queried the other way round: every algorithm's score for a given species. """ + project_accessor = "evaluation__occurrence_set__projects" + evaluation = models.ForeignKey(AlgorithmEvaluation, on_delete=models.CASCADE, related_name="taxa") taxon = models.ForeignKey("main.Taxon", on_delete=models.CASCADE, related_name="evaluations") diff --git a/ami/ml/reporting.py b/ami/ml/reporting.py index 03b3f3e78..fd46a102d 100644 --- a/ami/ml/reporting.py +++ b/ami/ml/reporting.py @@ -10,39 +10,55 @@ from django.db import models -from ami.main.models import TaxaList, Taxon +from ami.main.models import OccurrenceSet, Project, TaxaList, Taxon from ami.ml.models.algorithm import Algorithm from ami.ml.models.evaluation import AlgorithmEvaluation, TaxonEvaluation DEFAULT_EVALUATION_LIMIT = 5 +def visible_sets(project: Project | None): + """ + The evaluation sets a project may be shown: its own, plus the global ones. + + Taxa and algorithms are shared across the platform but evaluation sets are not, so + every read below has to be scoped or it reports another project's numbers. Passing + None means no scoping, which is only for a caller that has already scoped itself. + """ + if project is None: + return None + return OccurrenceSet.objects.for_project(project) + + # How "best" is decided, in one place so the single-list lookup and the list page cannot # drift. Ranked on the per-species average rather than the plain share: trap data is # long-tailed, so a model that only handles the common species would otherwise win. BEST_MODEL_ORDERING = ("-macro_accuracy", "-micro_accuracy") -def best_evaluation_for_taxa_list(taxa_list: TaxaList) -> AlgorithmEvaluation | None: +def best_evaluation_for_taxa_list(taxa_list: TaxaList, project: Project | None = None) -> AlgorithmEvaluation | None: """The algorithm that scores highest on the species in this list.""" # No .distinct(): the join repeats an evaluation once per species it scored in the list, # which cannot change which row sorts first. - return ( - AlgorithmEvaluation.objects.filter(taxa__taxon__lists=taxa_list) - .select_related("algorithm", "occurrence_set") - .order_by(*BEST_MODEL_ORDERING) - .first() - ) + sets = visible_sets(project) + rows = AlgorithmEvaluation.objects.filter(taxa__taxon__lists=taxa_list) + if sets is not None: + rows = rows.filter(occurrence_set__in=sets) + return rows.select_related("algorithm", "occurrence_set").order_by(*BEST_MODEL_ORDERING).first() -def annotate_best_model(taxa_lists: models.QuerySet) -> models.QuerySet: +def annotate_best_model(taxa_lists: models.QuerySet, project: Project | None = None) -> models.QuerySet: """ Attach each list's best-scoring evaluation, for a page of lists. Correlated subqueries rather than a lookup per row: every list on the page renders its best model, so a per-row lookup costs one query each. """ - best = AlgorithmEvaluation.objects.filter(taxa__taxon__lists=models.OuterRef("pk")).order_by(*BEST_MODEL_ORDERING) + best = AlgorithmEvaluation.objects.filter(taxa__taxon__lists=models.OuterRef("pk")) + sets = visible_sets(project) + if sets is not None: + best = best.filter(occurrence_set__in=sets) + best = best.order_by(*BEST_MODEL_ORDERING) return taxa_lists.annotate( best_algorithm_id=models.Subquery(best.values("algorithm_id")[:1]), best_algorithm_name=models.Subquery(best.values("algorithm__name")[:1]), @@ -52,18 +68,18 @@ def annotate_best_model(taxa_lists: models.QuerySet) -> models.QuerySet: ) -def performance_for_taxon(taxon: Taxon) -> list[dict[str, typing.Any]]: +def performance_for_taxon(taxon: Taxon, project: Project | None = None) -> list[dict[str, typing.Any]]: """ How each algorithm has done on one species. The breakdown the taxon page shows: one row per algorithm that has been scored on a set containing this species. """ - rows = ( - TaxonEvaluation.objects.filter(taxon=taxon) - .select_related("evaluation__algorithm", "evaluation__occurrence_set") - .order_by("-accuracy") - ) + rows = TaxonEvaluation.objects.filter(taxon=taxon) + sets = visible_sets(project) + if sets is not None: + rows = rows.filter(evaluation__occurrence_set__in=sets) + rows = rows.select_related("evaluation__algorithm", "evaluation__occurrence_set").order_by("-accuracy") return [ { "algorithm": { @@ -86,14 +102,21 @@ def performance_for_taxon(taxon: Taxon) -> list[dict[str, typing.Any]]: ] -def latest_evaluations(algorithm: Algorithm, limit: int = DEFAULT_EVALUATION_LIMIT) -> list[dict[str, typing.Any]]: +def latest_evaluations( + algorithm: Algorithm, limit: int = DEFAULT_EVALUATION_LIMIT, project: Project | None = None +) -> list[dict[str, typing.Any]]: """ What this algorithm has scored, most recent first. Shown on its details panel. Read through the related manager without filtering or reordering, so a list view that prefetched the evaluations is served from that cache instead of one query per row. """ - rows = sorted(algorithm.evaluations.all(), key=lambda row: row.created_at, reverse=True)[:limit] + rows = sorted(algorithm.evaluations.all(), key=lambda row: row.created_at, reverse=True) + if project is not None: + # Filtered in Python, not SQL, to keep using the prefetched cache above. + allowed = set(visible_sets(project).values_list("pk", flat=True)) + rows = [row for row in rows if row.occurrence_set_id in allowed] + rows = rows[:limit] return [ { "id": row.pk, diff --git a/ami/ml/serializers.py b/ami/ml/serializers.py index 38534c662..6b72f61af 100644 --- a/ami/ml/serializers.py +++ b/ami/ml/serializers.py @@ -60,10 +60,17 @@ class Meta: ] def get_evaluations(self, obj) -> list[dict]: - """How this algorithm has scored on each evaluation set. Empty until one is scored.""" + """ + How this algorithm has scored on each evaluation set. Empty until one is scored. + + Scoped to the project being viewed: an algorithm is shared across the platform but + an evaluation set is not. + """ + from ami.base.views import get_active_project from ami.ml import reporting - return reporting.latest_evaluations(obj) + project = get_active_project(request=self.context["request"], required=False) + return reporting.latest_evaluations(obj, project=project) class AlgorithmNestedSerializer(DefaultSerializer): diff --git a/ami/ml/tests.py b/ami/ml/tests.py index a871e86ee..c4836182f 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -13,6 +13,7 @@ from rest_framework.test import APIRequestFactory, APITestCase from ami.base.serializers import reverse_with_params +from ami.ml import reporting from ami.main.models import ( Classification, Deployment, @@ -3950,6 +3951,65 @@ def _evaluation(self, algorithm: Algorithm, micro: float, macro: float, per_taxo ) return evaluation + def _other_project_evaluation(self): + """An evaluation belonging to a different, private project, scoring the same taxa.""" + other = Project.objects.create(name="Someone Else's Project", draft=True) + their_set = OccurrenceSet.objects.create(name="Their private blind set") + their_set.projects.add(other) + algorithm = self._algorithm("their-model") + evaluation = AlgorithmEvaluation.objects.create( + algorithm=algorithm, + occurrence_set=their_set, + micro_accuracy=0.99, + macro_accuracy=0.99, + occurrences_scored=50, + species_scored=2, + ) + for taxon in (self.common, self.rare): + TaxonEvaluation.objects.create( + evaluation=evaluation, taxon=taxon, accuracy=0.99, occurrences_scored=25, correct=24 + ) + return evaluation + + def test_a_species_does_not_show_another_project_s_evaluations(self): + """ + Taxa are shared across projects, evaluation sets are not. + + Without scoping, a species page lists every evaluation that happened to score that + taxon anywhere on the platform, exposing another project's set name and numbers. + """ + self._other_project_evaluation() + mine = self._evaluation(self._algorithm("my-model"), micro=0.5, macro=0.5, per_taxon={self.common: (5, 10)}) + + rows = reporting.performance_for_taxon(self.common, project=self.project) + + self.assertEqual([r["algorithm"]["id"] for r in rows], [mine.algorithm_id]) + + def test_a_taxa_list_s_best_model_ignores_another_project_s_evaluation(self): + """Their model scores higher, but it was never scored on anything this project can see.""" + self._other_project_evaluation() + mine = self._evaluation(self._algorithm("my-model"), micro=0.5, macro=0.5, per_taxon={self.common: (5, 10)}) + + best = reporting.best_evaluation_for_taxa_list(self.taxa_list, project=self.project) + + self.assertEqual(best, mine) + + def test_an_algorithm_panel_shows_only_sets_this_project_can_see(self): + theirs = self._other_project_evaluation() + + rows = reporting.latest_evaluations(theirs.algorithm, project=self.project) + + self.assertEqual(rows, []) + + def test_a_global_set_is_visible_to_every_project(self): + """A set with no project is the platform-wide benchmark, so it must not be filtered out.""" + mine = self._evaluation(self._algorithm("my-model"), micro=0.5, macro=0.5, per_taxon={self.common: (5, 10)}) + + rows = reporting.performance_for_taxon(self.common, project=self.project) + + # self.occurrence_set belongs to no project, so it is global. + self.assertEqual([r["algorithm"]["id"] for r in rows], [mine.algorithm_id]) + def test_the_best_model_is_the_one_that_handles_the_rare_species(self): """Ranked on the per-species average, or a model that only knows the common one wins.""" from ami.ml import reporting From ad2e799367d8f82134d2b59607eb671828e2b808 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Mon, 21 Sep 2026 11:32:43 -0400 Subject: [PATCH 19/31] chore(lint): fix the import order that was failing the lint job isort groups the ami.ml imports together; the reporting import was added above the ami.main block, which put it in the wrong group. black and flake8 both passed, so only the isort hook caught it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/ml/tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ami/ml/tests.py b/ami/ml/tests.py index c4836182f..63889eb0e 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -13,7 +13,6 @@ from rest_framework.test import APIRequestFactory, APITestCase from ami.base.serializers import reverse_with_params -from ami.ml import reporting from ami.main.models import ( Classification, Deployment, @@ -30,7 +29,7 @@ TaxonRank, group_images_into_events, ) -from ami.ml import training_data +from ami.ml import reporting, training_data from ami.ml.models import ( Algorithm, AlgorithmCategoryMap, From cd3f2b31c7ec14cbd703585546b1005cae6fdc7c Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Mon, 21 Sep 2026 11:32:48 -0400 Subject: [PATCH 20/31] fix(jobs): stop the job form saying "A Evaluate algorithm job" The validation message built its article from a fixed "A", so two of the four job type names read wrong: "A ML pipeline job" and "A Evaluate algorithm job". A vowel check on the first letter would not fix it either, because "ML" starts with a consonant letter but is read as a vowel sound. Make the message plural instead, so it never has to choose: "Evaluate algorithm jobs need algorithm_key in their params." Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/jobs/serializers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ami/jobs/serializers.py b/ami/jobs/serializers.py index ba069c62f..979ba062f 100644 --- a/ami/jobs/serializers.py +++ b/ami/jobs/serializers.py @@ -184,14 +184,16 @@ def validate(self, attrs): missing_fields = [name for name in job_type.required_fields if not attrs.get(name)] if missing_fields: raise serializers.ValidationError( - {f"{name}_id": f"A {job_type.name} job needs a {name}." for name in missing_fields} + # Plural, so the message does not have to choose between "a" and "an": + # two of the four job type names start with a vowel sound. + {f"{name}_id": f"{job_type.name} jobs need a {name}." for name in missing_fields} ) params = attrs.get("params") or {} missing_params = [name for name in job_type.required_params if not params.get(name)] if missing_params: raise serializers.ValidationError( - {"params": f"A {job_type.name} job needs {', '.join(missing_params)} in its params."} + {"params": f"{job_type.name} jobs need {', '.join(missing_params)} in their params."} ) return attrs From cde69753b8b3136a0517182def77b35e740830a4 Mon Sep 17 00:00:00 2001 From: mohamedelabbas1996 Date: Wed, 23 Sep 2026 12:48:11 -0400 Subject: [PATCH 21/31] fix(permissions): let project managers run the retraining jobs, not only superusers Job.check_custom_permission builds its permission codename from the job type key, so a job type with no matching entry on Project matches nothing and has_perm returns False for every non-superuser. The three new types had no entry, which left a project manager able to create a retrain, an evaluation or an embedding run and then unable to start it. Declare the three permissions beside the other run_*_job entries and grant them to MLDataManager; ProjectManager inherits that set. This is the shape used when sync_deployment was granted to the same role (#1357). Backfill existing projects' role groups, as that change did. Both roles are backfilled here, not one: guardian rows are per group, and unlike sync_deployment neither role already held these. New projects pick them up through create_roles_for_project. Add a matrix over the six roles. It would pass on a fresh project either way once roles.py grants the permission, so the backfill was verified separately against a project created long before the migration, and the migration reverses and re-applies cleanly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CnKz4AS4iFrYj1GrgkZQbq --- ami/jobs/tests/test_jobs.py | 58 +++++++++ .../0098_job_permissions_for_retraining.py | 76 +++++++++++ .../0099_grant_retraining_job_permissions.py | 118 ++++++++++++++++++ ami/main/models.py | 6 + ami/users/roles.py | 5 + 5 files changed, 263 insertions(+) create mode 100644 ami/main/migrations/0098_job_permissions_for_retraining.py create mode 100644 ami/main/migrations/0099_grant_retraining_job_permissions.py diff --git a/ami/jobs/tests/test_jobs.py b/ami/jobs/tests/test_jobs.py index 68438e74c..60eca97d2 100644 --- a/ami/jobs/tests/test_jobs.py +++ b/ami/jobs/tests/test_jobs.py @@ -1800,3 +1800,61 @@ def test_browsable_page_renders_number_input(self): html = response.content.decode() self.assertNotIn('