From 81387f8335841ad7ae0b2ec0fa42f4ec119e2cae Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 7 Oct 2026 10:57:02 -0700 Subject: [PATCH 01/26] feat(projects): add station, capture, occurrence and member totals to the project list The list rolls up the cached counts on each deployment with one correlated subquery per column, makes them sortable, and stops prefetching every deployment of every project on the page. Co-Authored-By: Claude Opus 5.5 --- ami/main/api/serializers.py | 10 ++++- ami/main/api/views.py | 9 +++++ ami/main/models.py | 40 +++++++++++++++++++- ami/main/tests.py | 75 +++++++++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 3 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 1ca588980..0f07a7bfa 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -306,10 +306,14 @@ class Meta: class ProjectListSerializer(DefaultSerializer): deployments_count = serializers.IntegerField(read_only=True) + captures_count = serializers.IntegerField(read_only=True) + occurrences_count = serializers.IntegerField(read_only=True) + members_count = serializers.IntegerField(read_only=True) + last_capture_timestamp = serializers.DateTimeField(read_only=True, allow_null=True) class Meta: model = Project - fields = [ + base_fields = [ "id", "name", "description", @@ -320,6 +324,8 @@ class Meta: "image", "draft", ] + # Annotated by ProjectQuerySet.with_overview_counts() on the list action only. + fields = base_fields + ["captures_count", "occurrences_count", "members_count", "last_capture_timestamp"] class ProjectSettingsSerializer(DefaultSerializer): @@ -394,7 +400,7 @@ def get_is_member(self, obj): class Meta: model = Project - fields = ProjectListSerializer.Meta.fields + [ + fields = ProjectListSerializer.Meta.base_fields + [ "deployments", "summary_data", # Conditionally included based on with_charts query param "owner", diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 3d515b408..e6731eda5 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -195,6 +195,11 @@ class ProjectViewSet(DefaultViewSet, ProjectMixin): "last_capture_timestamp", "last_occurrence_updated_at", "last_job_updated_at", + # Annotated on the list action by ProjectQuerySet.with_overview_counts(). + "deployments_count", + "captures_count", + "occurrences_count", + "members_count", ] def get_queryset(self): @@ -212,6 +217,10 @@ def get_queryset(self): # default list stays cheap. Each is a correlated subquery returning one # row via a covering index, and only one is ever added per request. ordering = {field.lstrip("-") for field in self.request.query_params.get("ordering", "").split(",") if field} + if self.action == "list": + # The list shows rolled-up counts instead of nested deployments, so skip + # prefetching every deployment of every project on the page. + qs = qs.prefetch_related(None).with_overview_counts(last_capture="last_capture_timestamp" not in ordering) if "last_capture_timestamp" in ordering: # Live max capture time per project (Index Only Scan on # main_source_proj_ts_desc_idx); kept live rather than reading the diff --git a/ami/main/models.py b/ami/main/models.py index a4c0c82f1..da99a75d1 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -20,7 +20,7 @@ from django.core.exceptions import ValidationError from django.core.files.storage import default_storage from django.db import IntegrityError, models, transaction -from django.db.models import Exists, OuterRef, Q +from django.db.models import Exists, OuterRef, Q, Subquery from django.db.models.fields.files import ImageFieldFile from django.db.models.functions import Coalesce from django.db.models.signals import pre_delete @@ -230,6 +230,44 @@ def filter_by_user(self, user: User): """ return self.filter(members=user) + def with_overview_counts(self, last_capture: bool = True) -> "ProjectQuerySet": + """ + Annotate the per-project totals shown in the projects table. + + Totals are summed from the cached counts on each deployment, so they are only as + fresh as `Deployment.update_calculated_fields()`. Each is one correlated subquery + over the deployments or memberships of a single project. Pass `last_capture=False` + when the caller annotates its own `last_capture_timestamp`. + """ + + def deployment_rollup(aggregate): + return Subquery( + Deployment.objects.filter(project=OuterRef("pk")) + .order_by() + .values("project") + .annotate(value=aggregate) + .values("value")[:1] + ) + + qs = self.annotate( + deployments_count=Coalesce(deployment_rollup(models.Count("pk")), 0), + captures_count=Coalesce(deployment_rollup(models.Sum("captures_count")), 0), + occurrences_count=Coalesce(deployment_rollup(models.Sum("occurrences_count")), 0), + members_count=Coalesce( + Subquery( + UserProjectMembership.objects.filter(project=OuterRef("pk")) + .order_by() + .values("project") + .annotate(value=models.Count("pk")) + .values("value")[:1] + ), + 0, + ), + ) + if last_capture: + qs = qs.annotate(last_capture_timestamp=deployment_rollup(models.Max("last_capture_timestamp"))) + return qs + class ProjectManager(models.Manager.from_queryset(ProjectQuerySet)): pass diff --git a/ami/main/tests.py b/ami/main/tests.py index f24496256..c2914a4ac 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -2361,6 +2361,81 @@ def test_search_matches_name_and_description(self): self.assertEqual(self._search("beetle"), {self.beetle_project.pk}) +class TestProjectListOverviewCounts(APITestCase): + """The project list rolls up deployment cached counts per project, sortable, at a fixed query cost.""" + + endpoint = "/api/v2/projects/" + + def setUp(self) -> None: + self.owner = User.objects.create_user(email="overview-owner@insectai.org") + self.member = User.objects.create_user(email="overview-member@insectai.org") + self.busy = Project.objects.create(name="Busy project", owner=self.owner, create_defaults=False) + self.busy.members.add(self.member) + self.empty = Project.objects.create(name="Empty project", owner=self.owner, create_defaults=False) + last_capture = datetime.datetime(2026, 6, 1, 23, 0) + for name, captures, occurrences, timestamp in [ + ("Station A", 100, 10, last_capture), + ("Station B", 50, 5, last_capture - datetime.timedelta(days=3)), + ]: + deployment = Deployment.objects.create(name=name, project=self.busy) + # Set after create, which recalculates the cached counts from (no) captures. + Deployment.objects.filter(pk=deployment.pk).update( + captures_count=captures, occurrences_count=occurrences, last_capture_timestamp=timestamp + ) + self.last_capture = last_capture + return super().setUp() + + def _rows(self, **params) -> list[dict]: + response = self.client.get(self.endpoint, params) + self.assertEqual(response.status_code, status.HTTP_200_OK) + return response.json()["results"] + + def test_counts_are_rolled_up_from_deployments(self): + rows = {row["id"]: row for row in self._rows()} + busy, empty = rows[self.busy.pk], rows[self.empty.pk] + self.assertEqual( + (busy["deployments_count"], busy["captures_count"], busy["occurrences_count"], busy["members_count"]), + (2, 150, 15, 2), + ) + self.assertEqual(busy["last_capture_timestamp"][:16], self.last_capture.isoformat()[:16]) + self.assertEqual( + (empty["deployments_count"], empty["captures_count"], empty["occurrences_count"], empty["members_count"]), + (0, 0, 0, 1), + ) + self.assertIsNone(empty["last_capture_timestamp"]) + + def test_sort_by_rolled_up_count(self): + ids = [row["id"] for row in self._rows(ordering="-captures_count")] + self.assertEqual(ids[0], self.busy.pk) + ids = [row["id"] for row in self._rows(ordering="captures_count")] + self.assertEqual(ids[-1], self.busy.pk) + + def test_counts_are_read_inside_the_list_query(self): + """Deployments and memberships are only read by the one list query, never once per project. + + The per-row permission lookups the list serializer already makes are outside this check. + Cachalot is off so every query is seen. + """ + from cachalot.api import cachalot_disabled + + for i in range(5): + project = Project.objects.create(name=f"Extra {i}", owner=self.owner, create_defaults=False) + Deployment.objects.create(name=f"Extra station {i}", project=project) + disabled = cachalot_disabled() + disabled.__enter__() + try: + with CaptureQueriesContext(connection) as queries: + rows = self._rows() + finally: + # cachalot_disabled() does not restore itself when the block raises. + disabled.__exit__(None, None, None) + self.assertGreaterEqual(len(rows), 7) + rollup_queries = [ + q["sql"] for q in queries if '"main_deployment"' in q["sql"] or '"main_userprojectmembership"' in q["sql"] + ] + self.assertEqual(len(rollup_queries), 1, rollup_queries) + + class TestProjectPermissions(APITestCase): def _create_project(self, owner, member): self.project = Project.objects.create(name="T Project", description="Test Description", owner=owner) From b200a0122cea513d9f71afa404182e6a35c07f00 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 7 Oct 2026 10:57:06 -0700 Subject: [PATCH 02/26] feat(ui): add a sortable table view to the projects page Co-Authored-By: Claude Opus 5.5 --- ui/src/data-services/models/project.ts | 28 ++++++++ ui/src/pages/projects/project-columns.tsx | 84 +++++++++++++++++++++++ ui/src/pages/projects/projects.tsx | 72 +++++++++++++++++++ ui/src/utils/language.ts | 4 ++ 4 files changed, 188 insertions(+) create mode 100644 ui/src/pages/projects/project-columns.tsx diff --git a/ui/src/data-services/models/project.ts b/ui/src/data-services/models/project.ts index a266cd41c..b220715c9 100644 --- a/ui/src/data-services/models/project.ts +++ b/ui/src/data-services/models/project.ts @@ -18,6 +18,12 @@ export class Project { return this._project.user_permissions.includes(UserPermission.Delete) } + get createdAt(): Date | undefined { + return this._project.created_at + ? new Date(this._project.created_at) + : undefined + } + get canUpdate(): boolean { return this._project.user_permissions.includes(UserPermission.Update) } @@ -46,7 +52,29 @@ export class Project { return this._project.draft } + get lastCaptureDate(): Date | undefined { + return this._project.last_capture_timestamp + ? new Date(this._project.last_capture_timestamp) + : undefined + } + get name(): string { return this._project.name } + + get numCaptures(): number | undefined { + return this._project.captures_count + } + + get numDeployments(): number | undefined { + return this._project.deployments_count + } + + get numMembers(): number | undefined { + return this._project.members_count + } + + get numOccurrences(): number | undefined { + return this._project.occurrences_count + } } diff --git a/ui/src/pages/projects/project-columns.tsx b/ui/src/pages/projects/project-columns.tsx new file mode 100644 index 000000000..f8d6f5d86 --- /dev/null +++ b/ui/src/pages/projects/project-columns.tsx @@ -0,0 +1,84 @@ +import { Project } from 'data-services/models/project' +import { + BasicTableCell, + CellTheme, + DateTableCell, + TableColumn, + TextAlign, +} from 'nova-ui-kit' +import { Link } from 'react-router-dom' +import { APP_ROUTES } from 'utils/constants' +import { STRING, translate } from 'utils/language' + +const countColumn = ({ + id, + name, + sortField, + value, +}: { + id: string + name: string + sortField: string + value: (item: Project) => number | undefined +}): TableColumn => ({ + id, + name, + sortField, + styles: { textAlign: TextAlign.Right }, + renderCell: (item: Project) => , +}) + +export const columns: TableColumn[] = [ + { + id: 'name', + name: translate(STRING.FIELD_LABEL_NAME), + sortField: 'name', + renderCell: (item: Project) => ( + + + + ), + }, + countColumn({ + id: 'deployments', + name: translate(STRING.NAV_ITEM_DEPLOYMENTS), + sortField: 'deployments_count', + value: (item) => item.numDeployments, + }), + countColumn({ + id: 'captures', + name: translate(STRING.FIELD_LABEL_CAPTURES), + sortField: 'captures_count', + value: (item) => item.numCaptures, + }), + countColumn({ + id: 'occurrences', + name: translate(STRING.FIELD_LABEL_OCCURRENCES), + sortField: 'occurrences_count', + value: (item) => item.numOccurrences, + }), + countColumn({ + id: 'members', + name: translate(STRING.FIELD_LABEL_MEMBERS), + sortField: 'members_count', + value: (item) => item.numMembers, + }), + { + id: 'last-capture', + name: translate(STRING.FIELD_LABEL_LAST_CAPTURE), + sortField: 'last_capture_timestamp', + renderCell: (item: Project) => ( + + ), + }, + { + id: 'created-at', + name: translate(STRING.FIELD_LABEL_CREATED_AT), + sortField: 'created_at', + renderCell: (item: Project) => , + }, +] diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index a9816856b..99fabc014 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -1,23 +1,29 @@ import { SearchInput } from 'components/search-input/search-input' import { useProjects } from 'data-services/hooks/projects/useProjects' +import { Grid2X2Icon, TableIcon } from 'lucide-react' import { Button, + ColumnSettings, PageFooter, PageHeader, PaginationBar, SortControl, + Table, Tabs, + ToggleGroup, } from 'nova-ui-kit' import { NewProjectDialog } from 'pages/project-details/new-project-dialog' import { DOCS_LINKS } from 'utils/constants' import { STRING, translate } from 'utils/language' import { usePagination } from 'utils/usePagination' import { useSearch } from 'utils/useSearch' +import { useColumnSettings } from 'utils/useColumnSettings' import { UserPermission } from 'utils/user/types' import { useUser } from 'utils/user/userContext' import { useUserInfo } from 'utils/user/userInfoContext' import { useSelectedView } from 'utils/useSelectedView' import { useSort } from 'utils/useSort' +import { columns } from './project-columns' import { ProjectGallery } from './project-gallery' export const TABS = { @@ -44,6 +50,26 @@ const SORT_FIELDS = [ name: translate(STRING.SORT_JOBS_ACTIVITY), defaultSortOrder: 'desc' as const, }, + { + id: 'deployments_count', + name: translate(STRING.NAV_ITEM_DEPLOYMENTS), + defaultSortOrder: 'desc' as const, + }, + { + id: 'captures_count', + name: translate(STRING.FIELD_LABEL_CAPTURES), + defaultSortOrder: 'desc' as const, + }, + { + id: 'occurrences_count', + name: translate(STRING.FIELD_LABEL_OCCURRENCES), + defaultSortOrder: 'desc' as const, + }, + { + id: 'members_count', + name: translate(STRING.FIELD_LABEL_MEMBERS), + defaultSortOrder: 'desc' as const, + }, ] export const Projects = () => { @@ -51,6 +77,19 @@ export const Projects = () => { const { userInfo } = useUserInfo() const { selectedView: selectedTab, setSelectedView: setSelectedTab } = useSelectedView(user.loggedIn ? TABS.MY_PROJECTS : TABS.ALL_PROJECTS) + const { selectedView: layout, setSelectedView: setLayout } = useSelectedView( + 'gallery', + 'layout' + ) + const { columnSettings, setColumnSettings } = useColumnSettings('projects', { + name: true, + deployments: true, + captures: true, + occurrences: true, + members: true, + 'last-capture': true, + 'created-at': false, + }) const { sort, setSort } = useSort() const { pagination, setPage } = usePagination({ perPage: 40 }) const { search, setSearch } = useSearch() @@ -98,6 +137,22 @@ export const Projects = () => { value={search} onChange={setSearch} /> + {canCreate ? : null} ({ @@ -107,6 +162,13 @@ export const Projects = () => { setSort={setSort} sort={sort} /> + {layout === 'table' ? ( + + ) : null} {projects && projects.length === 0 && canCreate && !search ? (
@@ -126,6 +188,16 @@ export const Projects = () => {
+ ) : layout === 'table' ? ( + !!columnSettings[column.id])} + error={error} + isLoading={isLoading} + items={projects} + onSortSettingsChange={setSort} + sortable + sortSettings={sort} + /> ) : ( Date: Wed, 7 Oct 2026 12:56:54 -0700 Subject: [PATCH 03/26] feat(projects): load project totals and recent activity only for the projects table The project list adds station, capture, occurrence and member totals and the latest capture, occurrence update and job update only when asked with ?with_counts, or when sorted by one of them. The gallery request is the same as before. The three activity dates are read live through index-only scans. Co-Authored-By: Claude Opus 5.5 --- ami/main/api/serializers.py | 16 ++++++++-- ami/main/api/views.py | 62 ++++++++++--------------------------- ami/main/models.py | 29 ++++++++++++----- ami/main/tests.py | 26 ++++++++++------ 4 files changed, 68 insertions(+), 65 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 0f07a7bfa..54f9d6643 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -309,7 +309,9 @@ class ProjectListSerializer(DefaultSerializer): captures_count = serializers.IntegerField(read_only=True) occurrences_count = serializers.IntegerField(read_only=True) members_count = serializers.IntegerField(read_only=True) - last_capture_timestamp = serializers.DateTimeField(read_only=True, allow_null=True) + last_capture_timestamp = serializers.DateTimeField(read_only=True) + last_occurrence_updated_at = serializers.DateTimeField(read_only=True) + last_job_updated_at = serializers.DateTimeField(read_only=True) class Meta: model = Project @@ -324,8 +326,16 @@ class Meta: "image", "draft", ] - # Annotated by ProjectQuerySet.with_overview_counts() on the list action only. - fields = base_fields + ["captures_count", "occurrences_count", "members_count", "last_capture_timestamp"] + # Annotated by ProjectQuerySet.with_overview_counts() and with_recent_activity() when the + # list asks for them, and left out of the response otherwise. + fields = base_fields + [ + "captures_count", + "occurrences_count", + "members_count", + "last_capture_timestamp", + "last_occurrence_updated_at", + "last_job_updated_at", + ] class ProjectSettingsSerializer(DefaultSerializer): diff --git a/ami/main/api/views.py b/ami/main/api/views.py index e6731eda5..b0a794814 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -7,7 +7,7 @@ from django.core import exceptions from django.core.files.storage import default_storage from django.db import models -from django.db.models import OuterRef, Prefetch, Q, Subquery +from django.db.models import OuterRef, Prefetch, Q from django.db.models.query import QuerySet from django.forms import BooleanField, CharField, IntegerField from django.shortcuts import get_object_or_404, redirect @@ -38,6 +38,7 @@ from ami.main.models_future.occurrence import model_agreement_for_project, top_identifiers_for_project from ami.ml.models.algorithm import Algorithm from ami.ml.serializers import AlgorithmSerializer +from ami.utils.fields import url_boolean_param from ami.utils.requests import get_default_classification_threshold from ami.utils.storages import ConnectionTestResult @@ -185,22 +186,18 @@ class ProjectViewSet(DefaultViewSet, ProjectMixin): pagination_class = ProjectPagination permission_classes = [ObjectPermission] search_fields = ["name", "description"] - ordering_fields = [ - "name", - "created_at", - "updated_at", - # The three below are not Project fields; get_queryset annotates them on - # demand (see below). last_capture_timestamp mirrors the DeploymentViewSet - # ordering of the same name, but is a per-project rollup of capture times. - "last_capture_timestamp", - "last_occurrence_updated_at", - "last_job_updated_at", - # Annotated on the list action by ProjectQuerySet.with_overview_counts(). + # Not Project fields: annotated on the list action by ProjectQuerySet.with_overview_counts() + # and with_recent_activity(), which run only when asked for (see get_queryset). + overview_fields = [ "deployments_count", "captures_count", "occurrences_count", "members_count", + "last_capture_timestamp", + "last_occurrence_updated_at", + "last_job_updated_at", ] + ordering_fields = ["name", "created_at", "updated_at", *overview_fields] def get_queryset(self): qs: ProjectQuerySet = super().get_queryset() # type: ignore @@ -213,41 +210,14 @@ def get_queryset(self): if user: qs = qs.filter_by_user(user) - # Annotate "recent activity" fields only when sorting by them, so the - # default list stays cheap. Each is a correlated subquery returning one - # row via a covering index, and only one is ever added per request. + # The totals and activity dates feed the projects table, so they are added only when + # the table asks for them with ?with_counts, or when the list is sorted by one of them. ordering = {field.lstrip("-") for field in self.request.query_params.get("ordering", "").split(",") if field} - if self.action == "list": - # The list shows rolled-up counts instead of nested deployments, so skip - # prefetching every deployment of every project on the page. - qs = qs.prefetch_related(None).with_overview_counts(last_capture="last_capture_timestamp" not in ordering) - if "last_capture_timestamp" in ordering: - # Live max capture time per project (Index Only Scan on - # main_source_proj_ts_desc_idx); kept live rather than reading the - # denormalized Deployment field so the sort never lags ingestion. - # timestamp is nullable, and DESC sorts NULLs first, so exclude them - # explicitly — otherwise a single undated capture masks the real max. - qs = qs.annotate( - last_capture_timestamp=Subquery( - SourceImage.objects.filter(project=OuterRef("pk"), timestamp__isnull=False) - .order_by("-timestamp") - .values("timestamp")[:1] - ) - ) - if "last_occurrence_updated_at" in ordering: - qs = qs.annotate( - last_occurrence_updated_at=Subquery( - Occurrence.objects.filter(project=OuterRef("pk")).order_by("-updated_at").values("updated_at")[:1] - ) - ) - if "last_job_updated_at" in ordering: - from ami.jobs.models import Job - - qs = qs.annotate( - last_job_updated_at=Subquery( - Job.objects.filter(project=OuterRef("pk")).order_by("-updated_at").values("updated_at")[:1] - ) - ) + if self.action == "list" and ( + url_boolean_param(self.request, "with_counts") or ordering & set(self.overview_fields) + ): + # The annotated deployments_count replaces counting prefetched deployments, so skip the prefetch. + qs = qs.prefetch_related(None).with_overview_counts().with_recent_activity() return qs def get_serializer_class(self): diff --git a/ami/main/models.py b/ami/main/models.py index da99a75d1..16c5c615f 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -230,14 +230,13 @@ def filter_by_user(self, user: User): """ return self.filter(members=user) - def with_overview_counts(self, last_capture: bool = True) -> "ProjectQuerySet": + def with_overview_counts(self) -> "ProjectQuerySet": """ Annotate the per-project totals shown in the projects table. Totals are summed from the cached counts on each deployment, so they are only as fresh as `Deployment.update_calculated_fields()`. Each is one correlated subquery - over the deployments or memberships of a single project. Pass `last_capture=False` - when the caller annotates its own `last_capture_timestamp`. + over the deployments or memberships of a single project. """ def deployment_rollup(aggregate): @@ -249,7 +248,7 @@ def deployment_rollup(aggregate): .values("value")[:1] ) - qs = self.annotate( + return self.annotate( deployments_count=Coalesce(deployment_rollup(models.Count("pk")), 0), captures_count=Coalesce(deployment_rollup(models.Sum("captures_count")), 0), occurrences_count=Coalesce(deployment_rollup(models.Sum("occurrences_count")), 0), @@ -264,9 +263,25 @@ def deployment_rollup(aggregate): 0, ), ) - if last_capture: - qs = qs.annotate(last_capture_timestamp=deployment_rollup(models.Max("last_capture_timestamp"))) - return qs + + def with_recent_activity(self) -> "ProjectQuerySet": + """ + Annotate when each project last received a capture, an occurrence update and a job update. + + Read live rather than from cached fields so the projects table never lags ingestion. + Each is a correlated subquery answered by an index-only scan on a (project, time) index. + """ + from ami.jobs.models import Job + + def latest(queryset: models.QuerySet, field: str) -> Subquery: + return Subquery(queryset.filter(project=OuterRef("pk")).order_by(f"-{field}").values(field)[:1]) + + return self.annotate( + # timestamp is nullable and DESC sorts NULLs first, so undated captures are excluded. + last_capture_timestamp=latest(SourceImage.objects.filter(timestamp__isnull=False), "timestamp"), + last_occurrence_updated_at=latest(Occurrence.objects.all(), "updated_at"), + last_job_updated_at=latest(Job.objects.all(), "updated_at"), + ) class ProjectManager(models.Manager.from_queryset(ProjectQuerySet)): diff --git a/ami/main/tests.py b/ami/main/tests.py index c2914a4ac..a2fa516a3 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -2362,7 +2362,7 @@ def test_search_matches_name_and_description(self): class TestProjectListOverviewCounts(APITestCase): - """The project list rolls up deployment cached counts per project, sortable, at a fixed query cost.""" + """The projects table gets per-project totals and activity dates, sortable, at a fixed query cost.""" endpoint = "/api/v2/projects/" @@ -2378,15 +2378,14 @@ def setUp(self) -> None: ("Station B", 50, 5, last_capture - datetime.timedelta(days=3)), ]: deployment = Deployment.objects.create(name=name, project=self.busy) + SourceImage.objects.create(deployment=deployment, project=self.busy, timestamp=timestamp, path=name) # Set after create, which recalculates the cached counts from (no) captures. - Deployment.objects.filter(pk=deployment.pk).update( - captures_count=captures, occurrences_count=occurrences, last_capture_timestamp=timestamp - ) + Deployment.objects.filter(pk=deployment.pk).update(captures_count=captures, occurrences_count=occurrences) self.last_capture = last_capture return super().setUp() def _rows(self, **params) -> list[dict]: - response = self.client.get(self.endpoint, params) + response = self.client.get(self.endpoint, {"with_counts": "true", **params}) self.assertEqual(response.status_code, status.HTTP_200_OK) return response.json()["results"] @@ -2403,6 +2402,16 @@ def test_counts_are_rolled_up_from_deployments(self): (0, 0, 0, 1), ) self.assertIsNone(empty["last_capture_timestamp"]) + self.assertIsNone(empty["last_occurrence_updated_at"]) + self.assertIsNone(empty["last_job_updated_at"]) + + def test_totals_are_left_out_unless_asked_for(self): + response = self.client.get(self.endpoint) + self.assertEqual(response.status_code, status.HTTP_200_OK) + row = next(row for row in response.json()["results"] if row["id"] == self.busy.pk) + self.assertEqual(row["deployments_count"], 2) + for field in ["captures_count", "members_count", "last_capture_timestamp", "last_job_updated_at"]: + self.assertNotIn(field, row) def test_sort_by_rolled_up_count(self): ids = [row["id"] for row in self._rows(ordering="-captures_count")] @@ -2411,7 +2420,7 @@ def test_sort_by_rolled_up_count(self): self.assertEqual(ids[-1], self.busy.pk) def test_counts_are_read_inside_the_list_query(self): - """Deployments and memberships are only read by the one list query, never once per project. + """The tables behind the totals and dates are only read by the one list query, never once per project. The per-row permission lookups the list serializer already makes are outside this check. Cachalot is off so every query is seen. @@ -2430,9 +2439,8 @@ def test_counts_are_read_inside_the_list_query(self): # cachalot_disabled() does not restore itself when the block raises. disabled.__exit__(None, None, None) self.assertGreaterEqual(len(rows), 7) - rollup_queries = [ - q["sql"] for q in queries if '"main_deployment"' in q["sql"] or '"main_userprojectmembership"' in q["sql"] - ] + tables = ['"main_deployment"', '"main_userprojectmembership"', '"main_sourceimage"', '"jobs_job"'] + rollup_queries = [q["sql"] for q in queries if any(table in q["sql"] for table in tables)] self.assertEqual(len(rollup_queries), 1, rollup_queries) From c81d4ea0e7a6c04ca6cc77a6152a7b8dcecd9be4 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 7 Oct 2026 12:58:21 -0700 Subject: [PATCH 04/26] feat(ui): show cover images and recent activity in the projects table The table gains a cover image column and sortable columns for the latest occurrence update and job update, which replace the Sort by dropdown. Only the table asks the server for totals, so the gallery stays as light as before. Co-Authored-By: Claude Opus 5.5 --- ui/src/data-services/models/project.ts | 12 +++++ ui/src/pages/projects/project-columns.tsx | 29 ++++++++++++ ui/src/pages/projects/projects.tsx | 55 +++-------------------- ui/src/utils/language.ts | 10 ++--- 4 files changed, 50 insertions(+), 56 deletions(-) diff --git a/ui/src/data-services/models/project.ts b/ui/src/data-services/models/project.ts index b220715c9..432d1b5dd 100644 --- a/ui/src/data-services/models/project.ts +++ b/ui/src/data-services/models/project.ts @@ -58,6 +58,18 @@ export class Project { : undefined } + get lastJobUpdateDate(): Date | undefined { + return this._project.last_job_updated_at + ? new Date(this._project.last_job_updated_at) + : undefined + } + + get lastOccurrenceUpdateDate(): Date | undefined { + return this._project.last_occurrence_updated_at + ? new Date(this._project.last_occurrence_updated_at) + : undefined + } + get name(): string { return this._project.name } diff --git a/ui/src/pages/projects/project-columns.tsx b/ui/src/pages/projects/project-columns.tsx index f8d6f5d86..cbb187f0a 100644 --- a/ui/src/pages/projects/project-columns.tsx +++ b/ui/src/pages/projects/project-columns.tsx @@ -3,6 +3,8 @@ import { BasicTableCell, CellTheme, DateTableCell, + ImageCellTheme, + ImageTableCell, TableColumn, TextAlign, } from 'nova-ui-kit' @@ -29,6 +31,17 @@ const countColumn = ({ }) export const columns: TableColumn[] = [ + { + id: 'image', + name: translate(STRING.FIELD_LABEL_IMAGE), + renderCell: (item: Project) => ( + + ), + }, { id: 'name', name: translate(STRING.FIELD_LABEL_NAME), @@ -75,6 +88,22 @@ export const columns: TableColumn[] = [ ), }, + { + id: 'last-occurrence-update', + name: translate(STRING.FIELD_LABEL_LAST_OCCURRENCE_UPDATE), + sortField: 'last_occurrence_updated_at', + renderCell: (item: Project) => ( + + ), + }, + { + id: 'last-job-update', + name: translate(STRING.FIELD_LABEL_LAST_JOB_UPDATE), + sortField: 'last_job_updated_at', + renderCell: (item: Project) => ( + + ), + }, { id: 'created-at', name: translate(STRING.FIELD_LABEL_CREATED_AT), diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index 99fabc014..0da9ab9f4 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -7,7 +7,6 @@ import { PageFooter, PageHeader, PaginationBar, - SortControl, Table, Tabs, ToggleGroup, @@ -31,47 +30,6 @@ export const TABS = { ALL_PROJECTS: 'all-projects', } -const SORT_FIELDS = [ - { id: 'name', name: translate(STRING.FIELD_LABEL_NAME) }, - { id: 'created_at', name: translate(STRING.FIELD_LABEL_CREATED_AT) }, - { id: 'updated_at', name: translate(STRING.FIELD_LABEL_UPDATED_AT) }, - { - id: 'last_capture_timestamp', - name: translate(STRING.SORT_RECENT_CAPTURES), - defaultSortOrder: 'desc' as const, - }, - { - id: 'last_occurrence_updated_at', - name: translate(STRING.SORT_OCCURRENCE_UPDATES), - defaultSortOrder: 'desc' as const, - }, - { - id: 'last_job_updated_at', - name: translate(STRING.SORT_JOBS_ACTIVITY), - defaultSortOrder: 'desc' as const, - }, - { - id: 'deployments_count', - name: translate(STRING.NAV_ITEM_DEPLOYMENTS), - defaultSortOrder: 'desc' as const, - }, - { - id: 'captures_count', - name: translate(STRING.FIELD_LABEL_CAPTURES), - defaultSortOrder: 'desc' as const, - }, - { - id: 'occurrences_count', - name: translate(STRING.FIELD_LABEL_OCCURRENCES), - defaultSortOrder: 'desc' as const, - }, - { - id: 'members_count', - name: translate(STRING.FIELD_LABEL_MEMBERS), - defaultSortOrder: 'desc' as const, - }, -] - export const Projects = () => { const { user } = useUser() const { userInfo } = useUserInfo() @@ -82,12 +40,15 @@ export const Projects = () => { 'layout' ) const { columnSettings, setColumnSettings } = useColumnSettings('projects', { + image: true, name: true, deployments: true, captures: true, occurrences: true, members: true, 'last-capture': true, + 'last-occurrence-update': true, + 'last-job-update': true, 'created-at': false, }) const { sort, setSort } = useSort() @@ -98,6 +59,8 @@ export const Projects = () => { ? [{ field: 'user_id', value: userInfo?.id }] : []), ...(search ? [{ field: 'search', value: search }] : []), + // Totals and activity dates are only shown, and only computed, in the table. + ...(layout === 'table' ? [{ field: 'with_counts', value: 'true' }] : []), ] const { projects, total, userPermissions, isLoading, isFetching, error } = useProjects({ pagination, filters, sort }) @@ -154,14 +117,6 @@ export const Projects = () => { onValueChange={setLayout} /> {canCreate ? : null} - ({ - ...field, - sortField: field.id, - }))} - setSort={setSort} - sort={sort} - /> {layout === 'table' ? ( Date: Thu, 8 Oct 2026 00:57:58 -0700 Subject: [PATCH 05/26] refactor: name the projects overview fields once and show team size only to those who can see the team The totals and activity dates were listed separately in the viewset, the serializer and two queryset methods. ProjectQuerySet.OVERVIEW_FIELDS now names them next to the single with_overview() method that annotates them, the viewset and serializer read that list, and a test checks it against the annotations. A shared _per_project() helper builds each rollup. members_count now follows the permission on the team list: it is null unless the user is a member of the project or a superuser, so signed-out visitors no longer see team sizes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ami/main/api/serializers.py | 16 +++----- ami/main/api/views.py | 23 +++--------- ami/main/models.py | 75 ++++++++++++++++++------------------- ami/main/tests.py | 24 +++++++++++- 4 files changed, 70 insertions(+), 68 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 54f9d6643..90a2d57e6 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -28,6 +28,7 @@ Occurrence, Page, Project, + ProjectQuerySet, ProjectSettingsMixin, S3StorageSource, Site, @@ -320,22 +321,14 @@ class Meta: "name", "description", "details", - "deployments_count", "created_at", "updated_at", "image", "draft", ] - # Annotated by ProjectQuerySet.with_overview_counts() and with_recent_activity() when the - # list asks for them, and left out of the response otherwise. - fields = base_fields + [ - "captures_count", - "occurrences_count", - "members_count", - "last_capture_timestamp", - "last_occurrence_updated_at", - "last_job_updated_at", - ] + # Annotated by ProjectQuerySet.with_overview() when the list asks for them, and left out + # of the response otherwise, except deployments_count, which falls back to the model method. + fields = base_fields + list(ProjectQuerySet.OVERVIEW_FIELDS) class ProjectSettingsSerializer(DefaultSerializer): @@ -411,6 +404,7 @@ def get_is_member(self, obj): class Meta: model = Project fields = ProjectListSerializer.Meta.base_fields + [ + "deployments_count", "deployments", "summary_data", # Conditionally included based on with_charts query param "owner", diff --git a/ami/main/api/views.py b/ami/main/api/views.py index b0a794814..cee0acacc 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -158,9 +158,9 @@ class ProjectPagination(LimitOffsetPaginationWithPermissions): default_limit = 40 def get_count(self, queryset): - # The recent-activity orderings annotate correlated subqueries onto the - # queryset. They don't change the row count, so strip them (and ordering) - # before counting to keep the pagination COUNT query cheap. + # The overview totals and dates are correlated subqueries. They don't change + # the row count, so strip them (and ordering) before counting to keep the + # pagination COUNT query cheap. return super().get_count(queryset.order_by().values("pk")) @@ -186,18 +186,7 @@ class ProjectViewSet(DefaultViewSet, ProjectMixin): pagination_class = ProjectPagination permission_classes = [ObjectPermission] search_fields = ["name", "description"] - # Not Project fields: annotated on the list action by ProjectQuerySet.with_overview_counts() - # and with_recent_activity(), which run only when asked for (see get_queryset). - overview_fields = [ - "deployments_count", - "captures_count", - "occurrences_count", - "members_count", - "last_capture_timestamp", - "last_occurrence_updated_at", - "last_job_updated_at", - ] - ordering_fields = ["name", "created_at", "updated_at", *overview_fields] + ordering_fields = ["name", "created_at", "updated_at", *ProjectQuerySet.OVERVIEW_FIELDS] def get_queryset(self): qs: ProjectQuerySet = super().get_queryset() # type: ignore @@ -214,10 +203,10 @@ def get_queryset(self): # the table asks for them with ?with_counts, or when the list is sorted by one of them. ordering = {field.lstrip("-") for field in self.request.query_params.get("ordering", "").split(",") if field} if self.action == "list" and ( - url_boolean_param(self.request, "with_counts") or ordering & set(self.overview_fields) + url_boolean_param(self.request, "with_counts") or ordering & set(ProjectQuerySet.OVERVIEW_FIELDS) ): # The annotated deployments_count replaces counting prefetched deployments, so skip the prefetch. - qs = qs.prefetch_related(None).with_overview_counts().with_recent_activity() + qs = qs.prefetch_related(None).with_overview(self.request.user) return qs def get_serializer_class(self): diff --git a/ami/main/models.py b/ami/main/models.py index 16c5c615f..f2283c848 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -223,60 +223,59 @@ def get_or_create_default_project(user: User) -> "Project": return project +def _per_project(queryset: models.QuerySet, aggregate) -> Subquery: + """One aggregate over the rows of `queryset` that belong to the outer project, for a Project annotation.""" + return Subquery( + queryset.filter(project=OuterRef("pk")) + .order_by() + .values("project") + .annotate(value=aggregate) + .values("value")[:1] + ) + + class ProjectQuerySet(BaseQuerySet): + # Annotated by with_overview(). The projects API sorts and serializes by these names. + OVERVIEW_FIELDS = ( + "deployments_count", + "captures_count", + "occurrences_count", + "members_count", + "last_capture_timestamp", + "last_occurrence_updated_at", + "last_job_updated_at", + ) + def filter_by_user(self, user: User): """ Filters projects to include only those where the given user is a member. """ return self.filter(members=user) - def with_overview_counts(self) -> "ProjectQuerySet": + def with_overview(self, user: User | AnonymousUser) -> "ProjectQuerySet": """ - Annotate the per-project totals shown in the projects table. + Annotate the totals and latest activity shown in the projects table, named in OVERVIEW_FIELDS. - Totals are summed from the cached counts on each deployment, so they are only as - fresh as `Deployment.update_calculated_fields()`. Each is one correlated subquery - over the deployments or memberships of a single project. - """ - - def deployment_rollup(aggregate): - return Subquery( - Deployment.objects.filter(project=OuterRef("pk")) - .order_by() - .values("project") - .annotate(value=aggregate) - .values("value")[:1] - ) - - return self.annotate( - deployments_count=Coalesce(deployment_rollup(models.Count("pk")), 0), - captures_count=Coalesce(deployment_rollup(models.Sum("captures_count")), 0), - occurrences_count=Coalesce(deployment_rollup(models.Sum("occurrences_count")), 0), - members_count=Coalesce( - Subquery( - UserProjectMembership.objects.filter(project=OuterRef("pk")) - .order_by() - .values("project") - .annotate(value=models.Count("pk")) - .values("value")[:1] - ), - 0, - ), - ) - - def with_recent_activity(self) -> "ProjectQuerySet": - """ - Annotate when each project last received a capture, an occurrence update and a job update. - - Read live rather than from cached fields so the projects table never lags ingestion. - Each is a correlated subquery answered by an index-only scan on a (project, time) index. + Totals are summed from the cached counts on each deployment, so they are only as fresh as + `Deployment.update_calculated_fields()`. Activity dates are read live, each from a (project, time) + index. `members_count` follows the permission on the team list: it is null unless the user is + a member of the project or a superuser. """ from ami.jobs.models import Job def latest(queryset: models.QuerySet, field: str) -> Subquery: return Subquery(queryset.filter(project=OuterRef("pk")).order_by(f"-{field}").values(field)[:1]) + members_count = Coalesce(_per_project(UserProjectMembership.objects.all(), models.Count("pk")), 0) + if not user.is_superuser: + is_member = Exists(UserProjectMembership.objects.filter(project=OuterRef("pk"), user_id=user.pk)) + members_count = models.Case(models.When(is_member, then=members_count), default=None) + return self.annotate( + deployments_count=Coalesce(_per_project(Deployment.objects.all(), models.Count("pk")), 0), + captures_count=Coalesce(_per_project(Deployment.objects.all(), models.Sum("captures_count")), 0), + occurrences_count=Coalesce(_per_project(Deployment.objects.all(), models.Sum("occurrences_count")), 0), + members_count=members_count, # timestamp is nullable and DESC sorts NULLs first, so undated captures are excluded. last_capture_timestamp=latest(SourceImage.objects.filter(timestamp__isnull=False), "timestamp"), last_occurrence_updated_at=latest(Occurrence.objects.all(), "updated_at"), diff --git a/ami/main/tests.py b/ami/main/tests.py index a2fa516a3..d07f0dfd0 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -31,6 +31,7 @@ Identification, Occurrence, Project, + ProjectQuerySet, S3StorageSource, Site, SourceImage, @@ -2384,13 +2385,14 @@ def setUp(self) -> None: self.last_capture = last_capture return super().setUp() - def _rows(self, **params) -> list[dict]: + def _rows(self, user: User | None = None, **params) -> list[dict]: + self.client.force_authenticate(user) response = self.client.get(self.endpoint, {"with_counts": "true", **params}) self.assertEqual(response.status_code, status.HTTP_200_OK) return response.json()["results"] def test_counts_are_rolled_up_from_deployments(self): - rows = {row["id"]: row for row in self._rows()} + rows = {row["id"]: row for row in self._rows(self.owner)} busy, empty = rows[self.busy.pk], rows[self.empty.pk] self.assertEqual( (busy["deployments_count"], busy["captures_count"], busy["occurrences_count"], busy["members_count"]), @@ -2405,6 +2407,24 @@ def test_counts_are_rolled_up_from_deployments(self): self.assertIsNone(empty["last_occurrence_updated_at"]) self.assertIsNone(empty["last_job_updated_at"]) + def test_team_size_is_only_shown_to_those_who_can_see_the_team(self): + """members_count is null for projects whose team list the user may not open.""" + superuser = User.objects.create_superuser(email="overview-admin@insectai.org", password="unused") + for user, expected in [ + (None, (None, None)), + (self.member, (2, None)), + (self.owner, (2, 1)), + (superuser, (2, 1)), + ]: + rows = {row["id"]: row for row in self._rows(user)} + with self.subTest(user=user): + self.assertEqual((rows[self.busy.pk]["members_count"], rows[self.empty.pk]["members_count"]), expected) + + def test_overview_fields_name_every_annotation(self): + """The API sorts and serializes by OVERVIEW_FIELDS, so it must match what with_overview() adds.""" + annotated = Project.objects.with_overview(self.owner).query.annotations + self.assertEqual(set(annotated), set(ProjectQuerySet.OVERVIEW_FIELDS)) + def test_totals_are_left_out_unless_asked_for(self): response = self.client.get(self.endpoint) self.assertEqual(response.status_code, status.HTTP_200_OK) From 7e9c242bc0e0d9a402f91ddab959fd0db0318a03 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 01:16:18 -0700 Subject: [PATCH 06/26] test: keep one projects overview test per guarantee The field-list test repeated the totals test, which already reads every overview field by name. The sort test now sorts without asking for totals, the path that fails if the annotations are not added, and checks one direction instead of two. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ami/main/tests.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/ami/main/tests.py b/ami/main/tests.py index d07f0dfd0..d2e490f87 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -31,7 +31,6 @@ Identification, Occurrence, Project, - ProjectQuerySet, S3StorageSource, Site, SourceImage, @@ -2420,11 +2419,6 @@ def test_team_size_is_only_shown_to_those_who_can_see_the_team(self): with self.subTest(user=user): self.assertEqual((rows[self.busy.pk]["members_count"], rows[self.empty.pk]["members_count"]), expected) - def test_overview_fields_name_every_annotation(self): - """The API sorts and serializes by OVERVIEW_FIELDS, so it must match what with_overview() adds.""" - annotated = Project.objects.with_overview(self.owner).query.annotations - self.assertEqual(set(annotated), set(ProjectQuerySet.OVERVIEW_FIELDS)) - def test_totals_are_left_out_unless_asked_for(self): response = self.client.get(self.endpoint) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -2433,11 +2427,11 @@ def test_totals_are_left_out_unless_asked_for(self): for field in ["captures_count", "members_count", "last_capture_timestamp", "last_job_updated_at"]: self.assertNotIn(field, row) - def test_sort_by_rolled_up_count(self): - ids = [row["id"] for row in self._rows(ordering="-captures_count")] - self.assertEqual(ids[0], self.busy.pk) - ids = [row["id"] for row in self._rows(ordering="captures_count")] - self.assertEqual(ids[-1], self.busy.pk) + def test_sorting_by_a_total_works_without_asking_for_totals(self): + """A sort carried over from the table to the gallery still gets the totals it orders by.""" + response = self.client.get(self.endpoint, {"ordering": "-captures_count"}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json()["results"][0]["id"], self.busy.pk) def test_counts_are_read_inside_the_list_query(self): """The tables behind the totals and dates are only read by the one list query, never once per project. From 2d000686a97e72b279d6a0052b413eab99ae151b Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 01:23:08 -0700 Subject: [PATCH 07/26] feat(projects): add a taxa count to the projects table and trim the default columns The table shows each project's number of taxa, counted the way the project summary counts them: distinct identifications after the project's own default score threshold and taxa filters. Because every project has its own filters, the count is one grouped query over the page of projects rather than an annotation, and the column cannot be sorted. By default the table now shows the cover image, name, stations, captures, occurrences, taxa and last capture; members, the two update dates and the creation date can be turned on in the column settings. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ami/main/api/serializers.py | 8 ++++--- ami/main/api/views.py | 7 ++++++ ami/main/models.py | 28 +++++++++++++++++++++++ ami/main/tests.py | 28 ++++++++++++++++++++++- ui/src/data-services/models/project.ts | 4 ++++ ui/src/pages/projects/project-columns.tsx | 8 ++++++- ui/src/pages/projects/projects.tsx | 7 +++--- 7 files changed, 82 insertions(+), 8 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 90a2d57e6..50d41cf61 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -313,6 +313,8 @@ class ProjectListSerializer(DefaultSerializer): last_capture_timestamp = serializers.DateTimeField(read_only=True) last_occurrence_updated_at = serializers.DateTimeField(read_only=True) last_job_updated_at = serializers.DateTimeField(read_only=True) + # Set on the page by add_taxa_counts(); Project.taxa_count() counts something else. + taxa_count = serializers.IntegerField(read_only=True, source="observed_taxa_count") class Meta: model = Project @@ -326,9 +328,9 @@ class Meta: "image", "draft", ] - # Annotated by ProjectQuerySet.with_overview() when the list asks for them, and left out - # of the response otherwise, except deployments_count, which falls back to the model method. - fields = base_fields + list(ProjectQuerySet.OVERVIEW_FIELDS) + # Added when the list asks for them with ?with_counts, and left out of the response otherwise, + # except deployments_count, which falls back to the model method. + fields = base_fields + list(ProjectQuerySet.OVERVIEW_FIELDS) + ["taxa_count"] class ProjectSettingsSerializer(DefaultSerializer): diff --git a/ami/main/api/views.py b/ami/main/api/views.py index cee0acacc..0711b17f2 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -63,6 +63,7 @@ Taxon, TaxonRank, User, + add_taxa_counts, update_detection_counts, verified_taxon_counts, ) @@ -209,6 +210,12 @@ def get_queryset(self): qs = qs.prefetch_related(None).with_overview(self.request.user) return qs + def paginate_queryset(self, queryset): + page = super().paginate_queryset(queryset) + if page is not None and self.action == "list" and url_boolean_param(self.request, "with_counts"): + add_taxa_counts(page) + return page + def get_serializer_class(self): """ Return different serializers for list and detail views. diff --git a/ami/main/models.py b/ami/main/models.py index f2283c848..941f61668 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -3,6 +3,7 @@ import datetime import functools import logging +import operator import textwrap import time import typing @@ -234,6 +235,33 @@ def _per_project(queryset: models.QuerySet, aggregate) -> Subquery: ) +def add_taxa_counts(projects: list["Project"]) -> None: + """ + Set `observed_taxa_count` on each project: its distinct taxa after its own default filters, + the number the project summary shows. + + Each project has its own filters, so this is one grouped query over a page of projects + rather than an annotation, and the count cannot be sorted on. + """ + from ami.main.models_future.filters import build_occurrence_default_filters_q + + if not projects: + return + models.prefetch_related_objects(projects, "default_filters_include_taxa", "default_filters_exclude_taxa") + in_scope = functools.reduce( + operator.or_, (Q(project=project) & build_occurrence_default_filters_q(project) for project in projects) + ) + # The same conditions as OccurrenceQuerySet.unique_taxa(). + counts = dict( + Occurrence.objects.filter(in_scope, determination__isnull=False, event__isnull=False) + .values("project") + .annotate(taxa=models.Count("determination", distinct=True)) + .values_list("project", "taxa") + ) + for project in projects: + project.observed_taxa_count = counts.get(project.pk, 0) + + class ProjectQuerySet(BaseQuerySet): # Annotated by with_overview(). The projects API sorts and serializes by these names. OVERVIEW_FIELDS = ( diff --git a/ami/main/tests.py b/ami/main/tests.py index d2e490f87..fc6c0666e 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -2406,6 +2406,20 @@ def test_counts_are_rolled_up_from_deployments(self): self.assertIsNone(empty["last_occurrence_updated_at"]) self.assertIsNone(empty["last_job_updated_at"]) + def test_taxa_count_follows_each_projects_default_filters(self): + """Distinct taxa above the project's score threshold, as on the project summary.""" + station = Deployment.objects.filter(project=self.busy).first() + event = Event.objects.create( + project=self.busy, deployment=station, group_by="2026-06-01", start=self.last_capture + ) + moth, beetle, unsure = (Taxon.objects.create(name=f"Overview {name}") for name in ("moth", "beetle", "unsure")) + for taxon, score in [(moth, 0.9), (moth, 0.8), (beetle, 0.9), (unsure, 0.1)]: + Occurrence.objects.create( + project=self.busy, deployment=station, event=event, determination=taxon, determination_score=score + ) + rows = {row["id"]: row for row in self._rows(self.owner)} + self.assertEqual((rows[self.busy.pk]["taxa_count"], rows[self.empty.pk]["taxa_count"]), (2, 0)) + def test_team_size_is_only_shown_to_those_who_can_see_the_team(self): """members_count is null for projects whose team list the user may not open.""" superuser = User.objects.create_superuser(email="overview-admin@insectai.org", password="unused") @@ -2424,7 +2438,13 @@ def test_totals_are_left_out_unless_asked_for(self): self.assertEqual(response.status_code, status.HTTP_200_OK) row = next(row for row in response.json()["results"] if row["id"] == self.busy.pk) self.assertEqual(row["deployments_count"], 2) - for field in ["captures_count", "members_count", "last_capture_timestamp", "last_job_updated_at"]: + for field in [ + "captures_count", + "members_count", + "taxa_count", + "last_capture_timestamp", + "last_job_updated_at", + ]: self.assertNotIn(field, row) def test_sorting_by_a_total_works_without_asking_for_totals(self): @@ -2456,6 +2476,12 @@ def test_counts_are_read_inside_the_list_query(self): tables = ['"main_deployment"', '"main_userprojectmembership"', '"main_sourceimage"', '"jobs_job"'] rollup_queries = [q["sql"] for q in queries if any(table in q["sql"] for table in tables)] self.assertEqual(len(rollup_queries), 1, rollup_queries) + # Taxa are counted for the whole page in one grouped query, with each project's taxa filters prefetched. + for marker in [ + 'COUNT(DISTINCT "main_occurrence"."determination_id")', + '"main_project_default_filters_include_taxa"', + ]: + self.assertEqual(sum(marker in q["sql"] for q in queries), 1, marker) class TestProjectPermissions(APITestCase): diff --git a/ui/src/data-services/models/project.ts b/ui/src/data-services/models/project.ts index 432d1b5dd..3e0b8139a 100644 --- a/ui/src/data-services/models/project.ts +++ b/ui/src/data-services/models/project.ts @@ -89,4 +89,8 @@ export class Project { get numOccurrences(): number | undefined { return this._project.occurrences_count } + + get numTaxa(): number | undefined { + return this._project.taxa_count + } } diff --git a/ui/src/pages/projects/project-columns.tsx b/ui/src/pages/projects/project-columns.tsx index cbb187f0a..0f9d31013 100644 --- a/ui/src/pages/projects/project-columns.tsx +++ b/ui/src/pages/projects/project-columns.tsx @@ -20,7 +20,7 @@ const countColumn = ({ }: { id: string name: string - sortField: string + sortField?: string value: (item: Project) => number | undefined }): TableColumn => ({ id, @@ -74,6 +74,12 @@ export const columns: TableColumn[] = [ sortField: 'occurrences_count', value: (item) => item.numOccurrences, }), + // Not sortable: each project counts its taxa with its own default filters. + countColumn({ + id: 'taxa', + name: translate(STRING.FIELD_LABEL_TAXA), + value: (item) => item.numTaxa, + }), countColumn({ id: 'members', name: translate(STRING.FIELD_LABEL_MEMBERS), diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index 0da9ab9f4..393061d22 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -45,10 +45,11 @@ export const Projects = () => { deployments: true, captures: true, occurrences: true, - members: true, + taxa: true, + members: false, 'last-capture': true, - 'last-occurrence-update': true, - 'last-job-update': true, + 'last-occurrence-update': false, + 'last-job-update': false, 'created-at': false, }) const { sort, setSort } = useSort() From f170a8c0083c6f9e8117024fcce3a0bf0a03b680 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 01:24:30 -0700 Subject: [PATCH 08/26] feat(ui): show when each project was last updated in the projects table Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ui/src/data-services/models/project.ts | 6 ++++++ ui/src/pages/projects/project-columns.tsx | 6 ++++++ ui/src/pages/projects/projects.tsx | 1 + 3 files changed, 13 insertions(+) diff --git a/ui/src/data-services/models/project.ts b/ui/src/data-services/models/project.ts index 3e0b8139a..26d9a6d69 100644 --- a/ui/src/data-services/models/project.ts +++ b/ui/src/data-services/models/project.ts @@ -93,4 +93,10 @@ export class Project { get numTaxa(): number | undefined { return this._project.taxa_count } + + get updatedAt(): Date | undefined { + return this._project.updated_at + ? new Date(this._project.updated_at) + : undefined + } } diff --git a/ui/src/pages/projects/project-columns.tsx b/ui/src/pages/projects/project-columns.tsx index 0f9d31013..38041c8f5 100644 --- a/ui/src/pages/projects/project-columns.tsx +++ b/ui/src/pages/projects/project-columns.tsx @@ -116,4 +116,10 @@ export const columns: TableColumn[] = [ sortField: 'created_at', renderCell: (item: Project) => , }, + { + id: 'updated-at', + name: translate(STRING.FIELD_LABEL_UPDATED_AT), + sortField: 'updated_at', + renderCell: (item: Project) => , + }, ] diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index 393061d22..557acd29e 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -51,6 +51,7 @@ export const Projects = () => { 'last-occurrence-update': false, 'last-job-update': false, 'created-at': false, + 'updated-at': true, }) const { sort, setSort } = useSort() const { pagination, setPage } = usePagination({ perPage: 40 }) From c20c849adc5c6d8c5d4a9bc471a9df510deb0627 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 01:50:10 -0700 Subject: [PATCH 09/26] fix(ui): hide the projects table's Updated at column by default Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ui/src/pages/projects/projects.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index 557acd29e..264a7fb2a 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -51,7 +51,7 @@ export const Projects = () => { 'last-occurrence-update': false, 'last-job-update': false, 'created-at': false, - 'updated-at': true, + 'updated-at': false, }) const { sort, setSort } = useSort() const { pagination, setPage } = usePagination({ perPage: 40 }) From fd6bf93fed2c645afc4deec62663bd76ab879c87 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 02:10:31 -0700 Subject: [PATCH 10/26] feat(projects): always return activity dates, and name the taxa count taxa_observed_count The project list now always returns each project's latest capture, occurrence update and job update, since they come from index-only scans and cost a few milliseconds. ?with_counts gates only the totals. The queryset keeps the two apart, with_overview_counts() and with_recent_activity(), each with its own list of field names. The taxa count is returned as taxa_observed_count, leaving room for a verified-taxa count alongside it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ami/main/api/serializers.py | 16 ++++++--- ami/main/api/views.py | 22 +++++++++---- ami/main/models.py | 45 +++++++++++++------------- ami/main/tests.py | 16 ++++----- ui/src/data-services/models/project.ts | 2 +- 5 files changed, 57 insertions(+), 44 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 50d41cf61..c46806b48 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -313,8 +313,8 @@ class ProjectListSerializer(DefaultSerializer): last_capture_timestamp = serializers.DateTimeField(read_only=True) last_occurrence_updated_at = serializers.DateTimeField(read_only=True) last_job_updated_at = serializers.DateTimeField(read_only=True) - # Set on the page by add_taxa_counts(); Project.taxa_count() counts something else. - taxa_count = serializers.IntegerField(read_only=True, source="observed_taxa_count") + # Set on the page by add_taxa_counts(). + taxa_observed_count = serializers.IntegerField(read_only=True) class Meta: model = Project @@ -328,9 +328,15 @@ class Meta: "image", "draft", ] - # Added when the list asks for them with ?with_counts, and left out of the response otherwise, - # except deployments_count, which falls back to the model method. - fields = base_fields + list(ProjectQuerySet.OVERVIEW_FIELDS) + ["taxa_count"] + # The list always adds the activity dates. The totals are added when it asks for them with + # ?with_counts and left out otherwise, except deployments_count, which falls back to the + # model method. + fields = [ + *base_fields, + *ProjectQuerySet.RECENT_ACTIVITY_FIELDS, + *ProjectQuerySet.OVERVIEW_COUNT_FIELDS, + "taxa_observed_count", + ] class ProjectSettingsSerializer(DefaultSerializer): diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 0711b17f2..b8c0fdb88 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -159,8 +159,8 @@ class ProjectPagination(LimitOffsetPaginationWithPermissions): default_limit = 40 def get_count(self, queryset): - # The overview totals and dates are correlated subqueries. They don't change - # the row count, so strip them (and ordering) before counting to keep the + # The overview totals and activity dates are correlated subqueries. They don't + # change the row count, so strip them (and ordering) before counting to keep the # pagination COUNT query cheap. return super().get_count(queryset.order_by().values("pk")) @@ -187,7 +187,13 @@ class ProjectViewSet(DefaultViewSet, ProjectMixin): pagination_class = ProjectPagination permission_classes = [ObjectPermission] search_fields = ["name", "description"] - ordering_fields = ["name", "created_at", "updated_at", *ProjectQuerySet.OVERVIEW_FIELDS] + ordering_fields = [ + "name", + "created_at", + "updated_at", + *ProjectQuerySet.OVERVIEW_COUNT_FIELDS, + *ProjectQuerySet.RECENT_ACTIVITY_FIELDS, + ] def get_queryset(self): qs: ProjectQuerySet = super().get_queryset() # type: ignore @@ -200,14 +206,16 @@ def get_queryset(self): if user: qs = qs.filter_by_user(user) - # The totals and activity dates feed the projects table, so they are added only when - # the table asks for them with ?with_counts, or when the list is sorted by one of them. + if self.action == "list": + qs = qs.with_recent_activity() + # The totals feed the projects table, so they are added only when the table asks for them + # with ?with_counts, or when the list is sorted by one of them. ordering = {field.lstrip("-") for field in self.request.query_params.get("ordering", "").split(",") if field} if self.action == "list" and ( - url_boolean_param(self.request, "with_counts") or ordering & set(ProjectQuerySet.OVERVIEW_FIELDS) + url_boolean_param(self.request, "with_counts") or ordering & set(ProjectQuerySet.OVERVIEW_COUNT_FIELDS) ): # The annotated deployments_count replaces counting prefetched deployments, so skip the prefetch. - qs = qs.prefetch_related(None).with_overview(self.request.user) + qs = qs.prefetch_related(None).with_overview_counts(self.request.user) return qs def paginate_queryset(self, queryset): diff --git a/ami/main/models.py b/ami/main/models.py index 941f61668..ee22d5d77 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -237,7 +237,7 @@ def _per_project(queryset: models.QuerySet, aggregate) -> Subquery: def add_taxa_counts(projects: list["Project"]) -> None: """ - Set `observed_taxa_count` on each project: its distinct taxa after its own default filters, + Set `taxa_observed_count` on each project: its distinct taxa after its own default filters, the number the project summary shows. Each project has its own filters, so this is one grouped query over a page of projects @@ -259,20 +259,14 @@ def add_taxa_counts(projects: list["Project"]) -> None: .values_list("project", "taxa") ) for project in projects: - project.observed_taxa_count = counts.get(project.pk, 0) + project.taxa_observed_count = counts.get(project.pk, 0) class ProjectQuerySet(BaseQuerySet): - # Annotated by with_overview(). The projects API sorts and serializes by these names. - OVERVIEW_FIELDS = ( - "deployments_count", - "captures_count", - "occurrences_count", - "members_count", - "last_capture_timestamp", - "last_occurrence_updated_at", - "last_job_updated_at", - ) + # Annotated by with_overview_counts() and with_recent_activity(). The projects API sorts and + # serializes by these names. + OVERVIEW_COUNT_FIELDS = ("deployments_count", "captures_count", "occurrences_count", "members_count") + RECENT_ACTIVITY_FIELDS = ("last_capture_timestamp", "last_occurrence_updated_at", "last_job_updated_at") def filter_by_user(self, user: User): """ @@ -280,20 +274,14 @@ def filter_by_user(self, user: User): """ return self.filter(members=user) - def with_overview(self, user: User | AnonymousUser) -> "ProjectQuerySet": + def with_overview_counts(self, user: User | AnonymousUser) -> "ProjectQuerySet": """ - Annotate the totals and latest activity shown in the projects table, named in OVERVIEW_FIELDS. + Annotate the totals shown in the projects table, named in OVERVIEW_COUNT_FIELDS. Totals are summed from the cached counts on each deployment, so they are only as fresh as - `Deployment.update_calculated_fields()`. Activity dates are read live, each from a (project, time) - index. `members_count` follows the permission on the team list: it is null unless the user is - a member of the project or a superuser. + `Deployment.update_calculated_fields()`. `members_count` follows the permission on the team + list: it is null unless the user is a member of the project or a superuser. """ - from ami.jobs.models import Job - - def latest(queryset: models.QuerySet, field: str) -> Subquery: - return Subquery(queryset.filter(project=OuterRef("pk")).order_by(f"-{field}").values(field)[:1]) - members_count = Coalesce(_per_project(UserProjectMembership.objects.all(), models.Count("pk")), 0) if not user.is_superuser: is_member = Exists(UserProjectMembership.objects.filter(project=OuterRef("pk"), user_id=user.pk)) @@ -304,6 +292,19 @@ def latest(queryset: models.QuerySet, field: str) -> Subquery: captures_count=Coalesce(_per_project(Deployment.objects.all(), models.Sum("captures_count")), 0), occurrences_count=Coalesce(_per_project(Deployment.objects.all(), models.Sum("occurrences_count")), 0), members_count=members_count, + ) + + def with_recent_activity(self) -> "ProjectQuerySet": + """ + Annotate when each project last received a capture, an occurrence update and a job update, + named in RECENT_ACTIVITY_FIELDS. Each is read live from a (project, time) index. + """ + from ami.jobs.models import Job + + def latest(queryset: models.QuerySet, field: str) -> Subquery: + return Subquery(queryset.filter(project=OuterRef("pk")).order_by(f"-{field}").values(field)[:1]) + + return self.annotate( # timestamp is nullable and DESC sorts NULLs first, so undated captures are excluded. last_capture_timestamp=latest(SourceImage.objects.filter(timestamp__isnull=False), "timestamp"), last_occurrence_updated_at=latest(Occurrence.objects.all(), "updated_at"), diff --git a/ami/main/tests.py b/ami/main/tests.py index fc6c0666e..836b78cb5 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -2406,7 +2406,7 @@ def test_counts_are_rolled_up_from_deployments(self): self.assertIsNone(empty["last_occurrence_updated_at"]) self.assertIsNone(empty["last_job_updated_at"]) - def test_taxa_count_follows_each_projects_default_filters(self): + def test_taxa_observed_count_follows_each_projects_default_filters(self): """Distinct taxa above the project's score threshold, as on the project summary.""" station = Deployment.objects.filter(project=self.busy).first() event = Event.objects.create( @@ -2418,7 +2418,9 @@ def test_taxa_count_follows_each_projects_default_filters(self): project=self.busy, deployment=station, event=event, determination=taxon, determination_score=score ) rows = {row["id"]: row for row in self._rows(self.owner)} - self.assertEqual((rows[self.busy.pk]["taxa_count"], rows[self.empty.pk]["taxa_count"]), (2, 0)) + self.assertEqual( + (rows[self.busy.pk]["taxa_observed_count"], rows[self.empty.pk]["taxa_observed_count"]), (2, 0) + ) def test_team_size_is_only_shown_to_those_who_can_see_the_team(self): """members_count is null for projects whose team list the user may not open.""" @@ -2434,17 +2436,13 @@ def test_team_size_is_only_shown_to_those_who_can_see_the_team(self): self.assertEqual((rows[self.busy.pk]["members_count"], rows[self.empty.pk]["members_count"]), expected) def test_totals_are_left_out_unless_asked_for(self): + """The gallery gets the activity dates but none of the totals, apart from the station count.""" response = self.client.get(self.endpoint) self.assertEqual(response.status_code, status.HTTP_200_OK) row = next(row for row in response.json()["results"] if row["id"] == self.busy.pk) self.assertEqual(row["deployments_count"], 2) - for field in [ - "captures_count", - "members_count", - "taxa_count", - "last_capture_timestamp", - "last_job_updated_at", - ]: + self.assertEqual(row["last_capture_timestamp"][:16], self.last_capture.isoformat()[:16]) + for field in ["captures_count", "members_count", "taxa_observed_count"]: self.assertNotIn(field, row) def test_sorting_by_a_total_works_without_asking_for_totals(self): diff --git a/ui/src/data-services/models/project.ts b/ui/src/data-services/models/project.ts index 26d9a6d69..b86dc4f57 100644 --- a/ui/src/data-services/models/project.ts +++ b/ui/src/data-services/models/project.ts @@ -91,7 +91,7 @@ export class Project { } get numTaxa(): number | undefined { - return this._project.taxa_count + return this._project.taxa_observed_count } get updatedAt(): Date | undefined { From 06987f5820332969da9e62e5e557436a28b7ebc3 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 02:19:29 -0700 Subject: [PATCH 11/26] refactor(ui): ask for project totals through the shared withCounts fetch option The captures and capture sets pages already request counts this way; the projects page had passed the flag as a filter instead. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ui/src/pages/projects/projects.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index 264a7fb2a..cba3dcaee 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -61,11 +61,15 @@ export const Projects = () => { ? [{ field: 'user_id', value: userInfo?.id }] : []), ...(search ? [{ field: 'search', value: search }] : []), - // Totals and activity dates are only shown, and only computed, in the table. - ...(layout === 'table' ? [{ field: 'with_counts', value: 'true' }] : []), ] const { projects, total, userPermissions, isLoading, isFetching, error } = - useProjects({ pagination, filters, sort }) + useProjects({ + pagination, + filters, + sort, + // Totals are only shown, and only computed, in the table. + withCounts: layout === 'table', + }) const canCreate = userPermissions?.includes(UserPermission.Create) return ( From ac6b1acb762f72699e3efb9c0d348878d4870c08 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 02:25:23 -0700 Subject: [PATCH 12/26] test: build the projects overview test data once per class The seven tests now run in about 4.4 s instead of 8 s; each still runs in its own rolled-back transaction with its own copy of the shared objects. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ami/main/tests.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ami/main/tests.py b/ami/main/tests.py index 836b78cb5..8446a428f 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -2366,23 +2366,23 @@ class TestProjectListOverviewCounts(APITestCase): endpoint = "/api/v2/projects/" - def setUp(self) -> None: - self.owner = User.objects.create_user(email="overview-owner@insectai.org") - self.member = User.objects.create_user(email="overview-member@insectai.org") - self.busy = Project.objects.create(name="Busy project", owner=self.owner, create_defaults=False) - self.busy.members.add(self.member) - self.empty = Project.objects.create(name="Empty project", owner=self.owner, create_defaults=False) + @classmethod + def setUpTestData(cls) -> None: + cls.owner = User.objects.create_user(email="overview-owner@insectai.org") + cls.member = User.objects.create_user(email="overview-member@insectai.org") + cls.busy = Project.objects.create(name="Busy project", owner=cls.owner, create_defaults=False) + cls.busy.members.add(cls.member) + cls.empty = Project.objects.create(name="Empty project", owner=cls.owner, create_defaults=False) last_capture = datetime.datetime(2026, 6, 1, 23, 0) for name, captures, occurrences, timestamp in [ ("Station A", 100, 10, last_capture), ("Station B", 50, 5, last_capture - datetime.timedelta(days=3)), ]: - deployment = Deployment.objects.create(name=name, project=self.busy) - SourceImage.objects.create(deployment=deployment, project=self.busy, timestamp=timestamp, path=name) + deployment = Deployment.objects.create(name=name, project=cls.busy) + SourceImage.objects.create(deployment=deployment, project=cls.busy, timestamp=timestamp, path=name) # Set after create, which recalculates the cached counts from (no) captures. Deployment.objects.filter(pk=deployment.pk).update(captures_count=captures, occurrences_count=occurrences) - self.last_capture = last_capture - return super().setUp() + cls.last_capture = last_capture def _rows(self, user: User | None = None, **params) -> list[dict]: self.client.force_authenticate(user) From 7e4ff7882b18d30ff1b629f86ad23b0ff2b58951 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 02:44:14 -0700 Subject: [PATCH 13/26] fix(projects): show team size to anyone allowed to open the team list members_count used membership as the test, but the team list checks the view-membership object permission, which can also be granted directly. The count now uses the same permission through guardian's get_objects_for_user, and the test covers a directly granted outsider. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ami/main/models.py | 23 +++++++++++++++++------ ami/main/tests.py | 6 +++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/ami/main/models.py b/ami/main/models.py index ee22d5d77..e7697db58 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -279,13 +279,24 @@ def with_overview_counts(self, user: User | AnonymousUser) -> "ProjectQuerySet": Annotate the totals shown in the projects table, named in OVERVIEW_COUNT_FIELDS. Totals are summed from the cached counts on each deployment, so they are only as fresh as - `Deployment.update_calculated_fields()`. `members_count` follows the permission on the team - list: it is null unless the user is a member of the project or a superuser. + `Deployment.update_calculated_fields()`. `members_count` is null on projects whose team list + the user may not open, the same check as `UserProjectMembership.check_permission()`. """ - members_count = Coalesce(_per_project(UserProjectMembership.objects.all(), models.Count("pk")), 0) - if not user.is_superuser: - is_member = Exists(UserProjectMembership.objects.filter(project=OuterRef("pk"), user_id=user.pk)) - members_count = models.Case(models.When(is_member, then=members_count), default=None) + from guardian.shortcuts import get_objects_for_user + + can_view_team = get_objects_for_user( + user, + Project.Permissions.VIEW_USER_PROJECT_MEMBERSHIP, + klass=Project, + accept_global_perms=False, + ) + members_count = models.Case( + models.When( + pk__in=can_view_team.values("pk"), + then=Coalesce(_per_project(UserProjectMembership.objects.all(), models.Count("pk")), 0), + ), + default=None, + ) return self.annotate( deployments_count=Coalesce(_per_project(Deployment.objects.all(), models.Count("pk")), 0), diff --git a/ami/main/tests.py b/ami/main/tests.py index 8446a428f..7c67ea697 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -2423,10 +2423,14 @@ def test_taxa_observed_count_follows_each_projects_default_filters(self): ) def test_team_size_is_only_shown_to_those_who_can_see_the_team(self): - """members_count is null for projects whose team list the user may not open.""" + """members_count follows the permission to open the team list, whether held through a role or directly.""" superuser = User.objects.create_superuser(email="overview-admin@insectai.org", password="unused") + # Granted the team list on one project without joining it. + outsider = User.objects.create_user(email="overview-outsider@insectai.org") + assign_perm(Project.Permissions.VIEW_USER_PROJECT_MEMBERSHIP, outsider, self.busy) for user, expected in [ (None, (None, None)), + (outsider, (2, None)), (self.member, (2, None)), (self.owner, (2, 1)), (superuser, (2, 1)), From 2552c8b6fdb40979fe1ef597b9867a41dfc4e8f6 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 03:18:37 -0700 Subject: [PATCH 14/26] fix(ui): keep the projects toolbar in place when switching views The column chooser is shown in every view, as on the occurrences and other list pages, so the buttons next to it no longer shift when the table opens. Page header actions now wrap onto a second line whenever they run out of room, instead of only below the medium breakpoint. Between 768 px and a full row, the items used to shrink, which stacked the My projects / All projects tabs over each other. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- .../components/page-header/page-header.module.scss | 7 ++++++- ui/src/pages/projects/projects.tsx | 12 +++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ui/src/nova-ui-kit/components/page-header/page-header.module.scss b/ui/src/nova-ui-kit/components/page-header/page-header.module.scss index c51878ee0..308428046 100644 --- a/ui/src/nova-ui-kit/components/page-header/page-header.module.scss +++ b/ui/src/nova-ui-kit/components/page-header/page-header.module.scss @@ -16,6 +16,11 @@ gap: 8px; } +.actions { + flex-wrap: wrap; + justify-content: flex-end; +} + .title { @include body-large; font-weight: 600; @@ -39,7 +44,7 @@ .actions { align-items: flex-start; - flex-wrap: wrap; + justify-content: flex-start; } .details { diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index cba3dcaee..a8a44446c 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -123,13 +123,11 @@ export const Projects = () => { onValueChange={setLayout} /> {canCreate ? : null} - {layout === 'table' ? ( - - ) : null} + {projects && projects.length === 0 && canCreate && !search ? (
From f65d7bc21bf22489cc5f6bd0a2d6fec50bb38c57 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 03:31:41 -0700 Subject: [PATCH 15/26] fix(ui): stop the Create new button from blinking out while projects load The create permission arrives with each list response, so every new tab, search, page or view hid the button until the request finished and shifted the toolbar. The projects hook now keeps the last permissions it received while another query loads; signing out clears the query cache, and the anonymous response replaces them. Create new also moves next to the search box, left of the view chooser. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ui/src/data-services/hooks/projects/useProjects.ts | 11 +++++++++-- ui/src/pages/projects/projects.tsx | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ui/src/data-services/hooks/projects/useProjects.ts b/ui/src/data-services/hooks/projects/useProjects.ts index a42177a37..79fceedbe 100644 --- a/ui/src/data-services/hooks/projects/useProjects.ts +++ b/ui/src/data-services/hooks/projects/useProjects.ts @@ -2,7 +2,7 @@ import { API_ROUTES } from 'data-services/constants' import { Project, ServerProject } from 'data-services/models/project' import { FetchParams } from 'data-services/types' import { getFetchUrl } from 'data-services/utils' -import { useMemo } from 'react' +import { useMemo, useRef } from 'react' import { UserPermission } from 'utils/user/types' import { useAuthorizedQuery } from '../auth/useAuthorizedQuery' @@ -31,9 +31,16 @@ export const useProjects = ( const projects = useMemo(() => data?.results.map(convertServerRecord), [data]) + // Permissions do not depend on the filters, so keep the last known ones while + // another tab, search or view loads, instead of hiding and re-adding buttons. + const userPermissions = useRef() + if (data?.user_permissions) { + userPermissions.current = data.user_permissions + } + return { projects, - userPermissions: data?.user_permissions, + userPermissions: userPermissions.current, total: data?.count ?? 0, isLoading, isFetching, diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index a8a44446c..bf88eb73b 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -106,6 +106,7 @@ export const Projects = () => { value={search} onChange={setSearch} /> + {canCreate ? : null} { value={layout} onValueChange={setLayout} /> - {canCreate ? : null} Date: Wed, 7 Oct 2026 11:00:42 -0700 Subject: [PATCH 16/26] feat(projects): return each project's centre point in the project list The centre is the mean position of the project's stations, leaving out stations without coordinates and default stations placed at (0, 0). Co-Authored-By: Claude Opus 5.5 --- ami/main/api/serializers.py | 9 ++++++--- ami/main/api/views.py | 2 ++ ami/main/models.py | 18 ++++++++++++++++++ ami/main/tests.py | 16 +++++++++++++++- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index c46806b48..aac875172 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -315,6 +315,8 @@ class ProjectListSerializer(DefaultSerializer): last_job_updated_at = serializers.DateTimeField(read_only=True) # Set on the page by add_taxa_counts(). taxa_observed_count = serializers.IntegerField(read_only=True) + center_latitude = serializers.FloatField(read_only=True) + center_longitude = serializers.FloatField(read_only=True) class Meta: model = Project @@ -328,14 +330,15 @@ class Meta: "image", "draft", ] - # The list always adds the activity dates. The totals are added when it asks for them with - # ?with_counts and left out otherwise, except deployments_count, which falls back to the - # model method. + # The list always adds the activity dates. The totals and centres are added when it asks for + # them with ?with_counts or ?with_center and left out otherwise, except deployments_count, + # which falls back to the model method. fields = [ *base_fields, *ProjectQuerySet.RECENT_ACTIVITY_FIELDS, *ProjectQuerySet.OVERVIEW_COUNT_FIELDS, "taxa_observed_count", + *ProjectQuerySet.CENTER_FIELDS, ] diff --git a/ami/main/api/views.py b/ami/main/api/views.py index b8c0fdb88..4948abec1 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -216,6 +216,8 @@ def get_queryset(self): ): # The annotated deployments_count replaces counting prefetched deployments, so skip the prefetch. qs = qs.prefetch_related(None).with_overview_counts(self.request.user) + if self.action == "list" and url_boolean_param(self.request, "with_center"): + qs = qs.with_center() return qs def paginate_queryset(self, queryset): diff --git a/ami/main/models.py b/ami/main/models.py index e7697db58..2e8f7e28b 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -267,6 +267,8 @@ class ProjectQuerySet(BaseQuerySet): # serializes by these names. OVERVIEW_COUNT_FIELDS = ("deployments_count", "captures_count", "occurrences_count", "members_count") RECENT_ACTIVITY_FIELDS = ("last_capture_timestamp", "last_occurrence_updated_at", "last_job_updated_at") + # Annotated by with_center(). + CENTER_FIELDS = ("center_latitude", "center_longitude") def filter_by_user(self, user: User): """ @@ -322,6 +324,22 @@ def latest(queryset: models.QuerySet, field: str) -> Subquery: last_job_updated_at=latest(Job.objects.all(), "updated_at"), ) + def with_center(self) -> "ProjectQuerySet": + """ + Annotate CENTER_FIELDS: the mean position of the project's stations. + + Stations without coordinates, or at (0, 0) where default stations are created, are left + out; a project with none of its own gets nulls. A plain mean is wrong for stations on + both sides of the antimeridian. + """ + located = Deployment.objects.filter(latitude__isnull=False, longitude__isnull=False).exclude( + latitude=0, longitude=0 + ) + return self.annotate( + center_latitude=_per_project(located, models.Avg("latitude")), + center_longitude=_per_project(located, models.Avg("longitude")), + ) + class ProjectManager(models.Manager.from_queryset(ProjectQuerySet)): pass diff --git a/ami/main/tests.py b/ami/main/tests.py index 7c67ea697..7546d4f5e 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -2446,9 +2446,23 @@ def test_totals_are_left_out_unless_asked_for(self): row = next(row for row in response.json()["results"] if row["id"] == self.busy.pk) self.assertEqual(row["deployments_count"], 2) self.assertEqual(row["last_capture_timestamp"][:16], self.last_capture.isoformat()[:16]) - for field in ["captures_count", "members_count", "taxa_observed_count"]: + for field in ["captures_count", "members_count", "taxa_observed_count", "center_latitude"]: self.assertNotIn(field, row) + def test_center_ignores_stations_without_a_real_location(self): + stations = list(Deployment.objects.filter(project=self.busy).order_by("name")) + Deployment.objects.filter(pk=stations[0].pk).update(latitude=45.0, longitude=-73.0) + Deployment.objects.filter(pk=stations[1].pk).update(latitude=47.0, longitude=-71.0) + Deployment.objects.create(name="Placeholder", project=self.busy, latitude=0, longitude=0) + Deployment.objects.create(name="Unknown", project=self.busy) + + rows = {row["id"]: row for row in self._rows(with_center="true")} + busy, empty = rows[self.busy.pk], rows[self.empty.pk] + self.assertAlmostEqual(busy["center_latitude"], 46.0) + self.assertAlmostEqual(busy["center_longitude"], -72.0) + self.assertIsNone(empty["center_latitude"]) + self.assertIsNone(empty["center_longitude"]) + def test_sorting_by_a_total_works_without_asking_for_totals(self): """A sort carried over from the table to the gallery still gets the totals it orders by.""" response = self.client.get(self.endpoint, {"ordering": "-captures_count"}) From c4596f73bf410b2fff5ce999ecc7a4d083a04307 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 7 Oct 2026 11:00:47 -0700 Subject: [PATCH 17/26] feat(ui): show projects on a map on the projects page Co-Authored-By: Claude Opus 5.5 --- .../map/multi-marker-map/multi-marker-map.tsx | 7 ++- ui/src/data-services/models/project.ts | 9 +++ ui/src/pages/projects/projects-map.tsx | 57 +++++++++++++++++++ ui/src/pages/projects/projects.tsx | 10 +++- ui/src/utils/language.ts | 2 + 5 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 ui/src/pages/projects/projects-map.tsx diff --git a/ui/src/components/map/multi-marker-map/multi-marker-map.tsx b/ui/src/components/map/multi-marker-map/multi-marker-map.tsx index e5fd4420e..5ece631d6 100644 --- a/ui/src/components/map/multi-marker-map/multi-marker-map.tsx +++ b/ui/src/components/map/multi-marker-map/multi-marker-map.tsx @@ -1,3 +1,4 @@ +import classNames from 'classnames' import * as L from 'leaflet' import { LoadingSpinner } from 'nova-ui-kit' import { useEffect, useMemo, useRef } from 'react' @@ -16,9 +17,11 @@ import { MarkerPosition } from '../types' setup() export const MultiMarkerMap = ({ + className, markers, isLoading, }: { + className?: string markers: { position: MarkerPosition; popupContent?: JSX.Element }[] isLoading?: boolean }) => { @@ -43,7 +46,7 @@ export const MultiMarkerMap = ({ if (isLoading) { return ( -
+
) @@ -52,7 +55,7 @@ export const MultiMarkerMap = ({ return ( { + const markers = useMemo( + () => + projects.flatMap((project) => + project.center + ? [ + { + position: project.center, + popupContent: , + }, + ] + : [] + ), + [projects] + ) + + return ( + + ) +} + +const ProjectsMapPopupContent = ({ project }: { project: Project }) => ( + +) diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index bf88eb73b..dd2c350e2 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -1,6 +1,6 @@ import { SearchInput } from 'components/search-input/search-input' import { useProjects } from 'data-services/hooks/projects/useProjects' -import { Grid2X2Icon, TableIcon } from 'lucide-react' +import { Grid2X2Icon, MapIcon, TableIcon } from 'lucide-react' import { Button, ColumnSettings, @@ -24,6 +24,7 @@ import { useSelectedView } from 'utils/useSelectedView' import { useSort } from 'utils/useSort' import { columns } from './project-columns' import { ProjectGallery } from './project-gallery' +import { ProjectsMap } from './projects-map' export const TABS = { MY_PROJECTS: 'my-projects', @@ -119,6 +120,11 @@ export const Projects = () => { label: translate(STRING.TAB_ITEM_GALLERY), Icon: Grid2X2Icon, }, + { + value: 'map', + label: translate(STRING.TAB_ITEM_MAP), + Icon: MapIcon, + }, ]} value={layout} onValueChange={setLayout} @@ -157,6 +163,8 @@ export const Projects = () => { sortable sortSettings={sort} /> + ) : layout === 'map' ? ( + ) : ( Date: Wed, 7 Oct 2026 13:00:34 -0700 Subject: [PATCH 18/26] fix(ui): stop the map from crashing when a search leaves no markers The map is now created with its bounds, so it has a view before the minimap reads it. Before, switching from some markers to none showed "Set map center and zoom first" instead of the page. Co-Authored-By: Claude Opus 5.5 --- ui/src/components/map/multi-marker-map/multi-marker-map.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ui/src/components/map/multi-marker-map/multi-marker-map.tsx b/ui/src/components/map/multi-marker-map/multi-marker-map.tsx index 5ece631d6..73c4ec9b1 100644 --- a/ui/src/components/map/multi-marker-map/multi-marker-map.tsx +++ b/ui/src/components/map/multi-marker-map/multi-marker-map.tsx @@ -54,7 +54,9 @@ export const MultiMarkerMap = ({ return ( Date: Wed, 7 Oct 2026 13:00:35 -0700 Subject: [PATCH 19/26] feat(ui): ask for project centres only in the map view The map view requests station centres and totals for its popups; the gallery requests neither. Co-Authored-By: Claude Opus 5.5 --- ui/src/pages/projects/projects.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index dd2c350e2..d338d06fa 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -62,6 +62,7 @@ export const Projects = () => { ? [{ field: 'user_id', value: userInfo?.id }] : []), ...(search ? [{ field: 'search', value: search }] : []), + ...(layout === 'map' ? [{ field: 'with_center', value: 'true' }] : []), ] const { projects, total, userPermissions, isLoading, isFetching, error } = useProjects({ From 8459dbe8345245f83019f7783995824b0a7e6d90 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 7 Oct 2026 16:10:10 -0700 Subject: [PATCH 20/26] fix(ui): cap the map zoom so a single marker shows its region instead of crashing A search that leaves one project on the map produced zero-area bounds. Fitting them when the map is created, before the tile layer sets a maximum zoom, resolved to an infinite zoom and a NaN centre; fitting them afterwards zoomed to street level. Both fits now stop at the default zoom. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- .../components/map/multi-marker-map/multi-marker-map.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ui/src/components/map/multi-marker-map/multi-marker-map.tsx b/ui/src/components/map/multi-marker-map/multi-marker-map.tsx index 73c4ec9b1..16ca08b32 100644 --- a/ui/src/components/map/multi-marker-map/multi-marker-map.tsx +++ b/ui/src/components/map/multi-marker-map/multi-marker-map.tsx @@ -5,6 +5,7 @@ import { useEffect, useMemo, useRef } from 'react' import { MapContainer, Marker, Popup, TileLayer } from 'react-leaflet' import { ATTRIBUTION, + DEFAULT_ZOOM, MAX_BOUNDS, MIN_ZOOM, TILE_LAYER_URL, @@ -40,7 +41,7 @@ export const MultiMarkerMap = ({ useEffect(() => { requestAnimationFrame(() => { - mapRef.current?.fitBounds(bounds) + mapRef.current?.fitBounds(bounds, { maxZoom: DEFAULT_ZOOM }) }) }, [mapRef, bounds]) @@ -55,8 +56,10 @@ export const MultiMarkerMap = ({ return ( Date: Wed, 7 Oct 2026 16:12:38 -0700 Subject: [PATCH 21/26] fix(ui): keep station maps zooming to their marker, cap only the projects map The single-station map on a station's page should still zoom in to the station. The zoom cap after mounting is now a prop that only the projects map sets, since a project marker is a mean position. The cap at creation stays for every map, because the refit afterwards replaces it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- .../map/multi-marker-map/multi-marker-map.tsx | 10 ++++++---- ui/src/pages/projects/projects-map.tsx | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ui/src/components/map/multi-marker-map/multi-marker-map.tsx b/ui/src/components/map/multi-marker-map/multi-marker-map.tsx index 16ca08b32..9192f1880 100644 --- a/ui/src/components/map/multi-marker-map/multi-marker-map.tsx +++ b/ui/src/components/map/multi-marker-map/multi-marker-map.tsx @@ -21,10 +21,12 @@ export const MultiMarkerMap = ({ className, markers, isLoading, + maxZoom, }: { className?: string markers: { position: MarkerPosition; popupContent?: JSX.Element }[] isLoading?: boolean + maxZoom?: number }) => { const mapRef = useRef(null) @@ -41,9 +43,9 @@ export const MultiMarkerMap = ({ useEffect(() => { requestAnimationFrame(() => { - mapRef.current?.fitBounds(bounds, { maxZoom: DEFAULT_ZOOM }) + mapRef.current?.fitBounds(bounds, { maxZoom }) }) - }, [mapRef, bounds]) + }, [mapRef, bounds, maxZoom]) if (isLoading) { return ( @@ -56,8 +58,8 @@ export const MultiMarkerMap = ({ return ( ) } From f1b5d583458f7d2305601cf3c41d3d64c28b29ca Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 01:00:54 -0700 Subject: [PATCH 22/26] feat(ui): show every project on the map without loading the table totals The map plotted only the current page of 40 projects and asked for all the table totals to fill its popups. It now requests up to 1,000 projects with their centres only, hides the page footer, and its popups show the project and its number of stations, which the list already returns. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ami/main/tests.py | 5 ++++- ui/src/pages/projects/projects-map.tsx | 4 ---- ui/src/pages/projects/projects.tsx | 8 ++++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ami/main/tests.py b/ami/main/tests.py index 7546d4f5e..94bf99644 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -2456,8 +2456,11 @@ def test_center_ignores_stations_without_a_real_location(self): Deployment.objects.create(name="Placeholder", project=self.busy, latitude=0, longitude=0) Deployment.objects.create(name="Unknown", project=self.busy) - rows = {row["id"]: row for row in self._rows(with_center="true")} + # The map asks for centres without the totals. + response = self.client.get(self.endpoint, {"with_center": "true"}) + rows = {row["id"]: row for row in response.json()["results"]} busy, empty = rows[self.busy.pk], rows[self.empty.pk] + self.assertNotIn("captures_count", busy) self.assertAlmostEqual(busy["center_latitude"], 46.0) self.assertAlmostEqual(busy["center_longitude"], -72.0) self.assertIsNone(empty["center_latitude"]) diff --git a/ui/src/pages/projects/projects-map.tsx b/ui/src/pages/projects/projects-map.tsx index 46f717c84..f7ab32942 100644 --- a/ui/src/pages/projects/projects-map.tsx +++ b/ui/src/pages/projects/projects-map.tsx @@ -51,10 +51,6 @@ const ProjectsMapPopupContent = ({ project }: { project: Project }) => ( label: translate(STRING.NAV_ITEM_DEPLOYMENTS), value: project.numDeployments, }, - { - label: translate(STRING.FIELD_LABEL_CAPTURES), - value: project.numCaptures, - }, ]} /> ) diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index d338d06fa..fb9a872a3 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -26,6 +26,9 @@ import { columns } from './project-columns' import { ProjectGallery } from './project-gallery' import { ProjectsMap } from './projects-map' +// The map shows every project at once rather than a page of them. +const MAP_LIMIT = 1000 + export const TABS = { MY_PROJECTS: 'my-projects', ALL_PROJECTS: 'all-projects', @@ -66,7 +69,8 @@ export const Projects = () => { ] const { projects, total, userPermissions, isLoading, isFetching, error } = useProjects({ - pagination, + pagination: + layout === 'map' ? { page: 0, perPage: MAP_LIMIT } : pagination, filters, sort, // Totals are only shown, and only computed, in the table. @@ -174,7 +178,7 @@ export const Projects = () => { /> )} - {projects?.length ? ( + {projects?.length && layout !== 'map' ? ( Date: Thu, 8 Oct 2026 01:15:38 -0700 Subject: [PATCH 23/26] feat(ui): cap the projects map at 300 projects Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ui/src/pages/projects/projects.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index fb9a872a3..aad318d69 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -27,7 +27,7 @@ import { ProjectGallery } from './project-gallery' import { ProjectsMap } from './projects-map' // The map shows every project at once rather than a page of them. -const MAP_LIMIT = 1000 +const MAP_LIMIT = 300 export const TABS = { MY_PROJECTS: 'my-projects', From bb5ebeee1a92fb385cae5f29af9661a6b42f9fae Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 02:03:57 -0700 Subject: [PATCH 24/26] feat(ui): open map marker popups on hover as well as on click or tap Hovering a marker opens its popup and leaves it open, so the pointer can move up to the link inside it. Leaflet toggles a popup on click, and a tap also reports a hover, so a click reopens the popup after the toggle and a tap or click never closes it; the close button, a click on the map or opening another marker's popup still do. Applies to every marker map. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- .../components/map/multi-marker-map/multi-marker-map.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ui/src/components/map/multi-marker-map/multi-marker-map.tsx b/ui/src/components/map/multi-marker-map/multi-marker-map.tsx index 9192f1880..56d113e49 100644 --- a/ui/src/components/map/multi-marker-map/multi-marker-map.tsx +++ b/ui/src/components/map/multi-marker-map/multi-marker-map.tsx @@ -17,6 +17,13 @@ import { MarkerPosition } from '../types' setup() +// Hovering opens a marker's popup. Leaflet toggles the popup on click, and a tap also +// reports a hover, so a click reopens it after the toggle to leave it open either way. +const POPUP_EVENT_HANDLERS: L.LeafletEventHandlerFnMap = { + mouseover: (e) => e.target.openPopup(), + click: (e) => setTimeout(() => e.target.openPopup()), +} + export const MultiMarkerMap = ({ className, markers, @@ -74,6 +81,7 @@ export const MultiMarkerMap = ({ key={index} position={marker.position} interactive={!!marker.popupContent} + eventHandlers={marker.popupContent ? POPUP_EVENT_HANDLERS : undefined} > {marker.popupContent ? ( {marker.popupContent} From 71c1e6df3e65c9dda1b007a80008fed5798376b6 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 02:29:59 -0700 Subject: [PATCH 25/26] refactor(projects): return a project's position as a location object behind ?with_location The project list returns "location": {"latitude", "longitude"} when asked with ?with_location, so a later outline or region name can join the same object and flag. ProjectQuerySet.with_location() annotates centroid_latitude and centroid_longitude, listed in LOCATION_FIELDS; the location is null for a project with no station coordinates and left out of the response when not asked for. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ami/main/api/serializers.py | 23 ++++++++++++++++++----- ami/main/api/views.py | 4 ++-- ami/main/models.py | 12 ++++++------ ami/main/tests.py | 15 +++++++-------- ui/src/data-services/models/project.ts | 8 ++++---- ui/src/pages/projects/projects-map.tsx | 4 ++-- ui/src/pages/projects/projects.tsx | 2 +- 7 files changed, 40 insertions(+), 28 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index aac875172..a879d00e6 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -5,6 +5,7 @@ from drf_spectacular.utils import extend_schema_field from guardian.shortcuts import get_perms from rest_framework import serializers +from rest_framework.fields import SkipField from rest_framework.request import Request from ami.base.fields import DateStringField @@ -305,6 +306,19 @@ class Meta: ] +class ProjectLocationSerializer(serializers.Serializer): + """Where a project's stations are, from ProjectQuerySet.with_location(); null when none has coordinates.""" + + latitude = serializers.FloatField(source="centroid_latitude") + longitude = serializers.FloatField(source="centroid_longitude") + + def get_attribute(self, instance): + # Left out of the response unless the list asked for it with ?with_location. + if not hasattr(instance, "centroid_latitude"): + raise SkipField + return instance if instance.centroid_latitude is not None else None + + class ProjectListSerializer(DefaultSerializer): deployments_count = serializers.IntegerField(read_only=True) captures_count = serializers.IntegerField(read_only=True) @@ -315,8 +329,7 @@ class ProjectListSerializer(DefaultSerializer): last_job_updated_at = serializers.DateTimeField(read_only=True) # Set on the page by add_taxa_counts(). taxa_observed_count = serializers.IntegerField(read_only=True) - center_latitude = serializers.FloatField(read_only=True) - center_longitude = serializers.FloatField(read_only=True) + location = ProjectLocationSerializer(source="*", read_only=True) class Meta: model = Project @@ -330,15 +343,15 @@ class Meta: "image", "draft", ] - # The list always adds the activity dates. The totals and centres are added when it asks for - # them with ?with_counts or ?with_center and left out otherwise, except deployments_count, + # The list always adds the activity dates. The totals and location are added when it asks for + # them with ?with_counts or ?with_location and left out otherwise, except deployments_count, # which falls back to the model method. fields = [ *base_fields, *ProjectQuerySet.RECENT_ACTIVITY_FIELDS, *ProjectQuerySet.OVERVIEW_COUNT_FIELDS, "taxa_observed_count", - *ProjectQuerySet.CENTER_FIELDS, + "location", ] diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 4948abec1..4f8b19224 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -216,8 +216,8 @@ def get_queryset(self): ): # The annotated deployments_count replaces counting prefetched deployments, so skip the prefetch. qs = qs.prefetch_related(None).with_overview_counts(self.request.user) - if self.action == "list" and url_boolean_param(self.request, "with_center"): - qs = qs.with_center() + if self.action == "list" and url_boolean_param(self.request, "with_location"): + qs = qs.with_location() return qs def paginate_queryset(self, queryset): diff --git a/ami/main/models.py b/ami/main/models.py index 2e8f7e28b..0127371e3 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -267,8 +267,8 @@ class ProjectQuerySet(BaseQuerySet): # serializes by these names. OVERVIEW_COUNT_FIELDS = ("deployments_count", "captures_count", "occurrences_count", "members_count") RECENT_ACTIVITY_FIELDS = ("last_capture_timestamp", "last_occurrence_updated_at", "last_job_updated_at") - # Annotated by with_center(). - CENTER_FIELDS = ("center_latitude", "center_longitude") + # Annotated by with_location(). + LOCATION_FIELDS = ("centroid_latitude", "centroid_longitude") def filter_by_user(self, user: User): """ @@ -324,9 +324,9 @@ def latest(queryset: models.QuerySet, field: str) -> Subquery: last_job_updated_at=latest(Job.objects.all(), "updated_at"), ) - def with_center(self) -> "ProjectQuerySet": + def with_location(self) -> "ProjectQuerySet": """ - Annotate CENTER_FIELDS: the mean position of the project's stations. + Annotate LOCATION_FIELDS: the centroid of the project's stations, as the mean of their coordinates. Stations without coordinates, or at (0, 0) where default stations are created, are left out; a project with none of its own gets nulls. A plain mean is wrong for stations on @@ -336,8 +336,8 @@ def with_center(self) -> "ProjectQuerySet": latitude=0, longitude=0 ) return self.annotate( - center_latitude=_per_project(located, models.Avg("latitude")), - center_longitude=_per_project(located, models.Avg("longitude")), + centroid_latitude=_per_project(located, models.Avg("latitude")), + centroid_longitude=_per_project(located, models.Avg("longitude")), ) diff --git a/ami/main/tests.py b/ami/main/tests.py index 94bf99644..6c4a200de 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -2446,25 +2446,24 @@ def test_totals_are_left_out_unless_asked_for(self): row = next(row for row in response.json()["results"] if row["id"] == self.busy.pk) self.assertEqual(row["deployments_count"], 2) self.assertEqual(row["last_capture_timestamp"][:16], self.last_capture.isoformat()[:16]) - for field in ["captures_count", "members_count", "taxa_observed_count", "center_latitude"]: + for field in ["captures_count", "members_count", "taxa_observed_count", "location"]: self.assertNotIn(field, row) - def test_center_ignores_stations_without_a_real_location(self): + def test_location_ignores_stations_without_real_coordinates(self): stations = list(Deployment.objects.filter(project=self.busy).order_by("name")) Deployment.objects.filter(pk=stations[0].pk).update(latitude=45.0, longitude=-73.0) Deployment.objects.filter(pk=stations[1].pk).update(latitude=47.0, longitude=-71.0) Deployment.objects.create(name="Placeholder", project=self.busy, latitude=0, longitude=0) Deployment.objects.create(name="Unknown", project=self.busy) - # The map asks for centres without the totals. - response = self.client.get(self.endpoint, {"with_center": "true"}) + # The map asks for locations without the totals. + response = self.client.get(self.endpoint, {"with_location": "true"}) rows = {row["id"]: row for row in response.json()["results"]} busy, empty = rows[self.busy.pk], rows[self.empty.pk] self.assertNotIn("captures_count", busy) - self.assertAlmostEqual(busy["center_latitude"], 46.0) - self.assertAlmostEqual(busy["center_longitude"], -72.0) - self.assertIsNone(empty["center_latitude"]) - self.assertIsNone(empty["center_longitude"]) + self.assertAlmostEqual(busy["location"]["latitude"], 46.0) + self.assertAlmostEqual(busy["location"]["longitude"], -72.0) + self.assertIsNone(empty["location"]) def test_sorting_by_a_total_works_without_asking_for_totals(self): """A sort carried over from the table to the gallery still gets the totals it orders by.""" diff --git a/ui/src/data-services/models/project.ts b/ui/src/data-services/models/project.ts index f9980c710..e7d3e94b3 100644 --- a/ui/src/data-services/models/project.ts +++ b/ui/src/data-services/models/project.ts @@ -19,11 +19,11 @@ export class Project { return this._project.user_permissions.includes(UserPermission.Delete) } - get center(): MarkerPosition | undefined { - const { center_latitude, center_longitude } = this._project + get location(): MarkerPosition | undefined { + const { location } = this._project - return center_latitude !== null && center_latitude !== undefined - ? new MarkerPosition(center_latitude, center_longitude) + return location + ? new MarkerPosition(location.latitude, location.longitude) : undefined } diff --git a/ui/src/pages/projects/projects-map.tsx b/ui/src/pages/projects/projects-map.tsx index f7ab32942..714ba479c 100644 --- a/ui/src/pages/projects/projects-map.tsx +++ b/ui/src/pages/projects/projects-map.tsx @@ -16,10 +16,10 @@ export const ProjectsMap = ({ const markers = useMemo( () => projects.flatMap((project) => - project.center + project.location ? [ { - position: project.center, + position: project.location, popupContent: , }, ] diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index aad318d69..8c6ae9121 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -65,7 +65,7 @@ export const Projects = () => { ? [{ field: 'user_id', value: userInfo?.id }] : []), ...(search ? [{ field: 'search', value: search }] : []), - ...(layout === 'map' ? [{ field: 'with_center', value: 'true' }] : []), + ...(layout === 'map' ? [{ field: 'with_location', value: 'true' }] : []), ] const { projects, total, userPermissions, isLoading, isFetching, error } = useProjects({ From 4f754e42bc83c334c56e32437574e5c8497a4e9e Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 02:46:37 -0700 Subject: [PATCH 26/26] fix(ui): show an error on the projects map when the request fails A failed request left an empty world map, which looks the same as a search with no located projects. The map now shows the same error state as the table and gallery. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EfiDZMVyc8qjWsHS5iKzrP --- ui/src/pages/projects/projects-map.tsx | 7 +++++++ ui/src/pages/projects/projects.tsx | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/src/pages/projects/projects-map.tsx b/ui/src/pages/projects/projects-map.tsx index 714ba479c..c7c2832f6 100644 --- a/ui/src/pages/projects/projects-map.tsx +++ b/ui/src/pages/projects/projects-map.tsx @@ -1,3 +1,4 @@ +import { ErrorState } from 'components/error-state/error-state' import { DEFAULT_ZOOM } from 'components/map/config' import { MultiMarkerMap } from 'components/map/multi-marker-map/multi-marker-map' import { Project } from 'data-services/models/project' @@ -7,9 +8,11 @@ import { APP_ROUTES } from 'utils/constants' import { STRING, translate } from 'utils/language' export const ProjectsMap = ({ + error, isLoading, projects = [], }: { + error?: any isLoading: boolean projects?: Project[] }) => { @@ -28,6 +31,10 @@ export const ProjectsMap = ({ [projects] ) + if (error) { + return + } + return ( { sortSettings={sort} /> ) : layout === 'map' ? ( - + ) : (