diff --git a/docs/scripting/Endpoints.md b/docs/scripting/Endpoints.md index 8946e77f3..4c7dd6350 100644 --- a/docs/scripting/Endpoints.md +++ b/docs/scripting/Endpoints.md @@ -35,6 +35,11 @@ Operating on the Annotations for a specific DIVE Dataset Id. These incloude get - **Usage:** This endpoint is used to get detailed information about specific tracks within a dataset, including their attributes and associated detections. There are options to retrieve the annotations at specific revisions - **Query parameters:** `revision` (optional), `set` (optional string). Omit `set` for the default annotation collection. +#### `dive_annotation/track/authors` +- **Method:** GET +- **Usage:** Lists each track visible at a revision (latest by default) with the revision that saved it and that revision's author. Each row has `id`, `begin`, `end`, `featureCount`, `rev_created`, `author_id`, `author_name`, `created` and `description`. A `description` of `save` is a manual edit in the annotator; `Import ...` is an uploaded file or pipeline output. A save rewrites the whole track, so the author is the last user to change the track, not necessarily the creator of every detection. Paged with `limit`/`offset` and returns the total in the `Girder-Total-Count` header. For ops dumps that need full history, feature frames, or image-name mapping, see the Mongo sample script in `samples/scripts/userEdits/`. +- **Query parameters:** `revision` (optional), `set` (optional string), `limit`, `offset`, `sort`. + #### `dive_annotation/group` - **Method:** GET - **Usage:** Same as `dive_annotation/track`, but returns group annotations. Supports `revision` and `set` query parameters. diff --git a/samples/scripts/userEdits/README.md b/samples/scripts/userEdits/README.md new file mode 100644 index 000000000..90c4fcebc --- /dev/null +++ b/samples/scripts/userEdits/README.md @@ -0,0 +1,87 @@ +# DIVE Track Authors + +`trackAuthors.py` lists the current version of each track in a dataset (`trackItem`) along with the user who last saved it and its `rev_created` (from `revisionLogItem`). Pass `--all-versions` to include every replaced or deleted version as well. For image-sequence datasets it also maps frame numbers to image names, using the same ordering as the server. + +Each track version is joined to the revision log through `rev_created`. A save writes out the whole track again, so authorship is per track version, not per detection. + +The revision `description` shows where a version came from: + +| description | Source | +|-------------|--------| +| `save` | Manual edit in the DIVE annotator | +| `Import from ` | Uploaded annotation file or pipeline output | +| `initialize clone` | Dataset clone (earlier history lives on the source dataset) | + +## Prefer the HTTP API when you can + +For normal scripting against a running DIVE server, use +[`GET dive_annotation/track/authors`](../../../docs/scripting/Endpoints.md) +with a Girder token. That endpoint respects dataset ACLs, returns the author of +each visible track at a revision, and does not need Mongo access. + +Use this script only for ops/admin work that needs Mongo-only options the API +does not expose (`--all-versions`, `--features`, image-name mapping, full +revision log dump). It talks to Mongo directly and **bypasses Girder access +control**, so anyone with the URI can read every dataset. + +## Requirements + +- [uv](https://docs.astral.sh/uv/) (installs `click` and `pymongo` automatically) +- Network access to the Girder MongoDB + +The default `docker-compose.yml` does not publish the Mongo port, so `localhost:27017` is refused. Either pass `--container dive-mongo-1` (the script looks up the container IP with `docker inspect`), pass `--mongo-uri` with that IP, or add `ports: ["27017:27017"]` to the `mongo` service in an override file. + +## Usage + +From the repository root: + +```bash +# Current version of each track, with its author, rev_created and begin/end image names +uv run samples/scripts/userEdits/trackAuthors.py --container dive-mongo-1 + +# Full history (every saved version), each feature's frame and image name, and the revision log +uv run samples/scripts/userEdits/trackAuthors.py --all-versions --features --revisions + +# A different Mongo host, a named annotation set, output to a file +uv run samples/scripts/userEdits/trackAuthors.py \ + --mongo-uri mongodb://172.18.0.6:27017/girder --set mySet -o tracks.json +``` + +HTTP equivalent (token-authenticated, ACL-aware): + +```bash +curl -H "Girder-Token: $TOKEN" \ + "$GIRDER/api/v1/dive_annotation/track/authors?folderId=" +``` + +## Output + +```json +{ + "datasetId": "...", + "name": "...", + "type": "image-sequence", + "imageCount": 326, + "tracks": [ + { + "id": 0, + "begin": 238, + "end": 238, + "beginImage": "20171027.214850.033.029373.png", + "endImage": "20171027.214850.033.029373.png", + "featureCount": 1, + "types": ["Motion"], + "rev_created": 10, + "rev_deleted": null, + "author_name": "jdoe", + "author_id": "...", + "created": "...", + "description": "save" + } + ] +} +``` + +Track fields use the same names as `GET dive_annotation/track/authors` (`id`, `begin`, `end`, `featureCount`, `rev_created`, `author_id`, `author_name`, `created`, `description`), so output built on this script keeps working if you switch to the API. `beginImage`, `endImage`, `types`, `rev_deleted` and `features` are script-only extras. + +`rev_deleted` is always `null` by default; with `--all-versions` it is the revision that replaced or deleted that version. Image names are `null` for video datasets. Pass `--revisions` to add a `revisions` array with the dataset's full revision log (author, time, description, additions, deletions). diff --git a/samples/scripts/userEdits/trackAuthors.py b/samples/scripts/userEdits/trackAuthors.py new file mode 100644 index 000000000..9e14dc318 --- /dev/null +++ b/samples/scripts/userEdits/trackAuthors.py @@ -0,0 +1,266 @@ +# /// script +# requires-python = ">=3.8" +# dependencies = [ +# "click", +# "pymongo", +# ] +# /// +""" +List the trackItem versions of a DIVE dataset with who saved each one. + +Each saved version of a track is a document in the trackItem collection whose +rev_created points at a revisionLogItem entry (author, time, description). +For image-sequence datasets the frame numbers are mapped to image names using +the same ordering the server uses (server/dive_server/crud.py valid_images). + +Run with uv: + uv run samples/scripts/userEdits/trackAuthors.py + uv run samples/scripts/userEdits/trackAuthors.py --container dive-mongo-1 + uv run samples/scripts/userEdits/trackAuthors.py --all-versions --features --revisions + uv run samples/scripts/userEdits/trackAuthors.py --mongo-uri mongodb://localhost:27017/girder -o out.json +""" + +from __future__ import annotations + +import functools +import itertools +import json +import re +import subprocess +from typing import Any, Dict, List, Optional, Union + +import click +from bson import ObjectId +from bson.errors import InvalidId +from pymongo import MongoClient +from pymongo.errors import ServerSelectionTimeoutError + +# dive_utils/constants.py (safeImageRegex is case-insensitive) +WEB_VALID_IMAGE_FORMATS = ("png", "jpg", "jpeg") +SAFE_IMAGE_REGEX = r"(\." + r"|\.".join(WEB_VALID_IMAGE_FORMATS) + r")$" +IMAGE_SEQUENCE_TYPE = "image-sequence" +FOREIGN_MEDIA_ID_MARKER = "foreign_media_id" +NUMBERS_REGEX = re.compile(r"(\d+)") +FEATURE_COUNT = {"$size": {"$ifNull": ["$features", []]}} + + +def _maybe_int(value: str) -> Union[str, int]: + try: + return int(value) + except ValueError: + return value + + +def _str_chunks(value: str) -> List[Union[int, str]]: + return [_maybe_int(v) for v in NUMBERS_REGEX.split(value) if v != ""] + + +def str_numeric_compare(input1: str, input2: str) -> int: + """Mirror of dive_utils.strNumericCompare so frame order matches the server.""" + if input1 == input2: + return 0 + for a, b in itertools.zip_longest(_str_chunks(input1), _str_chunks(input2), fillvalue=None): + if a == b: + continue + if a is None: + return -1 + if b is None: + return 1 + if type(a) == int and type(b) == int: + return a - b + if type(a) == int: + return -1 + if type(b) == int: + return 1 + return 1 if str(a) > str(b) else -1 + return 0 + + +def get_clone_root(db, folder: Dict[str, Any]) -> Dict[str, Any]: + """Follow foreign_media_id so clones resolve to the folder holding the media.""" + next_id = folder.get(FOREIGN_MEDIA_ID_MARKER) + while next_id: + source = db.folder.find_one({"_id": ObjectId(str(next_id))}) + if source is None: + raise click.ClickException(f"Clone source folder {next_id} no longer exists") + folder = source + next_id = folder.get(FOREIGN_MEDIA_ID_MARKER) + return folder + + +def image_names(db, folder: Dict[str, Any]) -> List[str]: + media_folder = get_clone_root(db, folder) + items = db.item.find( + { + "folderId": media_folder["_id"], + "lowerName": {"$regex": SAFE_IMAGE_REGEX, "$options": "i"}, + }, + {"name": 1}, + ) + names = [item["name"] for item in items] + return sorted(names, key=functools.cmp_to_key(str_numeric_compare)) + + +def container_ip(container: str) -> str: + try: + result = subprocess.run( + [ + "docker", + "inspect", + "-f", + "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}", + container, + ], + capture_output=True, + text=True, + check=True, + ) + except (OSError, subprocess.CalledProcessError) as err: + stderr = getattr(err, "stderr", "") or str(err) + raise click.ClickException(f"Could not inspect container {container}: {stderr.strip()}") + ips = result.stdout.split() + if not ips: + raise click.ClickException(f"Container {container} has no network IP (is it running?)") + return ips[0] + + +def frame_name(images: Optional[List[str]], frame: int) -> Optional[str]: + if images is None or frame < 0 or frame >= len(images): + return None + return images[frame] + + +@click.command(name="trackAuthors", help="List trackItem versions and their authors for a dataset") +@click.argument("folder_id") +@click.option( + "--mongo-uri", + default="mongodb://localhost:27017/girder", + show_default=True, + help="Girder MongoDB URI (database name taken from the path)", +) +@click.option( + "--container", + default=None, + help="Docker container running Mongo (e.g. dive-mongo-1); its IP overrides --mongo-uri", +) +@click.option( + "--all-versions", + is_flag=True, + help="Include replaced and deleted versions (default: only the current version of each track)", +) +@click.option( + "--revisions", "include_revisions", is_flag=True, help="Include the dataset's full revision log" +) +@click.option("--features", is_flag=True, help="Include every feature frame (and image name)") +@click.option("--set", "annotation_set", default=None, help="Annotation set name (default set if omitted)") +@click.option("-o", "--output", type=click.Path(dir_okay=False), help="Write JSON here instead of stdout") +def main(folder_id, mongo_uri, container, all_versions, include_revisions, features, annotation_set, output): + if container: + mongo_uri = f"mongodb://{container_ip(container)}:27017/girder" + client = MongoClient(mongo_uri, serverSelectionTimeoutMS=5000) + try: + client.admin.command("ping") + except ServerSelectionTimeoutError: + raise click.ClickException( + f"Could not reach MongoDB at {mongo_uri}. The default docker-compose does not publish " + "port 27017; try --container dive-mongo-1 or pass --mongo-uri with the container IP." + ) + db = client.get_default_database() + try: + dataset_id = ObjectId(folder_id) + except InvalidId: + raise click.ClickException(f"Invalid folder id: {folder_id}") + + folder = db.folder.find_one({"_id": dataset_id}) + if folder is None: + raise click.ClickException(f"Folder {folder_id} not found") + dataset_type = folder.get("meta", {}).get("type") + images = image_names(db, folder) if dataset_type == IMAGE_SEQUENCE_TYPE else None + + # Revision numbers are unique per dataset across all sets; the default set is stored as None + track_query: Dict[str, Any] = {"dataset": dataset_id, "set": annotation_set or None} + if not all_versions: + track_query["rev_deleted"] = {"$exists": False} + + revisions = {r["revision"]: r for r in db.revisionLogItem.find({"dataset": dataset_id})} + if annotation_set: + revisions = {k: r for k, r in revisions.items() if r.get("set") == annotation_set} + else: + revisions = {k: r for k, r in revisions.items() if not r.get("set")} + + # Count features in Mongo so the heavy features array is only loaded with --features + track_fields: Dict[str, Any] = { + "_id": 0, + "id": 1, + "begin": 1, + "end": 1, + "confidencePairs": 1, + "rev_created": 1, + "rev_deleted": 1, + "featureCount": FEATURE_COUNT, + } + if features: + track_fields["features"] = 1 + + tracks = [] + for track in db.trackItem.find(track_query, track_fields).sort([("id", 1), ("rev_created", 1)]): + rev = revisions.get(track.get("rev_created"), {}) + entry = { + "id": track["id"], + "begin": track.get("begin"), + "end": track.get("end"), + "beginImage": frame_name(images, track.get("begin", -1)), + "endImage": frame_name(images, track.get("end", -1)), + "featureCount": track.get("featureCount", 0), + "types": [pair[0] for pair in track.get("confidencePairs", [])], + "rev_created": track.get("rev_created"), + "rev_deleted": track.get("rev_deleted"), + "author_name": rev.get("author_name"), + "author_id": str(rev["author_id"]) if rev.get("author_id") else None, + "created": rev["created"].isoformat() if rev.get("created") else None, + # "save" is a manual edit in the UI, "Import ..." is a file or pipeline import + "description": rev.get("description"), + } + if features: + entry["features"] = [ + { + "frame": f["frame"], + "image": frame_name(images, f["frame"]), + "keyframe": f.get("keyframe"), + "interpolate": f.get("interpolate"), + } + for f in track.get("features", []) + ] + tracks.append(entry) + + result = { + "datasetId": folder_id, + "name": folder.get("name"), + "type": dataset_type, + "imageCount": len(images) if images is not None else None, + "tracks": tracks, + } + if include_revisions: + result["revisions"] = [ + { + "revision": r["revision"], + "author_name": r.get("author_name"), + "created": r["created"].isoformat() if r.get("created") else None, + "description": r.get("description"), + "additions": r.get("additions"), + "deletions": r.get("deletions"), + } + for r in sorted(revisions.values(), key=lambda r: r["revision"]) + ] + + text = json.dumps(result, indent=2) + if output: + with open(output, "w") as f: + f.write(text) + click.echo(f"Wrote {len(tracks)} track versions to {output}", err=True) + else: + click.echo(text) + + +if __name__ == "__main__": + main() diff --git a/server/dive_server/crud_annotation.py b/server/dive_server/crud_annotation.py index a4a12f482..8c6c1a4d0 100644 --- a/server/dive_server/crud_annotation.py +++ b/server/dive_server/crud_annotation.py @@ -30,8 +30,20 @@ def list( offset=0, sort=DEFAULT_ANNOTATION_SORT, revision: Optional[int] = None, - set: Optional[int] = None, + set: Optional[str] = None, ) -> Cursor: + query = self.list_query(dsFolder, revision, set) + return self.find( + offset=offset, limit=limit, sort=sort, query=query, fields=self.PROJECT_FIELDS + ) + + def list_query( + self, + dsFolder: types.GirderModel, + revision: Optional[int] = None, + set: Optional[str] = None, + ) -> dict: + """Query for the annotations visible at revision (default latest)""" head: int = RevisionLogItem().latest(dsFolder, set) if revision is None else revision query = { DATASET: dsFolder['_id'], @@ -42,9 +54,7 @@ def list( query[SET] = set else: query[SET] = None - return self.find( - offset=offset, limit=limit, sort=sort, query=query, fields=self.PROJECT_FIELDS - ) + return query def initialize(self): self._indices = [ @@ -133,6 +143,61 @@ def sets( return cursor +def track_authors( + dsFolder: types.GirderModel, + limit=0, + offset=0, + sort=DEFAULT_ANNOTATION_SORT, + revision: Optional[int] = None, + set: Optional[str] = None, +) -> Tuple[List[dict], int]: + """ + List the visible version of each track with the revision log entry that saved it. + + A save rewrites the whole track, so the author is the last user to change the + track, not necessarily the user who created each detection. + """ + track_model = TrackItem() + query = track_model.list_query(dsFolder, revision, set) + # Leave out features: they are the bulk of each document and only the count is needed + fields = { + '_id': 0, + IDENTIFIER: 1, + 'begin': 1, + 'end': 1, + REVISION_CREATED: 1, + 'featureCount': {'$size': {'$ifNull': ['$features', []]}}, + } + cursor = track_model.find(query=query, offset=offset, limit=limit, sort=sort, fields=fields) + tracks = list(cursor) + total = track_model.collection.count_documents(query) + + revisions = { + entry[REVISION]: entry + for entry in RevisionLogItem().find( + query={ + DATASET: dsFolder['_id'], + REVISION: {'$in': list({track[REVISION_CREATED] for track in tracks})}, + }, + fields={ + '_id': 0, + REVISION: 1, + 'author_id': 1, + 'author_name': 1, + 'created': 1, + 'description': 1, + }, + ) + } + for track in tracks: + entry = revisions.get(track[REVISION_CREATED], {}) + track['author_id'] = entry.get('author_id') + track['author_name'] = entry.get('author_name') + track['created'] = entry.get('created') + track['description'] = entry.get('description') + return tracks, total + + def rollback(dsFolder: types.GirderModel, revision: int): """Reset to previous revision.""" # TODO implement immutabble forward-rollback (like git revert) diff --git a/server/dive_server/views_annotation.py b/server/dive_server/views_annotation.py index 913df8f37..74f32dcd2 100644 --- a/server/dive_server/views_annotation.py +++ b/server/dive_server/views_annotation.py @@ -20,13 +20,18 @@ } -GetAnnotationParams = ( - Description("Get annotations of a dataset") - .pagingParams("id", defaultLimit=0) - .modelParam("folderId", **DatasetModelParam, level=AccessType.READ) - .param('revision', 'revision', dataType='integer', required=False) - .param('set', 'set', dataType='string', required=False) -) +def annotation_params(summary: str) -> Description: + """Paging, dataset, revision and set params shared by the annotation list routes""" + return ( + Description(summary) + .pagingParams("id", defaultLimit=0) + .modelParam("folderId", **DatasetModelParam, level=AccessType.READ) + .param('revision', 'revision', dataType='integer', required=False) + .param('set', 'set', dataType='string', required=False) + ) + + +GetAnnotationParams = annotation_params("Get annotations of a dataset") class AnnotationResource(Resource): @@ -37,6 +42,7 @@ def __init__(self, resourceName): self.resourceName = resourceName self.route("GET", ("track",), self.get_tracks) + self.route("GET", ("track", "authors"), self.get_track_authors) self.route("GET", ("group",), self.get_groups) self.route("GET", ("revision",), self.get_revisions) self.route("GET", ("export",), self.export) @@ -52,6 +58,20 @@ def get_tracks(self, limit: int, offset: int, sort, folder, revision, set): folder, limit=limit, offset=offset, sort=sort, revision=revision, set=set ) + @access.user + @autoDescribeRoute( + annotation_params( + "List the user who last saved each track. A save rewrites the whole track, " + "so this is the last user to change the track, not the creator of each detection." + ) + ) + def get_track_authors(self, limit: int, offset: int, sort, folder, revision, set): + tracks, total = crud_annotation.track_authors( + folder, limit=limit, offset=offset, sort=sort, revision=revision, set=set + ) + cherrypy.response.headers['Girder-Total-Count'] = total + return tracks + @access.user @autoDescribeRoute(GetAnnotationParams) def get_groups(self, limit: int, offset: int, sort, folder, revision, set): diff --git a/server/tests/test_track_authors.py b/server/tests/test_track_authors.py new file mode 100644 index 000000000..4e3d8c58e --- /dev/null +++ b/server/tests/test_track_authors.py @@ -0,0 +1,84 @@ +from datetime import datetime +from unittest.mock import patch + +from dive_server import crud_annotation +from dive_server.crud_annotation import DATASET, IDENTIFIER, REVISION, REVISION_CREATED + +VISIBLE_QUERY = {DATASET: 'dataset-id', 'visible': True} + + +@patch('dive_server.crud_annotation.RevisionLogItem') +@patch('dive_server.crud_annotation.TrackItem') +def test_track_authors_joins_revision_log(track_item, revision_log): + """Each visible track carries the author of the revision that created it.""" + saved = datetime(2026, 1, 17, 18, 49) + track_item.return_value.list_query.return_value = VISIBLE_QUERY + track_item.return_value.find.return_value = [ + {IDENTIFIER: 1, 'begin': 0, 'end': 4, REVISION_CREATED: 2, 'featureCount': 5}, + {IDENTIFIER: 2, 'begin': 3, 'end': 3, REVISION_CREATED: 3, 'featureCount': 1}, + {IDENTIFIER: 3, 'begin': 7, 'end': 9, REVISION_CREATED: 2, 'featureCount': 3}, + ] + track_item.return_value.collection.count_documents.return_value = 3 + revision_log.return_value.find.return_value = [ + { + REVISION: 2, + 'author_id': 'user-a', + 'author_name': 'alice', + 'created': saved, + 'description': 'save', + }, + { + REVISION: 3, + 'author_id': 'user-b', + 'author_name': 'bob', + 'created': saved, + 'description': 'Import VIAME_CSV from detections.csv', + }, + ] + + tracks, total = crud_annotation.track_authors( + {'_id': 'dataset-id'}, limit=10, offset=0, revision=None, set=None + ) + + assert total == 3 + assert [(t[IDENTIFIER], t['author_name']) for t in tracks] == [ + (1, 'alice'), + (2, 'bob'), + (3, 'alice'), + ] + assert tracks[1]['description'] == 'Import VIAME_CSV from detections.csv' + assert tracks[0]['created'] == saved + + track_item.return_value.list_query.assert_called_once_with({'_id': 'dataset-id'}, None, None) + track_kwargs = track_item.return_value.find.call_args.kwargs + assert track_kwargs['query'] == VISIBLE_QUERY + # Features are counted in mongo rather than returned (find projection expression) + assert 'features' not in track_kwargs['fields'] + assert track_kwargs['fields']['featureCount'] == { + '$size': {'$ifNull': ['$features', []]}, + } + track_item.return_value.collection.count_documents.assert_called_once_with(VISIBLE_QUERY) + + # One revision log lookup covers every distinct revision on the page + revision_query = revision_log.return_value.find.call_args.kwargs['query'] + assert revision_query[DATASET] == 'dataset-id' + assert sorted(revision_query[REVISION]['$in']) == [2, 3] + + +@patch('dive_server.crud_annotation.RevisionLogItem') +@patch('dive_server.crud_annotation.TrackItem') +def test_track_authors_missing_revision_entry(track_item, revision_log): + """A track whose revision log entry is gone is still listed, without an author.""" + track_item.return_value.list_query.return_value = VISIBLE_QUERY + track_item.return_value.find.return_value = [ + {IDENTIFIER: 1, 'begin': 0, 'end': 0, REVISION_CREATED: 5, 'featureCount': 1}, + ] + track_item.return_value.collection.count_documents.return_value = 1 + revision_log.return_value.find.return_value = [] + + tracks, total = crud_annotation.track_authors({'_id': 'dataset-id'}) + + assert total == 1 + assert tracks[0][IDENTIFIER] == 1 + assert tracks[0]['author_name'] is None + assert tracks[0]['author_id'] is None