diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index acefa7785..9f11470f5 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -535,6 +535,7 @@ class Meta(DeploymentListSerializer.Meta): "data_source_subdir", "data_source_regex", "description", + "metadata", "example_captures", "manually_uploaded_captures", # "capture_images", @@ -1787,6 +1788,7 @@ class Meta: "details", "name", "description", + "metadata", "project", "created_at", "updated_at", diff --git a/ami/main/migrations/0096_deployment_metadata_device_metadata.py b/ami/main/migrations/0096_deployment_metadata_device_metadata.py new file mode 100644 index 000000000..883e56667 --- /dev/null +++ b/ami/main/migrations/0096_deployment_metadata_device_metadata.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.10 on 2026-09-07 19:31 + +import ami.main.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0095_grant_sync_deployment_to_mldatamanager"), + ] + + operations = [ + migrations.AddField( + model_name="deployment", + name="metadata", + field=models.JSONField( + blank=True, + default=dict, + help_text="Free-form JSON object holding attributes specific to this configuration, beyond the fields modelled natively. Camera height or habitat description, for example.", + validators=[ami.main.models.validate_metadata_object], + ), + ), + migrations.AddField( + model_name="device", + name="metadata", + field=models.JSONField( + blank=True, + default=dict, + help_text="Free-form JSON object holding attributes specific to this configuration, beyond the fields modelled natively. Camera height or habitat description, for example.", + validators=[ami.main.models.validate_metadata_object], + ), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index 78e67e275..4ccd8d037 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -632,6 +632,54 @@ class Meta: unique_together = ("user", "project") +# JSON type names used when telling an API client what shape it sent instead of an object. +_JSON_TYPE_NAMES: Final = { + bool: "a boolean", + dict: "an object", + float: "a number", + int: "a number", + list: "an array", + str: "a string", + type(None): "null", +} + + +def validate_metadata_object(value: typing.Any) -> None: + """ + Require metadata to be a JSON object, so its keys stay queryable and mappable. + + The contents are deliberately unconstrained, because the attributes worth + recording differ from one project to the next. The shape is not: a bare array, + string or number carries no field names, which leaves a Postgres key lookup + nothing to match and a published term nothing to map onto. + """ + if not isinstance(value, dict): + raise ValidationError( + "Metadata must be a JSON object of key/value pairs, not %(json_type)s.", + code="invalid_metadata", + params={"json_type": _JSON_TYPE_NAMES.get(type(value), "a value of an unsupported type")}, + ) + + +def metadata_field() -> models.JSONField: + """ + Build the free-form metadata column that a model exposes to its owners. + + Defined in one place so that the columns on different models cannot drift apart + in their default, their validation or the help text an API client reads. + """ + return models.JSONField( + default=dict, + blank=True, + validators=[validate_metadata_object], + help_text=( + "Free-form JSON object holding attributes specific to this configuration, " + "beyond the fields modelled natively. Camera height or habitat description, " + "for example." + ), + ) + + @final class Device(BaseModel): """ @@ -643,6 +691,7 @@ class Device(BaseModel): name = models.CharField(max_length=_POST_TITLE_MAX_LENGTH) description = models.TextField(blank=True) project = models.ForeignKey(Project, on_delete=models.SET_NULL, null=True, related_name="devices") + metadata = metadata_field() deployments: models.QuerySet["Deployment"] @@ -770,6 +819,7 @@ class Deployment(BaseModel): latitude = models.FloatField(null=True, blank=True) longitude = models.FloatField(null=True, blank=True) image = models.ImageField(upload_to="deployments", blank=True, null=True) + metadata = metadata_field() project = models.ForeignKey(Project, on_delete=models.SET_NULL, null=True, related_name="deployments") diff --git a/ami/main/tests.py b/ami/main/tests.py index e8f56c485..6ff49f8d1 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -1,5 +1,6 @@ import copy import datetime +import json import logging import typing from io import BytesIO @@ -8,7 +9,7 @@ from django.conf import settings from django.contrib.auth.models import AnonymousUser from django.core.files.uploadedfile import SimpleUploadedFile -from django.db import IntegrityError, connection, models +from django.db import IntegrityError, connection, models, transaction from django.test import TestCase, override_settings from django.test.utils import CaptureQueriesContext from django.utils import timezone @@ -6599,6 +6600,199 @@ def test_non_integer_id_returns_400_not_500(self): self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST, f"{endpoint}?{param}=abc") +class TestConfigurableMetadataFields(APITestCase): + """Free-form ``metadata`` on stations and device types. + + Every project records attributes that Antenna does not model natively: the height a + camera was mounted at, a description of the habitat, the make of a light. Deployments + and device types each carry a ``metadata`` column for those. Its contents are + unconstrained, but its shape is not — it must be a JSON object, so that individual keys + stay queryable in Postgres and can be mapped onto a term when records are published. + + These tests pin that an object survives a write and reads back unchanged, that a value + which is not an object is refused and leaves the stored value alone, that writing + metadata demands exactly the update permission the record already required, and that a + record created without metadata holds an empty object rather than null. + """ + + def setUp(self): + self.project = Project.objects.create(name="Metadata Project") + self.deployment = Deployment.objects.create(name="Metadata Deployment", project=self.project) + self.device = Device.objects.create(name="Metadata Device", project=self.project) + + self.project_manager = User.objects.create_user(email="metadata_manager@insectai.org") + ProjectManager.assign_user(self.project_manager, self.project) + self.basic_member = User.objects.create_user(email="metadata_member@insectai.org") + BasicMember.assign_user(self.basic_member, self.project) + + # Both models expose metadata on their detail endpoint, which is also the write path. + self.endpoints = { + "deployment": f"/api/v2/deployments/{self.deployment.pk}/", + "device": f"/api/v2/deployments/devices/{self.device.pk}/", + } + + def test_metadata_object_round_trips(self): + """A nested object written through the API reads back exactly as it was sent.""" + metadata = { + "camera_height_m": 2.5, + "habitat": "mixed deciduous woodland", + "lamp": {"type": "actinic", "watts": 15}, + "visits": ["2026-05-01", "2026-06-01"], + } + self.client.force_authenticate(user=self.project_manager) + + for name, endpoint in self.endpoints.items(): + response = self.client.patch(endpoint, {"metadata": metadata}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK, f"{name}: {response.content}") + self.assertEqual(response.json()["metadata"], metadata, name) + + # Read it back on a fresh request: nesting, numbers and lists must all survive + # the round trip through the database rather than only the write response. + read_back = self.client.get(endpoint) + self.assertEqual(read_back.status_code, status.HTTP_200_OK, name) + self.assertEqual(read_back.json()["metadata"], metadata, name) + + def test_non_object_metadata_is_rejected(self): + """An array, string, number or boolean is not metadata, and leaves the record untouched.""" + self.client.force_authenticate(user=self.project_manager) + + for name, endpoint in self.endpoints.items(): + for value in ([1, 2, 3], "camera height 2.5m", 42, True): + response = self.client.patch(endpoint, {"metadata": value}, format="json") + self.assertEqual( + response.status_code, + status.HTTP_400_BAD_REQUEST, + f"{name} accepted {value!r}", + ) + self.assertIn("metadata", response.json(), f"{name} did not blame the metadata field") + # The message has to say what is wrong, because the browsable API and the + # Django admin show it to a person with no form validation in front of them. + self.assertIn("object", str(response.json()["metadata"]).lower(), name) + + self.deployment.refresh_from_db() + self.device.refresh_from_db() + self.assertEqual(self.deployment.metadata, {}) + self.assertEqual(self.device.metadata, {}) + + def test_metadata_write_requires_the_records_update_permission(self): + """Metadata follows the update permission each record already had, and adds none of its own.""" + self.client.force_authenticate(user=self.basic_member) + + for name, endpoint in self.endpoints.items(): + # A basic member may read the record, so the refusal below is about writing. + self.assertEqual(self.client.get(endpoint).status_code, status.HTTP_200_OK, name) + + response = self.client.patch(endpoint, {"metadata": {"habitat": "hedgerow"}}, format="json") + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN, name) + + self.deployment.refresh_from_db() + self.device.refresh_from_db() + self.assertEqual(self.deployment.metadata, {}) + self.assertEqual(self.device.metadata, {}) + + def test_metadata_defaults_to_an_empty_object(self): + """A record created without metadata holds an empty object, and null is refused everywhere.""" + deployment = Deployment.objects.create(name="Station without metadata", project=self.project) + device = Device.objects.create(name="Device without metadata", project=self.project) + self.assertEqual(deployment.metadata, {}) + self.assertEqual(device.metadata, {}) + + self.client.force_authenticate(user=self.project_manager) + fresh_endpoints = { + "deployment": f"/api/v2/deployments/{deployment.pk}/", + "device": f"/api/v2/deployments/devices/{device.pk}/", + } + for name, endpoint in fresh_endpoints.items(): + self.assertEqual(self.client.get(endpoint).json()["metadata"], {}, name) + + # The column refuses null as well, so no code path can leave one behind for a + # reader to guard against. + for model in (Deployment, Device): + with self.assertRaises(IntegrityError, msg=model.__name__), transaction.atomic(): + model.objects.create(name="Null metadata", project=self.project, metadata=None) + + def test_null_is_refused_on_both_write_paths(self): + """Null is refused whichever way it arrives, by two different mechanisms. + + A client serialiser emitting ``None`` for an absent value is the likeliest way a + null reaches this field by accident. Sent as JSON it never gets as far as the + shape check, because the field is not nullable and the framework stops it first. + Sent through the station form it arrives as the text ``null``, is parsed into + ``None``, and the shape check is what refuses it. Both are pinned, because a + change to either mechanism would leave the other still looking correct. + """ + self.client.force_authenticate(user=self.project_manager) + + for name, endpoint in self.endpoints.items(): + response = self.client.patch(endpoint, {"metadata": None}, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, name) + self.assertIn("metadata", response.json(), name) + + response = self.client.patch( + self.endpoints["deployment"], + {"metadata": "null"}, + format="multipart", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.content) + self.assertIn("null", str(response.json()["metadata"]).lower()) + + self.deployment.refresh_from_db() + self.device.refresh_from_db() + self.assertEqual(self.deployment.metadata, {}) + self.assertEqual(self.device.metadata, {}) + + def test_station_metadata_survives_a_form_encoded_submission(self): + """A station is edited as a form, so its metadata arrives as JSON text and must be stored as an object. + + The station form is submitted as multipart because the record carries a cover + image, and multipart has no JSON types. The value therefore reaches the serializer + as a string of JSON. Storing that string verbatim would look like a success and + only surface later, when a reader finds text where a mapping should be. + """ + metadata = {"habitat": "mixed deciduous woodland", "camera_height_m": 2.5} + self.client.force_authenticate(user=self.project_manager) + + response = self.client.patch( + self.endpoints["deployment"], + {"metadata": json.dumps(metadata)}, + format="multipart", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK, response.content) + + self.deployment.refresh_from_db() + self.assertIsInstance(self.deployment.metadata, dict) + self.assertEqual(self.deployment.metadata, metadata) + + def test_form_encoded_submission_is_held_to_the_same_shape_rule(self): + """The shape rule is enforced on the server, not only by the form that usually feeds it. + + The station form refuses these five shapes before they leave the browser, but the + Django admin, the browsable API and any other client bypass that form entirely, so + each one is checked here against the endpoint itself. + """ + self.client.force_authenticate(user=self.project_manager) + + for value in ("[1, 2, 3]", '"a bare string"', "42", "true", "null"): + response = self.client.patch( + self.endpoints["deployment"], + {"metadata": value}, + format="multipart", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, f"accepted {value}") + self.assertIn("metadata", response.json(), f"{value} did not blame the metadata field") + + # Text that is not JSON at all is refused too, rather than being stored as a string. + response = self.client.patch( + self.endpoints["deployment"], + {"metadata": "habitat: woodland"}, + format="multipart", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.content) + + self.deployment.refresh_from_db() + self.assertEqual(self.deployment.metadata, {}) + + class TestDetectionNullMarker(TestCase): """ Covers the null-marker abstraction added for Issue #1310 follow-up: diff --git a/docs/claude/INDEX.md b/docs/claude/INDEX.md index 61f0980d8..54c76180c 100644 --- a/docs/claude/INDEX.md +++ b/docs/claude/INDEX.md @@ -40,6 +40,8 @@ archived. | `planning/celery-queue-split-rollout.md` | Rollout plan for the Celery queue split (`feat/celery-queue-split`) | | `planning/2026-05-28-captures-processed-filter-design.md` | Design: captures "Processed / Not processed" filter | | `planning/2026-05-28-captures-processed-filter-plan.md` | Implementation plan (checkbox tasks) for the captures processed filter | +| `planning/2026-09-07-configurable-metadata-fields.md` | Free-form `metadata` JSON object on Deployment & Device (issue #507): why the shape is validated but the contents are not, why it differs from `Project.feature_flags`, detail-vs-list exposure, and what #307 still asks for. Keywords: metadata, JSONField, jsonb, GBIF, deployments, devices | +| `planning/2026-09-07-configurable-metadata-fields-ui.md` | Frontend for issue #507: free-form JSON metadata field on the station and device-type forms. Text-not-object form value, validation helpers in `utils/fieldProcessors.ts`, multipart JSON seam. Keywords: metadata, JSON, forms, deployments, devices | ## Archive / session snapshots diff --git a/docs/claude/planning/2026-09-07-configurable-metadata-fields-ui.md b/docs/claude/planning/2026-09-07-configurable-metadata-fields-ui.md new file mode 100644 index 000000000..ce31757eb --- /dev/null +++ b/docs/claude/planning/2026-09-07-configurable-metadata-fields-ui.md @@ -0,0 +1,173 @@ +# Metadata fields on the station and device-type forms + +Date: 2026-09-07 +Issue: RolnickLab/antenna#507, "Add generic metadata JSON fields" +Scope: the frontend half only. The API field itself is being added on a separate branch. + +## What this gives an operator + +A project can now record properties of a station or a device type that the platform has +no field of its own for — the height of a mast, the wattage of a lamp, a local site code — +without waiting for those to become columns in the database. Both the station form and the +device-type form gained a "Metadata" box that holds free-form JSON. It reads back whatever +is already stored, and a mistake in the JSON is reported next to the box during editing +rather than coming back as a server error after saving. A record with broken JSON in +that box cannot be saved at all. + +The field is a monospace text box with inline errors. The syntax highlighting the ticket +offers as an optional extra is not part of it; see "What this does not do" below. + +## The contract this is built against + +The API exposes a writable `metadata` field on the station (deployment) and device-type +records. It is a free-form JSON object, and it is an empty object when the record has no +metadata. Nothing here assumes any particular key inside it. + +## How it is put together + +### The field itself + +`ui/src/components/form/metadata-field.tsx` is a textarea wired to react-hook-form through +the existing `FormController`, in the same shape as `form-field.tsx`. It reuses +`InputContent` from `nova-ui-kit` for the label, description and inline error, so it looks +like every other field on those forms. + +The form value is the JSON *text*, not a parsed object. That is the decision the rest of +the design follows from. A half-finished edit — an object with one quote missing — has no +object representation, so storing text is what lets the operator keep looking at their own +work while the error tells them what is wrong with it. It also preserves the key order and +indentation they chose rather than reformatting the box underneath them. + +### The three helpers + +All three live in `ui/src/utils/fieldProcessors.ts`, next to the existing +`parseIntegerList` / `formatIntegerList` / `validateIntegerList` trio, which solve exactly +the same "structured value edited as text" problem for comma-separated ID lists. + +- `formatMetadata(value)` renders a stored object as indented JSON for the field, and + renders a record with no metadata as an empty box rather than a bare `{}`. +- `parseMetadata(text)` turns the text back into the object the API stores. An empty box + becomes `{}`, which is how the API represents "no metadata". +- `validateMetadata(text)` is the `validate` rule in each form's field config. It accepts + an empty box, rejects text that does not parse, and rejects text that parses into + something other than an object — a list, a number, a bare string — because the API + stores an object and nothing else. + +`ui/src/utils/fieldProcessors.test.ts` covers all three, including the accept cases as +well as the reject cases, so a rule that silently stopped rejecting anything would fail. + +### The station form + +The station form is the multi-section dialog under +`ui/src/pages/deployment-details/deployment-details-form/`. The field sits in the General +section, full width beneath the two-column rows. + +- `config.ts` declares the label, the guidance text and the `validateMetadata` rule. +- `deployment-details-form.tsx` seeds the General section's stored value with + `formatMetadata(deployment.metadata)`. +- `section-general/section-general.tsx` renders the field. +- `data-services/models/deployment-details.ts` adds the `metadata` getter that reads the + stored object, and `metadata` to `DeploymentFieldValues` as the JSON text a submit + carries. + +Invalid JSON blocks the save because the dialog's Save button is already disabled while any +section reports itself invalid (`deployment-details-form.tsx`, the `allValid` check in +`SaveButton`), and `useSyncSectionStatus` feeds each section's validity into that. + +The two halves of that check run on different schedules, which is worth knowing before +reading a bug report about it. The Save button goes dead the moment the JSON stops being +valid, because react-hook-form keeps `isValid` current on every keystroke, but the message +under the box appears only when the field loses focus, because the General section +validates on blur. That difference is deliberate and is explained under "What this does +not do". + +The station endpoint takes a multipart request, because the form can also carry a cover +image. A multipart request can only carry text, so `data-services/hooks/deployments/utils.ts` +serialises the object and appends it as JSON. Django REST Framework's `JSONField` parses a +string back into an object when the request came in as form input, so the field arrives at +the serializer as the object it is meant to be. This is the one seam between the two halves +of the ticket that is worth checking on a running stack: it holds as long as the API's +`metadata` field is a plain DRF `JSONField`, which is what a `ModelSerializer` generates +for a `models.JSONField`. + +### The device-type form + +Device types are edited through the shared entity dialog, which picks a form per entity +type from `customFormMap` in +`ui/src/pages/project/entities/details-form/constants.ts`. Storage sources, capture sets, +exports and processing services already have their own forms there; device types were still +using the default name-and-description form. + +- `device-details-form.tsx` is a new entry in that map, keyed `device`. It is the default + entity form plus the metadata field. +- `data-services/models/device.ts` adds a `Device` model with the `metadata` getter, and + `data-services/hooks/entities/useEntities.ts` constructs it for the devices collection. + Reading the field through a model of its own keeps `Entity` — the base class sites, + exports and everything else share — free of a field only device types have. + +That endpoint takes a JSON body, so the object goes over the wire as an object, through the +existing `customFields` mechanism in `data-services/hooks/entities/utils.ts`. No change was +needed there. Invalid JSON blocks the save because react-hook-form only runs the submit +handler once every field has passed its rules. + +### Strings + +`FIELD_LABEL_METADATA`, `MESSAGE_METADATA_DESCRIPTION`, `MESSAGE_METADATA_INVALID` and +`MESSAGE_METADATA_NOT_OBJECT` were added to `ui/src/utils/language.ts`, in both the `STRING` +enum and the English display map. + +## What still needs to be verified + +Everything below needs the API field, so none of it could be checked while writing this. + +1. **A round trip through the station endpoint.** Save metadata on a station, reload the + dialog, confirm the box shows the same keys. This is what proves the multipart JSON + parsing described above, and it is the single most likely place for the two halves of + the ticket to disagree. +2. **A round trip through the device-type endpoint**, the same way. +3. **Clearing the box.** Emptying the field should leave the record with an empty object, + not a literal empty string and not the previous value. +4. **A station saved from the "new station" dialog**, which goes through + `useCreateDeployment` rather than `useUpdateDeployment`. + +## What this does not do + +Each of these was considered and left out on purpose. None of them is an oversight, and +none needs re-deciding unless the reason underneath it changes. + +- **Metadata is not displayed anywhere outside the edit dialogs.** It is absent from the + station's info view (`ui/src/pages/deployment-details/deployment-details-info.tsx`) and + from the tables that list device types, so someone browsing stations cannot tell which + ones carry metadata without opening each one to edit it. Putting raw JSON into a table + column is a layout question with several defensible answers — truncate the text, show a + key count, show a badge — and choosing one silently seemed worse than leaving it for + whoever picks up that decision. +- **The field does not highlight JSON syntax**, which the ticket offers as an optional + extra. Every route to it means adding a code-editor dependency such as CodeMirror or + Monaco to a bundle that has no code editor today, and that is too large a cost for an + optional nicety. The field uses a monospace face and reports its errors inline instead, + and only `ui/src/components/form/metadata-field.tsx` would have to change if the team + later decides the dependency is worth it. +- **The two forms do not report errors on the same schedule.** The station form shows the + message when the field loses focus and the device-type form shows it as you type, + because each section keeps the validation mode of the fields around it and the station's + General section validates on blur. Making metadata the one live-validating field there + would leave it the odd one out, and switching the whole section to on-change validation + would change the behaviour of fields this work has no business touching — the required + name would start flashing an error at somebody halfway through typing one. Both forms + disable Save on the keystroke that breaks the JSON, which is the half that actually + prevents a bad save. +- **Nothing here has an opinion about what goes inside the object.** The ticket mentions + that metadata should be queryable in PostgreSQL and publishable to GBIF. Both are + backend concerns and neither constrains what this form accepts. + +One thing that is a limit rather than a choice: rejecting a non-object is a guard on this +one client, not an enforcement of the contract. The form refuses a bare list, string or +number before it can be saved, but the admin, the browsable API and any future uploader +all bypass this form, so the API has to make the same check for itself. + +## Related + +- `docs/claude/reference/react-form-to-drf-values.md` — how form values reach DRF, and why + empty strings and nulls are not interchangeable. The `{}`-for-empty rule here is the + JSON-object version of the same question. diff --git a/docs/claude/planning/2026-09-07-configurable-metadata-fields.md b/docs/claude/planning/2026-09-07-configurable-metadata-fields.md new file mode 100644 index 000000000..d43058f25 --- /dev/null +++ b/docs/claude/planning/2026-09-07-configurable-metadata-fields.md @@ -0,0 +1,190 @@ +# Configurable metadata fields on stations and device types + +Date: 2026-09-07 +Status: backend implemented, frontend not started +Scope: the backend half of issue #507 ("Add generic metadata JSON fields"). The text +field with syntax validation and highlighting that the issue also asks for is a +frontend change and is not part of this work. + +## Goal + +Antenna models natively only the attributes that every project shares. Real deployments +carry more: the height a camera was mounted at, a description of the habitat, the make +of a light, the serial number of a controller. Today there is nowhere to record any of +that except free text in the description, where nothing can be searched or exported. + +This change gives a station (`Deployment`) and a device type (`Device`) a `metadata` +field, an open JSON object that the people who may already edit the record can write +whatever they need into. Because the column is stored as Postgres `jsonb`, individual +keys inside it can be queried in SQL, and because each entry is a named key rather than +a sentence, an entry can later be mapped onto a term when records are published to GBIF. + +## The one rule: metadata is an object + +The contents are deliberately unconstrained. No fixed schema would fit, because the +attributes worth recording differ from one project to the next, which is the premise of +the issue. What is constrained is the shape: the value must be a JSON object of +key/value pairs. A bare array, string, number or boolean is refused. + +The reason is that the shape is what makes the field useful rather than a matter of +taste. A value with no field names in it leaves a Postgres key lookup nothing to match +and a published term nothing to map onto, so accepting one would quietly produce rows +that no later feature can read. Refusing it at the point of writing keeps every stored +value queryable. + +Refusing an array is a decision, not an oversight. The discussion on issue #307 floated +modelling metadata as an array of `{title, value}` pairs, and that shape is deliberately +closed off. An object gives Postgres a key to index and look up, and gives a published +term something to map onto. An array of pairs gives neither unless the reader already +knows a private convention for what `title` and `value` mean. Anyone who genuinely needs +ordered pairs can still hold them in an array *under a key* — +`{"attributes": [{"title": ..., "value": ...}]}` — which keeps the outer shape queryable +and loses nothing. + +This is a deliberate departure from `Project.feature_flags`, the other JSON column on +these models' neighbours. That field is a `SchemaField` backed by the pydantic model +`ProjectFeatureFlags` (`ami/main/models.py`, `ProjectFeatureFlags`, around line 271), +because its keys are a closed set that the code itself reads by name. Metadata is the +opposite case — the keys belong to the operator, not to the code — so it is a plain +`models.JSONField` with a single shape check rather than a schema. + +## Where the code is + +- `ami/main/models.py`, `validate_metadata_object` (around line 647) — the shape check, + raising a `ValidationError` that names the JSON type it received. `_JSON_TYPE_NAMES` + just above it turns a Python type into the word a JSON author would recognise, so the + API error reads "not an array" rather than "not a list". +- `ami/main/models.py`, `metadata_field` (around line 664) — builds the column. Both + models call it rather than repeating the keyword arguments, so the two columns cannot + drift apart in their default, their validation or the help text an API client reads. +- `ami/main/models.py`, `Device.metadata` (around line 694) and `Deployment.metadata` + (around line 822) — the two columns. +- `ami/main/api/serializers.py`, `DeviceSerializer.Meta.fields` (around line 1791) and + `DeploymentSerializer.Meta.fields` (around line 538) — the exposure. +- `ami/main/migrations/0096_deployment_metadata_device_metadata.py` — one migration + adding both columns. + +**Known merge step:** three open branches each number their migration `0096` — this one, +the deployment status heartbeat work, and a model-coverage backfill. Whichever lands +second and third has to be renumbered onto the new head. Renumbering ahead of time only +moves the collision, so it is left for merge time. The heartbeat branch also adds columns +to `Deployment`, so it may merge cleanly against this file while still needing that +rename; a clean merge is not evidence of a correct one here. + +## Two decisions worth knowing about + +**The field is on the deployment detail endpoint, not the list.** `DeploymentSerializer` +extends `DeploymentListSerializer`, so adding a field to the parent would have put every +station's metadata blob into every row of the stations list. The field has no size limit +by design, and the list endpoints in this codebase have repeatedly been the ones with +performance problems, so the metadata is served from `GET /api/v2/deployments/{id}/` and +written there too. `Device` has a single serializer and no such split, so its metadata +appears on both its list and its detail response. + +**No new permission was introduced.** Both `DeploymentViewSet` and `DeviceViewSet` use +`ObjectPermission` (`ami/base/permissions.py`, `ObjectPermission`), which delegates to +`BaseModel.check_permission` and thence to the existing `update_deployment` and +`update_device` guardian permissions. A writable serializer field is therefore covered +by the same rule that already governs renaming a station: a project manager may write +metadata, a basic member may read it and gets a 403 if they try to write. The test suite +pins this rather than assuming it. + +One consequence worth stating, because it is easy to miss: Django model validators are +copied onto the serializer field by DRF's `get_field_kwargs`, so the single validator on +the model governs the API write path, the Django admin and any direct `full_clean()` +alike. There is no separate serializer-level check to keep in step. + +## The form-encoded seam, and why the field is left auto-generated + +A station is edited through a form submitted as `multipart/form-data`, because the record +carries a cover image. Multipart has no JSON types, so `metadata` reaches the server as a +string containing JSON rather than as an object. Storing that string verbatim would look +like a success at the time and surface much later, when something reads the row back and +finds text where a mapping should be. + +The field is therefore left for `ModelSerializer` to generate from the model, which makes +it a plain `serializers.JSONField`. That class marks a value taken from form input as a +JSON string and runs `json.loads` on it, so an object is what reaches the column. Anything +that overrides how the value is read — a `CharField`, or a subclass with its own +`get_value` — takes the plain-string branch instead and stores the text. Declaring +`metadata` explicitly is safe only as `serializers.JSONField`; declaring it as anything +else silently changes what is stored, which is why it is not declared at all. + +The shape rule is enforced on the server rather than in the form. The station form +refuses a bare array, a quoted string, a number, `true` and `null` before they leave the +browser, but the Django admin, the browsable API and any other client bypass that form +entirely. The validator on the model is what makes "an object, or nothing" an invariant +instead of one client's good manners, and the tests exercise all five shapes against the +endpoint itself. + +## Testing + +`ami/main/tests.py`, `TestConfigurableMetadataFields` (around line 6602), seven tests +covering both models: + +1. A nested object containing numbers, strings, a sub-object and an array is written + through the API and read back on a fresh request unchanged. +2. An array, a string, a number and a boolean are each refused with a 400 that blames + the `metadata` field, and the stored value is confirmed untouched afterwards. +3. A basic member can read the record but is refused the write with a 403, and the + stored value is again confirmed untouched. +4. A record created without metadata holds `{}`; `null` is refused by the API, and the + column itself raises `IntegrityError` on a null, so no code path can leave one behind + for a reader to guard against. +5. `null` is refused on both write paths, which reject it by different mechanisms: as + JSON it is stopped before the shape check because the field is not nullable, while + through the form it arrives as the text `null`, is parsed to `None`, and the shape + check refuses it. A change to either mechanism would leave the other looking correct, + so both are pinned. +6. A station update submitted as `multipart/form-data` with metadata as JSON text stores + an object, asserted by type rather than by value so that a stored string cannot pass. +7. The same form-encoded path refuses an array, a quoted string, a number, `true`, `null` + and text that is not JSON at all, and leaves the stored value untouched. + +Run them the way the rest of the suite is run against a worktree, per +`docs/claude/reference/worktree-testing.md`: + +``` +python manage.py test ami.main.tests.TestConfigurableMetadataFields +``` + +## Not done here + +- **The frontend text field.** Issue #507 asks for a text field that validates and + ideally highlights JSON syntax. Nothing in the UI has been touched. +- **A GIN index on either column.** `jsonb` is queryable without one, which satisfies + what the issue asks for. An index is a performance question that should be answered by + a real query pattern rather than added speculatively. +- **The GBIF mapping itself.** This change makes a published mapping possible by giving + the values names; it does not decide which keys map onto which terms. That decision + belongs with the export work. +- **`Site`.** Issue #507 names deployments and device types only. Sites are the natural + next candidate; see the section below for what issue #307 wants of them. + +## Relationship to issue #307 + +Issue #307 ("Allow meta data to be configured for Deployments") asks for the same +capability and supplies the motivating examples — hardware, habitat, camera height — +that this change serves. Those examples are now recordable, so the part of #307 that is +about deployment owners having somewhere to put this information is covered. + +Three things it asks for are not, and #307 should stay open for them rather than being +closed by #507. + +- **Standard fields alongside the generic object.** The discussion on #307 settles on + "some of both": a few standard fields that the interface can filter and chart on, plus + a generic object for everything else. This change ships only the generic object. Which + attributes graduate into native fields is a question for the standards and metadata + working group, not for the backend. +- **Ecological Metadata Language, and Site.** #307 proposes describing a research site's + ecology with EML, on the `Site` model. That is a schema question rather than a + free-form one, and this change adds no `Site` column and takes no position on EML. +- **Tagging as an alternative route.** #307 raises a tagging system as a possible answer + instead, one that would extend to occurrences and captures as well. Nothing here + forecloses that, and nothing here delivers it. + +One shape decision is worth flagging to anyone returning to #307, because a different +one was floated there: the discussion included an example that modelled metadata as an +array of `{title, value}` pairs. This change refuses arrays. Named keys in an object are +what make a Postgres key lookup and a term mapping possible, and an array of pairs would +have to be unpacked before either could work. diff --git a/ui/src/components/form/metadata-field.tsx b/ui/src/components/form/metadata-field.tsx new file mode 100644 index 000000000..7e964cdf2 --- /dev/null +++ b/ui/src/components/form/metadata-field.tsx @@ -0,0 +1,56 @@ +import { InputContent } from 'nova-ui-kit' +import { ControllerProps, FieldPath, FieldValues } from 'react-hook-form' +import { FormController } from './form-controller' +import { FormConfig } from './types' + +/* Tall enough to show a handful of metadata entries without taking over the form. */ +const VISIBLE_ROWS = 8 + +/** + * Free-form JSON entry for a record's metadata, so a project can keep fields + * the platform has no column of its own for. + * + * The form value is the text exactly as it was typed, not a parsed object. That + * keeps a half-finished edit on screen while the inline error explains what is + * wrong with it, and it preserves the key order and indentation the operator + * chose. Text that is not a JSON object is rejected by the `validate` rule in + * the field's config, and the form turns the text into an object on submit. + */ +export const MetadataField = < + TFieldValues extends FieldValues, + TName extends FieldPath +>({ + name, + control, + config, +}: Pick, 'name' | 'control'> & { + config: FormConfig +}) => { + const fieldConfig = config[name] + + return ( + ( + +