From e6c6e9fb1c60fe4c84d4372d70ef6cf74c9a39b6 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 29 Sep 2026 17:44:08 -0700 Subject: [PATCH 01/29] feat(jobs): describe creatable job types and validate job settings on create Adds GET /api/v2/jobs/types/?project_id=N, which lists the job types a project member may create, what each one runs on (scope), and a JSON Schema of its settings generated from the job type's pydantic model. Post-processing lists every registered task as a variant with its own schema, so a new task appears in the Create Job dialog without frontend work. The endpoint gates itself: anonymous requests are refused before the project is read, and only project members (or superusers) may list a project's types. Permissions are read once per request, so the query count does not grow with the number of types. Job params are now writable on create and checked by the job type through JobType.validate_params(project, user, params), which follows the same shape as the tracking branch's validate_post_processing_params so that branch can adopt it. Every id inside a post-processing config must belong to the job's project, tasks not on the member allowlist are staff-only, and settings a member may not change must keep their defaults. Params are fixed once a job exists. Platform-created job types (exports) are refused through the API, and capture sets, stations and captures from another project are refused. Class masking and the small size filter carry their labels, help text and picker hints on the pydantic fields, taken from the dialog spec. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NFUiikN95Y3yz4pBK9KPu1 --- ami/jobs/descriptors.py | 110 +++++++++ ami/jobs/models.py | 247 +++++++++++++++++++- ami/jobs/serializers.py | 119 +++++++++- ami/jobs/tests/test_job_types.py | 234 +++++++++++++++++++ ami/jobs/views.py | 26 ++- ami/ml/post_processing/class_masking.py | 36 ++- ami/ml/post_processing/registry.py | 17 ++ ami/ml/post_processing/small_size_filter.py | 12 +- 8 files changed, 787 insertions(+), 14 deletions(-) create mode 100644 ami/jobs/descriptors.py create mode 100644 ami/jobs/tests/test_job_types.py diff --git a/ami/jobs/descriptors.py b/ami/jobs/descriptors.py new file mode 100644 index 000000000..7fcfb3654 --- /dev/null +++ b/ami/jobs/descriptors.py @@ -0,0 +1,110 @@ +""" +Describe job types to the Create Job dialog, so a form can be generated for any of them. + +A job type is described by its docstring (the help text), the scope it needs (which pipeline, +capture set, station or sessions it runs on) and an optional pydantic ``config_schema`` for its +settings. The dialog renders all three without knowing the job type in advance, which is what lets +a new job type or post-processing task appear in the UI with no frontend work. See +``docs/claude/reference/jobs-panel.md``. +""" + +import copy +import dataclasses +import inspect +import typing + +import pydantic + +# Bumped when the shape of a normalized schema changes, so a deployed client can tell. +SCHEMA_VERSION = 1 + +# Widget hints a config field can carry, passed as extra keyword arguments to ``pydantic.Field``. +# Pydantic v1 copies unknown ``Field`` keywords into the field's JSON Schema unchanged. +WIDGET_KEY = "ami_widget" +ENTITY_KEY = "ami_entity" +ENTITY_FILTERS_KEY = "ami_entity_filters" + + +@dataclasses.dataclass(frozen=True) +class ScopeField: + """One thing a job runs on, chosen before its settings. + + ``field`` is the name the dialog sends. ``target`` says where: ``"job"`` fields are top-level + serializer fields backed by a Job column (``pipeline_id``, ``source_image_collection_id``), + ``"config"`` fields go inside ``params["config"]`` because the job has no column for them. + ``entity`` is the API list route the picker pages through, relative to ``/api/v2/``. + """ + + field: str + label: str + entity: str + required: bool = True + many: bool = False + target: typing.Literal["job", "config"] = "job" + entity_filters: dict[str, str] = dataclasses.field(default_factory=dict) + + def as_dict(self) -> dict: + return dataclasses.asdict(self) + + +PIPELINE_SCOPE = ScopeField(field="pipeline_id", label="Pipeline", entity="ml/pipelines") +CAPTURE_SET_SCOPE = ScopeField(field="source_image_collection_id", label="Capture set", entity="captures/collections") +STATION_SCOPE = ScopeField(field="deployment_id", label="Station", entity="deployments") + + +def describe_docstring(obj) -> str: + """Return the first paragraph of a class docstring, as the help text shown in the dialog.""" + doc = inspect.getdoc(obj) or "" + return doc.split("\n\n")[0].replace("\n", " ").strip() + + +def _sentence_case(name: str) -> str: + words = name.removesuffix("_ids").removesuffix("_id").replace("_", " ").strip() + return words[:1].upper() + words[1:] + + +def _inline_refs(node, definitions: dict): + """Replace ``$ref`` and single-element ``allOf`` with the definition they point to.""" + if isinstance(node, list): + return [_inline_refs(item, definitions) for item in node] + if not isinstance(node, dict): + return node + if "$ref" in node: + target = definitions[node["$ref"].split("/")[-1]] + merged = {**_inline_refs(copy.deepcopy(target), definitions), **{k: v for k, v in node.items() if k != "$ref"}} + return merged + if "allOf" in node and len(node["allOf"]) == 1: + rest = {k: v for k, v in node.items() if k != "allOf"} + return {**_inline_refs(node["allOf"][0], definitions), **rest} + return {key: _inline_refs(value, definitions) for key, value in node.items()} + + +def normalize_config_schema( + model: type[pydantic.BaseModel], + exclude: typing.Iterable[str] = (), +) -> dict: + """Turn a pydantic config model into the JSON Schema subset the dialog renders. + + Inlines definitions, drops the fields in ``exclude`` (scope fields the dialog asks for + separately, and settings the user may not change), and replaces pydantic's generated + title-case labels with a sentence-cased one when the task author did not write a title. + """ + raw = model.schema() + definitions = raw.pop("definitions", {}) + schema = _inline_refs(raw, definitions) + excluded = set(exclude) + properties = {} + for name, prop in schema.get("properties", {}).items(): + if name in excluded: + continue + field_info = model.__fields__[name].field_info + if field_info.title is None: + prop["title"] = _sentence_case(name) + properties[name] = prop + return { + "type": "object", + "title": schema.get("title", model.__name__), + "properties": properties, + "required": [name for name in schema.get("required", []) if name in properties], + "x-ami-schema-version": SCHEMA_VERSION, + } diff --git a/ami/jobs/models.py b/ami/jobs/models.py index ff65f31f2..678142439 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -1,3 +1,4 @@ +import dataclasses import datetime import logging import random @@ -13,13 +14,28 @@ from django.utils.text import slugify from django_pydantic_field import SchemaField from guardian.shortcuts import get_perms +from rest_framework import serializers from ami.base.models import BaseModel from ami.base.schemas import ConfigurableStage, ConfigurableStageParam +from ami.jobs.descriptors import ( + CAPTURE_SET_SCOPE, + ENTITY_KEY, + PIPELINE_SCOPE, + STATION_SCOPE, + ScopeField, + describe_docstring, + normalize_config_schema, +) from ami.jobs.tasks import cleanup_async_job_if_needed, run_job from ami.main.models import Deployment, Project, SourceImage, SourceImageCollection from ami.ml.models import Pipeline -from ami.ml.post_processing.registry import get_postprocessing_task +from ami.ml.post_processing.registry import ( + MEMBER_POST_PROCESSING_TASKS, + POSTPROCESSING_TASKS, + get_postprocessing_task, + staff_only_config_fields, +) from ami.utils.schemas import OrderedEnum logger = logging.getLogger(__name__) @@ -433,6 +449,82 @@ def emit(self, record: logging.LogRecord): logger.error(f"Failed to save log for job #{self.job.pk}: {e}") +def pydantic_messages(exc: pydantic.ValidationError) -> list[str]: + """Flatten a pydantic error into ``"field: message"`` lines for a 400 response.""" + messages = [] + for err in exc.errors(): + field = ".".join(str(part) for part in err.get("loc", ()) if part != "__root__") + messages.append(f"{field}: {err['msg']}" if field else err["msg"]) + return messages + + +def _entity_queryset(entity: str, project: Project | None): + """The rows of ``entity`` a job in ``project`` may refer to, or None when not project-scoped.""" + from django.db.models import Q + + from ami.main.models import Event, Occurrence, TaxaList + + scoped = { + "captures/collections": lambda: SourceImageCollection.objects.filter(project=project), + "deployments": lambda: Deployment.objects.filter(project=project), + "events": lambda: Event.objects.filter(project=project), + "occurrences": lambda: Occurrence.objects.filter(project=project), + # Public lists belong to no project and may be used by any. + "taxa/lists": lambda: TaxaList.objects.filter(Q(projects=project) | Q(projects__isnull=True)), + } + factory = scoped.get(entity) + return factory() if factory else None + + +def check_entities_in_project(values: dict, entities: dict[str, str], project: Project | None) -> None: + """Refuse ids in ``values`` that point outside ``project``. + + ``entities`` maps a field name to its API entity. The schema can only say an id is an + integer; this is the check that it names a row the job's project owns. + """ + errors = [] + for field, entity in entities.items(): + value = values.get(field) + if value in (None, [], ""): + continue + ids = set(value) if isinstance(value, (list, tuple)) else {value} + queryset = _entity_queryset(entity, project) + if queryset is None: + continue + found = set(queryset.filter(pk__in=ids).values_list("pk", flat=True).distinct()) + missing = sorted(ids - found) + if missing: + errors.append(f"{field}: {missing} not found in this project.") + if errors: + raise serializers.ValidationError({"params": {"config": errors}}) + + +def schema_entity_fields(model: type[pydantic.BaseModel]) -> dict[str, str]: + """Map each config field carrying an ``ami_entity`` hint to that entity.""" + return { + name: prop[ENTITY_KEY] + for name, prop in model.schema().get("properties", {}).items() + if isinstance(prop, dict) and ENTITY_KEY in prop + } + + +def _validate_config( + model_cls: type[pydantic.BaseModel], + config, + project: Project | None, + extra_entities: dict[str, str] | None = None, +) -> pydantic.BaseModel: + if not isinstance(config, dict): + raise serializers.ValidationError({"params": {"config": "Must be an object."}}) + try: + model = model_cls(**config) + except pydantic.ValidationError as exc: + raise serializers.ValidationError({"params": {"config": pydantic_messages(exc)}}) + entities = {**schema_entity_fields(model_cls), **(extra_entities or {})} + check_entities_in_project(model.dict(), entities, project) + return model + + @dataclass class JobType: """ @@ -444,6 +536,49 @@ class JobType: name: str key: str + # What a job of this type cannot run without. The API refuses to create one that is + # missing any of these, so a gap is a 400 when the job is made rather than a failure + # minutes later when it runs. ``required_params`` are keys inside ``Job.params``; + # ``required_fields`` are fields on the job itself. + required_fields: tuple[str, ...] = () + required_params: tuple[str, ...] = () + + # Whether a person can start one from the Create Job dialog. The rest are created by + # the platform for the user: an export from the exports page, for example. + user_creatable: bool = False + + # What the dialog asks for before the settings, and the pydantic model of the + # settings themselves (stored in ``Job.params["config"]``). See ami/jobs/descriptors.py. + scope_fields: tuple[ScopeField, ...] = () + config_schema: type[pydantic.BaseModel] | None = None + + # A job type whose work is chosen from a registry (post-processing tasks) names the + # ``params`` key that holds the choice, and lists the choices as variants. + variant_key: str | None = None + + @classmethod + def variants(cls, user=None) -> list[dict]: + return [] + + @classmethod + def validate_params(cls, project: Project | None, user, params) -> dict: + """Check a new job's ``params`` before it is saved and return what should be stored. + + Raises ``serializers.ValidationError`` (a 400) for a bad value. The default keeps + only ``config``, validated against ``config_schema`` when the type declares one, + plus any ``required_params``; job types that read nothing from params store none. + """ + if params in (None, {}): + params = {} + if not isinstance(params, dict): + raise serializers.ValidationError({"params": "Must be an object."}) + kept = {name: params[name] for name in cls.required_params if name in params} + if cls.config_schema is not None: + config = params.get("config") or {} + model = _validate_config(cls.config_schema, config, project) + kept["config"] = model.dict() + return kept + # @TODO Consider adding custom vocabulary for job types to be used in the UI # verb: str = "Sync" # present_participle: str = "syncing" @@ -458,8 +593,15 @@ def run(cls, job: "Job"): class MLJob(JobType): + """ + Run a processing pipeline over a capture set: detect, classify and create occurrences. + """ + name = "ML pipeline" key = "ml" + user_creatable = True + required_fields = ("pipeline",) + scope_fields = (PIPELINE_SCOPE, CAPTURE_SET_SCOPE) @classmethod def run(cls, job: "Job"): @@ -710,6 +852,9 @@ class DataStorageSyncJob(JobType): name = "Data storage sync" key = "data_storage_sync" + user_creatable = True + required_fields = ("deployment",) + scope_fields = (STATION_SCOPE,) regroup_stage_key = "regroup_sessions" regroup_stage_name = "Regroup sessions" @@ -802,8 +947,15 @@ def run(cls, job: "Job"): class SourceImageCollectionPopulateJob(JobType): + """ + Fill a capture set with the captures its sampling method selects. + """ + name = "Populate capture set" key = "populate_captures_collection" + user_creatable = True + required_fields = ("source_image_collection",) + scope_fields = (CAPTURE_SET_SCOPE,) @classmethod def run(cls, job: "Job"): @@ -889,8 +1041,98 @@ def run(cls, job: "Job"): class PostProcessingJob(JobType): + """ + Revise existing results with a post-processing method, such as masking classes or + filtering out detections too small to identify. + """ + name = "Post Processing" key = "post_processing" + user_creatable = True + variant_key = "task" + + # Config fields that say what a task runs on. The dialog asks for them as scope, so they + # are left out of the settings form. ``occurrence_id`` is the admin's single-occurrence + # path and is not offered in the dialog. + SCOPE_CONFIG_FIELDS = { + "source_image_collection_id": ScopeField( + field="source_image_collection_id", label="Capture set", entity="captures/collections", target="config" + ), + "event_ids": ScopeField(field="event_ids", label="Sessions", entity="events", many=True, target="config"), + } + HIDDEN_CONFIG_FIELDS = {"occurrence_id"} + + @classmethod + def member_may_run_task(cls, task_key: str) -> bool: + return task_key in MEMBER_POST_PROCESSING_TASKS + + @classmethod + def task_scope(cls, task_cls) -> list[ScopeField]: + fields = task_cls.config_schema.__fields__ + scope = [cls.SCOPE_CONFIG_FIELDS[name] for name in cls.SCOPE_CONFIG_FIELDS if name in fields] + if len(scope) > 1: + # The task's own root validator requires exactly one of them. + scope = [dataclasses.replace(field, required=False) for field in scope] + return scope + + @classmethod + def variants(cls, user=None) -> list[dict]: + is_superuser = bool(user and user.is_superuser) + variants = [] + for key, task_cls in POSTPROCESSING_TASKS.items(): + scope = cls.task_scope(task_cls) + exclude = {f.field for f in scope} | cls.HIDDEN_CONFIG_FIELDS + if not is_superuser: + member_fields = MEMBER_POST_PROCESSING_TASKS.get(key, frozenset()) + exclude |= set(task_cls.config_schema.__fields__) - member_fields + variants.append( + { + "key": key, + "name": task_cls.name, + "description": describe_docstring(task_cls), + "allowed_for_members": cls.member_may_run_task(key), + "scope": [f.as_dict() for f in scope], + "scope_rule": "exactly_one" if len(scope) > 1 else "all_required", + "config_schema": normalize_config_schema(task_cls.config_schema, exclude=exclude), + } + ) + return variants + + @classmethod + def validate_params(cls, project: Project | None, user, params) -> dict: + """Check a post-processing job's ``{"task": ..., "config": {...}}`` before it is saved. + + Returns the params with the config normalized by the task's schema, so the stored job + carries every default the worker will run with. Every id in the config must belong to + the job's project. Only superusers may start a task that is not on the member list, or + change a setting away from its default that members may not change. + """ + if not isinstance(params, dict) or set(params) - {"task", "config"}: + raise serializers.ValidationError( + {"params": 'Post-processing jobs take params of the form {"task": , "config": {...}}.'} + ) + task_key = params.get("task") + task_cls = get_postprocessing_task(task_key) if isinstance(task_key, str) else None + if task_cls is None: + raise serializers.ValidationError({"params": {"task": f"Unknown post-processing task {task_key!r}."}}) + is_superuser = bool(user and user.is_superuser) + if not is_superuser and not cls.member_may_run_task(task_key): + raise serializers.ValidationError( + {"params": {"task": f"The {task_cls.name} task can only be started by staff."}} + ) + config = params.get("config") or {} + if not isinstance(config, dict): + raise serializers.ValidationError({"params": {"config": "Must be an object."}}) + if not is_superuser: + staff_only = staff_only_config_fields(task_key, config) + if staff_only: + raise serializers.ValidationError( + {"params": {"config": [f"{name}: Only staff can change this setting." for name in staff_only]}} + ) + scope_entities = {f.field: f.entity for f in cls.task_scope(task_cls)} + scope_entities["occurrence_id"] = "occurrences" + model = _validate_config(task_cls.config_schema, config, project, extra_entities=scope_entities) + return {"task": task_key, "config": model.dict()} @classmethod def run(cls, job: "Job"): @@ -940,6 +1182,9 @@ class RegroupEventsJob(JobType): name = "Regroup sessions" key = "regroup_events" + user_creatable = True + required_fields = ("deployment",) + scope_fields = (STATION_SCOPE,) @classmethod def run(cls, job: "Job"): diff --git a/ami/jobs/serializers.py b/ami/jobs/serializers.py index f53199e73..6d20143bd 100644 --- a/ami/jobs/serializers.py +++ b/ami/jobs/serializers.py @@ -1,6 +1,7 @@ from django_pydantic_field.rest_framework import SchemaField from drf_spectacular.utils import extend_schema_field -from rest_framework import serializers +from guardian.shortcuts import get_perms +from rest_framework import exceptions, serializers from ami.exports.models import DataExport from ami.main.api.serializers import ( @@ -14,7 +15,18 @@ from ami.ml.schemas import PipelineProcessingTask, PipelineTaskResult, ProcessingServiceClientInfo from ami.ml.serializers import PipelineNestedSerializer -from .models import JOB_LOGS_DEFAULT_LIMIT, Job, JobProgress, MLJob, _legacy_logs_shape, serialize_job_logs +from .descriptors import describe_docstring, normalize_config_schema +from .models import ( + JOB_LOGS_DEFAULT_LIMIT, + VALID_JOB_TYPES, + Job, + JobProgress, + JobType, + MLJob, + _legacy_logs_shape, + get_job_type_by_key, + serialize_job_logs, +) from .schemas import QueuedTaskAcknowledgment @@ -41,6 +53,40 @@ class JobTypeSerializer(serializers.Serializer): key = serializers.SlugField(read_only=True) +def describe_job_types(project: Project, user) -> list[dict]: + """Describe every job type ``user`` may pick in the Create Job dialog for ``project``. + + ``allowed`` says whether the user may run a job of that type here; the dialog disables + rather than hides a type the user may not run, so they can see it exists. Permissions are + read once for the whole list. + """ + perms = set(get_perms(user, project)) + described = [] + for job_type in VALID_JOB_TYPES: + if not job_type.user_creatable: + continue + allowed = user.is_superuser or f"run_{job_type.key}_job" in perms + variants = job_type.variants(user=user) + for variant in variants: + allowed_for_members = variant.pop("allowed_for_members") + variant["allowed"] = allowed and (user.is_superuser or allowed_for_members) + described.append( + { + "key": job_type.key, + "name": job_type.name, + "description": describe_docstring(job_type), + "allowed": allowed, + "scope": [field.as_dict() for field in job_type.scope_fields], + "required_fields": list(job_type.required_fields), + "required_params": list(job_type.required_params), + "config_schema": (normalize_config_schema(job_type.config_schema) if job_type.config_schema else None), + "variant_key": job_type.variant_key, + "variants": variants, + } + ) + return described + + class JobListSerializer(DefaultSerializer): delay = serializers.IntegerField() project = JobProjectNestedSerializer(read_only=True) @@ -176,11 +222,80 @@ def get_logs(self, obj: Job) -> dict[str, list[str]]: class JobSerializer(JobListSerializer): # progress = serializers.JSONField(initial=Job.default_progress(), allow_null=False, required=False) + # A job's settings, checked by its job type when the job is created and fixed after that: + # a later update cannot swap in settings that were never validated. See JobType.validate_params. + params = serializers.JSONField(required=False, allow_null=True) + class Meta(JobListSerializer.Meta): fields = JobListSerializer.Meta.fields + [ "result", + "params", ] + def validate_job_type_key(self, value: str) -> str: + job_type = get_job_type_by_key(value) + if not job_type: + known = sorted(t.key for t in VALID_JOB_TYPES if t.user_creatable) + raise serializers.ValidationError(f"Unknown job type '{value}'. Known types: {known}") + if self.instance is None and not job_type.user_creatable: + raise serializers.ValidationError( + f"{job_type.name} jobs are created by the platform, not through this API." + ) + return value + + def validate(self, attrs: dict) -> dict: + attrs = super().validate(attrs) + if self.instance is not None: + # Settings are fixed once the job exists. + attrs.pop("params", None) + return attrs + + job_type = get_job_type_by_key(attrs.get("job_type_key", MLJob.key)) + if job_type is None: # validate_job_type_key refuses unknown keys; this guards the default + raise serializers.ValidationError({"job_type_key": "Unknown job type."}) + project = attrs.get("project") + self._check_scope_in_project(attrs, project) + + missing_fields = [name for name in job_type.required_fields if not attrs.get(name)] + if missing_fields: + raise serializers.ValidationError( + {f"{name}_id": f"{job_type.name} jobs need a {name}." for name in missing_fields} + ) + params = attrs.get("params") or {} + missing_params = [name for name in job_type.required_params if not params.get(name)] + if missing_params: + raise serializers.ValidationError( + {"params": f"{job_type.name} jobs need {', '.join(missing_params)} in their params."} + ) + + request = self.context.get("request") + user = getattr(request, "user", None) + attrs["params"] = job_type.validate_params(project, user, attrs.get("params")) + if job_type.variant_key: + self._check_may_run(job_type, project, attrs["params"], user) + return attrs + + def _check_scope_in_project(self, attrs: dict, project: Project | None) -> None: + errors = {} + for field in ("deployment", "source_image_collection", "source_image_single"): + obj = attrs.get(field) + if obj is not None and obj.project_id != getattr(project, "pk", None): + errors[f"{field}_id"] = "Not found in this project." + if errors: + raise serializers.ValidationError(errors) + + def _check_may_run(self, job_type: type[JobType], project: Project | None, params: dict, user) -> None: + # Creating a job whose type runs registered methods (post-processing) takes the + # permission to run it, so a role that cannot start one is refused before a job it + # could never run is stored. + if user is None: + return + job = Job(job_type_key=job_type.key, project=project, params=params) + if not job.check_custom_permission(user, "run"): + raise exceptions.PermissionDenied( + f"You do not have permission to run {job_type.name} jobs in this project." + ) + class MinimalJobSerializer(DefaultSerializer): """Minimal serializer returning only essential job fields.""" diff --git a/ami/jobs/tests/test_job_types.py b/ami/jobs/tests/test_job_types.py new file mode 100644 index 000000000..f2a4fc465 --- /dev/null +++ b/ami/jobs/tests/test_job_types.py @@ -0,0 +1,234 @@ +from cachalot.api import cachalot_disabled +from rest_framework import status +from rest_framework.test import APITestCase + +from ami.base.serializers import reverse_with_params +from ami.jobs.descriptors import normalize_config_schema +from ami.jobs.models import Job, PostProcessingJob +from ami.main.models import Occurrence, Project, SourceImageCollection, TaxaList +from ami.ml.models import Algorithm +from ami.ml.post_processing import registry +from ami.ml.post_processing.class_masking import ClassMaskingConfig +from ami.users.models import User +from ami.users.roles import BasicMember, MLDataManager + + +def types_url(project_id=None): + params = {"project_id": project_id} if project_id is not None else {} + return reverse_with_params("api:job-types", params=params) + + +class TestJobTypesEndpoint(APITestCase): + """GET /jobs/types/ describes the job types a project member may create, and nobody else may read it.""" + + def setUp(self): + self.owner = User.objects.create_user(email="owner@insectai.org") + self.project = Project.objects.create(name="Job types project", owner=self.owner) + self.other_project = Project.objects.create(name="Other project", owner=self.owner) + self.basic = User.objects.create_user(email="basic@insectai.org") + BasicMember.assign_user(self.basic, self.project) + self.ml_manager = User.objects.create_user(email="ml@insectai.org") + MLDataManager.assign_user(self.ml_manager, self.project) + self.outsider = User.objects.create_user(email="outsider@insectai.org") + self.superuser = User.objects.create_user(email="super@insectai.org", is_staff=True, is_superuser=True) + + def get_types(self, user, project_id=None): + self.client.force_authenticate(user=user) + return self.client.get(types_url(self.project.pk if project_id is None else project_id)) + + def test_anonymous_is_refused_before_the_project_is_read(self): + self.client.force_authenticate(user=None) + response = self.client.get(types_url(self.project.pk)) + self.assertIn(response.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) + + def test_non_member_is_refused(self): + response = self.get_types(self.outsider) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_project_id_is_required_and_validated(self): + self.client.force_authenticate(user=self.basic) + self.assertEqual(self.client.get(types_url()).status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(self.client.get(types_url("abc")).status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(self.client.get(types_url(999999)).status_code, status.HTTP_404_NOT_FOUND) + + def test_member_sees_creatable_types_with_permission_resolved(self): + response = self.get_types(self.basic) + self.assertEqual(response.status_code, status.HTTP_200_OK) + by_key = {t["key"]: t for t in response.json()["results"]} + # Exports are created from the exports page, never offered here. + self.assertNotIn("data_export", by_key) + self.assertNotIn("unknown", by_key) + self.assertEqual( + set(by_key), + {"ml", "data_storage_sync", "populate_captures_collection", "post_processing", "regroup_events"}, + ) + # A basic member may create jobs but not run a pipeline over a capture set. + self.assertFalse(by_key["ml"]["allowed"]) + self.assertEqual([s["field"] for s in by_key["ml"]["scope"]], ["pipeline_id", "source_image_collection_id"]) + self.assertTrue(by_key["ml"]["description"]) + + def test_ml_data_manager_may_run_ml_jobs(self): + by_key = {t["key"]: t for t in self.get_types(self.ml_manager).json()["results"]} + self.assertTrue(by_key["ml"]["allowed"]) + + def test_staff_only_tasks_are_disabled_and_their_settings_hidden_for_members(self): + by_key = {t["key"]: t for t in self.get_types(self.ml_manager).json()["results"]} + variants = {v["key"]: v for v in by_key["post_processing"]["variants"]} + masking = variants["class_masking"] + self.assertFalse(masking["allowed"]) + self.assertEqual(masking["config_schema"]["properties"], {}) + + def test_superuser_sees_every_post_processing_setting_except_scope(self): + by_key = {t["key"]: t for t in self.get_types(self.superuser).json()["results"]} + post_processing = by_key["post_processing"] + self.assertEqual(post_processing["variant_key"], "task") + masking = {v["key"]: v for v in post_processing["variants"]}["class_masking"] + self.assertTrue(masking["allowed"]) + self.assertEqual([s["field"] for s in masking["scope"]], ["source_image_collection_id"]) + self.assertEqual(masking["scope"][0]["target"], "config") + properties = masking["config_schema"]["properties"] + self.assertEqual(set(properties), {"taxa_list_id", "algorithm_id", "reweight"}) + self.assertEqual(properties["taxa_list_id"]["title"], "Taxa list to keep") + self.assertEqual(properties["algorithm_id"]["ami_entity"], "ml/algorithms") + self.assertTrue(masking["description"].startswith("Masks out classes")) + + def test_query_count_does_not_grow_with_job_types(self): + self.client.force_authenticate(user=self.ml_manager) + with cachalot_disabled(): + # Project, membership, user and group permissions, plus the request's savepoint pair: + # fixed however many job types and post-processing tasks are listed. + with self.assertNumQueries(6): + response = self.client.get(types_url(self.project.pk)) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertGreater(len(response.json()["results"]), 3) + + +class TestSchemaNormalizer(APITestCase): + def test_scope_fields_are_dropped_and_titles_filled(self): + schema = normalize_config_schema(ClassMaskingConfig, exclude={"source_image_collection_id", "occurrence_id"}) + self.assertNotIn("source_image_collection_id", schema["properties"]) + self.assertEqual(schema["required"], ["taxa_list_id", "algorithm_id"]) + self.assertEqual(schema["x-ami-schema-version"], 1) + + +class TestCreateJobWithParams(APITestCase): + """POST /jobs/ checks a job's settings against its job type before the job is stored.""" + + def setUp(self): + self.owner = User.objects.create_user(email="owner@insectai.org") + self.project = Project.objects.create(name="Params project", owner=self.owner) + self.other_project = Project.objects.create(name="Other params project", owner=self.owner) + self.collection = SourceImageCollection.objects.create(name="Mine", project=self.project) + self.other_collection = SourceImageCollection.objects.create(name="Theirs", project=self.other_project) + self.taxa_list = TaxaList.objects.create(name="Keep") + self.taxa_list.projects.add(self.project) + self.algorithm = Algorithm.objects.create(name="Classifier", key="classifier") + self.superuser = User.objects.create_user(email="super@insectai.org", is_staff=True, is_superuser=True) + self.ml_manager = User.objects.create_user(email="ml@insectai.org") + MLDataManager.assign_user(self.ml_manager, self.project) + + def post_job(self, user, **body): + self.client.force_authenticate(user=user) + payload = {"name": "Job", "delay": 0, "project_id": self.project.pk, **body} + return self.client.post(reverse_with_params("api:job-list"), payload, format="json") + + def masking_params(self, collection_id): + return { + "task": "class_masking", + "config": { + "source_image_collection_id": collection_id, + "taxa_list_id": self.taxa_list.pk, + "algorithm_id": self.algorithm.pk, + }, + } + + def test_superuser_creates_a_post_processing_job_with_defaults_filled(self): + response = self.post_job( + self.superuser, job_type_key="post_processing", params=self.masking_params(self.collection.pk) + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.json()) + job = Job.objects.get(pk=response.json()["id"]) + self.assertEqual(job.params["task"], "class_masking") + self.assertTrue(job.params["config"]["reweight"]) + self.assertIsNone(job.params["config"]["occurrence_id"]) + + def test_capture_set_from_another_project_is_refused(self): + response = self.post_job( + self.superuser, job_type_key="post_processing", params=self.masking_params(self.other_collection.pk) + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("source_image_collection_id", str(response.json())) + + def test_occurrence_from_another_project_is_refused(self): + occurrence = Occurrence.objects.create(project=self.other_project) + params = self.masking_params(None) + params["config"]["occurrence_id"] = occurrence.pk + del params["config"]["source_image_collection_id"] + response = self.post_job(self.superuser, job_type_key="post_processing", params=params) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_schema_errors_come_back_per_field(self): + params = self.masking_params(self.collection.pk) + del params["config"]["taxa_list_id"] + response = self.post_job(self.superuser, job_type_key="post_processing", params=params) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("taxa_list_id: field required", response.json()["params"]["config"]) + + def test_member_cannot_start_a_staff_only_task(self): + response = self.post_job( + self.ml_manager, job_type_key="post_processing", params=self.masking_params(self.collection.pk) + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("staff", str(response.json())) + + def test_member_on_the_allowlist_still_cannot_change_staff_only_settings(self): + original = dict(registry.MEMBER_POST_PROCESSING_TASKS) + registry.MEMBER_POST_PROCESSING_TASKS["class_masking"] = frozenset( + {"source_image_collection_id", "taxa_list_id", "algorithm_id"} + ) + try: + params = self.masking_params(self.collection.pk) + params["config"]["reweight"] = False + response = self.post_job(self.ml_manager, job_type_key="post_processing", params=params) + finally: + registry.MEMBER_POST_PROCESSING_TASKS.clear() + registry.MEMBER_POST_PROCESSING_TASKS.update(original) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("reweight: Only staff", str(response.json())) + + def test_platform_job_types_cannot_be_created_through_the_api(self): + response = self.post_job(self.superuser, job_type_key="data_export") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("job_type_key", response.json()) + + def test_params_are_ignored_for_job_types_without_settings(self): + response = self.post_job( + self.superuser, + job_type_key="populate_captures_collection", + source_image_collection_id=self.collection.pk, + params={"anything": 1}, + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.json()) + self.assertEqual(Job.objects.get(pk=response.json()["id"]).params, {}) + + def test_capture_set_column_from_another_project_is_refused(self): + response = self.post_job( + self.superuser, + job_type_key="populate_captures_collection", + source_image_collection_id=self.other_collection.pk, + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("source_image_collection_id", response.json()) + + def test_params_cannot_be_changed_after_creation(self): + response = self.post_job( + self.superuser, job_type_key="post_processing", params=self.masking_params(self.collection.pk) + ) + job_id = response.json()["id"] + detail = reverse_with_params("api:job-detail", args=[job_id]) + self.client.patch(detail, {"params": {"task": "small_size_filter", "config": {}}}, format="json") + self.assertEqual(Job.objects.get(pk=job_id).params["task"], "class_masking") + + def test_post_processing_scope_is_described_by_the_task(self): + scope = PostProcessingJob.task_scope(registry.POSTPROCESSING_TASKS["small_size_filter"]) + self.assertEqual([f.field for f in scope], ["source_image_collection_id"]) diff --git a/ami/jobs/views.py b/ami/jobs/views.py index 380d9f959..4a82b77d4 100644 --- a/ami/jobs/views.py +++ b/ami/jobs/views.py @@ -12,7 +12,7 @@ from drf_spectacular.utils import extend_schema, extend_schema_view from rest_framework import serializers from rest_framework.decorators import action -from rest_framework.exceptions import PermissionDenied, ValidationError +from rest_framework.exceptions import NotAuthenticated, PermissionDenied, ValidationError from rest_framework.filters import BaseFilterBackend from rest_framework.response import Response @@ -41,7 +41,7 @@ from ami.utils.fields import url_boolean_param from .models import Job, JobDispatchMode, JobState -from .serializers import JobListSerializer, JobSerializer, MinimalJobSerializer +from .serializers import JobListSerializer, JobSerializer, MinimalJobSerializer, describe_job_types logger = logging.getLogger(__name__) @@ -257,6 +257,28 @@ def get_serializer_context(self): ) return context + @extend_schema(parameters=[project_id_doc_param]) + @action(detail=False, methods=["get"], name="types") + def types(self, request): + """ + List the job types the Create Job dialog can offer for a project, with what each one + runs on (``scope``) and a JSON Schema of its settings (``config_schema``). + + Only members of the project may read it. See docs/claude/reference/jobs-panel.md. + """ + # ObjectPermission.has_permission allows every request, and a list-style action never + # reaches the object check, so this action gates itself. + if not request.user.is_authenticated: + raise NotAuthenticated() + self.require_project = True + project = self.get_active_project() + if project is None: # get_active_project already raises 400/404 when required + raise ValidationError({"project_id": "This parameter is required."}) + user = request.user + if not (user.is_superuser or project.owner_id == user.pk or project.members.filter(pk=user.pk).exists()): + raise PermissionDenied("Only members of this project can list its job types.") + return Response({"results": describe_job_types(project, user)}) + @action(detail=True, methods=["post"], name="run") def run(self, request, pk=None): """ diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index 2da2001b7..a6e1cfe86 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -21,14 +21,29 @@ class ClassMaskingConfig(pydantic.BaseModel): # discriminated-scope shape — the shared pattern for per-occurrence triggers. source_image_collection_id: int | None = None occurrence_id: int | None = None - # The taxa list to keep: classes whose taxon is not in this list are masked out. - taxa_list_id: int - # The source classifier whose terminal classifications are re-scored. - algorithm_id: int - # When True (default), renormalise the kept classes' scores to sum to 1 after - # masking. When False, the kept classes retain their original absolute scores and - # the excluded classes are zeroed; the chosen species is identical either way. - reweight: bool = True + taxa_list_id: int = pydantic.Field( + ..., + title="Taxa list to keep", + description="Classes outside this list are masked out.", + ami_widget="entity", + ami_entity="taxa/lists", + ) + algorithm_id: int = pydantic.Field( + ..., + title="Source classifier", + description="Its terminal predictions are the ones re-scored.", + ami_widget="entity", + ami_entity="ml/algorithms", + ami_entity_filters={"task_type": "classification"}, + ) + reweight: bool = pydantic.Field( + True, + title="Reweight scores", + description=( + "Renormalise the kept classes to sum to 1. Off keeps raw absolute scores; " + "the chosen species is the same either way." + ), + ) @pydantic.root_validator(skip_on_failure=True) def _exactly_one_scope(cls, values: dict) -> dict: @@ -235,6 +250,11 @@ def make_classifications_filtered_by_taxa_list( class ClassMaskingTask(BasePostProcessingTask): + """ + Masks out classes whose taxon is not on the chosen list and renormalises each prediction over what + remains. The original classification is kept and demoted. + """ + key = "class_masking" name = "Class masking" config_schema = ClassMaskingConfig diff --git a/ami/ml/post_processing/registry.py b/ami/ml/post_processing/registry.py index 308be18ae..4679fa01d 100644 --- a/ami/ml/post_processing/registry.py +++ b/ami/ml/post_processing/registry.py @@ -11,3 +11,20 @@ def get_postprocessing_task(key: str): """Return a post-processing task class by key.""" return POSTPROCESSING_TASKS.get(key) + + +# Post-processing tasks a project member may start through the jobs API, with the config +# fields a member may set. Every other field must keep its schema default. Tasks not listed +# here are staff tools: superusers can still start them from the Create Job dialog. +MEMBER_POST_PROCESSING_TASKS: dict[str, frozenset[str]] = {} + + +def staff_only_config_fields(task_key: str, config: dict) -> list[str]: + """Return the fields in ``config`` that only staff may set away from their default.""" + schema_fields = POSTPROCESSING_TASKS[task_key].config_schema.__fields__ + allowed = MEMBER_POST_PROCESSING_TASKS.get(task_key, frozenset()) + return sorted( + name + for name, value in config.items() + if name in schema_fields and name not in allowed and value != schema_fields[name].default + ) diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index 6a39af780..34b280055 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -14,7 +14,13 @@ class SmallSizeFilterConfig(pydantic.BaseModel): # post-processing tasks copy when they gain per-occurrence / per-event triggers. source_image_collection_id: int | None = None occurrence_id: int | None = None - size_threshold: float = 0.0008 + size_threshold: float = pydantic.Field( + 0.0008, + title="Size threshold", + description="Detections smaller than this fraction of the image area are marked as not identifiable.", + gt=0.0, + lt=1.0, + ) @pydantic.validator("size_threshold") def _threshold_in_unit_interval(cls, v: float) -> float: @@ -34,6 +40,10 @@ class Config: class SmallSizeFilterTask(BasePostProcessingTask): + """ + Marks detections that are too small to identify, so they stop counting towards species totals. + """ + key = "small_size_filter" name = "Small size filter" config_schema = SmallSizeFilterConfig From b40050f446696ca6d2e743b42436c3d55e896eea Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 29 Sep 2026 17:47:11 -0700 Subject: [PATCH 02/29] feat(ui): add job type discovery and schema-to-form mapping helpers The Create job dialog is driven by the new jobs/types endpoint, which describes each creatable job type, its methods, scope pickers and a JSON schema for method settings. This adds the typed server models, a React Query hook for the endpoint, and two pure helpers that the dialog builds on. schema-to-fields turns a JSON schema property into a field descriptor (integer and number bounds, boolean, select, text, entity picker, integer list, JSON fallback). build-job-payload turns the dialog state into the POST body, placing scope fields at the top level or inside params.config according to their target, and omitting empty optional values. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NFUiikN95Y3yz4pBK9KPu1 --- .../form/schema-form/build-job-payload.ts | 121 +++++++++++++++ .../form/schema-form/schema-to-fields.ts | 126 +++++++++++++++ .../tests/build-job-payload.test.ts | 146 ++++++++++++++++++ .../tests/schema-to-fields.test.ts | 124 +++++++++++++++ ui/src/data-services/constants.ts | 1 + .../data-services/hooks/jobs/useJobTypes.ts | 21 +++ ui/src/data-services/models/job-type.ts | 55 +++++++ 7 files changed, 594 insertions(+) create mode 100644 ui/src/components/form/schema-form/build-job-payload.ts create mode 100644 ui/src/components/form/schema-form/schema-to-fields.ts create mode 100644 ui/src/components/form/schema-form/tests/build-job-payload.test.ts create mode 100644 ui/src/components/form/schema-form/tests/schema-to-fields.test.ts create mode 100644 ui/src/data-services/hooks/jobs/useJobTypes.ts create mode 100644 ui/src/data-services/models/job-type.ts diff --git a/ui/src/components/form/schema-form/build-job-payload.ts b/ui/src/components/form/schema-form/build-job-payload.ts new file mode 100644 index 000000000..c0967e720 --- /dev/null +++ b/ui/src/components/form/schema-form/build-job-payload.ts @@ -0,0 +1,121 @@ +import { + ServerJobType, + ServerJobTypeVariant, +} from 'data-services/models/job-type' +import { parseIntegerList } from 'utils/fieldProcessors' +import { + FieldDescriptor, + schemaToFields, + scopeToFields, +} from './schema-to-fields' + +export interface CreateJobState { + projectId: string + jobType: ServerJobType + variant?: ServerJobTypeVariant + scopeValues: { [field: string]: unknown } + scopeLabels?: { [field: string]: string } + configValues: { [field: string]: unknown } + name?: string + delay?: string | number + startNow?: boolean + today?: string +} + +export const coerceValue = (field: FieldDescriptor, raw: unknown): unknown => { + if (raw === undefined || raw === null || raw === '') { + return undefined + } + switch (field.kind) { + case 'integer': + case 'number': + case 'entity': + return Number.isNaN(Number(raw)) ? raw : Number(raw) + case 'integer-list': + return parseIntegerList(`${raw}`) ?? undefined + case 'json': + try { + return JSON.parse(`${raw}`) + } catch { + return raw + } + default: + return raw + } +} + +const collect = ( + fields: FieldDescriptor[], + values: { [field: string]: unknown } +) => { + const result: { [field: string]: unknown } = {} + fields.forEach((field) => { + const value = coerceValue(field, values[field.name]) + if (value !== undefined) { + result[field.name] = value + } + }) + return result +} + +const getScopeFields = (state: CreateJobState) => { + const { jobType, variant } = state + return [ + ...scopeToFields(jobType.scope), + ...scopeToFields(variant?.scope ?? [], { + optional: variant?.scope_rule === 'exactly_one', + }), + ] +} + +export const buildJobPayload = (state: CreateJobState) => { + const { jobType, variant, projectId } = state + const scopeItems = [ + ...jobType.scope.map((item) => ({ ...item })), + ...(variant?.scope ?? []), + ] + const scopeFields = getScopeFields(state) + const scopeValues = collect(scopeFields, state.scopeValues) + + const jobScope: { [field: string]: unknown } = {} + const configScope: { [field: string]: unknown } = {} + scopeItems.forEach((item) => { + if (item.field in scopeValues) { + const target = item.target === 'config' ? configScope : jobScope + target[item.field] = scopeValues[item.field] + } + }) + + const schema = variant?.config_schema ?? jobType.config_schema + const config = { + ...collect(schemaToFields(schema), state.configValues), + ...configScope, + } + + let params: { [key: string]: unknown } | undefined + if (jobType.variant_key && variant) { + params = { [jobType.variant_key]: variant.key, config } + } else if (jobType.config_schema) { + params = { config } + } + + const label = variant?.name ?? jobType.name + const scopeLabel = Object.values(state.scopeLabels ?? {}).find(Boolean) + const name = + state.name?.trim() || + `${label} – ${ + scopeLabel ?? state.today ?? new Date().toISOString().slice(0, 10) + }` + + return { + body: { + name, + delay: Number(state.delay) || 0, + project_id: projectId, + job_type_key: jobType.key, + ...jobScope, + ...(params ? { params } : {}), + }, + startNow: !!state.startNow, + } +} diff --git a/ui/src/components/form/schema-form/schema-to-fields.ts b/ui/src/components/form/schema-form/schema-to-fields.ts new file mode 100644 index 000000000..32a950861 --- /dev/null +++ b/ui/src/components/form/schema-form/schema-to-fields.ts @@ -0,0 +1,126 @@ +import { + ServerConfigSchema, + ServerScopeField, +} from 'data-services/models/job-type' + +export type FieldKind = + | 'integer' + | 'number' + | 'boolean' + | 'select' + | 'text' + | 'entity' + | 'integer-list' + | 'json' + +export interface FieldDescriptor { + name: string + label: string + description?: string + kind: FieldKind + required: boolean + defaultValue?: unknown + min?: number + max?: number + exclusiveMin?: number + exclusiveMax?: number + options?: (string | number)[] + entity?: string + entityFilters?: { [key: string]: string | number | boolean } +} + +export const schemaToFields = ( + schema?: ServerConfigSchema | null +): FieldDescriptor[] => + Object.entries(schema?.properties ?? {}).map(([name, prop]) => { + const base = { + name, + label: prop.title ?? name, + description: prop.description, + required: !!schema?.required?.includes(name), + defaultValue: prop.default, + } + + if (prop.ami_widget === 'entity' && prop.ami_entity) { + return { + ...base, + kind: 'entity', + entity: prop.ami_entity, + entityFilters: prop.ami_entity_filters, + } + } + if (prop.type === 'boolean') { + return { ...base, kind: 'boolean' } + } + if (prop.type === 'integer' || prop.type === 'number') { + return { + ...base, + kind: prop.type, + min: prop.minimum, + max: prop.maximum, + exclusiveMin: prop.exclusiveMinimum, + exclusiveMax: prop.exclusiveMaximum, + } + } + if (prop.type === 'string' && prop.enum?.length) { + return { ...base, kind: 'select', options: prop.enum } + } + if (prop.type === 'string') { + return { ...base, kind: 'text' } + } + if (prop.type === 'array' && prop.items?.type === 'integer') { + return { ...base, kind: 'integer-list' } + } + return { ...base, kind: 'json' } + }) + +export const scopeToFields = ( + scope: ServerScopeField[], + { optional }: { optional?: boolean } = {} +): FieldDescriptor[] => + scope.map((item) => ({ + name: item.field, + label: item.label, + kind: item.many ? 'integer-list' : 'entity', + required: item.required && !optional, + entity: item.entity, + entityFilters: item.entity_filters, + })) + +export const getInitialValue = (field: FieldDescriptor): unknown => + field.defaultValue === undefined || field.defaultValue === null + ? undefined + : field.kind === 'boolean' || field.kind === 'select' + ? field.defaultValue + : field.kind === 'json' + ? JSON.stringify(field.defaultValue) + : `${field.defaultValue}` + +export const validateNumber = ( + field: FieldDescriptor, + raw: unknown +): string | undefined => { + if (raw === undefined || raw === null || raw === '') { + return undefined + } + const value = Number(raw) + if (Number.isNaN(value)) { + return 'Enter a number' + } + if (field.kind === 'integer' && !Number.isInteger(value)) { + return 'Enter a whole number' + } + if (field.min !== undefined && value < field.min) { + return `Must be at least ${field.min}` + } + if (field.max !== undefined && value > field.max) { + return `Must be at most ${field.max}` + } + if (field.exclusiveMin !== undefined && value <= field.exclusiveMin) { + return `Must be greater than ${field.exclusiveMin}` + } + if (field.exclusiveMax !== undefined && value >= field.exclusiveMax) { + return `Must be less than ${field.exclusiveMax}` + } + return undefined +} diff --git a/ui/src/components/form/schema-form/tests/build-job-payload.test.ts b/ui/src/components/form/schema-form/tests/build-job-payload.test.ts new file mode 100644 index 000000000..3a1d2728a --- /dev/null +++ b/ui/src/components/form/schema-form/tests/build-job-payload.test.ts @@ -0,0 +1,146 @@ +import { ServerJobType, ServerScopeField } from 'data-services/models/job-type' +import { buildJobPayload } from '../build-job-payload' + +const scope = ( + field: string, + overrides: Partial = {} +): ServerScopeField => ({ + field, + label: field, + entity: 'x', + required: true, + many: false, + target: 'job', + entity_filters: {}, + ...overrides, +}) + +const mlType: ServerJobType = { + key: 'ml', + name: 'ML pipeline', + allowed: true, + scope: [scope('pipeline_id'), scope('source_image_collection_id')], + required_fields: [], + required_params: [], + config_schema: null, + variant_key: null, + variants: [], +} + +const postProcessing: ServerJobType = { + ...mlType, + key: 'post_processing', + name: 'Post Processing', + scope: [], + variant_key: 'task', + variants: [ + { + key: 'class_masking', + name: 'Class masking', + allowed: true, + scope: [scope('source_image_collection_id', { target: 'config' })], + scope_rule: 'all_required', + config_schema: { + required: ['taxa_list_id'], + properties: { + taxa_list_id: { + type: 'integer', + ami_widget: 'entity', + ami_entity: 'taxa/lists', + }, + reweight: { type: 'boolean', default: true }, + note: { type: 'string' }, + }, + }, + }, + ], +} + +describe('buildJobPayload', () => { + test('ml job puts scope fields at the top level and omits params', () => { + const { body, startNow } = buildJobPayload({ + projectId: '7', + jobType: mlType, + scopeValues: { pipeline_id: '3', source_image_collection_id: '12' }, + scopeLabels: { source_image_collection_id: 'Night 1' }, + configValues: {}, + name: 'My job', + delay: '5', + startNow: true, + }) + expect(body).toEqual({ + name: 'My job', + delay: 5, + project_id: '7', + job_type_key: 'ml', + pipeline_id: 3, + source_image_collection_id: 12, + }) + expect(startNow).toBe(true) + }) + + test('post processing nests variant, config values and config-target scope', () => { + const { body } = buildJobPayload({ + projectId: '7', + jobType: postProcessing, + variant: postProcessing.variants[0], + scopeValues: { source_image_collection_id: '12' }, + configValues: { taxa_list_id: '4', reweight: true, note: '' }, + }) + expect(body.params).toEqual({ + task: 'class_masking', + config: { + taxa_list_id: 4, + reweight: true, + source_image_collection_id: 12, + }, + }) + expect(body).not.toHaveProperty('source_image_collection_id') + }) + + test('defaults: name from method and scope label, delay 0, no start', () => { + const { body, startNow } = buildJobPayload({ + projectId: '7', + jobType: postProcessing, + variant: postProcessing.variants[0], + scopeValues: {}, + scopeLabels: { source_image_collection_id: 'Night 1' }, + configValues: {}, + }) + expect(body.name).toBe('Class masking – Night 1') + expect(body.delay).toBe(0) + expect(startNow).toBe(false) + }) + + test('name falls back to the date without a scope label', () => { + const { body } = buildJobPayload({ + projectId: '7', + jobType: mlType, + scopeValues: {}, + configValues: {}, + name: ' ', + today: '2026-09-30', + }) + expect(body.name).toBe('ML pipeline – 2026-09-30') + }) + + test('own config schema wraps config; empty optional values are omitted', () => { + const type: ServerJobType = { + ...mlType, + scope: [], + config_schema: { + properties: { + size: { type: 'number' }, + ids: { type: 'array', items: { type: 'integer' } }, + }, + }, + } + const { body } = buildJobPayload({ + projectId: '1', + jobType: type, + scopeValues: {}, + configValues: { size: '', ids: '1, 2' }, + }) + expect(body.params).toEqual({ config: { ids: [1, 2] } }) + }) +}) diff --git a/ui/src/components/form/schema-form/tests/schema-to-fields.test.ts b/ui/src/components/form/schema-form/tests/schema-to-fields.test.ts new file mode 100644 index 000000000..18043c65c --- /dev/null +++ b/ui/src/components/form/schema-form/tests/schema-to-fields.test.ts @@ -0,0 +1,124 @@ +import { + getInitialValue, + schemaToFields, + scopeToFields, + validateNumber, +} from '../schema-to-fields' + +const schema = { + required: ['taxa_list_id'], + properties: { + taxa_list_id: { + title: 'Taxa list to keep', + description: 'Help', + type: 'integer', + ami_widget: 'entity', + ami_entity: 'taxa/lists', + }, + algorithm_id: { + type: 'integer', + ami_widget: 'entity', + ami_entity: 'ml/algorithms', + ami_entity_filters: { task_type: 'classification' }, + }, + reweight: { type: 'boolean', default: true }, + size_threshold: { + type: 'number', + default: 0.0008, + exclusiveMinimum: 0, + exclusiveMaximum: 1, + }, + count: { type: 'integer', minimum: 1, maximum: 5 }, + mode: { type: 'string', enum: ['a', 'b'] }, + note: { type: 'string' }, + ids: { type: 'array', items: { type: 'integer' } }, + other: { type: 'object' }, + }, +} + +describe('schemaToFields', () => { + const fields = Object.fromEntries( + schemaToFields(schema).map((f) => [f.name, f]) + ) + + test('maps entity widgets with route, filters and required flag', () => { + expect(fields.taxa_list_id).toMatchObject({ + kind: 'entity', + entity: 'taxa/lists', + required: true, + label: 'Taxa list to keep', + description: 'Help', + }) + expect(fields.algorithm_id.entityFilters).toEqual({ + task_type: 'classification', + }) + expect(fields.algorithm_id.label).toBe('algorithm_id') + expect(fields.algorithm_id.required).toBe(false) + }) + + test('maps primitives and bounds', () => { + expect(fields.reweight).toMatchObject({ + kind: 'boolean', + defaultValue: true, + }) + expect(fields.size_threshold).toMatchObject({ + kind: 'number', + exclusiveMin: 0, + exclusiveMax: 1, + }) + expect(fields.count).toMatchObject({ kind: 'integer', min: 1, max: 5 }) + expect(fields.mode).toMatchObject({ kind: 'select', options: ['a', 'b'] }) + expect(fields.note.kind).toBe('text') + expect(fields.ids.kind).toBe('integer-list') + expect(fields.other.kind).toBe('json') + }) + + test('returns no fields for a missing schema', () => { + expect(schemaToFields(null)).toEqual([]) + expect(schemaToFields({ properties: {} })).toEqual([]) + }) +}) + +describe('scopeToFields', () => { + const scope = [ + { + field: 'event_ids', + label: 'Sessions', + entity: 'events', + required: true, + many: true, + target: 'config' as const, + entity_filters: {}, + }, + ] + + test('many scope becomes an integer list, optional on request', () => { + expect(scopeToFields(scope)[0]).toMatchObject({ + kind: 'integer-list', + required: true, + }) + expect(scopeToFields(scope, { optional: true })[0].required).toBe(false) + }) +}) + +describe('getInitialValue / validateNumber', () => { + test('defaults are stringified except booleans', () => { + const [reweight, threshold, count] = schemaToFields(schema).filter((f) => + ['reweight', 'size_threshold', 'count'].includes(f.name) + ) + expect(getInitialValue(reweight)).toBe(true) + expect(getInitialValue(threshold)).toBe('0.0008') + expect(getInitialValue(count)).toBeUndefined() + }) + + test('validates bounds and integers', () => { + const f = Object.fromEntries(schemaToFields(schema).map((x) => [x.name, x])) + expect(validateNumber(f.size_threshold, '0')).toBeDefined() + expect(validateNumber(f.size_threshold, '1')).toBeDefined() + expect(validateNumber(f.size_threshold, '0.5')).toBeUndefined() + expect(validateNumber(f.count, '0')).toBeDefined() + expect(validateNumber(f.count, '6')).toBeDefined() + expect(validateNumber(f.count, '2.5')).toBeDefined() + expect(validateNumber(f.count, '')).toBeUndefined() + }) +}) diff --git a/ui/src/data-services/constants.ts b/ui/src/data-services/constants.ts index 423a931b0..56aa92f93 100644 --- a/ui/src/data-services/constants.ts +++ b/ui/src/data-services/constants.ts @@ -10,6 +10,7 @@ export const API_ROUTES = { DEVICES: 'deployments/devices', EXPORTS: 'exports', IDENTIFICATIONS: 'identifications', + JOB_TYPES: 'jobs/types', JOBS: 'jobs', LOGIN: 'auth/token/login', LOGOUT: 'auth/token/logout', diff --git a/ui/src/data-services/hooks/jobs/useJobTypes.ts b/ui/src/data-services/hooks/jobs/useJobTypes.ts new file mode 100644 index 000000000..5a7d8fa4f --- /dev/null +++ b/ui/src/data-services/hooks/jobs/useJobTypes.ts @@ -0,0 +1,21 @@ +import { API_ROUTES, API_URL } from 'data-services/constants' +import { ServerJobType } from 'data-services/models/job-type' +import { useAuthorizedQuery } from '../auth/useAuthorizedQuery' + +export const useJobTypes = ( + projectId?: string +): { + jobTypes?: ServerJobType[] + isLoading: boolean + error?: unknown +} => { + const { data, isLoading, error } = useAuthorizedQuery<{ + results: ServerJobType[] + }>({ + enabled: !!projectId, + queryKey: [API_ROUTES.JOB_TYPES, projectId], + url: `${API_URL}/${API_ROUTES.JOB_TYPES}/?project_id=${projectId}`, + }) + + return { jobTypes: data?.results, isLoading, error } +} diff --git a/ui/src/data-services/models/job-type.ts b/ui/src/data-services/models/job-type.ts new file mode 100644 index 000000000..b430b0828 --- /dev/null +++ b/ui/src/data-services/models/job-type.ts @@ -0,0 +1,55 @@ +export interface ServerScopeField { + field: string + label: string + entity: string + required: boolean + many: boolean + target: 'job' | 'config' + entity_filters: { [key: string]: string | number | boolean } +} + +export interface ServerConfigSchemaProperty { + title?: string + description?: string + type?: string + default?: unknown + enum?: (string | number)[] + minimum?: number + maximum?: number + exclusiveMinimum?: number + exclusiveMaximum?: number + items?: { type?: string } + ami_widget?: string + ami_entity?: string + ami_entity_filters?: { [key: string]: string | number | boolean } +} + +export interface ServerConfigSchema { + type?: string + title?: string + required?: string[] + properties?: { [name: string]: ServerConfigSchemaProperty } +} + +export interface ServerJobTypeVariant { + key: string + name: string + description?: string + allowed: boolean + scope: ServerScopeField[] + scope_rule?: 'all_required' | 'exactly_one' + config_schema: ServerConfigSchema | null +} + +export interface ServerJobType { + key: string + name: string + description?: string + allowed: boolean + scope: ServerScopeField[] + required_fields: string[] + required_params: string[] + config_schema: ServerConfigSchema | null + variant_key: string | null + variants: ServerJobTypeVariant[] +} From 06d9bb8e251b1c88c964cb474a992bbc7b7f7b2b Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 29 Sep 2026 17:47:13 -0700 Subject: [PATCH 03/29] docs: explain how a job type appears in the generated Create Job dialog [skip ci] Adds a reference note on the GET /jobs/types/ contract and the attributes a job type or post-processing task declares to get a working form, with the files that implement each part, and indexes it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NFUiikN95Y3yz4pBK9KPu1 --- docs/claude/INDEX.md | 1 + docs/claude/reference/jobs-panel.md | 65 +++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 docs/claude/reference/jobs-panel.md diff --git a/docs/claude/INDEX.md b/docs/claude/INDEX.md index 61f0980d8..de78a6778 100644 --- a/docs/claude/INDEX.md +++ b/docs/claude/INDEX.md @@ -9,6 +9,7 @@ archived. | File | Description | |---|---| +| `reference/jobs-panel.md` | How a job type or post-processing task appears in the generated Create Job dialog: `GET /jobs/types/` contract, ScopeField, pydantic Field hints (`ami_widget`/`ami_entity`), `validate_params`, member allowlist, gating gotcha. Indexed 2026-09-29. Keywords: jobs panel, job types, config_schema, post-processing, params | | `reference/canonical-patterns.md` | Existing helpers/patterns to reuse before writing new ones, with file:line refs (SingleParamSerializer, ProjectMixin, permissions, schemas, fixtures). Keywords: reuse, helpers, conventions, DRF | | `reference/query-patterns.md` | DB model relationship table, composite indexes, prefetch/select_related patterns, full custom QuerySet method catalog, query anti-patterns. Keywords: N+1, indexes, ORM, performance | | `reference/api-stats-pattern.md` | How to add aggregate/leaderboard/chart endpoints (`//stats//`): GenericViewSet + @action, pure querysets in models_future. Keywords: stats, charts, aggregation | diff --git a/docs/claude/reference/jobs-panel.md b/docs/claude/reference/jobs-panel.md new file mode 100644 index 000000000..1f6ba86b8 --- /dev/null +++ b/docs/claude/reference/jobs-panel.md @@ -0,0 +1,65 @@ +# Jobs panel: how a job type appears in the Create Job dialog + +The Create Job dialog is generated from `GET /api/v2/jobs/types/?project_id=N`. A job type or +post-processing task shows up there, with a working form, once it declares the attributes below. +No frontend change is needed. Design history: branch `feat/jobs-panel-design`, +`docs/claude/planning/2026-09-18-jobs-panel-schema-driven-design.md`. PR #1447. + +## Where things live + +| What | File | +|---|---| +| `ScopeField`, schema normalizer, docstring → description | `ami/jobs/descriptors.py` | +| `JobType` attributes, `validate_params`, project-scope id checks | `ami/jobs/models.py` (`class JobType`, `PostProcessingJob`, `check_entities_in_project`) | +| Response builder `describe_job_types`; `params` validation on create | `ami/jobs/serializers.py` (`JobSerializer.validate`) | +| The gated `types` action | `ami/jobs/views.py` (`JobViewSet.types`) | +| Member allowlist and staff-only settings | `ami/ml/post_processing/registry.py` (`MEMBER_POST_PROCESSING_TASKS`, `staff_only_config_fields`) | +| Tests (permission matrix, query count, params validation) | `ami/jobs/tests/test_job_types.py` | + +## Making a job type creatable + +On the `JobType` subclass: + +- **Docstring**: its first paragraph is the help text under the job type select. +- `user_creatable = True`: without it the type is not listed and `POST /jobs/` refuses it. +- `scope_fields`: what the job runs on, as `ScopeField`s (`PIPELINE_SCOPE`, `CAPTURE_SET_SCOPE`, + `STATION_SCOPE`, or your own). `target="job"` means a top-level serializer field backed by a Job + column. `target="config"` means the value goes inside `params["config"]`. +- `required_fields` / `required_params`: checked on create (same shape as #1407). +- `config_schema`: a pydantic (v1) model for `params["config"]`. It is rendered as a form and + validated by `JobType.validate_params(project, user, params)`. +- `variant_key` + `variants()`: for types whose work is picked from a registry (post-processing). + +## Making a post-processing task appear + +Register it in `POSTPROCESSING_TASKS` and give the task class a docstring. Its `config_schema` +fields become the form: + +```python +taxa_list_id: int = pydantic.Field( + ..., title="Taxa list to keep", description="Classes outside this list are masked out.", + ami_widget="entity", ami_entity="taxa/lists", +) +``` + +- `title` / `description` are the label and help text (English only, not translated). +- `ami_widget="entity"` + `ami_entity=""` (+ `ami_entity_filters`) renders a picker + that pages that list endpoint. The same hint makes the server check that the id belongs to the + job's project (`_entity_queryset` in `ami/jobs/models.py`). Add a route there if the entity is + new; an unmapped entity is not project-checked. +- `gt` / `lt` / `ge` / `le` become the form's numeric bounds. +- `source_image_collection_id` and `event_ids` are treated as scope, not settings + (`PostProcessingJob.SCOPE_CONFIG_FIELDS`). If a config has both, the dialog shows both and the + task's own root validator enforces "exactly one". `occurrence_id` is hidden (admin-only path). +- Members can start a task only if it is in `MEMBER_POST_PROCESSING_TASKS`, and can change only + the listed fields. Superusers can start any task and change any setting. Non-superusers do not + receive the other fields in the schema. + +## Gotchas + +- `ObjectPermission.has_permission` returns True for every request, and a `detail=False` action + never reaches the object check. `types` gates itself (authenticated project member or superuser). +- Pydantic is v1 here (`Model.schema()`, `__fields__[...].field_info`). The normalizer is the one + place that shapes schemas; `x-ami-schema-version` marks its output. +- `params` is fixed after creation: `JobSerializer.validate` drops it on update. +- `assertNumQueries` for `types` counts the request's savepoint pair (6 total). From b2ae0fc00df97bf2ed87ff3e60dda5077ae4ffa6 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 29 Sep 2026 17:49:52 -0700 Subject: [PATCH 04/29] feat(ui): add Create job strings and server error mapping Adds the static interface strings for the Create job dialog and a helper that maps DRF validation errors onto the generated form fields. Config errors arrive as ": message" strings under params.config and scope errors are keyed by field name; anything that does not match a known field is returned as a general message. Labels and help text generated from server schemas are shown as received, so ui/AGENTS.md now records that they are an English-only exception to the translation rule. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NFUiikN95Y3yz4pBK9KPu1 --- ui/AGENTS.md | 1 + .../form/schema-form/map-server-errors.ts | 46 +++++++++++++++++++ .../tests/map-server-errors.test.ts | 42 +++++++++++++++++ ui/src/utils/language.ts | 16 +++++++ 4 files changed, 105 insertions(+) create mode 100644 ui/src/components/form/schema-form/map-server-errors.ts create mode 100644 ui/src/components/form/schema-form/tests/map-server-errors.test.ts diff --git a/ui/AGENTS.md b/ui/AGENTS.md index 461e19975..38785d477 100644 --- a/ui/AGENTS.md +++ b/ui/AGENTS.md @@ -11,6 +11,7 @@ in `src/design-system/` — check there before writing a new component. - All user-facing strings go through the translation layer: `translate(STRING.KEY)` from `src/utils/language.ts`. Add new keys to the `STRING` enum and `ENGLISH_STRINGS` map. Never hardcode UI copy in components. +- Exception: labels and help text generated from server-provided schemas (the Create job dialog's scope and settings fields) are rendered as received and are English-only. - UI copy uses sentence case: "Taxa list", not "Taxa List". ## Data services & types diff --git a/ui/src/components/form/schema-form/map-server-errors.ts b/ui/src/components/form/schema-form/map-server-errors.ts new file mode 100644 index 000000000..33c190f82 --- /dev/null +++ b/ui/src/components/form/schema-form/map-server-errors.ts @@ -0,0 +1,46 @@ +// Server validation errors arrive as DRF JSON. Config problems are strings of +// the form ": message" under params.config; scope problems are keyed by +// the scope field name. +export const mapServerErrors = ( + data: unknown, + { + configFields, + scopeFields, + }: { configFields: string[]; scopeFields: string[] } +) => { + const fieldErrors: { [formName: string]: string } = {} + const general: string[] = [] + + const asList = (value: unknown): string[] => + (Array.isArray(value) ? value : [value]) + .filter((item) => item !== undefined && item !== null) + .map((item) => (typeof item === 'string' ? item : JSON.stringify(item))) + + if (!data || typeof data !== 'object') { + return { fieldErrors, general } + } + + Object.entries(data as { [key: string]: unknown }).forEach(([key, value]) => { + if (key === 'params' && value && typeof value === 'object') { + Object.entries(value as { [key: string]: unknown }).forEach( + ([paramKey, paramValue]) => { + asList(paramValue).forEach((message) => { + const separator = message.indexOf(': ') + const field = separator > 0 ? message.slice(0, separator) : '' + if (paramKey === 'config' && configFields.includes(field)) { + fieldErrors[`config.${field}`] ??= message.slice(separator + 2) + } else { + general.push(message) + } + }) + } + ) + } else if (scopeFields.includes(key)) { + fieldErrors[`scope.${key}`] = asList(value)[0] + } else { + general.push(...asList(value)) + } + }) + + return { fieldErrors, general } +} diff --git a/ui/src/components/form/schema-form/tests/map-server-errors.test.ts b/ui/src/components/form/schema-form/tests/map-server-errors.test.ts new file mode 100644 index 000000000..8e72d6213 --- /dev/null +++ b/ui/src/components/form/schema-form/tests/map-server-errors.test.ts @@ -0,0 +1,42 @@ +import { mapServerErrors } from '../map-server-errors' + +const known = { + configFields: ['taxa_list_id', 'reweight'], + scopeFields: ['source_image_collection_id'], +} + +describe('mapServerErrors', () => { + test('maps ": message" config errors onto generated fields', () => { + const { fieldErrors, general } = mapServerErrors( + { + params: { + config: [ + 'taxa_list_id: field required', + 'reweight: Only staff can change this setting.', + ], + }, + }, + known + ) + expect(fieldErrors).toEqual({ + 'config.taxa_list_id': 'field required', + 'config.reweight': 'Only staff can change this setting.', + }) + expect(general).toEqual([]) + }) + + test('maps scope errors and leaves unmatched messages general', () => { + const { fieldErrors, general } = mapServerErrors( + { + source_image_collection_id: ['Not found in this project.'], + params: { task: 'Unknown task', config: ['other: bad'] }, + detail: 'Nope', + }, + known + ) + expect(fieldErrors).toEqual({ + 'scope.source_image_collection_id': 'Not found in this project.', + }) + expect(general).toEqual(['Unknown task', 'other: bad', 'Nope']) + }) +}) diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index feb83bed6..45020e3e6 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -1,4 +1,12 @@ export enum STRING { + JOB_CREATE, + JOB_FIELD_TYPE, + JOB_FIELD_METHOD, + JOB_SETTINGS_DIVIDER, + JOB_ADVANCED, + JOB_START_IMMEDIATELY, + JOB_NOT_PERMITTED, + JOB_LOADING_TYPES, /* BUTTON */ ADD, ADMIN, @@ -382,6 +390,14 @@ export enum STRING { } const ENGLISH_STRINGS: { [key in STRING]: string } = { + [STRING.JOB_CREATE]: 'Create job', + [STRING.JOB_FIELD_TYPE]: 'Job type', + [STRING.JOB_FIELD_METHOD]: 'Method', + [STRING.JOB_SETTINGS_DIVIDER]: '{{method}} settings', + [STRING.JOB_ADVANCED]: 'Advanced', + [STRING.JOB_START_IMMEDIATELY]: 'Start immediately', + [STRING.JOB_NOT_PERMITTED]: 'Not permitted for your role', + [STRING.JOB_LOADING_TYPES]: 'Loading job types', /* BUTTON */ [STRING.ADD]: 'Add', [STRING.ADMIN]: 'Admin', From 992e30a831dcd4f07ff341d7d513a687b34f575d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 29 Sep 2026 17:49:52 -0700 Subject: [PATCH 05/29] feat(ui): replace the New job dialog with a schema-driven Create job dialog The Jobs page now opens a Create job dialog built from the jobs/types endpoint. Choosing a job type (and a method, for types that have them) reveals the scope pickers the server declares, followed by a settings section generated from the method's JSON schema. Job name and delay live under a collapsed Advanced section, and Start immediately sits beside the Cancel and Create job buttons. Options the user's role may not use are shown disabled rather than hidden. Server validation errors are placed on the matching field, with the rest shown in a general error block. The previous dialog is kept as a fallback that is rendered only if the job types request fails. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NFUiikN95Y3yz4pBK9KPu1 --- .../form/schema-form/entity-select.tsx | 71 ++++ .../form/schema-form/schema-field.tsx | 142 +++++++ .../hooks/jobs/useCreateTypedJob.ts | 33 ++ .../pages/job-details/create-job-dialog.tsx | 348 ++++++++++++++++++ ui/src/pages/job-details/new-job-dialog.tsx | 1 + ui/src/pages/jobs/jobs.tsx | 4 +- 6 files changed, 597 insertions(+), 2 deletions(-) create mode 100644 ui/src/components/form/schema-form/entity-select.tsx create mode 100644 ui/src/components/form/schema-form/schema-field.tsx create mode 100644 ui/src/data-services/hooks/jobs/useCreateTypedJob.ts create mode 100644 ui/src/pages/job-details/create-job-dialog.tsx diff --git a/ui/src/components/form/schema-form/entity-select.tsx b/ui/src/components/form/schema-form/entity-select.tsx new file mode 100644 index 000000000..dcb844c14 --- /dev/null +++ b/ui/src/components/form/schema-form/entity-select.tsx @@ -0,0 +1,71 @@ +import { API_URL } from 'data-services/constants' +import { useAuthorizedQuery } from 'data-services/hooks/auth/useAuthorizedQuery' +import { Select } from 'nova-ui-kit' +import { STRING, translate } from 'utils/language' + +const PAGE_SIZE = 100 + +interface EntityOption { + id: string + label: string +} + +const getLabel = (record: any): string => { + const name = record.name ?? `${record.id}` + return typeof record.source_images_count === 'number' + ? `${name} (${record.source_images_count.toLocaleString()})` + : name +} + +export const EntitySelect = ({ + entity, + entityFilters, + projectId, + value, + onValueChange, +}: { + entity: string + entityFilters?: { [key: string]: string | number | boolean } + projectId: string + value?: string + onValueChange: (value: string | undefined, label?: string) => void +}) => { + const params = new URLSearchParams({ + project_id: projectId, + limit: `${PAGE_SIZE}`, + ...Object.fromEntries( + Object.entries(entityFilters ?? {}).map(([k, v]) => [k, `${v}`]) + ), + }) + const { data, isLoading } = useAuthorizedQuery<{ results: any[] }>({ + queryKey: [entity, 'options', params.toString()], + url: `${API_URL}/${entity}/?${params.toString()}`, + }) + const options: EntityOption[] = (data?.results ?? []).map((record) => ({ + id: `${record.id}`, + label: getLabel(record), + })) + const selected = options.some((option) => option.id === value) ? value : '' + + return ( + + onValueChange(id, options.find((option) => option.id === id)?.label) + } + value={selected} + > + + + + + {options.map((option) => ( + + {option.label} + + ))} + + + ) +} diff --git a/ui/src/components/form/schema-form/schema-field.tsx b/ui/src/components/form/schema-form/schema-field.tsx new file mode 100644 index 000000000..88a57866f --- /dev/null +++ b/ui/src/components/form/schema-form/schema-field.tsx @@ -0,0 +1,142 @@ +import { Checkbox, Input, InputContent, Select } from 'nova-ui-kit' +import { Control, Controller } from 'react-hook-form' +import { STRING, translate } from 'utils/language' +import { EntitySelect } from './entity-select' +import { FieldDescriptor, validateNumber } from './schema-to-fields' + +// Labels and help text come from the server schema and are shown as received. +export const SchemaField = ({ + control, + field, + formName, + projectId, + onLabelChange, +}: { + control: Control + field: FieldDescriptor + formName: string + projectId: string + onLabelChange?: (label?: string) => void +}) => ( + { + if (field.kind === 'integer' || field.kind === 'number') { + return validateNumber(field, value) + } + if (field.kind === 'json' && value) { + try { + JSON.parse(`${value}`) + } catch { + return translate(STRING.MESSAGE_VALUE_INVALID) + } + } + return undefined + }, + }} + render={({ field: controller, fieldState }) => { + const label = field.required ? `${field.label} *` : field.label + const error = fieldState.error?.message + + switch (field.kind) { + case 'boolean': + return ( + + + + ) + case 'entity': + return ( + + { + controller.onChange(value) + onLabelChange?.(optionLabel) + }} + /> + + ) + case 'select': + return ( + + + + + + + {field.options?.map((option) => ( + + {option} + + ))} + + + + ) + case 'json': + return ( + +