From 3b0708a95faac5c4a0096d2e1d1285810b5cb9c5 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 23 Sep 2026 21:47:18 +0100 Subject: [PATCH 01/23] api/v3_0/sources: search the data sources listing, and narrow it to one type Context: - The listing returned every data source the caller may read, with no way to search it or to ask for one kind. A picker offering a choice of forecasters therefore had to fetch them all and sift through them in the browser. Change: - Add a `filter` of space-separated search terms, matched against a source's name, its model and its id prefix, the way the sensors listing already searches, and a `type`, which narrows the listing to one source type. - Move the rule about which sources a user may read into the data-sources service, as `get_readable_source_account_ids` and `user_may_read_source`, so that it can be applied outside this endpoint without being restated. Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/change_log.rst | 4 ++ flexmeasures/api/v3_0/sources.py | 69 +++++++++++++------ .../api/v3_0/tests/test_sources_api.py | 51 ++++++++++++++ flexmeasures/data/services/data_sources.py | 32 +++++++++ flexmeasures/ui/static/openapi-specs.json | 22 +++++- 5 files changed, 155 insertions(+), 23 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 1e34a3079a..65c0b29451 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -5,6 +5,10 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL (e.g. `/api/v3_0`), allowing developers to upgrade at their own pace. +v3.0-38 | September 23, 2026 +"""""""""""""""""""""""""""" +- ``GET /api/v3_0/sources`` now accepts a ``filter`` of space-separated search terms, matched against a source's name, its model and its id prefix, and a ``type``, which narrows the listing to one source type, such as ``forecaster``. + v3.0-37 | September 15, 2026 """""""""""""""""""""""""""" - Added ``POST /api/v3_0/assets//automations``, ``PATCH /api/v3_0/assets//automations/`` and ``DELETE /api/v3_0/assets//automations/`` for managing an asset's automations. They require the same permission as writing data under the asset, and an automation may only involve sensors that its creator can access: read access to the sensors it reads data from, and permission to record data on the sensors it writes to (a ``403`` otherwise). Both the creation and the update accept a ``timezone``, in which the automation's cron expression is interpreted; it defaults to the asset's own timezone, taken from the asset's timezone attribute or one of its sensors, and to the server's ``FLEXMEASURES_TIMEZONE`` if the asset has neither. The automation endpoints also report a ``next_run``: the next scheduled run, null while the automation is inactive, and excluding any catch-up run still pending. Both ``next_run`` and ``cursor`` are now reported as clock times in the automation's own timezone, where ``cursor`` was previously reported in UTC, since a recurrence is read in that timezone. The automation endpoints' last two snake_case fields, ``job_stats`` and ``redis_connection_err``, are now ``job-stats`` and ``redis-connection-err``, like the rest. ``GET /api/v3_0/assets//jobs`` renames its ``redis_connection_err`` to ``redis-connection-err`` as well, so that one field is not spelled two ways; that field shipped in v1.0.0, and is read by FlexMeasures' own UI rather than by the FlexMeasures client. diff --git a/flexmeasures/api/v3_0/sources.py b/flexmeasures/api/v3_0/sources.py index 2d0efbfc57..ac5154ae4c 100644 --- a/flexmeasures/api/v3_0/sources.py +++ b/flexmeasures/api/v3_0/sources.py @@ -3,15 +3,20 @@ from flask import current_app from flask_classful import FlaskView, route from flask_json import as_json -from flask_security import current_user, auth_required +from flask_security import auth_required from marshmallow import fields, Schema from packaging.version import Version, InvalidVersion from sqlalchemy import select, or_, and_ from webargs.flaskparser import use_kwargs -from flexmeasures.auth.policy import user_has_admin_access, CONSULTANT_ROLE +from flexmeasures.api.common.schemas.search import SearchFilterField from flexmeasures.data import db from flexmeasures.data.models.data_sources import DataSource, DEFAULT_DATASOURCE_TYPES +from flexmeasures.data.queries.utils import id_prefix_filter +from flexmeasures.data.services.data_sources import ( + get_readable_source_account_ids, + user_may_read_source, +) """ API endpoint to list accessible data sources and defined source types. @@ -30,21 +35,31 @@ class SourceQuerySchema(Schema): ) }, ) + filter = SearchFilterField( + required=False, + metadata={ + "description": "Search terms, separated by spaces, matched against the source's name, its model and its id prefix. A source matching any of the terms is returned.", + "example": "TrainPredictPipeline", + }, + ) + type = fields.Str( + required=False, + metadata={ + "description": "Only return sources of this type, such as forecaster, scheduler or reporter.", + "example": "forecaster", + }, + ) -def _get_accessible_account_ids() -> list[int] | None: - """Return account IDs whose sources the current user may read. - - Returns None to indicate "all accounts" (admin access). - """ - if user_has_admin_access(current_user, "read"): - return None # all sources - - accessible_ids = [current_user.account_id] - if current_user.has_role(CONSULTANT_ROLE): - for client_account in current_user.account.consultancy_client_accounts: - accessible_ids.append(client_account.id) - return accessible_ids +def source_search_term_filter(term: str): + """Match a search term against what identifies a source: its name, its model, its description and its id.""" + filters = [ + DataSource.name.ilike(f"%{term}%"), + DataSource.model.ilike(f"%{term}%"), + ] + if term.isdecimal(): + filters.append(id_prefix_filter(DataSource.id, term)) + return or_(*filters) def _filter_sources_to_latest(sources: list[DataSource]) -> list[DataSource]: @@ -92,7 +107,12 @@ class SourceAPI(FlaskView): @route("", methods=["GET"]) @use_kwargs(SourceQuerySchema, location="query") @as_json - def index(self, only_latest: bool = True): + def index( + self, + only_latest: bool = True, + filter: list[str] | None = None, + type: str | None = None, + ): """List accessible data sources and defined source types. .. :quickref: Sources; List accessible data sources and defined source types. @@ -104,6 +124,9 @@ def index(self, only_latest: bool = True): Returns the list of data sources accessible to the current user and the defined source types. + The ``filter`` parameter searches the sources by name, by model and by id prefix, + and the ``type`` parameter narrows the list to one source type, such as ``forecaster``. + **Access rules:** - Admins see all data sources. @@ -148,7 +171,7 @@ def index(self, only_latest: bool = True): tags: - Sources """ - accessible_account_ids = _get_accessible_account_ids() + accessible_account_ids = get_readable_source_account_ids() query = select(DataSource) if accessible_account_ids is not None: @@ -163,6 +186,12 @@ def index(self, only_latest: bool = True): ), ) ) + if type is not None: + query = query.where(DataSource.type == type) + if filter is not None: + query = query.where( + or_(*(source_search_term_filter(term) for term in filter)) + ) sources: list[DataSource] = list(db.session.scalars(query).all()) @@ -234,11 +263,7 @@ def get(self, id: int): source = db.session.get(DataSource, id) if source is None: return {"message": f"No data source found with id {id}."}, 404 - accessible_account_ids = _get_accessible_account_ids() - if accessible_account_ids is not None and not ( - source.account_id in accessible_account_ids - or (source.account_id is None and source.user_id is None) - ): + if not user_may_read_source(source): return {"message": "You cannot read this data source."}, 403 return _serialize_source(source, with_attributes=True), 200 diff --git a/flexmeasures/api/v3_0/tests/test_sources_api.py b/flexmeasures/api/v3_0/tests/test_sources_api.py index 23c8a979a2..4751291334 100644 --- a/flexmeasures/api/v3_0/tests/test_sources_api.py +++ b/flexmeasures/api/v3_0/tests/test_sources_api.py @@ -366,3 +366,54 @@ def test_get_source_auth( def test_get_nonexistent_source(client, setup_api_test_data, requesting_user): response = client.get(url_for("SourceAPI:get", id=99999)) assert response.status_code == 404 + + +@pytest.mark.parametrize( + "requesting_user", + ["test_admin_user@seita.nl"], + indirect=True, +) +def test_get_sources_filtered_by_type_and_search_term( + client, setup_api_test_data, requesting_user, db +): + """The listing can be narrowed to one source type, and searched by name, by model and by id.""" + forecaster = DataSource( + name="SearchableSeita", + type="forecaster", + model="TrainPredictPipeline", + ) + reporter = DataSource( + name="SearchableSeita", + type="reporter", + model="PandasReporter", + ) + db.session.add_all([forecaster, reporter]) + db.session.flush() + + by_type = client.get(url_for("SourceAPI:index"), query_string={"type": "reporter"}) + assert by_type.status_code == 200 + assert {source["type"] for source in by_type.json["sources"]} == {"reporter"} + assert reporter.id in {source["id"] for source in by_type.json["sources"]} + assert forecaster.id not in {source["id"] for source in by_type.json["sources"]} + + by_model = client.get( + url_for("SourceAPI:index"), query_string={"filter": "TrainPredictPipeline"} + ) + assert by_model.status_code == 200 + assert forecaster.id in {source["id"] for source in by_model.json["sources"]} + assert reporter.id not in {source["id"] for source in by_model.json["sources"]} + + by_name = client.get( + url_for("SourceAPI:index"), query_string={"filter": "searchableseita"} + ) + assert by_name.status_code == 200 + assert {forecaster.id, reporter.id} <= { + source["id"] for source in by_name.json["sources"] + } + + by_id = client.get( + url_for("SourceAPI:index"), + query_string={"filter": str(forecaster.id), "only_latest": "false"}, + ) + assert by_id.status_code == 200 + assert forecaster.id in {source["id"] for source in by_id.json["sources"]} diff --git a/flexmeasures/data/services/data_sources.py b/flexmeasures/data/services/data_sources.py index 2cac829595..2bc8eddf2f 100644 --- a/flexmeasures/data/services/data_sources.py +++ b/flexmeasures/data/services/data_sources.py @@ -131,3 +131,35 @@ def get_data_generator( data_generator._save_config = save_config return data_generator + + +def get_readable_source_account_ids() -> list[int] | None: + """Return the ids of the accounts whose data sources the current user may read. + + Returns None to say that every account's sources are readable, which is what admin access amounts to. + """ + from flask_security import current_user + + from flexmeasures.auth.policy import user_has_admin_access, CONSULTANT_ROLE + + if user_has_admin_access(current_user, "read"): + return None # all sources + readable_ids = [current_user.account_id] + if current_user.has_role(CONSULTANT_ROLE): + for client_account in current_user.account.consultancy_client_accounts: + readable_ids.append(client_account.id) + return readable_ids + + +def user_may_read_source(source: DataSource) -> bool: + """Whether the current user may read the given data source. + + A source belonging to one of the user's own accounts is readable, and so is a system source, + which is one that names neither an account nor a user. + """ + readable_account_ids = get_readable_source_account_ids() + if readable_account_ids is None: + return True + if source.account_id in readable_account_ids: + return True + return source.account_id is None and source.user_id is None diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 653f93acbe..07a07110ee 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -5643,7 +5643,7 @@ "/api/v3_0/sources": { "get": { "summary": "List accessible data sources and defined source types.", - "description": "Returns the list of data sources accessible to the current user and\nthe defined source types.\n\nAccess rules:\n\n- Admins see all data sources.\n- Users with the consultant role see sources belonging to their\n own account and to any consultancy-client accounts for which their\n account is the consultancy.\n- All other authenticated users see only sources belonging to their\n own account, plus sources that have neither a user_id nor an\n account_id (i.e. system/public sources).\n", + "description": "Returns the list of data sources accessible to the current user and\nthe defined source types.\n\nThe filter parameter searches the sources by name, by model and by id prefix,\nand the type parameter narrows the list to one source type, such as forecaster.\n\nAccess rules:\n\n- Admins see all data sources.\n- Users with the consultant role see sources belonging to their\n own account and to any consultancy-client accounts for which their\n account is the consultancy.\n- All other authenticated users see only sources belonging to their\n own account, plus sources that have neither a user_id nor an\n account_id (i.e. system/public sources).\n", "security": [ { "ApiKeyAuth": [] @@ -5659,6 +5659,26 @@ "default": true }, "required": false + }, + { + "in": "query", + "name": "filter", + "description": "Search terms, separated by spaces, matched against the source's name, its model and its id prefix. A source matching any of the terms is returned.", + "schema": { + "type": "string", + "example": "TrainPredictPipeline" + }, + "required": false + }, + { + "in": "query", + "name": "type", + "description": "Only return sources of this type, such as forecaster, scheduler or reporter.", + "schema": { + "type": "string", + "example": "forecaster" + }, + "required": false } ], "responses": { From e332a30b6252d6001f29379d3120fc093a30ba7e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 23 Sep 2026 21:47:50 +0100 Subject: [PATCH 02/23] api/v3_0/assets: let an automation name the data source it computes under Context: - A data source stores a data generator and the configuration it runs under, and the results computed under it are recorded on it. `flexmeasures add automation` has taken `--source` since the command was written, so several automations can share one generator and one lineage of data, but the API could only be handed a class and a config, which reaches an existing source by accident, when what it builds happens to hash to the same attributes. Change: - Accept a `source` on the creation endpoint, deserialized to the DataSource the service already knows how to take. It cannot be combined with `data-generator` or `config`, which the source itself determines, and a schedule automation cannot name one, since it re-resolves its source from the asset and the flex config on every run and would move off the named one at the next run. - Require that the caller may read the source, by the same rule the sources listing applies. A source is otherwise a way to compute under, and record on, a generator configuration belonging to another organisation. The CLI runs without a user and stays trusted, as it already is for sensors. - Let DataSourceIdField dump None, so that the field's absent default serializes rather than failing the OpenAPI generation. Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/change_log.rst | 1 + documentation/features/automations.rst | 7 + flexmeasures/api/v3_0/assets.py | 12 ++ .../api/v3_0/tests/test_automations_api.py | 170 ++++++++++++++++++ flexmeasures/data/schemas/automations.py | 47 ++++- flexmeasures/data/schemas/sources.py | 8 +- flexmeasures/data/services/automations.py | 13 +- flexmeasures/ui/static/openapi-specs.json | 22 +++ 8 files changed, 276 insertions(+), 4 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 65c0b29451..b643fb9f89 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -7,6 +7,7 @@ API change log v3.0-38 | September 23, 2026 """""""""""""""""""""""""""" +- ``POST /api/v3_0/assets//automations`` now accepts a ``source``: the id of an existing data source whose data generator and configuration the automation reuses, and under which it records what it computes. It cannot be combined with ``data-generator`` or ``config``, which a data source already determines, and a schedule automation cannot name one, since it resolves its data source from the asset and the flex config on every run (a ``422`` in both cases). A data source that the caller cannot read is refused with a ``403``. - ``GET /api/v3_0/sources`` now accepts a ``filter`` of space-separated search terms, matched against a source's name, its model and its id prefix, and a ``type``, which narrows the listing to one source type, such as ``forecaster``. v3.0-37 | September 15, 2026 diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index de6d73c718..9f82ca4270 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -25,6 +25,12 @@ The remaining options are the ones the task itself needs: a forecast automation The forecaster and its configuration are stored on a data source, so you can also pass ``--source`` to reuse the data source of an existing forecaster, in which case ``--forecaster`` and ``--config`` (and the individual configuration options) are not needed — the data source already determines them. That data source is required while the automation exists, so it cannot be deleted until the automation is removed. +The API takes the same choice as the ``source`` field of `[POST] /assets/(id)/automations <../api/v3_0.html#post--api-v3_0-assets-id-automations>`_, which cannot be combined with ``data-generator`` or ``config``. +In the UI, the asset's *Automations* page searches the data sources you may read, by id, by name and by data generator class, and shows the configuration the picked one stores before you create the automation. +A data source is only yours to name if it is yours to read. +Reusing one means the results of both automations are recorded under the same source, so they share one lineage of data. +A schedule automation cannot name a data source, because it works its own out on every run, as described under `Automating schedules`_. + The recurrence is defined by a standard five-field cron string (minute, hour, day of month, month, and day of week), which defaults to ``"0 0 * * *"`` (daily at midnight). It is interpreted in the automation's IANA timezone. If ``--timezone`` is omitted, the current ``FLEXMEASURES_TIMEZONE`` value is copied to the automation. @@ -115,6 +121,7 @@ A schedule automation has a data generator too, but you do not name it separatel It is put together from choices you have already made: the flex config in the trigger message, the flex config saved on the asset tree, and the scheduler that the asset resolves to. Because those live in two places, and the asset can be edited without touching the automation, the runner puts the generator together again on every run, and moves the automation to another data source when the combination has changed. Editing an asset's flex-model is therefore a configuration change, and shows up as one: the schedules computed before and after it carry different data sources. +That is also why a schedule automation cannot be pinned to a data source you pick: the next run would move it off that source again. Because the schedule is recomputed on every run, the flex config may only describe the site and its devices, not one moment. A field with a fixed moment in it, such as ``soc-at-start`` or a ``soc-targets`` entry with a ``datetime``, is refused when the automation is created, and the error names the field. diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index b0dfac91f7..a6dbee602a 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -1730,6 +1730,18 @@ def post_automation(self, automation_data: dict, id: int, asset: GenericAsset): parameters: start-offset: "1D,DB" duration: P1D + forecasts_on_an_existing_source: + summary: Forecasts computed by an existing data source + description: >- + Reuses the forecaster and the configuration stored on data source 6, + and records the forecasts under that same source. + value: + name: Day-ahead PV forecasts + cron: "0 6 * * *" + type: forecasting + source: 6 + parameters: + sensor: 2092 responses: 201: description: CREATED diff --git a/flexmeasures/api/v3_0/tests/test_automations_api.py b/flexmeasures/api/v3_0/tests/test_automations_api.py index 695159334e..12ce6591e3 100644 --- a/flexmeasures/api/v3_0/tests/test_automations_api.py +++ b/flexmeasures/api/v3_0/tests/test_automations_api.py @@ -930,3 +930,173 @@ def test_a_forecast_automation_names_the_data_generator_it_runs( fresh_db.session.delete(automation) fresh_db.session.flush() + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_post_automation_reusing_a_data_source( + app, + fresh_db, + add_battery_assets_fresh_db, + requesting_user, +): + """Naming an existing data source reuses the data generator and the configuration stored on it.""" + battery = add_battery_assets_fresh_db["Test battery"] + sensor_id = battery.sensors[0].id + with app.test_client() as client: + first_response = client.post( + url_for("AssetAPI:post_automation", id=battery.id), + json={ + "name": "Forecasts defining a generator", + "cron": "0 6 * * *", + "type": "forecasting", + "parameters": {"sensor": sensor_id}, + }, + ) + assert first_response.status_code == 201, first_response.json + source_id = fresh_db.session.get( + Automation, first_response.json["id"] + ).generator_id + + second_response = client.post( + url_for("AssetAPI:post_automation", id=battery.id), + json={ + "name": "Forecasts reusing that generator", + "cron": "0 18 * * *", + "type": "forecasting", + "source": source_id, + "parameters": {"sensor": sensor_id}, + }, + ) + assert second_response.status_code == 201, second_response.json + reusing_automation = fresh_db.session.get(Automation, second_response.json["id"]) + assert reusing_automation.generator_id == source_id + + +@pytest.mark.parametrize( + "extra_fields, expected_message", + [ + ( + {"data-generator": "TrainPredictPipeline"}, + "data-generator cannot be combined with source", + ), + ({"config": {"model": "CustomLGBM"}}, "config cannot be combined with source"), + ], +) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_post_automation_refuses_a_source_beside_a_generator( + app, + fresh_db, + add_battery_assets_fresh_db, + requesting_user, + extra_fields, + expected_message, +): + """A data source already stores the data generator and its config, so naming both ways at once is refused.""" + battery = add_battery_assets_fresh_db["Test battery"] + source = DataSource( + name="reusable forecaster", + type="forecaster", + model="TrainPredictPipeline", + attributes={"data_generator": {"config": {}}}, + ) + fresh_db.session.add(source) + fresh_db.session.flush() + with app.test_client() as client: + response = client.post( + url_for("AssetAPI:post_automation", id=battery.id), + json={ + "name": "Over-specified forecasts", + "cron": "0 6 * * *", + "type": "forecasting", + "source": source.id, + "parameters": {"sensor": battery.sensors[0].id}, + **extra_fields, + }, + ) + assert response.status_code == 422 + assert expected_message in str(response.json) + assert ( + fresh_db.session.execute( + select(Automation).filter_by(name="Over-specified forecasts") + ).scalar_one_or_none() + is None + ) + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_post_schedule_automation_refuses_a_source( + app, + fresh_db, + add_battery_assets_fresh_db, + requesting_user, +): + """A schedule automation resolves its data source on every run, so it cannot be pinned to one.""" + battery = add_battery_assets_fresh_db["Test battery"] + source = DataSource( + name="reusable forecaster", + type="forecaster", + model="TrainPredictPipeline", + attributes={"data_generator": {"config": {}}}, + ) + fresh_db.session.add(source) + fresh_db.session.flush() + with app.test_client() as client: + response = client.post( + url_for("AssetAPI:post_automation", id=battery.id), + json={ + "name": "Pinned schedules", + "cron": "0 0 * * *", + "type": "scheduling", + "source": source.id, + "parameters": {"duration": "PT12H"}, + }, + ) + assert response.status_code == 422 + assert "schedule automation cannot name a source" in str(response.json) + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_post_automation_with_a_source_of_another_account( + app, + fresh_db, + setup_accounts_fresh_db, + add_battery_assets_fresh_db, + requesting_user, +): + """A data source the caller cannot read is not theirs to compute under, whatever it stores.""" + battery = add_battery_assets_fresh_db["Test battery"] + foreign_source = DataSource( + name="foreign forecaster", + type="forecaster", + model="TrainPredictPipeline", + attributes={"data_generator": {"config": {}}}, + account=setup_accounts_fresh_db["Dummy"], + ) + fresh_db.session.add(foreign_source) + fresh_db.session.flush() + with app.test_client() as client: + response = client.post( + url_for("AssetAPI:post_automation", id=battery.id), + json={ + "name": "Borrowed forecasts", + "cron": "0 6 * * *", + "type": "forecasting", + "source": foreign_source.id, + "parameters": {"sensor": battery.sensors[0].id}, + }, + ) + assert response.status_code == 403 + assert ( + fresh_db.session.execute( + select(Automation).filter_by(name="Borrowed forecasts") + ).scalar_one_or_none() + is None + ) diff --git a/flexmeasures/data/schemas/automations.py b/flexmeasures/data/schemas/automations.py index 4a070c9641..20259e9686 100644 --- a/flexmeasures/data/schemas/automations.py +++ b/flexmeasures/data/schemas/automations.py @@ -5,11 +5,19 @@ from croniter import croniter from croniter.croniter import CroniterBadDateError -from marshmallow import fields, validate, validates, Schema, ValidationError +from marshmallow import ( + fields, + validate, + validates, + validates_schema, + Schema, + ValidationError, +) from pytz import all_timezones_set from flexmeasures.data import ma, db from flexmeasures.data.models.automations import Automation +from flexmeasures.data.schemas.sources import DataSourceIdField from flexmeasures.data.schemas.utils import ( FMValidationError, MarshmallowClickMixin, @@ -109,6 +117,43 @@ class AutomationCreationSchema(Schema): "example": {}, }, ) + source = DataSourceIdField( + load_default=None, + allow_none=True, + metadata={ + "description": "Id of an existing data source to reuse, instead of naming a `data-generator` and its `config`." + " The data generator class and the configuration it runs under are read from that source," + " and the automation records its results under it, so several automations can share one generator and one lineage of data." + " A schedule automation cannot name one, as it resolves its data source from the asset and the flex config on every run.", + "example": 6, + }, + ) + + @validates_schema + def validate_generator_is_named_once(self, data: dict, **kwargs): + """A data source already determines the data generator and its config, so the two ways of naming one are exclusive.""" + if data.get("source") is None: + return + named_alongside = [ + data_key + for data_key, value in ( + ("data-generator", data.get("generator_class")), + ("config", data.get("config")), + ) + if value + ] + if named_alongside: + raise ValidationError( + f"{' and '.join(named_alongside)} cannot be combined with source:" + " a data source already stores the data generator and the configuration it runs under.", + field_name="source", + ) + if data.get("automation_type") == "scheduling": + raise ValidationError( + "A schedule automation cannot name a source:" + " its data source follows from the asset and the flex config, and is resolved afresh on every run.", + field_name="source", + ) class AutomationUpdateSchema(Schema): diff --git a/flexmeasures/data/schemas/sources.py b/flexmeasures/data/schemas/sources.py index ef384b2e9b..f0bebe2335 100644 --- a/flexmeasures/data/schemas/sources.py +++ b/flexmeasures/data/schemas/sources.py @@ -22,5 +22,11 @@ def _deserialize(self, value, attr, obj, **kwargs) -> DataSource: return source def _serialize(self, source, attr, data, **kwargs): - """Turn a DataSource into a source id.""" + """Turn a DataSource into a source id. + + A field that is allowed to be left out serializes its absence, rather than failing on it, + which is what dumping an optional field's default asks for. + """ + if source is None: + return None return source.id diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index 120e2fcf5d..50e341bfdc 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -1226,17 +1226,26 @@ def create_automation( An audit log record is added to the asset. :param check_permissions: whether to require that the current user may read the sensors that the automation reads from, - and record data on the sensors it writes to. + record data on the sensors it writes to, and read the data source they name, if they name one. Set this for automations created by a user (through the API or the UI); the CLI runs without a user, and is trusted. :raises marshmallow.ValidationError: if the parameters are invalid. :raises ValueError: if the data generator cannot be set up. - :raises werkzeug.exceptions.Forbidden: if a sensor is not accessible to the user. + :raises werkzeug.exceptions.Forbidden: if a sensor, or the named data source, is not accessible to the user. :returns: the automation and a list of warnings. """ from marshmallow import ValidationError + from werkzeug.exceptions import Forbidden from flexmeasures.data.models.audit_log import AssetAuditLog + from flexmeasures.data.services.data_sources import user_may_read_source + + # A named source hands over whatever configuration it stores, and the automation's results are recorded under it, + # so it is only the user's to name if it is theirs to read. + if check_permissions and source is not None and not user_may_read_source(source): + exception = Forbidden() + exception.api_message = f"You cannot define an automation on data source {source.id}, which you cannot read yourself." + raise exception parameters = parameters or {} timezone = timezone or get_default_automation_timezone(asset) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 07a07110ee..9b863c8537 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -3707,6 +3707,19 @@ "duration": "P1D" } } + }, + "forecasts_on_an_existing_source": { + "summary": "Forecasts computed by an existing data source", + "description": "Reuses the forecaster and the configuration stored on data source 6, and records the forecasts under that same source.", + "value": { + "name": "Day-ahead PV forecasts", + "cron": "0 6 * * *", + "type": "forecasting", + "source": 6, + "parameters": { + "sensor": 2092 + } + } } } } @@ -6868,6 +6881,15 @@ "description": "Configuration stored on the data generator, as opposed to the `parameters` it runs with. Used by a forecast or report automation.", "example": {}, "additionalProperties": {} + }, + "source": { + "type": [ + "integer", + "null" + ], + "default": null, + "description": "Id of an existing data source to reuse, instead of naming a `data-generator` and its `config`. The data generator class and the configuration it runs under are read from that source, and the automation records its results under it, so several automations can share one generator and one lineage of data. A schedule automation cannot name one, as it resolves its data source from the asset and the flex config on every run.", + "example": 6 } }, "required": [ From c247d303478de67d191402a6850120155461d1bc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 23 Sep 2026 21:48:37 +0100 Subject: [PATCH 03/23] ui/assets: pick a data source for a new automation, and see what it stores Context: - The creation form asked for a data generator class and a config as free text, with no way to say "the one that automation already uses", and nothing showed what a generator would be configured with until after the automation existed. Change: - Add a search box above those two fields, which lists the data sources of the automation's own type, matched by id, by name and by generator class, and lists them on focus so they can be browsed without guessing a term. - Show the picked source's description and the configuration it stores, and disable the class and config fields while it is picked, since the source already determines both. Clearing it hands them back. - Drop the whole section for a schedule automation, which has no generator to name, and clear a selection when the type changes, as the source type does. Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 2 + .../templates/assets/asset_automations.html | 256 ++++++++++++++++-- .../tests/js/test_automation_source_picker.py | 97 +++++++ 3 files changed, 328 insertions(+), 27 deletions(-) create mode 100644 flexmeasures/ui/tests/js/test_automation_source_picker.py diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 84a3dfea50..140b0c5be3 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -33,6 +33,7 @@ New features * Changing the selected time range on an asset or sensor chart now only loads the data that is actually new, instead of reloading the whole range, which makes stepping through or extending a long period much faster; reloading the page, or leaving it open for five minutes, still fetches everything afresh [see `PR #2433 `_] * The statistics table on a sensor page now shows all data sources together by default, as the graph does [see `PR #2462 `_] * The asset lists on an organisation's page and on the asset overview now show only top-level assets per default, so the sites you are looking for are no longer buried among their sub-assets, and both offer the same two checkboxes to widen the list again: untick *Top-level only* for the whole tree, or tick *Include public assets*, which is off per default [see `PR #2523 `_] +* A new automation can now be pointed at a data source that already holds a forecaster or a reporter, instead of naming that class and its configuration again, in the UI and the API as well as in the CLI. See `Automations, in detail`_ [see `PR #XXXX `_] Infrastructure / Support ------------------------- @@ -76,6 +77,7 @@ Automations arrived over several pull requests. This is what each of them contri * Reports as well as forecasts and schedules: a report automation stores report parameters, with its reporter and the reporter's configuration on a data source, and reports on a period resolved afresh on each run, either from ``start-offset`` and ``end-offset`` applied to the run time in the automation's timezone, or since the last successful report ended, while a fixed ``start`` or ``end`` is refused; a report job records only on the sensors the automation was checked against [see `PR #2297 `_] * Every automation times its runs the same way: a fixed ``start``, ``end`` or ``prior`` in its parameters is refused, as every run would share that moment, and two of ``start-offset``, ``end-offset`` and ``duration`` describe the period each run covers instead, with the offsets applied to the time the run was due on the automation's own clock, so that, for instance, a schedule automation can plan the whole of the next day [see `PR #2551 `_] * Look up automations from the command line with ``flexmeasures show automations``, which lists them all (inactive ones included) with the IDs that the edit, delete and run commands expect, and, with ``--id``, shows a single automation's recurrence, cursor, parameters and the sensors it reads from and writes to [see `PR #2533 `_] +* A forecast or report automation can now reuse an existing data source, in the API and the UI as well as in the CLI, so that several automations compute under one data generator, one configuration and one lineage of data. The *Automations* page searches the data sources you may read, by id, by name and by data generator class, and shows the configuration the picked one stores before the automation is created [see `PR #XXXX `_] v1.0.1 | September 9, 2026 diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index 2262e9192f..2c79b8b318 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -66,6 +66,21 @@ background: var(--bs-btn-hover-bg); border-color: var(--bs-btn-hover-border-color); } + /* The search results sit under their input, and a long list scrolls rather than pushing the rest of the form down. */ + .automation-source-results { + max-height: 12rem; + overflow-y: auto; + } + .automation-source-results .list-group-item { + cursor: pointer; + } + /* A data generator config can be wide, so it wraps rather than scrolling the modal sideways. */ + #automationSourceConfig { + white-space: pre-wrap; + word-break: break-word; + max-height: 12rem; + overflow-y: auto; + }
@@ -130,16 +145,40 @@
New automation for {{ asset.name }}
The local clock the recurrence is read in. For example, Europe/Amsterdam keeps 06:00 at 06:00 locally when daylight saving changes. Defaults to this asset's timezone.
-
- - -
- Forecaster class (defaults to TrainPredictPipeline) or reporter class (required for type reporting, e.g. PandasReporter). Not used for scheduling. +
+
+ +
+ + +
+
+
+ Optional. Pick a data source to take its data generator and configuration from, instead of naming a class and a config below. + The automation then records what it computes under that same source, alongside the data it already holds. + Not used for scheduling, which works its source out from the asset and the flex config on every run. +
+
+
+
Selected data source
+

+
Configuration
+ 

+                      

+                    

+                  

+                  

+                    
+                    
+                    

+                      Forecaster class (defaults to TrainPredictPipeline) or reporter class (required for type reporting, e.g. PandasReporter). Not used for scheduling.
+                    

+                  

+                  

+                    
+                    
                   

-                

-                

-                  
-                  
                 

                 

                   
@@ -283,6 +322,98 @@ 
Edit automation

     $("#automations_msg").removeClass("d-none").text(message);
   }
 
+  // A data source stores a data generator and the configuration it runs under, and the results computed under it are recorded on it,
+  // so picking one here is an alternative to naming a class and a config by hand.
+  // Only forecast and report automations can name one: a schedule automation works its source out from the asset and the flex config on every run.
+  const SOURCE_TYPE_PER_AUTOMATION_TYPE = {
+    forecasting: "forecaster",
+    reporting: "reporter",
+  };
+
+  // How many search results the list shows at once, so that a broad search stays readable.
+  const MAX_SOURCE_RESULTS = 25;
+
+  // The data source the new automation will reuse, or null while none is picked.
+  let selectedAutomationSource = null;
+
+  function sourceSearchUrl(automationType, searchTerm) {
+    const parameters = new URLSearchParams();
+    // Every version of a source can be picked, so the listing is not collapsed to the latest one per generator.
+    parameters.append("only_latest", "false");
+    const sourceType = SOURCE_TYPE_PER_AUTOMATION_TYPE[automationType];
+    if (sourceType) {
+      parameters.append("type", sourceType);
+    }
+    if (searchTerm) {
+      parameters.append("filter", searchTerm);
+    }
+    return `/api/v3_0/sources?${parameters.toString()}`;
+  }
+
+  function sourceLabel(source) {
+    return `${source.description || source.name} (data source ${source.id})`;
+  }
+
+  function sourceResultsHtml(sources) {
+    if (!sources || !sources.length) {
+      return '
No matching data sources.
';
+    }
+    const items = sources.slice(0, MAX_SOURCE_RESULTS).map(source =>
+      ``
+    );
+    if (sources.length > MAX_SOURCE_RESULTS) {
+      items.push(
+        `
${sources.length - MAX_SOURCE_RESULTS} more match; narrow the search to see them.
`
+      );
+    }
+    return items.join("");
+  }
+
+  // The configuration a data generator stores on its source, as the source's own record spells it.
+  function sourceConfigText(source) {
+    const config = source?.attributes?.data_generator?.config;
+    if (config === undefined || config === null) {
+      return "This data source stores no data generator configuration.";
+    }
+    return JSON.stringify(config, null, 4);
+  }
+
+  // While a source is picked, it determines the class and the config, so those two fields cannot also be filled in.
+  function setGeneratorFieldsEnabled(enabled) {
+    $("#automationGenerator, #automationConfig").prop("disabled", !enabled);
+  }
+
+  function showSelectedSource(source) {
+    selectedAutomationSource = source;
+    $("#automationSourceResults").addClass("d-none").empty();
+    $("#automationSourceSearch").val(sourceLabel(source));
+    $("#automationSourceClear").removeClass("d-none");
+    $("#automationSourceSelected").text(sourceLabel(source));
+    $("#automationSourceConfig").text(sourceConfigText(source));
+    $("#automationSourcePreview").removeClass("d-none");
+    setGeneratorFieldsEnabled(false);
+  }
+
+  function clearSelectedSource() {
+    selectedAutomationSource = null;
+    $("#automationSourceSearch").val("");
+    $("#automationSourceClear").addClass("d-none");
+    $("#automationSourceResults").addClass("d-none").empty();
+    $("#automationSourcePreview").addClass("d-none");
+    $("#automationSourceSelected").text("");
+    $("#automationSourceConfig").text("");
+    setGeneratorFieldsEnabled(true);
+  }
+
+  // A schedule automation has no data generator to name, so the whole section goes away for one,
+  // and a switch between forecasting and reporting drops a selection made for the other type.
+  function applyAutomationTypeToSourcePicker(automationType) {
+    clearSelectedSource();
+    $("#automationGeneratorSection").toggleClass("d-none", !SOURCE_TYPE_PER_AUTOMATION_TYPE[automationType]);
+  }
+
   // Run this automation once, now, on top of its recurring runs.
   function runButton(automation) {
     if (!userCanRunAutomations) {
@@ -563,6 +694,71 @@ 
Recently created jobs

       });
     });
 
+    // Search the data sources whose data generator a new automation can reuse
+    let sourceSearchTimer = null;
+    function searchSources(searchTerm, delayMs) {
+      window.clearTimeout(sourceSearchTimer);
+      sourceSearchTimer = window.setTimeout(function () {
+        $.ajax({
+          url: sourceSearchUrl($("#automationType").val(), searchTerm),
+          method: "GET",
+          success: function (res) {
+            $("#automationSourceResults").html(sourceResultsHtml(res.sources)).removeClass("d-none");
+          },
+          error: function (xhr) {
+            $("#automationSourceResults")
+              .html(`
Could not search data sources: ${esc(automationAjaxErrorMessage(xhr))}.
`)
+              .removeClass("d-none");
+          },
+        });
+      }, delayMs);
+    }
+
+    $("#automationSourceSearch").on("input", function () {
+      if (selectedAutomationSource) {
+        // Typing over the name of the picked source starts a new search, rather than editing a label.
+        clearSelectedSource();
+        $("#automationSourceSearch").val(this.value);
+      }
+      // Typing waits for a pause, so that a word does not send a request per letter.
+      searchSources(this.value.trim(), 250);
+    });
+
+    // Clicking into the empty box lists what there is to pick, so the sources can be browsed without guessing a search term.
+    $("#automationSourceSearch").on("focus", function () {
+      if (!selectedAutomationSource) {
+        searchSources(this.value.trim(), 0);
+      }
+    });
+
+    // Clicking away closes the list, which is otherwise left hanging over the rest of the form.
+    $(document).on("click", function (event) {
+      if (!$(event.target).closest("#automationSourceSearch, #automationSourceResults").length) {
+        $("#automationSourceResults").addClass("d-none");
+      }
+    });
+
+    // Read the picked source's own record, which is what carries the configuration its data generator stores.
+    $(document).on("click", ".automation-source-option", function () {
+      const sourceId = $(this).data("source-id");
+      $.ajax({
+        url: `/api/v3_0/sources/${sourceId}`,
+        method: "GET",
+        success: (source) => showSelectedSource(source),
+        error: function (xhr) {
+          $("#newAutomationErr").removeClass("d-none").text(
+            `Could not read data source ${sourceId}: ${automationAjaxErrorMessage(xhr)}.`
+          );
+        },
+      });
+    });
+
+    $("#automationSourceClear").on("click", () => clearSelectedSource());
+
+    $("#automationType").on("change", function () {
+      applyAutomationTypeToSourcePicker(this.value);
+    });
+
     // Create an automation
     $("#newAutomationForm").submit(function (event) {
       event.preventDefault();
@@ -576,30 +772,36 @@ 
Recently created jobs

           return;
         }
       }
-      let config = {};
-      const configText = $("#automationConfig").val().trim();
-      if (configText) {
-        try {
-          config = JSON.parse(configText);
-        } catch (e) {
-          $("#newAutomationErr").removeClass("d-none").text("The data generator config is not valid JSON.");
-          return;
+      const automationData = {
+        name: $("#automationName").val(),
+        type: $("#automationType").val(),
+        cron: $("#automationCron").val(),
+        timezone: $("#automationTimezone").val(),
+        active: $("#automationActive").is(":checked"),
+        parameters: parameters,
+      };
+      if (selectedAutomationSource) {
+        // The source already stores the data generator and its configuration, which is why the two fields are disabled here.
+        automationData.source = selectedAutomationSource.id;
+      } else {
+        let config = {};
+        const configText = $("#automationConfig").val().trim();
+        if (configText) {
+          try {
+            config = JSON.parse(configText);
+          } catch (e) {
+            $("#newAutomationErr").removeClass("d-none").text("The data generator config is not valid JSON.");
+            return;
+          }
         }
+        automationData["data-generator"] = $("#automationGenerator").val().trim() || null;
+        automationData.config = config;
       }
       $.ajax({
         url: `/api/v3_0/assets/${assetId}/automations`,
         method: "POST",
         contentType: "application/json",
-        data: JSON.stringify({
-          name: $("#automationName").val(),
-          type: $("#automationType").val(),
-          cron: $("#automationCron").val(),
-          timezone: $("#automationTimezone").val(),
-          active: $("#automationActive").is(":checked"),
-          "data-generator": $("#automationGenerator").val().trim() || null,
-          config: config,
-          parameters: parameters,
-        }),
+        data: JSON.stringify(automationData),
         success: () => location.reload(),
         error: function (xhr) {
           $("#newAutomationErr").removeClass("d-none").text(apiError(xhr, "Could not create the automation."));
diff --git a/flexmeasures/ui/tests/js/test_automation_source_picker.py b/flexmeasures/ui/tests/js/test_automation_source_picker.py
new file mode 100644
index 0000000000..fe4578fb8a
--- /dev/null
+++ b/flexmeasures/ui/tests/js/test_automation_source_picker.py
@@ -0,0 +1,97 @@
+"""Browser checks for the data source picker in the automation page's creation modal."""
+
+import json
+import re
+from pathlib import Path
+
+import pytest
+
+TEMPLATE = (
+    Path(__file__).resolve().parents[2] / "templates/assets/asset_automations.html"
+)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def setup_ui_test_data():
+    """The inline-script checks do not need the UI suite's database fixtures."""
+
+
+def picker_script() -> str:
+    """Render the page's inline JavaScript, with the three Jinja values it reads filled in."""
+    match = re.search(r"", TEMPLATE.read_text(), re.DOTALL)
+    assert match is not None
+    return (
+        match.group(1)
+        .replace("{{ asset.id }}", "3")
+        .replace("{{ user_can_manage_automations | tojson }}", "true")
+        .replace("{{ user_can_create_children | tojson }}", "true")
+    )
+
+
+def test_source_search_asks_for_the_sources_of_the_automations_own_type(assert_js):
+    """A forecast automation searches forecasters, a report automation reporters, and a schedule automation has no generator to pick."""
+    assert_js(f"""
+        window.$ = () => ({{ready: () => {{}}}});
+        const page = new Function({json.dumps(picker_script())}
+            + "\\nreturn {{ sourceSearchUrl, sourceLabel, sourceResultsHtml, sourceConfigText }};")();
+
+        const forecastUrl = new URL(page.sourceSearchUrl("forecasting", "pipeline"), "http://x");
+        eq("a forecast automation searches forecasters", forecastUrl.searchParams.get("type"), "forecaster");
+        eq("the search term is passed on", forecastUrl.searchParams.get("filter"), "pipeline");
+        eq("every version can be picked, not just the latest", forecastUrl.searchParams.get("only_latest"), "false");
+
+        const reportUrl = new URL(page.sourceSearchUrl("reporting", ""), "http://x");
+        eq("a report automation searches reporters", reportUrl.searchParams.get("type"), "reporter");
+        eq("an empty search term is left out", reportUrl.searchParams.get("filter"), null);
+
+        const scheduleUrl = new URL(page.sourceSearchUrl("scheduling", "x"), "http://x");
+        eq("a schedule automation narrows to no source type", scheduleUrl.searchParams.get("type"), null);
+    """)
+
+
+def test_search_results_name_each_source_and_cap_a_broad_search(assert_js):
+    """Each result carries the id the picker reads the source by, and a long list says how much it leaves out."""
+    assert_js(f"""
+        window.$ = () => ({{ready: () => {{}}}});
+        const page = new Function({json.dumps(picker_script())}
+            + "\\nreturn {{ sourceSearchUrl, sourceLabel, sourceResultsHtml, sourceConfigText, MAX_SOURCE_RESULTS }};")();
+
+        eq("a source is named by its description and its id",
+           page.sourceLabel({{id: 6, name: "Seita", description: "Seita's TrainPredictPipeline model v1"}}),
+           "Seita's TrainPredictPipeline model v1 (data source 6)");
+        eq("a source without a description falls back to its name",
+           page.sourceLabel({{id: 7, name: "Seita"}}), "Seita (data source 7)");
+
+        const empty = page.sourceResultsHtml([]);
+        check("an empty search says so rather than showing nothing", empty.includes("No matching data sources"), empty);
+
+        const one = page.sourceResultsHtml([{{id: 6, name: "Seita", description: "Seita's forecaster"}}]);
+        check("a result carries the id the picker reads the source by", one.includes('data-source-id="6"'), one);
+        check("a result is a button, so it cannot submit the form", one.includes('type="button"'), one);
+
+        const many = Array.from({{length: page.MAX_SOURCE_RESULTS + 3}}, (_, i) => ({{id: i + 1, name: "Seita"}}));
+        const capped = page.sourceResultsHtml(many);
+        eq("a broad search shows at most the cap",
+           (capped.match(/automation-source-option/g) || []).length, page.MAX_SOURCE_RESULTS);
+        check("and says how many it leaves out", capped.includes("3 more match"), capped);
+
+        const hostile = page.sourceResultsHtml([{{id: 1, name: ""}}]);
+        check("a source name is escaped rather than rendered", !hostile.includes(" ({{ready: () => {{}}}});
+        const page = new Function({json.dumps(picker_script())}
+            + "\\nreturn {{ sourceConfigText }};")();
+
+        eq("the stored config is shown as it is stored",
+           page.sourceConfigText({{attributes: {{data_generator: {{config: {{model: "CustomLGBM"}}}}}}}}),
+           JSON.stringify({{model: "CustomLGBM"}}, null, 4));
+        const none = page.sourceConfigText({{attributes: {{}}}});
+        check("a source storing no config says so", none.includes("no data generator configuration"), none);
+        const bare = page.sourceConfigText({{}});
+        check("and so does a source with no attributes at all", bare.includes("no data generator configuration"), bare);
+    """)

From 3c3f822638783d94bf6a6f24312a198dde7a9bdf Mon Sep 17 00:00:00 2001
From: Mohamed Belhsan Hmida 
Date: Wed, 23 Sep 2026 22:20:43 +0100
Subject: [PATCH 04/23] ui/assets: copy an automation into the creation form

The Automations page could create an automation from scratch or edit one's
recurrence, but a variation on an existing automation, such as the same
forecast on a different schedule, had to be typed out again from its details.

Each row's Actions menu now offers Copy, which opens the creation form with
the original's type, recurrence, timezone and parameters filled in. Nothing
is created until the form is submitted, so the copy is a starting point that
can still be changed, rather than a command of its own.

The copy is filled in inactive, so that a variation being worked on does not
start running while it is still being edited, and so that two automations
writing the same results never appear from a single click. Its name carries a
(copy) suffix, cut to the 80 characters the field and the API allow, so that a
long name loses its tail rather than the suffix that marks it.

A forecast or report copy reuses the data source the original computes under,
which already stores the data generator and its configuration, rather than
naming that class and config again. A schedule automation resolves its source
afresh on every run and cannot name one, so a copy of one names none.

The parameters are carried over exactly as stored, which is safe here because
the copy stays on the same asset, so the sensors they name stay valid and the
creation endpoint checks them against the user's permissions as it would for
any new automation.

The creation form is now shared between New automation and Copy, so opening it
blank clears what a copy left in it, rather than inheriting the last copy's
settings.

Signed-off-by: Mohamed Belhsan Hmida 
---
 documentation/changelog.rst                   |   1 +
 documentation/features/automations.rst        |   4 +
 .../templates/assets/asset_automations.html   |  97 +++++++-
 .../ui/tests/js/test_automation_actions.py    |   6 +-
 .../ui/tests/js/test_automation_copy.py       | 207 ++++++++++++++++++
 5 files changed, 311 insertions(+), 4 deletions(-)
 create mode 100644 flexmeasures/ui/tests/js/test_automation_copy.py

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index 140b0c5be3..3586d6d0af 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -76,6 +76,7 @@ Automations arrived over several pull requests. This is what each of them contri
 * An automation's output sensors are checked against its creator's permissions when the automation is created, and the schedules it computes are held to exactly those sensors: a scheduler that returns results for any other sensor is refused, rather than recording on a sensor that was only ever checked for read access [see `PR #2536 `_]
 * Reports as well as forecasts and schedules: a report automation stores report parameters, with its reporter and the reporter's configuration on a data source, and reports on a period resolved afresh on each run, either from ``start-offset`` and ``end-offset`` applied to the run time in the automation's timezone, or since the last successful report ended, while a fixed ``start`` or ``end`` is refused; a report job records only on the sensors the automation was checked against [see `PR #2297 `_]
 * Every automation times its runs the same way: a fixed ``start``, ``end`` or ``prior`` in its parameters is refused, as every run would share that moment, and two of ``start-offset``, ``end-offset`` and ``duration`` describe the period each run covers instead, with the offsets applied to the time the run was due on the automation's own clock, so that, for instance, a schedule automation can plan the whole of the next day [see `PR #2551 `_]
+* An automation can be copied from the asset's *Automations* page: *Copy* opens the creation form with the original's type, recurrence, timezone and parameters filled in, and the data source it computes under already selected, so that a variation on an existing automation does not have to be typed out again. The copy is created inactive, so that it does not start running while it is still being worked on, and it is named after the original with a ``(copy)`` suffix [see `PR #XXXX `_]
 * Look up automations from the command line with ``flexmeasures show automations``, which lists them all (inactive ones included) with the IDs that the edit, delete and run commands expect, and, with ``--id``, shows a single automation's recurrence, cursor, parameters and the sensors it reads from and writes to [see `PR #2533 `_]
 * A forecast or report automation can now reuse an existing data source, in the API and the UI as well as in the CLI, so that several automations compute under one data generator, one configuration and one lineage of data. The *Automations* page searches the data sources you may read, by id, by name and by data generator class, and shows the configuration the picked one stores before the automation is created [see `PR #XXXX `_]
 
diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst
index 9f82ca4270..58e0d09366 100644
--- a/documentation/features/automations.rst
+++ b/documentation/features/automations.rst
@@ -223,6 +223,10 @@ The page shows the next scheduled run for each automation (excluding any pending
 An automation's details show the sensors it reads from and writes to, linking to each sensor's page.
 Conversely, a sensor's page lists the automations that write data to it.
 
+Each row's *Actions* menu also offers *Copy*, which opens the creation form with that automation's type, recurrence, timezone and parameters filled in, and its data source already selected.
+It is a starting point for a variation on an existing automation, such as the same forecast on a different recurrence, rather than a command of its own: nothing is created until you submit the form, and you can change anything in it first.
+The copy is created inactive, so that it does not start running while you are still working on it, and it is named after the original with a ``(copy)`` suffix.
+
 .. _automation_cursor:
 
 Appendix: how the runner decides what is due
diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html
index 2c79b8b318..ed8a14f914 100644
--- a/flexmeasures/ui/templates/assets/asset_automations.html
+++ b/flexmeasures/ui/templates/assets/asset_automations.html
@@ -105,7 +105,7 @@ 

 
         {% if user_can_manage_automations %}
         

-          
+          
         

 
         
@@ -286,6 +286,8 @@ 
Edit automation

   // Running an automation writes data under the asset, so the same permission gates the button and the endpoint.
   const userCanRunAutomations = {{ user_can_create_children | tojson }};
   const automationsById = new Map();
+  // What the details endpoint said about each automation, kept so that copying one does not have to ask again.
+  const automationDetailsById = new Map();
 
   // Escape user-controlled strings before interpolating them into HTML
   function esc(value) {
@@ -414,6 +416,56 @@ 
Edit automation

     $("#automationGeneratorSection").toggleClass("d-none", !SOURCE_TYPE_PER_AUTOMATION_TYPE[automationType]);
   }
 
+  /* Empty the creation form, so that the next automation starts from the defaults the form was rendered with.
+     The form is shared with the Copy action, which leaves the previous automation's settings in it,
+     so opening it blank has to clear them rather than trust that they were never there. */
+  function resetNewAutomationForm() {
+    document.getElementById("newAutomationForm").reset();
+    // Resetting restores the type the form was rendered with, and the picker follows whichever type that is.
+    applyAutomationTypeToSourcePicker($("#automationType").val());
+    $("#newAutomationErr").addClass("d-none").text("");
+  }
+
+  function prefillNewAutomationForm(values) {
+    resetNewAutomationForm();
+    $("#automationName").val(values.name);
+    $("#automationType").val(values.type);
+    // The type decides whether a data generator can be named at all, so the picker follows it before anything is filled in.
+    applyAutomationTypeToSourcePicker(values.type);
+    $("#automationCron").val(values.cron);
+    $("#automationTimezone").val(values.timezone);
+    $("#automationParameters").val(values.parameters);
+    $("#automationActive").prop("checked", values.active);
+  }
+
+  // The name column allows 80 characters, and so does the API, so a long name loses its tail rather than the suffix that marks the copy.
+  const MAX_AUTOMATION_NAME_LENGTH = 80;
+  const AUTOMATION_COPY_SUFFIX = " (copy)";
+
+  function copiedAutomationName(name) {
+    const room = MAX_AUTOMATION_NAME_LENGTH - AUTOMATION_COPY_SUFFIX.length;
+    return `${String(name ?? "").slice(0, room)}${AUTOMATION_COPY_SUFFIX}`;
+  }
+
+  /* What the creation form should hold to recreate an automation on this same asset.
+     The copy starts inactive, so that a variation being worked on does not start running while it is still being edited,
+     and so that two automations writing the same results never appear by a single click.
+     The sensors the parameters name stay valid, as the copy stays on this asset, so they are carried over as they are stored.
+     A forecast or report automation reuses the original's data source, which already stores the data generator and its configuration;
+     a schedule automation resolves its own source on every run, so it names none. */
+  function automationCopyValues(automation, details) {
+    const sourceId = details?.source?.id;
+    return {
+      name: copiedAutomationName(automation.name),
+      type: automation.type,
+      cron: automation["cron"],
+      timezone: automation.timezone,
+      parameters: JSON.stringify(details?.parameters ?? {}, null, 4),
+      active: false,
+      sourceId: SOURCE_TYPE_PER_AUTOMATION_TYPE[automation.type] && sourceId ? sourceId : null,
+    };
+  }
+
   // Run this automation once, now, on top of its recurring runs.
   function runButton(automation) {
     if (!userCanRunAutomations) {
@@ -430,6 +482,8 @@ 
Edit automation

     ` : "";
     const manageItems = userCanManage ? `
       
    • 
+      
    • 
       
    • 
@@ -519,6 +573,9 @@ 
    Automation '${esc(automation.name)}' (ID: ${automation.i
       url: `/api/v3_0/assets/${assetId}/automations/${automationId}`,
       method: "GET",
       success: function (res) {
+        // Copying an automation reads its parameters and its data source from here, rather than asking for them again.
+        automationDetailsById.set(automationId, res);
+
         // Show job stats in the table row
         const jobStats = res["job-stats"] || {};
         const statsStr = Object.keys(jobStats).length
@@ -647,6 +704,44 @@ 
    Recently created jobs
    
       bootstrap.Modal.getOrCreateInstance(document.getElementById("editAutomationModal")).show();
     });
 
+    // Open the creation form filled in from an existing automation, to create a variation on it.
+    $(document).on("click", ".automation-copy", function () {
+      const automationId = Number($(this).data("id"));
+      const automation = automationsById.get(automationId);
+      const details = automationDetailsById.get(automationId);
+      if (!automation) {
+        showToast("Could not find the automation to copy. Refresh the page to try again.", "error");
+        return;
+      }
+      if (!details) {
+        // The parameters and the data source come from the details request, which every row starts on page load.
+        showToast("This automation's details are still loading. Try again in a moment.", "error");
+        return;
+      }
+      const values = automationCopyValues(automation, details);
+      prefillNewAutomationForm(values);
+      bootstrap.Modal.getOrCreateInstance(document.getElementById("newAutomationModal")).show();
+      if (values.sourceId) {
+        // The form is already open while this loads, so a slow read of the source does not hold up the rest of the copy.
+        $.ajax({
+          url: `/api/v3_0/sources/${values.sourceId}`,
+          method: "GET",
+          success: (source) => showSelectedSource(source),
+          error: function (xhr) {
+            // Without the source, the copy would fall back to a default data generator rather than the original's, so say so plainly.
+            $("#newAutomationErr").removeClass("d-none").text(
+              `Could not read data source ${values.sourceId}, which this automation computes under:`
+              + ` ${automationAjaxErrorMessage(xhr)}.`
+              + " Name a data generator and its configuration below, or the copy will use the default one."
+            );
+          },
+        });
+      }
+    });
+
+    // Opening the form from the New automation button starts from the defaults, not from whatever a copy left in it.
+    $("#newAutomationButton").on("click", () => resetNewAutomationForm());
+
     $("#editAutomationForm").submit(function (event) {
       event.preventDefault();
       const automationId = $("#editAutomationId").val();
diff --git a/flexmeasures/ui/tests/js/test_automation_actions.py b/flexmeasures/ui/tests/js/test_automation_actions.py
index 3ff36634cd..0195becaf3 100644
--- a/flexmeasures/ui/tests/js/test_automation_actions.py
+++ b/flexmeasures/ui/tests/js/test_automation_actions.py
@@ -50,7 +50,7 @@ def test_automation_actions_are_grouped_and_permission_gated(assert_js):
             recurrence_description: "At 06:00", next_run: "2026-09-15T04:00:00+00:00",
         }};
         const row = manager.AutomationRow(automation);
-        check("one menu holds all four controls", ["run-automation", "automation-edit", "automation-toggle", "automation-delete"]
+        check("one menu holds all five controls", ["run-automation", "automation-edit", "automation-copy", "automation-toggle", "automation-delete"]
               .every(name => row.actions.includes(name)), row.actions);
         check("run now is no longer mixed with details", !row.details.includes("run-automation"), row.details);
         check("details stays its own button", row.details.includes("automation-details"), row.details);
@@ -60,8 +60,8 @@ def test_automation_actions_are_grouped_and_permission_gated(assert_js):
         const toggle = holder.querySelector(".automation-actions .dropdown-toggle");
         check("the row offers a single Actions toggle", toggle !== null
               && holder.querySelectorAll(".automation-actions > .btn").length === 1, row.actions);
-        check("the four actions sit in its menu as items",
-              holder.querySelectorAll(".automation-actions .dropdown-menu .dropdown-item").length === 4,
+        check("the five actions sit in its menu as items",
+              holder.querySelectorAll(".automation-actions .dropdown-menu .dropdown-item").length === 5,
               row.actions);
         const inactive = manager.AutomationRow({{...automation, active: false, next_run: null}});
         check("inactive action says Activate", inactive.actions.includes("Activate") && !inactive.actions.includes("Deactivate"), inactive.actions);
diff --git a/flexmeasures/ui/tests/js/test_automation_copy.py b/flexmeasures/ui/tests/js/test_automation_copy.py
new file mode 100644
index 0000000000..913669e124
--- /dev/null
+++ b/flexmeasures/ui/tests/js/test_automation_copy.py
@@ -0,0 +1,207 @@
+"""Browser checks for copying an automation into the automation page's creation form."""
+
+import json
+import re
+from pathlib import Path
+
+import pytest
+
+TEMPLATE = (
+    Path(__file__).resolve().parents[2] / "templates/assets/asset_automations.html"
+)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def setup_ui_test_data():
+    """The inline-script checks do not need the UI suite's database fixtures."""
+
+
+def copy_script(can_manage: bool = True) -> str:
+    """Render the page's inline JavaScript, with the three Jinja values it reads filled in."""
+    match = re.search(r"", TEMPLATE.read_text(), re.DOTALL)
+    assert match is not None
+    return (
+        match.group(1)
+        .replace("{{ asset.id }}", "3")
+        .replace("{{ user_can_manage_automations | tojson }}", str(can_manage).lower())
+        .replace("{{ user_can_create_children | tojson }}", "true")
+    )
+
+
+def creation_form_html() -> str:
+    """The creation form as the page renders it, so the checks run against the real fields."""
+    match = re.search(
+        r'(
    .*?
    )', TEMPLATE.read_text(), re.DOTALL
+    )
+    assert match is not None
+    return match.group(1).replace("{{ asset.timezone }}", "Europe/Amsterdam")
+
+
+# A stand-in for the handful of jQuery calls the form helpers make, backed by the real DOM.
+JQUERY_STUB = """
+    window.$ = function (selector) {
+        const nodes = typeof selector === "string" ? [...document.querySelectorAll(selector)] : [];
+        const self = {
+            ready: () => {},
+            on: () => self,
+            val: function (value) {
+                if (value === undefined) return nodes.length ? nodes[0].value : undefined;
+                nodes.forEach(node => { node.value = value; });
+                return self;
+            },
+            prop: function (name, value) {
+                nodes.forEach(node => { node[name] = value; });
+                return self;
+            },
+            is: (what) => what === ":checked" && nodes.length > 0 && nodes[0].checked,
+            text: function (value) {
+                nodes.forEach(node => { node.textContent = value; });
+                return self;
+            },
+            empty: function () {
+                nodes.forEach(node => { node.innerHTML = ""; });
+                return self;
+            },
+            addClass: function (name) {
+                nodes.forEach(node => node.classList.add(name));
+                return self;
+            },
+            removeClass: function (name) {
+                nodes.forEach(node => node.classList.remove(name));
+                return self;
+            },
+            toggleClass: function (name, on) {
+                nodes.forEach(node => node.classList.toggle(name, on));
+                return self;
+            },
+        };
+        return self;
+    };
+"""
+
+
+def test_a_copy_carries_the_settings_over_and_starts_inactive(assert_js):
+    """A copy recreates the original on the same asset, except that it does not start running."""
+    assert_js(f"""
+        window.$ = () => ({{ready: () => {{}}}});
+        const page = new Function({json.dumps(copy_script())}
+            + "\\nreturn {{ automationCopyValues, copiedAutomationName }};")();
+
+        const automation = {{
+            id: 7, name: "Campus forecast", type: "forecasting", active: true,
+            "cron": "0 6 * * *", timezone: "Europe/Amsterdam",
+        }};
+        const details = {{parameters: {{sensor: 2092, "start-offset": "1D,DB"}}, source: {{id: 6, description: "Seita's forecaster"}}}};
+        const values = page.automationCopyValues(automation, details);
+
+        eq("the copy is marked as one in its name", values.name, "Campus forecast (copy)");
+        eq("the recurrence is carried over", values.cron, "0 6 * * *");
+        eq("and so is the timezone it is read in", values.timezone, "Europe/Amsterdam");
+        eq("the type is carried over", values.type, "forecasting");
+        eq("the parameters are carried over as stored", JSON.parse(values.parameters), details.parameters);
+        eq("a copy of an active automation still starts inactive", values.active, false);
+        eq("a forecast copy reuses the original's data source", values.sourceId, 6);
+    """)
+
+
+def test_a_copy_only_reuses_a_data_source_where_one_can_be_named(assert_js):
+    """A schedule automation resolves its source on every run, so a copy of one must not name the original's."""
+    assert_js(f"""
+        window.$ = () => ({{ready: () => {{}}}});
+        const page = new Function({json.dumps(copy_script())}
+            + "\\nreturn {{ automationCopyValues }};")();
+
+        const source = {{id: 6, description: "Seita's generator"}};
+        const scheduleCopy = page.automationCopyValues(
+            {{id: 8, name: "Battery schedule", type: "scheduling", "cron": "0 * * * *", timezone: "UTC"}},
+            {{parameters: {{}}, source: source}});
+        eq("a schedule copy names no data source", scheduleCopy.sourceId, null);
+
+        const reportCopy = page.automationCopyValues(
+            {{id: 9, name: "Weekly report", type: "reporting", "cron": "0 0 * * 1", timezone: "UTC"}},
+            {{parameters: {{}}, source: source}});
+        eq("a report copy reuses the original's data source", reportCopy.sourceId, 6);
+
+        const sourceless = page.automationCopyValues(
+            {{id: 10, name: "Orphan", type: "forecasting", "cron": "0 0 * * *", timezone: "UTC"}},
+            {{parameters: {{}}}});
+        eq("an automation whose details carry no source names none", sourceless.sourceId, null);
+        eq("and its parameters still default to an empty object", JSON.parse(sourceless.parameters), {{}});
+    """)
+
+
+def test_a_long_name_keeps_the_suffix_that_marks_the_copy(assert_js):
+    """The name field and the API both stop at 80 characters, so the tail goes rather than the suffix."""
+    assert_js(f"""
+        window.$ = () => ({{ready: () => {{}}}});
+        const page = new Function({json.dumps(copy_script())}
+            + "\\nreturn {{ copiedAutomationName }};")();
+
+        const long = page.copiedAutomationName("x".repeat(120));
+        eq("a long name is cut to what the field holds", long.length, 80);
+        check("and still says it is a copy", long.endsWith(" (copy)"), long);
+        eq("a short name is left alone apart from the suffix",
+           page.copiedAutomationName("Campus forecast"), "Campus forecast (copy)");
+    """)
+
+
+def test_opening_a_blank_form_clears_what_a_copy_left_in_it(assert_js):
+    """The creation form is shared with the Copy action, so New automation has to clear it rather than inherit a copy."""
+    assert_js(f"""
+        // Appended rather than assigned to the body, which holds the harness's own results element.
+        const holder = document.createElement("div");
+        holder.innerHTML = {json.dumps(creation_form_html())};
+        document.body.appendChild(holder);
+        {JQUERY_STUB}
+        const page = new Function({json.dumps(copy_script())}
+            + "\\nreturn {{ prefillNewAutomationForm, resetNewAutomationForm, automationCopyValues }};")();
+
+        const values = page.automationCopyValues(
+            {{id: 7, name: "Campus forecast", type: "reporting", "cron": "30 7 * * *", timezone: "Africa/Tunis"}},
+            {{parameters: {{sensor: 2092}}, source: {{id: 6}}}});
+        page.prefillNewAutomationForm(values);
+
+        eq("the copy fills the name in", document.getElementById("automationName").value, "Campus forecast (copy)");
+        eq("and the recurrence", document.getElementById("automationCron").value, "30 7 * * *");
+        eq("and the timezone", document.getElementById("automationTimezone").value, "Africa/Tunis");
+        eq("and the type", document.getElementById("automationType").value, "reporting");
+        eq("and the parameters", JSON.parse(document.getElementById("automationParameters").value), {{sensor: 2092}});
+        eq("and leaves the copy inactive", document.getElementById("automationActive").checked, false);
+
+        page.resetNewAutomationForm();
+
+        eq("opening a blank form clears the copied name", document.getElementById("automationName").value, "");
+        eq("and the copied recurrence", document.getElementById("automationCron").value, "");
+        eq("and the copied parameters", document.getElementById("automationParameters").value, "");
+        eq("and restores the asset's own timezone",
+           document.getElementById("automationTimezone").value, "Europe/Amsterdam");
+        eq("and the default type", document.getElementById("automationType").value, "forecasting");
+        eq("and a new automation is active again, as the form is rendered",
+           document.getElementById("automationActive").checked, true);
+        check("and the data generator fields are usable again, whatever the copy did to them",
+              !document.getElementById("automationGenerator").disabled
+              && !document.getElementById("automationConfig").disabled, "still disabled");
+    """)
+
+
+def test_copy_is_offered_to_managers_only(assert_js):
+    """Copying creates an automation, so it sits behind the same permission as creating one."""
+    manager_script = copy_script(can_manage=True)
+    viewer_script = copy_script(can_manage=False)
+    assert_js(f"""
+        window.$ = () => ({{ready: () => {{}}}});
+        const manager = new Function({json.dumps(manager_script)} + "\\nreturn {{ AutomationRow }};")();
+        const viewer = new Function({json.dumps(viewer_script)} + "\\nreturn {{ AutomationRow }};")();
+        const automation = {{
+            id: 7, name: "Campus forecast", type: "forecasting", active: true,
+            "created-at": null, "cron": "0 6 * * *", timezone: "Europe/Amsterdam",
+            "recurrence-description": "At 06:00", "next-run": null,
+        }};
+        const managerRow = manager.AutomationRow(automation);
+        check("a manager is offered Copy", managerRow.actions.includes("automation-copy"), managerRow.actions);
+        check("and it carries the id to copy", managerRow.actions.includes('class="dropdown-item automation-copy" data-id="7"'),
+              managerRow.actions);
+        const viewerRow = viewer.AutomationRow(automation);
+        check("someone who may not manage automations is not offered Copy",
+              !viewerRow.actions.includes("automation-copy"), viewerRow.actions);
+    """)

From e4e36bad53c3291e4bdcd8e20dbb09076b0bd76b Mon Sep 17 00:00:00 2001
From: Mohamed Belhsan Hmida 
Date: Thu, 24 Sep 2026 10:30:43 +0100
Subject: [PATCH 05/23] docs/changelog: point the data source picker entries at
 PR #2584

Signed-off-by: Mohamed Belhsan Hmida 
---
 documentation/changelog.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index 42a4b49b3c..ebfa80e2b5 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -36,7 +36,7 @@ New features
 * Changing the selected time range on an asset or sensor chart now only loads the data that is actually new, instead of reloading the whole range, which makes stepping through or extending a long period much faster; reloading the page, or leaving it open for five minutes, still fetches everything afresh [see `PR #2433 `_]
 * The statistics table on a sensor page now shows all data sources together by default, as the graph does [see `PR #2462 `_]
 * The asset lists on an organisation's page and on the asset overview now show only top-level assets per default, so the sites you are looking for are no longer buried among their sub-assets, and both offer the same two checkboxes to widen the list again: untick *Top-level only* for the whole tree, or tick *Include public assets*, which is off per default [see `PR #2523 `_]
-* A new automation can now be pointed at a data source that already holds a forecaster or a reporter, instead of naming that class and its configuration again, in the UI and the API as well as in the CLI. See `Automations, in detail`_ [see `PR #XXXX `_]
+* A new automation can now be pointed at a data source that already holds a forecaster or a reporter, instead of naming that class and its configuration again, in the UI and the API as well as in the CLI. See `Automations, in detail`_ [see `PR #2584 `_]
 
 Infrastructure / Support
 -------------------------
@@ -91,7 +91,7 @@ Automations arrived over several pull requests. This is what each of them contri
 * Every automation times its runs the same way: a fixed ``start``, ``end`` or ``prior`` in its parameters is refused, as every run would share that moment, and two of ``start-offset``, ``end-offset`` and ``duration`` describe the period each run covers instead, with the offsets applied to the time the run was due on the automation's own clock, so that, for instance, a schedule automation can plan the whole of the next day [see `PR #2551 `_]
 * Look up automations from the command line with ``flexmeasures show automations``, which lists them all (inactive ones included) with the IDs that the edit, delete and run commands expect, and, with ``--id``, shows a single automation's recurrence, cursor, parameters and the sensors it reads from and writes to [see `PR #2533 `_]
 * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_]
-* A forecast or report automation can now reuse an existing data source, in the API and the UI as well as in the CLI, so that several automations compute under one data generator, one configuration and one lineage of data. The *Automations* page searches the data sources you may read, by id, by name and by data generator class, and shows the configuration the picked one stores before the automation is created [see `PR #XXXX `_]
+* A forecast or report automation can now reuse an existing data source, in the API and the UI as well as in the CLI, so that several automations compute under one data generator, one configuration and one lineage of data. The *Automations* page searches the data sources you may read, by id, by name and by data generator class, and shows the configuration the picked one stores before the automation is created [see `PR #2584 `_]
 
 
 v1.0.1 | September 9, 2026

From 81db15c996862a14e4a85a2fcea8fe63535d5069 Mon Sep 17 00:00:00 2001
From: Mohamed Belhsan Hmida 
Date: Fri, 25 Sep 2026 02:18:00 +0100
Subject: [PATCH 06/23] docs/changelog: fold the copy button into the
 automations UI entry

Automations have not shipped, so the changelog says what the feature is
rather than how it got there: the entry that already covers creating and
managing automations in the UI names Copy among the row's controls and
carries this PR's number, instead of a bullet of its own.

Signed-off-by: Mohamed Belhsan Hmida 
---
 documentation/changelog.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index 19bf2efef4..ad2fd2efd9 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -85,11 +85,10 @@ Automations arrived over several pull requests. This is what each of them contri
 * Schedules as well as forecasts: a schedule automation stores what the schedule trigger endpoint accepts, and schedules from each run's own time [see `PR #2293 `_]
 * A scheduler's data source now also records the flex config the scheduler computed under, so a schedule can be traced back to the configuration that produced it, and a schedule automation points at such a data source, the way a forecast automation points at its forecaster's [see `PR #2444 `_]
 * A single automation can now be run on demand, from the CLI (``flexmeasures jobs run-automation``), the API (``POST /assets//automations//trigger``) and the asset's *Automations* page (a *Run now* button), which is useful to try out a new automation, to re-run one after fixing what made it fail, or to refresh its results after late input data arrived [see `PR #2460 `_]
-* Automations can be created, edited and deleted in the UI and through new API endpoints (``[POST|PATCH|DELETE] /assets/(id)/automations``), by whoever may add data under the asset, with their recurrence expressed in a selectable IANA timezone, and only involving sensors they can access themselves (read access to the sensors an automation reads, and permission to record data on the sensors it writes to). The *Automations* page shows when each active automation is due to run next, as a clock time in the automation's own timezone, so a recurrence no longer has to be read back from its cron string; the same time is available as ``next_run`` on the automations API endpoints, and is null while an automation is inactive. It is the next scheduled clock time, so it excludes catch-up work still pending, and it follows the dispatcher's daylight-saving rules, taking the first fold of a repeated local time and the first valid minute after a skipped one. Each row's *Run now*, *Edit*, *Activate*/*Deactivate* and *Delete* controls are collected into a single *Actions* menu, leaving *Details* beside it, so a long listing carries two controls per row instead of five [see `PR #2294 `_]
+* Automations can be created, edited and deleted in the UI and through new API endpoints (``[POST|PATCH|DELETE] /assets/(id)/automations``), by whoever may add data under the asset, with their recurrence expressed in a selectable IANA timezone, and only involving sensors they can access themselves (read access to the sensors an automation reads, and permission to record data on the sensors it writes to). The *Automations* page shows when each active automation is due to run next, as a clock time in the automation's own timezone, so a recurrence no longer has to be read back from its cron string; the same time is available as ``next_run`` on the automations API endpoints, and is null while an automation is inactive. It is the next scheduled clock time, so it excludes catch-up work still pending, and it follows the dispatcher's daylight-saving rules, taking the first fold of a repeated local time and the first valid minute after a skipped one. Each row's *Run now*, *Edit*, *Copy*, *Activate*/*Deactivate* and *Delete* controls are collected into a single *Actions* menu, leaving *Details* beside it, so a long listing carries two controls per row instead of five. *Copy* opens the creation form filled in from that automation, and leaves the copy inactive [see `PR #2294 `_ and `PR #2591 `_]
 * An automation's output sensors are checked against its creator's permissions when the automation is created, and the schedules it computes are held to exactly those sensors: a scheduler that returns results for any other sensor is refused, rather than recording on a sensor that was only ever checked for read access [see `PR #2536 `_]
 * Reports as well as forecasts and schedules: a report automation stores report parameters, with its reporter and the reporter's configuration on a data source, and reports on a period resolved afresh on each run, either from ``start-offset`` and ``end-offset`` applied to the run time in the automation's timezone, or since the last successful report ended, while a fixed ``start`` or ``end`` is refused; a report job records only on the sensors the automation was checked against [see `PR #2297 `_]
 * Every automation times its runs the same way: a fixed ``start``, ``end`` or ``prior`` in its parameters is refused, as every run would share that moment, and two of ``start-offset``, ``end-offset`` and ``duration`` describe the period each run covers instead, with the offsets applied to the time the run was due on the automation's own clock, so that, for instance, a schedule automation can plan the whole of the next day [see `PR #2551 `_]
-* An automation can be copied from the asset's *Automations* page: *Copy* opens the creation form with the original's type, recurrence, timezone and parameters filled in, and the data source it computes under already selected, so that a variation on an existing automation does not have to be typed out again. The copy is created inactive, so that it does not start running while it is still being worked on, and it is named after the original with a ``(copy)`` suffix [see `PR #XXXX `_]
 * Look up automations from the command line with ``flexmeasures show automations``, which lists them all (inactive ones included) with the IDs that the edit, delete and run commands expect, and, with ``--id``, shows a single automation's recurrence, cursor, parameters and the sensors it reads from and writes to [see `PR #2533 `_]
 * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_]
 * A forecast or report automation can now reuse an existing data source, in the API and the UI as well as in the CLI, so that several automations compute under one data generator, one configuration and one lineage of data. The *Automations* page searches the data sources you may read, by id, by name and by data generator class, and shows the configuration the picked one stores before the automation is created [see `PR #2584 `_]

From 93f3c1eb0f481fd5627fdc931081f54cac0e82d0 Mon Sep 17 00:00:00 2001
From: Mohamed Belhsan Hmida 
Date: Fri, 25 Sep 2026 09:00:36 +0100
Subject: [PATCH 07/23] docs/changelog: carry this work on the automations
 entry that already covers it

The automations feature has not shipped yet, so the changelog describes where it
ends up rather than the steps it took to get there. Someone reading the v1.1.0
notes never saw a version of the *New automation* form without a data source to
pick, and a bullet announcing that one arrived tells them about a past they have
no memory of.

The per-PR list under *Automations, in detail* is the place where a contribution
is still worth naming, so this PR's number joins the entry that already describes
the creation form and the data source it records under.

Signed-off-by: Mohamed Belhsan Hmida 
---
 documentation/changelog.rst | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index 774aa74000..89884c3ad0 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -36,7 +36,6 @@ New features
 * Changing the selected time range on an asset or sensor chart now only loads the data that is actually new, instead of reloading the whole range, which makes stepping through or extending a long period much faster; reloading the page, or leaving it open for five minutes, still fetches everything afresh [see `PR #2433 `_]
 * The statistics table on a sensor page now shows all data sources together by default, as the graph does [see `PR #2462 `_]
 * The asset lists on an organisation's page and on the asset overview now show only top-level assets per default, so the sites you are looking for are no longer buried among their sub-assets, and both offer the same two checkboxes to widen the list again: untick *Top-level only* for the whole tree, or tick *Include public assets*, which is off per default [see `PR #2523 `_]
-* A new automation can now be pointed at a data source that already holds a forecaster or a reporter, instead of naming that class and its configuration again, in the UI and the API as well as in the CLI. See `Automations, in detail`_ [see `PR #2584 `_]
 
 Infrastructure / Support
 -------------------------
@@ -91,8 +90,7 @@ Automations arrived over several pull requests. This is what each of them contri
 * Reports as well as forecasts and schedules: a report automation stores report parameters, with its reporter and the reporter's configuration on a data source, and reports on a period resolved afresh on each run, either from ``start-offset`` and ``end-offset`` applied to the run time in the automation's timezone, or since the last successful report ended, while a fixed ``start`` or ``end`` is refused; a report job records only on the sensors the automation was checked against [see `PR #2297 `_]
 * Every automation times its runs the same way: a fixed ``start``, ``end`` or ``prior`` in its parameters is refused, as every run would share that moment, and two of ``start-offset``, ``end-offset`` and ``duration`` describe the period each run covers instead, with the offsets applied to the time the run was due on the automation's own clock, so that, for instance, a schedule automation can plan the whole of the next day [see `PR #2551 `_]
 * Look up automations from the command line with ``flexmeasures show automations``, which lists them all (inactive ones included) with the IDs that the edit, delete and run commands expect, and, with ``--id``, shows a single automation's recurrence, cursor, parameters and the sensors it reads from and writes to [see `PR #2533 `_]
-* The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_]
-* A forecast or report automation can now reuse an existing data source, in the API and the UI as well as in the CLI, so that several automations compute under one data generator, one configuration and one lineage of data. The *Automations* page searches the data sources you may read, by id, by name and by data generator class, and shows the configuration the picked one stores before the automation is created [see `PR #2584 `_]
+* The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2584 `_]
 
 
 v1.0.1 | September 9, 2026

From 6d14ae96faf989f0e7c5aebe4a10a84fb5959fc7 Mon Sep 17 00:00:00 2001
From: Mohamed Belhsan Hmida 
Date: Fri, 25 Sep 2026 09:00:49 +0100
Subject: [PATCH 08/23] docs/api: leave the automations endpoint to the release
 that introduces it

``POST /assets//automations`` arrived at v3.0-37, inside this same unreleased
cycle, so no API consumer has ever called it without a ``source``. The release
will simply introduce the endpoint, with that field on it, and an entry saying it
"now accepts" one describes a revision nobody integrated against.

The ``GET /sources`` entry stays. That listing predates this cycle and is already
in users' hands, so a new way to search and narrow it is a change they need told
about.

Signed-off-by: Mohamed Belhsan Hmida 
---
 documentation/api/change_log.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index 2c25d67258..c6a5042890 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -7,7 +7,6 @@ API change log
 
 v3.0-39 | September 23, 2026
 """"""""""""""""""""""""""""
-- ``POST /api/v3_0/assets//automations`` now accepts a ``source``: the id of an existing data source whose data generator and configuration the automation reuses, and under which it records what it computes. It cannot be combined with ``data-generator`` or ``config``, which a data source already determines, and a schedule automation cannot name one, since it resolves its data source from the asset and the flex config on every run (a ``422`` in both cases). A data source that the caller cannot read is refused with a ``403``.
 - ``GET /api/v3_0/sources`` now accepts a ``filter`` of space-separated search terms, matched against a source's name, its model and its id prefix, and a ``type``, which narrows the listing to one source type, such as ``forecaster``.
 
 v3.0-38 | September 16, 2026

From 54902ef3a8c2ff08caf1770d6f132b8ca09f3674 Mon Sep 17 00:00:00 2001
From: Mohamed Belhsan Hmida 
Date: Fri, 25 Sep 2026 09:01:11 +0100
Subject: [PATCH 09/23] data/schemas/automations: say what the source field is,
 not what it implies

The field's description had grown into a paragraph, restating what a data source
stores, how the results are attributed to it, and why a schedule automation may
not name one. All three are already in the automations documentation, where they
have room to be explained, and none of them help someone reading the OpenAPI
reference to decide what to put in the field.

One sentence is left: it is the ID of a data source to reuse, in place of naming
a generator and a config.

Signed-off-by: Mohamed Belhsan Hmida 
---
 flexmeasures/data/schemas/automations.py  | 5 +----
 flexmeasures/ui/static/openapi-specs.json | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/flexmeasures/data/schemas/automations.py b/flexmeasures/data/schemas/automations.py
index 20259e9686..f29a921a36 100644
--- a/flexmeasures/data/schemas/automations.py
+++ b/flexmeasures/data/schemas/automations.py
@@ -121,10 +121,7 @@ class AutomationCreationSchema(Schema):
         load_default=None,
         allow_none=True,
         metadata={
-            "description": "Id of an existing data source to reuse, instead of naming a `data-generator` and its `config`."
-            " The data generator class and the configuration it runs under are read from that source,"
-            " and the automation records its results under it, so several automations can share one generator and one lineage of data."
-            " A schedule automation cannot name one, as it resolves its data source from the asset and the flex config on every run.",
+            "description": "ID of an existing data source to reuse, instead of naming a `data-generator` and its `config`.",
             "example": 6,
         },
     )
diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json
index 4c2945f109..c6c6cea2ad 100644
--- a/flexmeasures/ui/static/openapi-specs.json
+++ b/flexmeasures/ui/static/openapi-specs.json
@@ -6658,7 +6658,7 @@
               "null"
             ],
             "default": null,
-            "description": "Id of an existing data source to reuse, instead of naming a `data-generator` and its `config`. The data generator class and the configuration it runs under are read from that source, and the automation records its results under it, so several automations can share one generator and one lineage of data. A schedule automation cannot name one, as it resolves its data source from the asset and the flex config on every run.",
+            "description": "ID of an existing data source to reuse, instead of naming a `data-generator` and its `config`.",
             "example": 6
           }
         },

From b9ac7dd57783ebe9f28134bead9ddf061658ba25 Mon Sep 17 00:00:00 2001
From: Mohamed Belhsan Hmida 
Date: Fri, 25 Sep 2026 09:03:34 +0100
Subject: [PATCH 10/23] data/schemas: let the source field be plainly optional,
 and leave DataSourceIdField alone

Giving the field a ``load_default`` of None put a default into the OpenAPI spec,
which the generator then pushed back through the field's own serializer. A field
that turns an ID into a DataSource has nothing to do with None, so it raised, and
the way out taken at the time was to teach it to swallow None -- changing the
field for every schema that uses it, to accommodate one field's setup.

The field is simply not required, so say that instead. Nothing then asks for a
default, the serializer goes back to doing one thing, and the published schema
improves as well: ``source`` is now an integer that may be omitted, rather than a
nullable integer defaulting to null.

Signed-off-by: Mohamed Belhsan Hmida 
---
 flexmeasures/data/schemas/automations.py  | 3 +--
 flexmeasures/data/schemas/sources.py      | 8 +-------
 flexmeasures/ui/static/openapi-specs.json | 6 +-----
 3 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/flexmeasures/data/schemas/automations.py b/flexmeasures/data/schemas/automations.py
index f29a921a36..fd7e5d75ce 100644
--- a/flexmeasures/data/schemas/automations.py
+++ b/flexmeasures/data/schemas/automations.py
@@ -118,8 +118,7 @@ class AutomationCreationSchema(Schema):
         },
     )
     source = DataSourceIdField(
-        load_default=None,
-        allow_none=True,
+        required=False,
         metadata={
             "description": "ID of an existing data source to reuse, instead of naming a `data-generator` and its `config`.",
             "example": 6,
diff --git a/flexmeasures/data/schemas/sources.py b/flexmeasures/data/schemas/sources.py
index f0bebe2335..ef384b2e9b 100644
--- a/flexmeasures/data/schemas/sources.py
+++ b/flexmeasures/data/schemas/sources.py
@@ -22,11 +22,5 @@ def _deserialize(self, value, attr, obj, **kwargs) -> DataSource:
         return source
 
     def _serialize(self, source, attr, data, **kwargs):
-        """Turn a DataSource into a source id.
-
-        A field that is allowed to be left out serializes its absence, rather than failing on it,
-        which is what dumping an optional field's default asks for.
-        """
-        if source is None:
-            return None
+        """Turn a DataSource into a source id."""
         return source.id
diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json
index c6c6cea2ad..50734da3e0 100644
--- a/flexmeasures/ui/static/openapi-specs.json
+++ b/flexmeasures/ui/static/openapi-specs.json
@@ -6653,11 +6653,7 @@
             "additionalProperties": {}
           },
           "source": {
-            "type": [
-              "integer",
-              "null"
-            ],
-            "default": null,
+            "type": "integer",
             "description": "ID of an existing data source to reuse, instead of naming a `data-generator` and its `config`.",
             "example": 6
           }

From a52da13355b435f3e980dcc671fefb386a9e093f Mon Sep 17 00:00:00 2001
From: Mohamed Belhsan Hmida 
Date: Wed, 30 Sep 2026 21:22:35 +0100
Subject: [PATCH 11/23] ui/assets: move the automation form's reference text
 onto info icons

The creation form carried nine blocks of help text, several of them three to
five lines, so the fields it was meant to explain were spread thin between
paragraphs and the modal read as a page of prose with inputs in it.

Each field's label now carries the info icon the sensor creation form already
uses, with the same text behind it on hover. Two hints stay on the page, because
they are the ones you act on while typing rather than read once: that the
generator config can be left empty, and the offset example worth copying into
the parameters. Bootstrap only turns those icons into tooltips when asked, so the
page now initialises them, as the chart pages do.

Signed-off-by: Mohamed Belhsan Hmida 
---
 .../templates/assets/asset_automations.html   | 72 +++++++++++--------
 1 file changed, 41 insertions(+), 31 deletions(-)

diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html
index 39e006c42e..e107285d39 100644
--- a/flexmeasures/ui/templates/assets/asset_automations.html
+++ b/flexmeasures/ui/templates/assets/asset_automations.html
@@ -168,37 +168,42 @@ 
    New automation for {{ asset.name }}
    
                   
                 
    
                 
    
-                  
+                  
                   
-                  
    Create plugin-defined automation types with the CLI. They can be viewed and managed here.
    
                 
    
                 
    
-                  
+                  
                   
-                  
    Use five fields: minute, hour, day of month, month, day of week. 0 6 * * * runs daily at 06:00; */15 * * * * runs every 15 minutes. Times use the timezone below.
    
                 
    
                 
    
-                  
+                  
                   
-                  
    The local clock the recurrence is read in. For example, Europe/Amsterdam keeps 06:00 at 06:00 locally when daylight saving changes. Defaults to this asset's timezone.
    
                 
    
                 
                 
    
-                  
+                  
                   
    
                     
                     
                   
    
                   
    
-                  
    
-                    Optional. Pick a data source to take its data generator and configuration from, instead of setting the two fields below.
-                    The automation then records what it computes under that same source, alongside the data it already holds.
-                  
    
                   
    
                     
    
                       
    Selected data source
    
@@ -209,30 +214,29 @@ 
    Configuration
    
                   
    
                 
    
                 
    
-                  
+                  
                   
-                  
    
-                    Forecaster class (defaults to TrainPredictPipeline) or reporter class (required for type reporting, e.g. PandasReporter).
-                  
    
                 
    
                 
    
-                  
+                  
                   
-                  
    
-                    How the generator is set up: for a forecaster, its model, training period and regressors. Leave empty for its defaults.
-                    These settings are what distinguish one data source from another, so the results record under a data source of their own.
-                    They cannot be changed afterwards.
-                  
    
+                  
    Leave empty for the generator's defaults.
    
                 
    
                 
    
-                  
+                  
                   
                   
    
-                    Forecast parameters (for type forecasting), a schedule trigger message (for type scheduling), or report parameters (for type reporting).
-                    A fixed "start", "end" or "prior" is refused, as every run would share that moment.
-                    Say how the period each run covers relates to that run with two of "start-offset", "end-offset" and "duration",
-                    such as {"start-offset": "1D,DB", "duration": "P1D"} for the whole of the next day,
-                    or leave the timing out, so that a forecast or schedule starts at the run time, and a report continues from the last successful one.
+                    Say the period each run covers with two of "start-offset", "end-offset" and "duration",
+                    such as {"start-offset": "1D,DB", "duration": "P1D"} for the whole of the next day.
                   
    
                 
    
                 
    
@@ -264,14 +268,18 @@ 
    Edit automation
    
                   
                 
    
                 
    
-                  
+                  
                   
-                  
    Five fields: minute, hour, day of month, month, day of week. 0 6 * * * runs daily at 06:00 in the timezone below. Changing the schedule recalculates the next eligible run.
    
                 
    
                 
    
-                  
+                  
                   
-                  
    The cron time follows this local clock, including daylight-saving changes. For example, choose Europe/Amsterdam to run at 06:00 Amsterdam time year-round.
    
                 
    
                 
    
                   
@@ -826,6 +834,8 @@ 
    Recently created jobs
    
 
   $(document).ready(function () {
     $.fn.dataTable.ext.errMode = 'none';
+    // The field labels carry their reference text on an info icon, which Bootstrap only turns into a tooltip once asked.
+    document.querySelectorAll('[data-bs-toggle="tooltip"]').forEach(el => new bootstrap.Tooltip(el));
     $('button[data-bs-toggle="tab"]').on("shown.bs.tab", function () {
       const table = $($(this).attr("data-bs-target")).find("table");
       if ($.fn.dataTable.isDataTable(table)) {

From 423d385e0d119da4f626e258c597c43184cd2983 Mon Sep 17 00:00:00 2001
From: Mohamed Belhsan Hmida 
Date: Wed, 30 Sep 2026 21:54:50 +0100
Subject: [PATCH 12/23] ui/assets: raise the form's tooltips above the modal
 they belong to

Bootstrap appends a tooltip to the document body, where it sits at a z-index of
1080. The theme gives a modal 999993. A tooltip opened from a field inside the
creation form was therefore built, shown and positioned correctly, and painted
behind the modal, so hovering an info icon appeared to do nothing at all.

Anchoring each tooltip to the modal that holds its icon puts the two in one
stacking context, and the tooltip comes out on top. Icons outside a modal keep
the body as their container.

Signed-off-by: Mohamed Belhsan Hmida 
---
 flexmeasures/ui/templates/assets/asset_automations.html | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html
index e107285d39..a5cfacd571 100644
--- a/flexmeasures/ui/templates/assets/asset_automations.html
+++ b/flexmeasures/ui/templates/assets/asset_automations.html
@@ -835,7 +835,11 @@ 
    Recently created jobs
    
   $(document).ready(function () {
     $.fn.dataTable.ext.errMode = 'none';
     // The field labels carry their reference text on an info icon, which Bootstrap only turns into a tooltip once asked.
-    document.querySelectorAll('[data-bs-toggle="tooltip"]').forEach(el => new bootstrap.Tooltip(el));
+    // A tooltip is appended to the body by default, at a z-index far below the one the theme gives a modal,
+    // so one raised from inside a modal is painted behind it. Anchor it to the modal to put it in the same stacking context.
+    document.querySelectorAll('[data-bs-toggle="tooltip"]').forEach(function (el) {
+      new bootstrap.Tooltip(el, { container: el.closest(".modal") || "body" });
+    });
     $('button[data-bs-toggle="tab"]').on("shown.bs.tab", function () {
       const table = $($(this).attr("data-bs-target")).find("table");
       if ($.fn.dataTable.isDataTable(table)) {

From fbb6efa85ed2aa911d552f83ea93ec89b892868f Mon Sep 17 00:00:00 2001
From: Mohamed Belhsan Hmida 
Date: Thu, 1 Oct 2026 00:24:38 +0100
Subject: [PATCH 13/23] ui/assets: say that the job counts reach back only as
 far as Redis keeps jobs

The jobs column counts what the job cache returns, and that cache holds only the
jobs Redis still has: FLEXMEASURES_JOB_TTL expires them, and the cache drops the
ids it can no longer fetch. A daily automation two weeks old therefore shows a
week of jobs on an instance that keeps them for a week, which reads as runs that
never happened rather than as records that have been cleaned up.

The column and the info panel now say so, naming the window this instance is
configured for rather than pointing at the setting. The phrasing works for either
shape the value takes, since a one-day retention reads as "a day" and not "1 day".

The browser checks render the page's script through a bare Jinja environment, so
that environment now carries the filter and the configuration value the page
reads, keeping its promise that nothing is left unrendered.

Signed-off-by: Mohamed Belhsan Hmida 
---
 .../templates/assets/asset_automations.html   | 17 +++++++++--
 .../ui/tests/js/test_automation_actions.py    | 30 ++++++++++---------
 2 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html
index a5cfacd571..6c75cb9642 100644
--- a/flexmeasures/ui/templates/assets/asset_automations.html
+++ b/flexmeasures/ui/templates/assets/asset_automations.html
@@ -325,6 +325,10 @@ 
    Edit automation
    
 
 ", template, re.DOTALL)
     assert match is not None
-    script = (
-        Environment(autoescape=True, undefined=StrictUndefined)
-        .from_string(match.group(1))
-        .render(
-            asset={"id": 3, "child_assets": [object()] if has_children else []},
-            include_child_assets=include_children,
-            user_can_manage_automations=can_manage,
-            user_can_create_children=can_run,
-            automation_types={
-                "forecasting": "Forecasts",
-                "scheduling": "Schedules",
-                "mock-ingestion": "Mock ingestion",
-            },
-        )
+    environment = Environment(autoescape=True, undefined=StrictUndefined)
+    # The page reads a configured value through a filter the app registers, so both are supplied here.
+    environment.filters["naturalized_timedelta"] = naturaldelta
+    script = environment.from_string(match.group(1)).render(
+        asset={"id": 3, "child_assets": [object()] if has_children else []},
+        include_child_assets=include_children,
+        user_can_manage_automations=can_manage,
+        user_can_create_children=can_run,
+        automation_types={
+            "forecasting": "Forecasts",
+            "scheduling": "Schedules",
+            "mock-ingestion": "Mock ingestion",
+        },
+        config={"FLEXMEASURES_JOB_TTL": timedelta(days=7)},
     )
     # An unrendered value would reach the browser as a syntax error, which reads as every check failing at once.
     assert (

From 0d7cb823449905ee1b902e24d37c88d735a75e19 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" 
Date: Thu, 1 Oct 2026 11:12:02 +0200
Subject: [PATCH 14/23] Order the data source listing, let a client limit it,
 and keep its types unnarrowed

A search from the automation form asked for every source a user may read, in whatever order Postgres returned them,
so an instance with many sources sent its whole source table to the browser,
and the sources shown could differ between two searches for the same term.
The listing is now ordered with the most recently created source first, and takes a limit.

The source types in the response are read from every source the user may read, rather than from the sources the call returns,
so that narrowing the listing no longer narrows the types a client can offer to narrow it by.

Co-Authored-By: Claude Opus 5 
Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC
Signed-off-by: F.N. Claessen 
---
 flexmeasures/api/v3_0/sources.py              | 60 +++++++++++++-----
 .../api/v3_0/tests/test_sources_api.py        | 63 +++++++++++++++++++
 flexmeasures/ui/static/openapi-specs.json     | 13 +++-
 3 files changed, 121 insertions(+), 15 deletions(-)

diff --git a/flexmeasures/api/v3_0/sources.py b/flexmeasures/api/v3_0/sources.py
index ac5154ae4c..c095ca4462 100644
--- a/flexmeasures/api/v3_0/sources.py
+++ b/flexmeasures/api/v3_0/sources.py
@@ -4,7 +4,7 @@
 from flask_classful import FlaskView, route
 from flask_json import as_json
 from flask_security import auth_required
-from marshmallow import fields, Schema
+from marshmallow import fields, Schema, validate
 from packaging.version import Version, InvalidVersion
 from sqlalchemy import select, or_, and_
 from webargs.flaskparser import use_kwargs
@@ -49,10 +49,20 @@ class SourceQuerySchema(Schema):
             "example": "forecaster",
         },
     )
+    limit = fields.Int(
+        required=False,
+        validate=validate.Range(min=1),
+        metadata={
+            "description": "Maximum number of sources to return, the most recently created ones first."
+            " Without it, every accessible source is returned."
+            " Note that `only_latest` collapses the sources which this limit let through, so the response can hold fewer sources than the limit allows.",
+            "example": 25,
+        },
+    )
 
 
 def source_search_term_filter(term: str):
-    """Match a search term against what identifies a source: its name, its model, its description and its id."""
+    """Match a search term against what identifies a source: its name, its model and its id."""
     filters = [
         DataSource.name.ilike(f"%{term}%"),
         DataSource.model.ilike(f"%{term}%"),
@@ -112,6 +122,7 @@ def index(
         only_latest: bool = True,
         filter: list[str] | None = None,
         type: str | None = None,
+        limit: int | None = None,
     ):
         """List accessible data sources and defined source types.
 
@@ -125,7 +136,12 @@ def index(
             the defined source types.
 
             The ``filter`` parameter searches the sources by name, by model and by id prefix,
-            and the ``type`` parameter narrows the list to one source type, such as ``forecaster``.
+            the ``type`` parameter narrows the list to one source type, such as ``forecaster``,
+            and the ``limit`` parameter returns only the most recently created ones.
+
+            The ``types`` in the response are those of every source the user may read,
+            rather than only those of the sources this call returns,
+            so that one call can both search the sources and offer the types to search by.
 
             **Access rules:**
 
@@ -173,25 +189,32 @@ def index(
         """
         accessible_account_ids = get_readable_source_account_ids()
 
-        query = select(DataSource)
+        readable_sources = None
         if accessible_account_ids is not None:
             # Sources owned by one of the accessible accounts, OR sources
             # with no account_id AND no user_id (system / public sources).
-            query = query.where(
-                or_(
-                    DataSource.account_id.in_(accessible_account_ids),
-                    and_(
-                        DataSource.account_id.is_(None),
-                        DataSource.user_id.is_(None),
-                    ),
-                )
+            readable_sources = or_(
+                DataSource.account_id.in_(accessible_account_ids),
+                and_(
+                    DataSource.account_id.is_(None),
+                    DataSource.user_id.is_(None),
+                ),
             )
+
+        query = select(DataSource)
+        if readable_sources is not None:
+            query = query.where(readable_sources)
         if type is not None:
             query = query.where(DataSource.type == type)
         if filter is not None:
             query = query.where(
                 or_(*(source_search_term_filter(term) for term in filter))
             )
+        # The listing is ordered so that a limited one holds the same sources each time it is asked for,
+        # and so that a source which was just created is in it rather than behind the limit.
+        query = query.order_by(DataSource.id.desc())
+        if limit is not None:
+            query = query.limit(limit)
 
         sources: list[DataSource] = list(db.session.scalars(query).all())
 
@@ -200,8 +223,17 @@ def index(
 
         serialized = [_serialize_source(s) for s in sources]
 
-        # Collect any extra types present in the DB but not in the defaults
-        db_types = {s.type for s in sources if s.type}
+        # Collect any extra types present in the DB but not in the defaults.
+        # These are read from every source the user may read, rather than from the sources returned above,
+        # so that narrowing the listing does not also narrow the types a client can offer to narrow it by.
+        type_query = select(DataSource.type).distinct()
+        if readable_sources is not None:
+            type_query = type_query.where(readable_sources)
+        db_types = {
+            source_type
+            for source_type in db.session.scalars(type_query).all()
+            if source_type
+        }
         all_types = list(DEFAULT_DATASOURCE_TYPES) + sorted(
             db_types - set(DEFAULT_DATASOURCE_TYPES)
         )
diff --git a/flexmeasures/api/v3_0/tests/test_sources_api.py b/flexmeasures/api/v3_0/tests/test_sources_api.py
index 4751291334..55d4d948c2 100644
--- a/flexmeasures/api/v3_0/tests/test_sources_api.py
+++ b/flexmeasures/api/v3_0/tests/test_sources_api.py
@@ -417,3 +417,66 @@ def test_get_sources_filtered_by_type_and_search_term(
     )
     assert by_id.status_code == 200
     assert forecaster.id in {source["id"] for source in by_id.json["sources"]}
+
+
+@pytest.mark.parametrize(
+    "requesting_user",
+    ["test_prosumer_user@seita.nl"],
+    indirect=True,
+)
+def test_get_sources_limit_returns_the_newest_ones(
+    client, setup_api_test_data, requesting_user, db
+):
+    """A limited listing holds that many sources, the most recently created ones, in a stable order."""
+    sources = [
+        DataSource(name=f"LimitedSeita{index}", type="forecaster", model="Pipeline")
+        for index in range(4)
+    ]
+    db.session.add_all(sources)
+    db.session.flush()
+    newest_two = [source.id for source in sorted(sources, key=lambda s: -s.id)][:2]
+
+    limited = client.get(
+        url_for("SourceAPI:index"),
+        query_string={"filter": "LimitedSeita", "only_latest": "false", "limit": 2},
+    )
+    assert limited.status_code == 200
+    assert [source["id"] for source in limited.json["sources"]] == newest_two
+
+    unlimited = client.get(
+        url_for("SourceAPI:index"),
+        query_string={"filter": "LimitedSeita", "only_latest": "false"},
+    )
+    assert unlimited.status_code == 200
+    assert len(unlimited.json["sources"]) == 4
+
+
+@pytest.mark.parametrize(
+    "requesting_user",
+    ["test_prosumer_user@seita.nl"],
+    indirect=True,
+)
+def test_get_sources_types_are_not_narrowed_by_the_filters(
+    client, setup_api_test_data, requesting_user, db
+):
+    """The types cover every source the user may read, so that one call can both search and offer the types to search by."""
+    db.session.add(
+        DataSource(
+            name="TypedSeita",
+            type="soothsayer",
+            model="CrystalBall",
+            account_id=requesting_user.account_id,
+        )
+    )
+    db.session.flush()
+
+    unfiltered = client.get(url_for("SourceAPI:index"))
+    assert unfiltered.status_code == 200
+    assert "soothsayer" in unfiltered.json["types"]
+
+    filtered = client.get(
+        url_for("SourceAPI:index"), query_string={"type": "scheduler"}
+    )
+    assert filtered.status_code == 200
+    assert "soothsayer" not in {source["type"] for source in filtered.json["sources"]}
+    assert "soothsayer" in filtered.json["types"]
diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json
index bae273472e..82510d2419 100644
--- a/flexmeasures/ui/static/openapi-specs.json
+++ b/flexmeasures/ui/static/openapi-specs.json
@@ -5539,7 +5539,7 @@
     "/api/v3_0/sources": {
       "get": {
         "summary": "List accessible data sources and defined source types.",
-        "description": "Returns the list of data sources accessible to the current user and\nthe defined source types.\n\nThe filter parameter searches the sources by name, by model and by id prefix,\nand the type parameter narrows the list to one source type, such as forecaster.\n\nAccess rules:\n\n- Admins see all data sources.\n- Users with the consultant role see sources belonging to their\n  own account and to any consultancy-client accounts for which their\n  account is the consultancy.\n- All other authenticated users see only sources belonging to their\n  own account, plus sources that have neither a user_id nor an\n  account_id (i.e. system/public sources).\n",
+        "description": "Returns the list of data sources accessible to the current user and\nthe defined source types.\n\nThe filter parameter searches the sources by name, by model and by id prefix,\nthe type parameter narrows the list to one source type, such as forecaster,\nand the limit parameter returns only the most recently created ones.\n\nThe types in the response are those of every source the user may read,\nrather than only those of the sources this call returns,\nso that one call can both search the sources and offer the types to search by.\n\nAccess rules:\n\n- Admins see all data sources.\n- Users with the consultant role see sources belonging to their\n  own account and to any consultancy-client accounts for which their\n  account is the consultancy.\n- All other authenticated users see only sources belonging to their\n  own account, plus sources that have neither a user_id nor an\n  account_id (i.e. system/public sources).\n",
         "security": [
           {
             "ApiKeyAuth": []
@@ -5575,6 +5575,17 @@
               "example": "forecaster"
             },
             "required": false
+          },
+          {
+            "in": "query",
+            "name": "limit",
+            "description": "Maximum number of sources to return, the most recently created ones first. Without it, every accessible source is returned. Note that `only_latest` collapses the sources which this limit let through, so the response can hold fewer sources than the limit allows.",
+            "schema": {
+              "type": "integer",
+              "minimum": 1,
+              "example": 25
+            },
+            "required": false
           }
         ],
         "responses": {

From ea9a9c42a143ca8dc5d4e482afc131e684e2b09e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" 
Date: Thu, 1 Oct 2026 11:12:20 +0200
Subject: [PATCH 15/23] Bound the source search, and open the automation form
 on a clean slate

The search now asks for one source more than the list shows, which is what tells the list that there are more to narrow down to,
and which keeps an instance with many sources from sending all of them to the browser on every pause in typing.
The list therefore says that it leaves sources out rather than how many, which is no longer a number it has.

A source picked and then left behind by closing the form used to still disable the data generator fields when the form was opened again,
with nothing on screen to say why, so closing the form now clears the picked source.

Co-Authored-By: Claude Opus 5 
Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC
Signed-off-by: F.N. Claessen 
---
 .../ui/templates/assets/asset_automations.html   | 10 +++++++++-
 .../ui/tests/js/test_automation_source_picker.py | 16 +++++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html
index 6c75cb9642..347ae543b9 100644
--- a/flexmeasures/ui/templates/assets/asset_automations.html
+++ b/flexmeasures/ui/templates/assets/asset_automations.html
@@ -412,6 +412,9 @@ 
    Edit automation
    
     if (searchTerm) {
       parameters.append("filter", searchTerm);
     }
+    // One more source than the list shows, which is what tells the list that there are more to narrow down to,
+    // and which keeps a search on an instance with many sources from sending all of them to the browser.
+    parameters.append("limit", String(MAX_SOURCE_RESULTS + 1));
     return `/api/v3_0/sources?${parameters.toString()}`;
   }
 
@@ -429,8 +432,9 @@ 
    Edit automation
    
       `
     );
     if (sources.length > MAX_SOURCE_RESULTS) {
+      // The search asks for one source more than it shows, so this says that there are more, not how many.
       items.push(
-        `
    ${sources.length - MAX_SOURCE_RESULTS} more match; narrow the search to see them.
    `
+        '
    More data sources match; narrow the search to see them.
    '
       );
     }
     return items.join("");
@@ -1026,6 +1030,10 @@ 
    Recently created jobs
 
     $("#automationSourceClear").on("click", () => clearSelectedSource());
 
+    // A source picked and then left behind by closing the form would still disable the generator fields when the form is opened again,
+    // with nothing on screen to say why, so the form opens on a clean slate.
+    $("#newAutomationModal").on("hidden.bs.modal", () => clearSelectedSource());
+
     // Read a JSON textarea, reporting where it is malformed rather than failing silently.
     function readJsonField(selector, label) {
       const text = $(selector).val().trim();
diff --git a/flexmeasures/ui/tests/js/test_automation_source_picker.py b/flexmeasures/ui/tests/js/test_automation_source_picker.py
index 7cb4931f9f..d44ef93904 100644
--- a/flexmeasures/ui/tests/js/test_automation_source_picker.py
+++ b/flexmeasures/ui/tests/js/test_automation_source_picker.py
@@ -40,6 +40,19 @@ def test_source_search_asks_for_the_sources_of_the_automations_own_type(assert_j
     """)
 
 
+def test_source_search_asks_for_no_more_sources_than_it_shows(assert_js):
+    """A search on an instance with many sources is bounded by the server, rather than sent to the browser in full."""
+    assert_js(f"""
+        window.$ = () => ({{ready: () => {{}}}});
+        const page = new Function({json.dumps(picker_script())}
+            + "\\nreturn {{ sourceSearchUrl, MAX_SOURCE_RESULTS }};")();
+
+        const url = new URL(page.sourceSearchUrl("forecasting", "pipeline"), "http://x");
+        eq("the search asks for one source more than the list shows",
+           url.searchParams.get("limit"), String(page.MAX_SOURCE_RESULTS + 1));
+    """)
+
+
 def test_search_results_name_each_source_and_cap_a_broad_search(assert_js):
     """Each result carries the id the picker reads the source by, and a long list says how much it leaves out."""
     assert_js(f"""
@@ -64,7 +77,8 @@ def test_search_results_name_each_source_and_cap_a_broad_search(assert_js):
         const capped = page.sourceResultsHtml(many);
         eq("a broad search shows at most the cap",
            (capped.match(/automation-source-option/g) || []).length, page.MAX_SOURCE_RESULTS);
-        check("and says how many it leaves out", capped.includes("3 more match"), capped);
+        check("and says that it leaves some out, which the count behind the limit cannot say",
+              capped.includes("More data sources match"), capped);
 
         const hostile = page.sourceResultsHtml([{{id: 1, name: ""}}]);
         check("a source name is escaped rather than rendered", !hostile.includes("
Date: Thu, 1 Oct 2026 11:12:41 +0200
Subject: [PATCH 16/23] Read an empty source on an automation as no source at
 all

A client which fills in the whole creation form and leaves the source field empty sends a null source,
which was refused with 'Field may not be null' rather than taken as the automation setting up its own data generator.
Our own form works around this by leaving the field out altogether.

Co-Authored-By: Claude Opus 5 
Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC
Signed-off-by: F.N. Claessen 
---
 .../api/v3_0/tests/test_automations_api.py    | 29 +++++++++++++++++++
 flexmeasures/data/schemas/automations.py      |  3 ++
 flexmeasures/ui/static/openapi-specs.json     |  5 +++-
 3 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/api/v3_0/tests/test_automations_api.py b/flexmeasures/api/v3_0/tests/test_automations_api.py
index 44aba94e06..f9c133c061 100644
--- a/flexmeasures/api/v3_0/tests/test_automations_api.py
+++ b/flexmeasures/api/v3_0/tests/test_automations_api.py
@@ -1803,3 +1803,32 @@ def test_post_automation_with_a_source_of_another_account(
         ).scalar_one_or_none()
         is None
     )
+
+
+@pytest.mark.parametrize(
+    "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True
+)
+def test_post_automation_accepts_an_empty_source(
+    app,
+    fresh_db,
+    add_battery_assets_fresh_db,
+    requesting_user,
+):
+    """Leaving the source field empty means the automation sets up its own data generator, as leaving it out does."""
+    battery = add_battery_assets_fresh_db["Test battery"]
+    with app.test_client() as client:
+        response = client.post(
+            url_for("AssetAPI:post_automation", id=battery.id),
+            json={
+                "name": "Forecasts without a source",
+                "cron": "0 6 * * *",
+                "type": "forecasting",
+                "source": None,
+                "parameters": {"sensor": battery.sensors[0].id},
+            },
+        )
+    assert response.status_code == 201, response.json
+    automation = fresh_db.session.execute(
+        select(Automation).filter_by(name="Forecasts without a source")
+    ).scalar_one()
+    assert automation.generator_id is not None
diff --git a/flexmeasures/data/schemas/automations.py b/flexmeasures/data/schemas/automations.py
index 777c151f47..458e617413 100644
--- a/flexmeasures/data/schemas/automations.py
+++ b/flexmeasures/data/schemas/automations.py
@@ -124,6 +124,9 @@ class AutomationCreationSchema(Schema):
     )
     source = DataSourceIdField(
         required=False,
+        # Leaving the source out and sending a null one both mean that the automation sets up its own data generator,
+        # which is what a client sends when it fills in the rest of the form and leaves the source field empty.
+        allow_none=True,
         metadata={
             "description": "ID of an existing data source to reuse, instead of naming a `data-generator` and its `config`.",
             "example": 6,
diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json
index 82510d2419..f9591ed0fa 100644
--- a/flexmeasures/ui/static/openapi-specs.json
+++ b/flexmeasures/ui/static/openapi-specs.json
@@ -6768,7 +6768,10 @@
             "additionalProperties": {}
           },
           "source": {
-            "type": "integer",
+            "type": [
+              "integer",
+              "null"
+            ],
             "description": "ID of an existing data source to reuse, instead of naming a `data-generator` and its `config`.",
             "example": 6
           }

From 68fa7d88e17bac79c0708256744ae405b4e4cd4c Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" 
Date: Thu, 1 Oct 2026 11:12:54 +0200
Subject: [PATCH 17/23] Changelog: the source an automation reuses, and
 searching the data sources

Co-Authored-By: Claude Opus 5 
Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC
Signed-off-by: F.N. Claessen 
---
 documentation/api/change_log.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index bc1ffaf416..25a698eb80 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -5,11 +5,12 @@ API change log
 
 .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL (e.g. `/api/v3_0`), allowing developers to upgrade at their own pace.
 
-
 v3.0-40 | September XX, 2026
 """"""""""""""""""""""""""""
 - Added ``DELETE /api/v3_0/sensors//annotations``, which deletes a sensor's annotations, optionally only those of one ``source`` or lying entirely between ``start`` and ``until``. An annotation that is also registered on another sensor, an asset or an account is only removed from this sensor. It requires permission to delete the sensor, like ``DELETE /api/v3_0/sensors//data``, and reports how many annotations it removed from the sensor as ``deleted``.
 - ``POST /api/v3_0/accounts//annotations``, ``POST /api/v3_0/assets//annotations`` and ``POST /api/v3_0/sensors//annotations`` now accept an ``end`` equal to the ``start``, which makes an instantaneous annotation, where they used to reply with a ``422``. An ``end`` before the ``start`` is still refused.
+- ``POST /api/v3_0/assets//automations`` now accepts ``source``, the ID of an existing data source to reuse, instead of a ``data-generator`` and its ``config``. The source hands over the data generator it stores, and the automation records its results under that same source, so naming one is refused with a ``403`` unless the caller may read it, and with a ``422`` where it is combined with ``data-generator`` or ``config``, where it stores no data generator of the automation's type, or where the automation schedules (a schedule automation's source follows from the asset and the flex config).
+- ``GET /api/v3_0/sources`` now accepts ``filter``, matching search terms against a source's name, its model and its id prefix, ``type``, narrowing the list to one source type, and ``limit``, returning only that many sources, the most recently created ones first. The ``types`` in the response still cover every source the caller may read, so that one call can both search the sources and offer the types to search by.
 
 v3.0-39 | September 22, 2026
 """"""""""""""""""""""""""""

From cf2177cf63834b289f2a5a3006ed06082588447a Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" 
Date: Thu, 1 Oct 2026 11:12:55 +0200
Subject: [PATCH 18/23] Drop a local import of ValidationError which the module
 already has

Co-Authored-By: Claude Opus 5 
Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC
Signed-off-by: F.N. Claessen 
---
 flexmeasures/data/services/automations.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py
index 9ae1370440..a85996eff1 100644
--- a/flexmeasures/data/services/automations.py
+++ b/flexmeasures/data/services/automations.py
@@ -1884,7 +1884,6 @@ def _create_builtin_automation(
     :raises werkzeug.exceptions.Forbidden: if a sensor, or the named data source, is not accessible to the user.
     :returns: the automation and a list of warnings.
     """
-    from marshmallow import ValidationError
     from werkzeug.exceptions import Forbidden
 
     from flexmeasures.data.models.audit_log import AssetAuditLog

From d8d77a5ab3beb04056a33f7dada4dc7ff19baeca Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" 
Date: Thu, 1 Oct 2026 12:20:04 +0200
Subject: [PATCH 19/23] Decide who may read a data source per source, rather
 than by its organisation alone

Almost every data generator's source belongs to no organisation, because nothing on the creation path passes one,
and a source naming neither an organisation nor a user was readable by every authenticated user.
The read check this pull request added therefore hardly ever refused anything.

A source is now readable when it belongs to an organisation the user may read,
when an automation they may read computes under it,
or when it has recorded data on a sensor they may read, which is what makes it possible to ask what produced a number one is allowed to see.
The first two are what makes a source one to work with: those are the sources the listing holds,
because reusing a source means recording under its lineage, which is more than reading what it computed.
The stored configuration follows the stricter rule as well, since a data generator's configuration names the sensors it runs on,
which can be sensors of an organisation whose data the user cannot see at all.

Two tests created sources belonging to no organisation to exercise version collapsing rather than visibility;
their fixtures name an organisation now.

Co-Authored-By: Claude Opus 5 
Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC
Signed-off-by: F.N. Claessen 
---
 flexmeasures/api/v3_0/sources.py              |  59 ++++----
 .../api/v3_0/tests/test_sources_api.py        | 131 +++++++++++++++++-
 flexmeasures/data/services/data_sources.py    | 115 +++++++++++++--
 flexmeasures/ui/static/openapi-specs.json     |   4 +-
 4 files changed, 268 insertions(+), 41 deletions(-)

diff --git a/flexmeasures/api/v3_0/sources.py b/flexmeasures/api/v3_0/sources.py
index c095ca4462..6ec7bbe162 100644
--- a/flexmeasures/api/v3_0/sources.py
+++ b/flexmeasures/api/v3_0/sources.py
@@ -6,7 +6,7 @@
 from flask_security import auth_required
 from marshmallow import fields, Schema, validate
 from packaging.version import Version, InvalidVersion
-from sqlalchemy import select, or_, and_
+from sqlalchemy import select, or_
 from webargs.flaskparser import use_kwargs
 
 from flexmeasures.api.common.schemas.search import SearchFilterField
@@ -14,8 +14,9 @@
 from flexmeasures.data.models.data_sources import DataSource, DEFAULT_DATASOURCE_TYPES
 from flexmeasures.data.queries.utils import id_prefix_filter
 from flexmeasures.data.services.data_sources import (
-    get_readable_source_account_ids,
+    usable_source_filter,
     user_may_read_source,
+    user_may_use_source,
 )
 
 """
@@ -139,19 +140,20 @@ def index(
             the ``type`` parameter narrows the list to one source type, such as ``forecaster``,
             and the ``limit`` parameter returns only the most recently created ones.
 
-            The ``types`` in the response are those of every source the user may read,
+            The ``types`` in the response are those of every source this listing can hold,
             rather than only those of the sources this call returns,
             so that one call can both search the sources and offer the types to search by.
 
             **Access rules:**
 
+            This lists the sources which are the user's to work with, which is what a source has to be to be reused.
+            That is a stricter rule than the one for reading one source with ``GET /api/v3_0/sources/``,
+            which also covers a source that recorded data on a sensor the user may read.
+
             - Admins see all data sources.
-            - Users with the ``consultant`` role see sources belonging to their
-              own account and to any consultancy-client accounts for which their
-              account is the consultancy.
-            - All other authenticated users see only sources belonging to their
-              own account, plus sources that have neither a ``user_id`` nor an
-              ``account_id`` (i.e. system/public sources).
+            - Everyone else sees the sources of their own organisation, of any organisation they consult for,
+              and the sources an automation they may read computes under.
+            - A source which belongs to no organisation is not listed for that reason alone.
 
           security:
             - ApiKeyAuth: []
@@ -187,23 +189,13 @@ def index(
           tags:
             - Sources
         """
-        accessible_account_ids = get_readable_source_account_ids()
-
-        readable_sources = None
-        if accessible_account_ids is not None:
-            # Sources owned by one of the accessible accounts, OR sources
-            # with no account_id AND no user_id (system / public sources).
-            readable_sources = or_(
-                DataSource.account_id.in_(accessible_account_ids),
-                and_(
-                    DataSource.account_id.is_(None),
-                    DataSource.user_id.is_(None),
-                ),
-            )
+        # The listing holds the sources which are the user's to work with, rather than every source they may read:
+        # it is what a client picks a source to reuse from, and reusing one is a stronger thing to be allowed than reading what it computed.
+        usable_sources = usable_source_filter()
 
         query = select(DataSource)
-        if readable_sources is not None:
-            query = query.where(readable_sources)
+        if usable_sources is not None:
+            query = query.where(usable_sources)
         if type is not None:
             query = query.where(DataSource.type == type)
         if filter is not None:
@@ -227,8 +219,8 @@ def index(
         # These are read from every source the user may read, rather than from the sources returned above,
         # so that narrowing the listing does not also narrow the types a client can offer to narrow it by.
         type_query = select(DataSource.type).distinct()
-        if readable_sources is not None:
-            type_query = type_query.where(readable_sources)
+        if usable_sources is not None:
+            type_query = type_query.where(usable_sources)
         db_types = {
             source_type
             for source_type in db.session.scalars(type_query).all()
@@ -255,7 +247,12 @@ def get(self, id: int):
             data generators (such as forecasters, schedulers and reporters) store their
             configuration.
 
-            The access rules are the same as for listing data sources.
+            A source is readable when it belongs to an organisation the user may read,
+            when an automation they may read computes under it,
+            or when it has recorded data on a sensor they may read, so that they can ask what computed a number they see.
+
+            The ``attributes``, which hold the configuration a data generator was set up with, are only included for the first two:
+            a configuration names the sensors it runs on, which can be sensors the user cannot see at all.
           security:
             - ApiKeyAuth: []
           parameters:
@@ -295,9 +292,13 @@ def get(self, id: int):
         source = db.session.get(DataSource, id)
         if source is None:
             return {"message": f"No data source found with id {id}."}, 404
-        if not user_may_read_source(source):
+        # A source which recorded on one of the user's sensors is theirs to read, so that they can ask what computed a number they see.
+        # Its attributes are another matter: a data generator's configuration names the sensors it runs on,
+        # which can be sensors of an organisation whose data the user cannot see at all, so those are kept for the sources which are theirs to work with.
+        may_use = user_may_use_source(source)
+        if not may_use and not user_may_read_source(source):
             return {"message": "You cannot read this data source."}, 403
-        return _serialize_source(source, with_attributes=True), 200
+        return _serialize_source(source, with_attributes=may_use), 200
 
 
 def _serialize_source(source: DataSource, with_attributes: bool = False) -> dict:
diff --git a/flexmeasures/api/v3_0/tests/test_sources_api.py b/flexmeasures/api/v3_0/tests/test_sources_api.py
index 55d4d948c2..7ac71d0a72 100644
--- a/flexmeasures/api/v3_0/tests/test_sources_api.py
+++ b/flexmeasures/api/v3_0/tests/test_sources_api.py
@@ -153,18 +153,20 @@ def test_get_sources_consultant_sees_client_sources(
 )
 def test_get_sources_only_latest(client, setup_api_test_data, requesting_user, db):
     """The endpoint should default to latest-only and allow opting out via only_latest=false."""
-    # Create two versioned sources in the same group, both public (no account_id / user_id)
+    # Create two versioned sources in the same group, both of the organisation whose sources the user may read
     source_v1 = DataSource(
         name="VersionedScheduler",
         type="scheduler",
         model="TestModel",
         version="1.0",
+        account_id=requesting_user.account_id,
     )
     source_v2 = DataSource(
         name="VersionedScheduler",
         type="scheduler",
         model="TestModel",
         version="2.0",
+        account_id=requesting_user.account_id,
     )
     db.session.add_all([source_v1, source_v2])
     db.session.flush()
@@ -278,12 +280,14 @@ def test_get_sources_only_latest_tie_break_by_id(
         type="scheduler",
         model="TieModel",
         version="1.0",
+        account_id=requesting_user.account_id,
     )
     source_higher_id = DataSource(
         name="TieBreakScheduler",
         type="scheduler",
         model="TieModel",
         version="1.0",
+        account_id=requesting_user.account_id,
     )
     db.session.add(source_lower_id)
     db.session.flush()
@@ -429,7 +433,12 @@ def test_get_sources_limit_returns_the_newest_ones(
 ):
     """A limited listing holds that many sources, the most recently created ones, in a stable order."""
     sources = [
-        DataSource(name=f"LimitedSeita{index}", type="forecaster", model="Pipeline")
+        DataSource(
+            name=f"LimitedSeita{index}",
+            type="forecaster",
+            model="Pipeline",
+            account_id=requesting_user.account_id,
+        )
         for index in range(4)
     ]
     db.session.add_all(sources)
@@ -480,3 +489,121 @@ def test_get_sources_types_are_not_narrowed_by_the_filters(
     assert filtered.status_code == 200
     assert "soothsayer" not in {source["type"] for source in filtered.json["sources"]}
     assert "soothsayer" in filtered.json["types"]
+
+
+@pytest.mark.parametrize(
+    "requesting_user",
+    ["test_prosumer_user@seita.nl"],
+    indirect=True,
+)
+def test_belonging_to_no_organisation_does_not_make_a_source_readable(
+    client, setup_api_test_data, requesting_user, db
+):
+    """A data generator's source used to belong to no organisation, which made every one of them readable by everybody."""
+    orphan = DataSource(
+        name="OrphanForecaster",
+        type="forecaster",
+        model="TrainPredictPipeline",
+        attributes={"data_generator": {"config": {"model": "CustomLGBM"}}},
+    )
+    db.session.add(orphan)
+    db.session.flush()
+
+    listing = client.get(
+        url_for("SourceAPI:index"), query_string={"filter": "OrphanForecaster"}
+    )
+    assert listing.status_code == 200
+    assert listing.json["sources"] == []
+
+    detail = client.get(url_for("SourceAPI:get", id=orphan.id))
+    assert detail.status_code == 403
+
+
+@pytest.mark.parametrize(
+    "requesting_user",
+    ["test_supplier_user_4@seita.nl"],
+    indirect=True,
+)
+def test_a_source_recorded_on_a_readable_sensor_hands_over_no_configuration(
+    client, setup_api_test_data, requesting_user, db
+):
+    """What computed a number one may see is a fair question, but the answer must not name another organisation's sensors."""
+    from flexmeasures.data.models.data_sources import SensorDataSource
+
+    gas_sensor = setup_api_test_data["some gas sensor"]
+    prosumer_user = find_user_by_email("test_prosumer_user@seita.nl")
+    foreign = DataSource(
+        name="ForeignForecaster",
+        type="forecaster",
+        model="TrainPredictPipeline",
+        account=prosumer_user.account,
+        attributes={"data_generator": {"config": {"sensor": 404}}},
+    )
+    db.session.add(foreign)
+    db.session.flush()
+    db.session.add(SensorDataSource(sensor_id=gas_sensor.id, source_id=foreign.id))
+    db.session.flush()
+
+    detail = client.get(url_for("SourceAPI:get", id=foreign.id))
+    assert detail.status_code == 200
+    assert detail.json["id"] == foreign.id
+    assert "attributes" not in detail.json
+
+    # Recording on a sensor is not what makes a source one to reuse, so it stays out of the listing.
+    listing = client.get(
+        url_for("SourceAPI:index"), query_string={"filter": "ForeignForecaster"}
+    )
+    assert listing.status_code == 200
+    assert listing.json["sources"] == []
+
+
+@pytest.mark.parametrize(
+    "requesting_user",
+    ["test_prosumer_user@seita.nl"],
+    indirect=True,
+)
+def test_an_automation_of_ones_own_makes_its_source_readable(
+    client, setup_api_test_data, requesting_user, db
+):
+    """A source one's own automation computes under is one's own to work with, before it has recorded anything."""
+    from flexmeasures.data.models.automations import Automation
+    from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
+
+    asset = GenericAsset(
+        name="automated site",
+        generic_asset_type=db.session.scalars(select(GenericAssetType)).first(),
+        account_id=requesting_user.account_id,
+    )
+    db.session.add(asset)
+    db.session.flush()
+    source = DataSource(
+        name="AutomatedForecaster",
+        type="forecaster",
+        model="TrainPredictPipeline",
+        attributes={"data_generator": {"config": {"model": "CustomLGBM"}}},
+    )
+    db.session.add(source)
+    db.session.flush()
+    db.session.add(
+        Automation(
+            asset_id=asset.id,
+            type="forecasting",
+            name="Hourly forecasts",
+            cronstr="0 * * * *",
+            timezone="Europe/Amsterdam",
+            generator_id=source.id,
+        )
+    )
+    db.session.flush()
+
+    listing = client.get(
+        url_for("SourceAPI:index"), query_string={"filter": "AutomatedForecaster"}
+    )
+    assert listing.status_code == 200
+    assert [s["id"] for s in listing.json["sources"]] == [source.id]
+
+    detail = client.get(url_for("SourceAPI:get", id=source.id))
+    assert detail.status_code == 200
+    assert detail.json["attributes"] == {
+        "data_generator": {"config": {"model": "CustomLGBM"}}
+    }
diff --git a/flexmeasures/data/services/data_sources.py b/flexmeasures/data/services/data_sources.py
index 9078a6d920..cfae23b498 100644
--- a/flexmeasures/data/services/data_sources.py
+++ b/flexmeasures/data/services/data_sources.py
@@ -3,7 +3,8 @@
 import logging
 
 from flask import current_app
-from sqlalchemy import select
+from sqlalchemy import or_, select
+from sqlalchemy.sql import ColumnElement
 from typing import Type, TypeVar
 
 from flexmeasures import Account, Source, User
@@ -158,15 +159,113 @@ def get_readable_source_account_ids() -> list[int] | None:
     return readable_ids
 
 
-def user_may_read_source(source: DataSource) -> bool:
-    """Whether the current user may read the given data source.
+def _readable_asset_condition(readable_account_ids: list[int]) -> ColumnElement[bool]:
+    """The assets the current user may read, as the asset's own access rules spell it out:
+    those of an organisation they may read, and public ones, which every logged-in user may read.
+    """
+    from flexmeasures.data.models.generic_assets import GenericAsset
+
+    return or_(
+        GenericAsset.account_id.in_(readable_account_ids),
+        GenericAsset.account_id.is_(None),
+    )
+
 
-    A source belonging to one of the user's own accounts is readable, and so is a system source,
-    which is one that names neither an account nor a user.
+def _usable_source_conditions(
+    readable_account_ids: list[int],
+) -> list[ColumnElement[bool]]:
+    """The two conditions under which a data source is the current user's to work with.
+
+    A source is theirs when it belongs to an organisation they may read,
+    and theirs by proxy when an automation they may read computes under it,
+    which is how a source they set up becomes theirs before it has recorded anything.
+    """
+    from flexmeasures.data.models.automations import Automation
+    from flexmeasures.data.models.generic_assets import GenericAsset
+
+    return [
+        DataSource.account_id.in_(readable_account_ids),
+        select(1)
+        .select_from(Automation)
+        .join(GenericAsset, Automation.asset_id == GenericAsset.id)
+        .where(
+            Automation.generator_id == DataSource.id,
+            _readable_asset_condition(readable_account_ids),
+        )
+        .exists(),
+    ]
+
+
+def usable_source_filter() -> ColumnElement[bool] | None:
+    """A condition selecting the data sources which are the current user's to work with.
+
+    These are the sources they may reuse for an automation of their own, and the ones whose stored configuration they may read,
+    which is a stronger thing to be allowed than reading what a source has computed: see `readable_source_filter`.
+
+    Returns None to say that every source qualifies, which is what admin access amounts to.
     """
     readable_account_ids = get_readable_source_account_ids()
     if readable_account_ids is None:
+        return None
+    return or_(*_usable_source_conditions(readable_account_ids))
+
+
+def readable_source_filter() -> ColumnElement[bool] | None:
+    """A condition selecting the data sources the current user may read.
+
+    On top of the sources which are theirs to work with (see `usable_source_filter`),
+    this covers the sources which have recorded data on a sensor they may read,
+    because "what computed this number?" is a fair question about data one is allowed to see.
+
+    Returns None to say that every source is readable, which is what admin access amounts to.
+    """
+    from flexmeasures.data.models.data_sources import SensorDataSource
+    from flexmeasures.data.models.generic_assets import GenericAsset
+    from flexmeasures.data.models.time_series import Sensor
+
+    readable_account_ids = get_readable_source_account_ids()
+    if readable_account_ids is None:
+        return None
+    return or_(
+        *_usable_source_conditions(readable_account_ids),
+        select(1)
+        .select_from(SensorDataSource)
+        .join(Sensor, SensorDataSource.sensor_id == Sensor.id)
+        .join(GenericAsset, Sensor.generic_asset_id == GenericAsset.id)
+        .where(
+            SensorDataSource.source_id == DataSource.id,
+            _readable_asset_condition(readable_account_ids),
+        )
+        .exists(),
+    )
+
+
+def _source_matches(source: DataSource, condition: ColumnElement[bool] | None) -> bool:
+    """Whether the given source is one of those the condition selects."""
+    if condition is None:
         return True
-    if source.account_id in readable_account_ids:
-        return True
-    return source.account_id is None and source.user_id is None
+    return bool(
+        db.session.scalar(
+            select(
+                select(DataSource.id)
+                .where(DataSource.id == source.id, condition)
+                .exists()
+            )
+        )
+    )
+
+
+def user_may_read_source(source: DataSource) -> bool:
+    """Whether the current user may read the given data source.
+
+    See `readable_source_filter` for what makes a source readable.
+    """
+    return _source_matches(source, readable_source_filter())
+
+
+def user_may_use_source(source: DataSource) -> bool:
+    """Whether the given data source is the current user's to work with.
+
+    See `usable_source_filter` for what that means, and why it is a stronger thing to be allowed than reading the source.
+    """
+    return _source_matches(source, usable_source_filter())
diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json
index f9591ed0fa..3fe4063e21 100644
--- a/flexmeasures/ui/static/openapi-specs.json
+++ b/flexmeasures/ui/static/openapi-specs.json
@@ -5476,7 +5476,7 @@
     "/api/v3_0/sources/{id}": {
       "get": {
         "summary": "Get one data source.",
-        "description": "Returns the full record of one data source, including the attributes in which\ndata generators (such as forecasters, schedulers and reporters) store their\nconfiguration.\n\nThe access rules are the same as for listing data sources.\n",
+        "description": "Returns the full record of one data source, including the attributes in which\ndata generators (such as forecasters, schedulers and reporters) store their\nconfiguration.\n\nA source is readable when it belongs to an organisation the user may read,\nwhen an automation they may read computes under it,\nor when it has recorded data on a sensor they may read, so that they can ask what computed a number they see.\n\nThe attributes, which hold the configuration a data generator was set up with, are only included for the first two:\na configuration names the sensors it runs on, which can be sensors the user cannot see at all.\n",
         "security": [
           {
             "ApiKeyAuth": []
@@ -5539,7 +5539,7 @@
     "/api/v3_0/sources": {
       "get": {
         "summary": "List accessible data sources and defined source types.",
-        "description": "Returns the list of data sources accessible to the current user and\nthe defined source types.\n\nThe filter parameter searches the sources by name, by model and by id prefix,\nthe type parameter narrows the list to one source type, such as forecaster,\nand the limit parameter returns only the most recently created ones.\n\nThe types in the response are those of every source the user may read,\nrather than only those of the sources this call returns,\nso that one call can both search the sources and offer the types to search by.\n\nAccess rules:\n\n- Admins see all data sources.\n- Users with the consultant role see sources belonging to their\n  own account and to any consultancy-client accounts for which their\n  account is the consultancy.\n- All other authenticated users see only sources belonging to their\n  own account, plus sources that have neither a user_id nor an\n  account_id (i.e. system/public sources).\n",
+        "description": "Returns the list of data sources accessible to the current user and\nthe defined source types.\n\nThe filter parameter searches the sources by name, by model and by id prefix,\nthe type parameter narrows the list to one source type, such as forecaster,\nand the limit parameter returns only the most recently created ones.\n\nThe types in the response are those of every source this listing can hold,\nrather than only those of the sources this call returns,\nso that one call can both search the sources and offer the types to search by.\n\nAccess rules:\n\nThis lists the sources which are the user's to work with, which is what a source has to be to be reused.\nThat is a stricter rule than the one for reading one source with GET /api/v3_0/sources/,\nwhich also covers a source that recorded data on a sensor the user may read.\n\n- Admins see all data sources.\n- Everyone else sees the sources of their own organisation, of any organisation they consult for,\n  and the sources an automation they may read computes under.\n- A source which belongs to no organisation is not listed for that reason alone.\n",
         "security": [
           {
             "ApiKeyAuth": []

From 424f09384e7ad66726e5a0f0136a0dcd632408a9 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" 
Date: Thu, 1 Oct 2026 12:20:17 +0200
Subject: [PATCH 20/23] Changelog and docs: which data sources are yours to
 read, and which to work with

Co-Authored-By: Claude Opus 5 
Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC
Signed-off-by: F.N. Claessen 
---
 documentation/api/change_log.rst       | 1 +
 documentation/changelog.rst            | 1 +
 documentation/features/automations.rst | 9 ++++++---
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index 25a698eb80..a4020055b6 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -11,6 +11,7 @@ v3.0-40 | September XX, 2026
 - ``POST /api/v3_0/accounts//annotations``, ``POST /api/v3_0/assets//annotations`` and ``POST /api/v3_0/sensors//annotations`` now accept an ``end`` equal to the ``start``, which makes an instantaneous annotation, where they used to reply with a ``422``. An ``end`` before the ``start`` is still refused.
 - ``POST /api/v3_0/assets//automations`` now accepts ``source``, the ID of an existing data source to reuse, instead of a ``data-generator`` and its ``config``. The source hands over the data generator it stores, and the automation records its results under that same source, so naming one is refused with a ``403`` unless the caller may read it, and with a ``422`` where it is combined with ``data-generator`` or ``config``, where it stores no data generator of the automation's type, or where the automation schedules (a schedule automation's source follows from the asset and the flex config).
 - ``GET /api/v3_0/sources`` now accepts ``filter``, matching search terms against a source's name, its model and its id prefix, ``type``, narrowing the list to one source type, and ``limit``, returning only that many sources, the most recently created ones first. The ``types`` in the response still cover every source the caller may read, so that one call can both search the sources and offer the types to search by.
+- ``GET /api/v3_0/sources`` and ``GET /api/v3_0/sources/`` now decide access per data source, rather than only by the organisation a source names. A source is readable when it belongs to an organisation the caller may read, when an automation they may read computes under it, or when it has recorded data on a sensor they may read. The listing holds only the first two, which are the sources the caller may reuse for an automation of their own, and ``attributes`` is returned only for those two as well, because a data generator's configuration names the sensors it runs on. A data source which names no organisation is no longer readable for that reason alone, where it used to be readable by every authenticated user.
 
 v3.0-39 | September 22, 2026
 """"""""""""""""""""""""""""
diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index a068ec8b4b..ae142f988d 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -33,6 +33,7 @@ New features
 
 Infrastructure / Support
 -------------------------
+* Who may read a data source is now decided per source, rather than only by the organisation the source names. A source is readable when it belongs to an organisation you may read, when an automation you may read computes under it, or when it has recorded data on a sensor you may read. A source which names no organisation, which is what most data generators' sources look like today, is no longer readable by everyone for that reason alone. The configuration a data generator stores on its source is handed over only for the first two cases, because such a configuration names the sensors it runs on [see `PR #2584 `_]
 * The endpoints supporting the UI moved from ``/api/dev`` to ``/api/ui``, where the old prefix keeps working until FlexMeasures v2 [see `PR #2578 `_]
 * Find the built-in schedulers, reporters and forecasters from an explicit list, instead of importing every module under ``flexmeasures.data.models`` at start-up to look for them. Shortens boot time for every process and keeps an unrelated broken module from stopping the app; plugins keep being discovered as before [see `PR #2566 `_]
 * ``MetaStorageScheduler``, an internal base class that was never meant to be selected, is no longer registered and can no longer be named as a custom scheduler; name ``StorageScheduler`` instead [see `PR #2566 `_]
diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst
index afeeb4e27d..d27f4df6bb 100644
--- a/documentation/features/automations.rst
+++ b/documentation/features/automations.rst
@@ -29,9 +29,12 @@ The forecaster and its configuration are stored on a data source, so you can als
 That data source is required while the automation exists, so it cannot be deleted until the automation is removed.
 
 The API takes the same choice as the ``source`` field of `[POST] /assets/(id)/automations <../api/v3_0.html#post--api-v3_0-assets-id-automations>`_, which cannot be combined with ``data-generator`` or ``config``.
-In the UI, the asset's *Automations* page searches the data sources you may read, by id, by name and by data generator class, and shows the configuration the picked one stores before you create the automation.
-A data source is only yours to name if it is yours to read.
-Reusing one means the results of both automations are recorded under the same source, so they share one lineage of data.
+In the UI, the asset's *Automations* page searches the data sources you may work with, by id, by name and by data generator class, and shows the configuration the picked one stores before you create the automation.
+A data source is yours to work with when it belongs to your organisation, or when an automation you may read already computes under it, which is how a source becomes yours before it has recorded anything.
+Those are the sources you may name, and the only ones whose stored configuration you are shown.
+Reading a data source is allowed more widely: one that has recorded data on a sensor you may read answers the question of what computed a number you can see.
+A source you reach only that way hands over no configuration, because a data generator's configuration names the sensors it runs on, which may be sensors you cannot see at all.
+Reusing a source means the results of both automations are recorded under it, so they share one lineage of data.
 A schedule automation cannot name a data source, because it works its own out on every run, as described under `Automating schedules`_.
 
 The recurrence is defined by a standard five-field cron string (minute, hour, day of month, month, and day of week), which defaults to ``"0 0 * * *"`` (daily at midnight).

From eb687f245227b57cd6c88ef52c5a2160d3f7f270 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" 
Date: Thu, 1 Oct 2026 12:24:47 +0200
Subject: [PATCH 21/23] Naming a source for an automation takes more than being
 allowed to read it

The listing a source is picked from holds the sources the user may work with,
while reading one source is allowed more widely, for a source which recorded data on a sensor they may read.
Creating the automation checked the wider of the two, so a source that had recorded on one of the user's own sensors
could be named for an automation of theirs, which records under that source and so writes into another organisation's lineage.

Co-Authored-By: Claude Opus 5 
Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC
Signed-off-by: F.N. Claessen 
---
 .../api/v3_0/tests/test_automations_api.py    | 53 +++++++++++++++++++
 flexmeasures/data/services/automations.py     |  8 +--
 2 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/api/v3_0/tests/test_automations_api.py b/flexmeasures/api/v3_0/tests/test_automations_api.py
index f9c133c061..b8526b800a 100644
--- a/flexmeasures/api/v3_0/tests/test_automations_api.py
+++ b/flexmeasures/api/v3_0/tests/test_automations_api.py
@@ -1832,3 +1832,56 @@ def test_post_automation_accepts_an_empty_source(
         select(Automation).filter_by(name="Forecasts without a source")
     ).scalar_one()
     assert automation.generator_id is not None
+
+
+@pytest.mark.parametrize(
+    "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True
+)
+def test_post_automation_refuses_a_source_one_may_only_read(
+    app,
+    fresh_db,
+    add_battery_assets_fresh_db,
+    setup_accounts_fresh_db,
+    requesting_user,
+):
+    """Having seen what a source computed on one's own sensor does not make it a source to record under."""
+    from flexmeasures.data.models.data_sources import SensorDataSource
+
+    battery = add_battery_assets_fresh_db["Test battery"]
+    sensor = battery.sensors[0]
+    foreign_source = DataSource(
+        name="foreign forecaster",
+        type="forecaster",
+        model="TrainPredictPipeline",
+        attributes={"data_generator": {"config": {}}},
+        account=setup_accounts_fresh_db["Dummy"],
+    )
+    fresh_db.session.add(foreign_source)
+    fresh_db.session.flush()
+    # The source has recorded on a sensor of the user's own asset, which makes it readable, but not theirs to work with.
+    fresh_db.session.add(
+        SensorDataSource(sensor_id=sensor.id, source_id=foreign_source.id)
+    )
+    fresh_db.session.flush()
+
+    with app.test_client() as client:
+        readable = client.get(url_for("SourceAPI:get", id=foreign_source.id))
+        response = client.post(
+            url_for("AssetAPI:post_automation", id=battery.id),
+            json={
+                "name": "Forecasts under a source only seen",
+                "cron": "0 6 * * *",
+                "type": "forecasting",
+                "source": foreign_source.id,
+                "parameters": {"sensor": sensor.id},
+            },
+        )
+    assert readable.status_code == 200
+    assert response.status_code == 403
+    assert "not yours to work with" in str(response.json)
+    assert (
+        fresh_db.session.execute(
+            select(Automation).filter_by(name="Forecasts under a source only seen")
+        ).scalar_one_or_none()
+        is None
+    )
diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py
index a85996eff1..8714406513 100644
--- a/flexmeasures/data/services/automations.py
+++ b/flexmeasures/data/services/automations.py
@@ -1887,13 +1887,13 @@ def _create_builtin_automation(
     from werkzeug.exceptions import Forbidden
 
     from flexmeasures.data.models.audit_log import AssetAuditLog
-    from flexmeasures.data.services.data_sources import user_may_read_source
+    from flexmeasures.data.services.data_sources import user_may_use_source
 
     # A named source hands over whatever configuration it stores, and the automation's results are recorded under it,
-    # so it is only the user's to name if it is theirs to read.
-    if check_permissions and source is not None and not user_may_read_source(source):
+    # so naming one takes more than being allowed to read what it computed: it has to be a source the user may work with.
+    if check_permissions and source is not None and not user_may_use_source(source):
         exception = Forbidden()
-        exception.api_message = f"You cannot define an automation on data source {source.id}, which you cannot read yourself."
+        exception.api_message = f"You cannot define an automation on data source {source.id}, which is not yours to work with."
         raise exception
 
     parameters = parameters or {}

From 926cca25c7da5cc876575e45a5af841c96504f3e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" 
Date: Thu, 1 Oct 2026 12:50:49 +0200
Subject: [PATCH 22/23] Address review: say which sources the types come from,
 and unwrap a comment

Co-Authored-By: Claude Opus 5 
Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC
Signed-off-by: F.N. Claessen 
---
 flexmeasures/api/v3_0/sources.py                        | 2 +-
 flexmeasures/ui/templates/assets/asset_automations.html | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/api/v3_0/sources.py b/flexmeasures/api/v3_0/sources.py
index 6ec7bbe162..6ea6e7d942 100644
--- a/flexmeasures/api/v3_0/sources.py
+++ b/flexmeasures/api/v3_0/sources.py
@@ -216,7 +216,7 @@ def index(
         serialized = [_serialize_source(s) for s in sources]
 
         # Collect any extra types present in the DB but not in the defaults.
-        # These are read from every source the user may read, rather than from the sources returned above,
+        # These are read from every source this listing can hold, rather than from the sources returned above,
         # so that narrowing the listing does not also narrow the types a client can offer to narrow it by.
         type_query = select(DataSource.type).distinct()
         if usable_sources is not None:
diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html
index 347ae543b9..26cb9b2a4f 100644
--- a/flexmeasures/ui/templates/assets/asset_automations.html
+++ b/flexmeasures/ui/templates/assets/asset_automations.html
@@ -1073,8 +1073,7 @@ 
    Recently created jobs
       if (reusesSource) {
         automationData.source = selectedAutomationSource.id;
       } else {
-        // Read from the gated values, so a schedule automation sends no generator
-        // left behind in the hidden fields by an earlier choice of type.
+        // Read from the gated values, so a schedule automation sends no generator left behind in the hidden fields by an earlier choice of type.
         automationData["data-generator"] = generator || null;
         automationData.config = config;
       }

From edec10f66a6ffad964eed703a4e106c03e33a7dd Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" 
Date: Sat, 3 Oct 2026 11:13:00 +0200
Subject: [PATCH 23/23] Changelog: one entry for the automations page's
 actions, not two

The merge of main brought in the entry PR #2563 had extended, beside this branch's own edit of the same line.
Neither conflicted with the other, so both survived: the section carried the same 900-character bullet twice,
differing only in whether it mentioned Copy and which pull requests it cited.

Co-Authored-By: Claude Opus 5 
Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC
Signed-off-by: F.N. Claessen 
---
 documentation/changelog.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index f1c600ea07..f8784aaaa1 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -94,8 +94,7 @@ Automations arrived over several pull requests. This is what each of them contri
 * Schedules as well as forecasts: a schedule automation stores what the schedule trigger endpoint accepts, and schedules from each run's own time [see `PR #2293 `_]
 * A scheduler's data source now also records the flex config the scheduler computed under, so a schedule can be traced back to the configuration that produced it, and a schedule automation points at such a data source, the way a forecast automation points at its forecaster's [see `PR #2464 `_]
 * A single automation can now be run on demand, from the CLI (``flexmeasures jobs run-automation``), the API (``POST /assets//automations//trigger``) and the asset's *Automations* page (a *Run now* button), which is useful to try out a new automation, to re-run one after fixing what made it fail, or to refresh its results after late input data arrived [see `PR #2460 `_]
-* Automations can be created, edited and deleted in the UI and through new API endpoints (``[POST|PATCH|DELETE] /assets/(id)/automations``), by whoever may add data under the asset, with their recurrence expressed in a selectable IANA timezone, and only involving sensors they can access themselves (read access to the sensors an automation reads, and permission to record data on the sensors it writes to). The *Automations* page shows when each active automation is due to run next, as a clock time in the automation's own timezone, so a recurrence no longer has to be read back from its cron string; the same time is available as ``next_run`` on the automations API endpoints, and is null while an automation is inactive. It is the next scheduled clock time, so it excludes catch-up work still pending, and it follows the dispatcher's daylight-saving rules, taking the first fold of a repeated local time and the first valid minute after a skipped one. Each row's *Run now*, *Edit*, *Activate*/*Deactivate* and *Delete* controls are collected into a single *Actions* menu, leaving *Details* beside it, so a long listing carries two controls per row instead of five [see `PR #2294 `_ and `PR #2563 `_]
-* Automations can be created, edited and deleted in the UI and through new API endpoints (``[POST|PATCH|DELETE] /assets/(id)/automations``), by whoever may add data under the asset, with their recurrence expressed in a selectable IANA timezone, and only involving sensors they can access themselves (read access to the sensors an automation reads, and permission to record data on the sensors it writes to). The *Automations* page shows when each active automation is due to run next, as a clock time in the automation's own timezone, so a recurrence no longer has to be read back from its cron string; the same time is available as ``next_run`` on the automations API endpoints, and is null while an automation is inactive. It is the next scheduled clock time, so it excludes catch-up work still pending, and it follows the dispatcher's daylight-saving rules, taking the first fold of a repeated local time and the first valid minute after a skipped one. Each row's *Run now*, *Edit*, *Copy*, *Activate*/*Deactivate* and *Delete* controls are collected into a single *Actions* menu, leaving *Details* beside it, so a long listing carries two controls per row instead of five. *Copy* opens the creation form filled in from that automation, and leaves the copy inactive [see `PR #2294 `_ and `PR #2591 `_]
+* Automations can be created, edited and deleted in the UI and through new API endpoints (``[POST|PATCH|DELETE] /assets/(id)/automations``), by whoever may add data under the asset, with their recurrence expressed in a selectable IANA timezone, and only involving sensors they can access themselves (read access to the sensors an automation reads, and permission to record data on the sensors it writes to). The *Automations* page shows when each active automation is due to run next, as a clock time in the automation's own timezone, so a recurrence no longer has to be read back from its cron string; the same time is available as ``next_run`` on the automations API endpoints, and is null while an automation is inactive. It is the next scheduled clock time, so it excludes catch-up work still pending, and it follows the dispatcher's daylight-saving rules, taking the first fold of a repeated local time and the first valid minute after a skipped one. Each row's *Run now*, *Edit*, *Copy*, *Activate*/*Deactivate* and *Delete* controls are collected into a single *Actions* menu, leaving *Details* beside it, so a long listing carries two controls per row instead of five. *Copy* opens the creation form filled in from that automation, and leaves the copy inactive [see `PR #2294 `_, `PR #2563 `_ and `PR #2591 `_]
 * An automation's output sensors are checked against its creator's permissions when the automation is created, and the schedules it computes are held to exactly those sensors: a scheduler that returns results for any other sensor is refused, rather than recording on a sensor that was only ever checked for read access [see `PR #2536 `_]
 * Reports as well as forecasts and schedules: a report automation stores report parameters, with its reporter and the reporter's configuration on a data source, and reports on a period resolved afresh on each run, either from ``start-offset`` and ``end-offset`` applied to the run time in the automation's timezone, or since the last successful report ended, while a fixed ``start`` or ``end`` is refused; a report job records only on the sensors the automation was checked against [see `PR #2297 `_]
 * Every automation times its runs the same way: a fixed ``start``, ``end`` or ``prior`` in its parameters is refused, as every run would share that moment, and two of ``start-offset``, ``end-offset`` and ``duration`` describe the period each run covers instead, with the offsets applied to the time the run was due on the automation's own clock, so that, for instance, a schedule automation can plan the whole of the next day [see `PR #2551 `_]