From 24d97439c18a981449db4c6d13c0f0bca9295c0b Mon Sep 17 00:00:00 2001 From: kparasch Date: Mon, 28 Sep 2026 17:27:21 +0200 Subject: [PATCH 01/10] template variation 2 --- examples/template/build_template_example.py | 5 +++ examples/template/templated_config.yaml | 29 +++++++++++++ pyaml/accelerator.py | 4 ++ pyaml/configuration/fileloader.py | 43 +++++++++++++++++++ pyaml/configuration/template.py | 47 +++++++++++++++++++++ 5 files changed, 128 insertions(+) create mode 100644 examples/template/build_template_example.py create mode 100644 examples/template/templated_config.yaml create mode 100644 pyaml/configuration/template.py diff --git a/examples/template/build_template_example.py b/examples/template/build_template_example.py new file mode 100644 index 00000000..b863f500 --- /dev/null +++ b/examples/template/build_template_example.py @@ -0,0 +1,5 @@ +from pyaml.accelerator import Accelerator + +sr = Accelerator.load("templated_config.yaml") +for dev in sr._devices: + print(dev) diff --git a/examples/template/templated_config.yaml b/examples/template/templated_config.yaml new file mode 100644 index 00000000..71be2cc3 --- /dev/null +++ b/examples/template/templated_config.yaml @@ -0,0 +1,29 @@ +class: pyaml.accelerator.Accelerator +machine: sr +facility: PETRAIII +energy: 6e9 +controls: + - class: tango.pyaml.controlsystem.TangoControlSystem + tango_host: ebs-simu-3:10000 + name: live + catalog: + class: tango.pyaml.tango_catalog.TangoCatalog + disconnected: True +devices: +- ${template:SHI,SHI,01} +- ${template:SHI,SHI,02} +- ${template:SHI,SHI,03} +templates: + - name: SHI + parameters: + - sh_name + - ii + config: + class: pyaml.magnet.hcorrector.HCorrector + name: "{sh_name}-{ii}" + model: + class: pyaml.magnet.linear_model.LinearMagnetModel + unit: rad + hardware_unit: str + calibration_factor: 1.0 + powerconverter: tango/MAGNET/{sh_name}-{ii}/B1L diff --git a/pyaml/accelerator.py b/pyaml/accelerator.py index 0bc27301..58e4da5b 100644 --- a/pyaml/accelerator.py +++ b/pyaml/accelerator.py @@ -392,6 +392,10 @@ def load(filename: str, include_locations: bool = False, ignore_external=False, """ manager = ConfigurationManager() + # ensure TemplateManager is clean before loading a new accelerator config + from .configuration.template import TemplateManager + + TemplateManager.clear() if not validate and include_locations: warnings.warn( diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index ef5f9f0e..05261258 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -293,6 +293,39 @@ def resolve_file(value: str, context: LoadContext | None = None) -> Any: return _load(value, context) +@resolver("template") +def resolve_template(value: str, _context: LoadContext | None = None) -> Any: + """ + Resolve a configuration by template. + + Parameters + ---------- + value : str + Name and arguments of the template. + Must be in the format: NAME,ARG1,ARG2,... + _context : LoadContext or None, optional + Unused loading context retained for resolver compatibility. + + Returns + ------- + object + Parsed and expanded configuration data. + + Raises + ------ + PyAMLException + If the environment variable is not set. + """ + try: + from .template import TemplateManager + + name, args = value.split(",", maxsplit=1) + arguments = args.split(",") + return TemplateManager.generate(name, *arguments) + except KeyError as exc: + raise PyAMLException(f"Invalid template resolver call {value}.") from exc + + def load(filename: str, include_locations: bool = False) -> Union[dict, list]: """ Load a configuration file. @@ -363,6 +396,16 @@ def expand(self, obj: Union[dict, list, Any]) -> Union[dict, list, Any]: """ if isinstance(obj, dict): + if "templates" in obj: + from .template import TemplateManager + + if "templates" in obj: + templates = obj.pop("templates") + for template in templates: + TemplateManager.add( + name=template["name"], parameters=template["parameters"], config=template["config"] + ) + return self._expand_dict(obj) if isinstance(obj, list): return self._expand_list(obj) diff --git a/pyaml/configuration/template.py b/pyaml/configuration/template.py new file mode 100644 index 00000000..1ca29aa3 --- /dev/null +++ b/pyaml/configuration/template.py @@ -0,0 +1,47 @@ +import yaml + +from ..common.element import Element +from ..common.exception import PyAMLConfigException + + +class TemplateManager: + template_names = [] + template_parameters = {} + template_codes = {} + + @classmethod + def add(cls, name: str, parameters: list[str], config: dict): + if name in cls.template_names: + raise PyAMLConfigException("Template '{name}' has already been registered.") + + cls.template_names.append(name) + cls.template_parameters[name] = parameters + cls.template_codes[name] = yaml.safe_dump(config, sort_keys=False) # serialize into yaml, retain order + + @classmethod + def generate(cls, name: str, *args): + number_of_parameters = len(cls.template_parameters[name]) + number_of_arguments = len(args) + + if number_of_parameters != number_of_arguments: + raise PyAMLConfigException( + f"Invalid number of arguments ({args}: {number_of_arguments}) passed to template {name}." + f" Expected {number_of_parameters}." + ) + + # name the arguments by position + arguments_dict = {} + for arg_name, arg_value in zip(cls.template_parameters[name], args, strict=True): + arguments_dict[arg_name] = arg_value + + # use standard str function format to replace. + new_code = cls.template_codes[name].format(**arguments_dict) + # code str must already be in yaml format + config = yaml.safe_load(new_code) + return config + + @classmethod + def clear(cls): + cls.template_names = [] + cls.template_parameters = {} + cls.template_codes = {} From 2144ec4d9b81a2551a70ea95841da2278b830e1c Mon Sep 17 00:00:00 2001 From: kparasch Date: Wed, 30 Sep 2026 14:06:02 +0200 Subject: [PATCH 02/10] no longer serializing into a dictionary. Instead walking through dictionary recursively and replacing text to avoid edge cases which produce invalid yaml or json strings. Also raise a warning if an argument containts {...} text --- pyaml/configuration/template.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/pyaml/configuration/template.py b/pyaml/configuration/template.py index 1ca29aa3..fc98093f 100644 --- a/pyaml/configuration/template.py +++ b/pyaml/configuration/template.py @@ -1,8 +1,24 @@ -import yaml +import copy +import logging +import re from ..common.element import Element from ..common.exception import PyAMLConfigException +logger = logging.getLogger(__name__) + + +def substitute(obj, arguments): + if isinstance(obj, dict): + return {key: substitute(value, arguments) for key, value in obj.items()} + if isinstance(obj, list): + return [substitute(value, arguments) for value in obj] + if isinstance(obj, str): + for name, value in arguments.items(): + obj = obj.replace("{" + name + "}", str(value)) + return obj + return obj + class TemplateManager: template_names = [] @@ -16,7 +32,7 @@ def add(cls, name: str, parameters: list[str], config: dict): cls.template_names.append(name) cls.template_parameters[name] = parameters - cls.template_codes[name] = yaml.safe_dump(config, sort_keys=False) # serialize into yaml, retain order + cls.template_codes[name] = config @classmethod def generate(cls, name: str, *args): @@ -32,12 +48,16 @@ def generate(cls, name: str, *args): # name the arguments by position arguments_dict = {} for arg_name, arg_value in zip(cls.template_parameters[name], args, strict=True): + # check if {...} is included in any of the arguments, and a raise a warning if so. + if re.search(r"\{[^{}]+\}", str(arg_value)): + logger.warning( + f"Argument {arg_name!r} for template {name!r} contains a placeholder: {arg_value!r}. " + "Sequential replacement may substitute placeholders inside this argument.", + ) arguments_dict[arg_name] = arg_value - # use standard str function format to replace. - new_code = cls.template_codes[name].format(**arguments_dict) - # code str must already be in yaml format - config = yaml.safe_load(new_code) + config = substitute(copy.deepcopy(cls.template_codes[name]), arguments_dict) + return config @classmethod From e9d1cf5b5223afc2a3878b8f772933030ac8c678 Mon Sep 17 00:00:00 2001 From: kparasch Date: Wed, 30 Sep 2026 15:01:06 +0200 Subject: [PATCH 03/10] Cleaner way of registering templates --- pyaml/configuration/fileloader.py | 40 ++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index 05261258..9e7b1965 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -15,7 +15,7 @@ from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Union +from typing import Any, Optional, Union import yaml from yaml import CLoader @@ -23,6 +23,7 @@ from yaml.loader import SafeLoader from .. import PyAMLException +from .template import TemplateManager logger = logging.getLogger(__name__) @@ -147,6 +148,8 @@ class LoadContext: include_locations: bool = False include_stack: list[Path] = field(default_factory=list) + expand: Optional[Callable] = None + # Is populated with ConfigLoader's expand function for use in template resolver @contextmanager def loading(self, path: Path): @@ -321,7 +324,8 @@ def resolve_template(value: str, _context: LoadContext | None = None) -> Any: name, args = value.split(",", maxsplit=1) arguments = args.split(",") - return TemplateManager.generate(name, *arguments) + generated = TemplateManager.generate(name, *arguments) + return _context.expand(generated) except KeyError as exc: raise PyAMLException(f"Invalid template resolver call {value}.") from exc @@ -353,6 +357,7 @@ def _load(filename: str, context: LoadContext) -> Union[dict, list]: else: raise PyAMLException(f"{filename} File format not supported (only .yaml .yml or .json)") + context.expand = loader.expand return loader.load() @@ -396,16 +401,6 @@ def expand(self, obj: Union[dict, list, Any]) -> Union[dict, list, Any]: """ if isinstance(obj, dict): - if "templates" in obj: - from .template import TemplateManager - - if "templates" in obj: - templates = obj.pop("templates") - for template in templates: - TemplateManager.add( - name=template["name"], parameters=template["parameters"], config=template["config"] - ) - return self._expand_dict(obj) if isinstance(obj, list): return self._expand_list(obj) @@ -558,6 +553,17 @@ def _expand_list(self, items: list) -> list: return expanded + def register_templates(self, config): + if not isinstance(config, dict): + return + + for template in config.pop("templates", []): + TemplateManager.add( + name=template["name"], + parameters=template["parameters"], + config=template["config"], + ) + @abstractmethod def load(self) -> Union[dict, list]: """Load and parse the current configuration file.""" @@ -593,7 +599,10 @@ def load(self) -> Union[dict, list]: logger.log(logging.DEBUG, f"Loading YAML file '{self.path}'") with open(self.path) as file: try: - return self.expand(yaml.load(file, Loader=self._loader)) + parsed_config = yaml.load(file, Loader=self._loader) + # "templates" is popped out here if it exists + self.register_templates(parsed_config) + return self.expand(parsed_config) except yaml.YAMLError as exc: raise PyAMLException(f"{self.path}: {exc}") from exc @@ -626,7 +635,10 @@ def load(self) -> Union[dict, list]: logger.log(logging.DEBUG, f"Loading JSON file '{self.path}'") with open(self.path) as file: try: - return self.expand(json.load(file)) + parsed_config = json.load(file) + # "templates" is popped out here if it exists + self.register_templates(parsed_config) + return self.expand(parsed_config) except json.JSONDecodeError as exc: raise PyAMLException(f"{self.path}: {exc}") from exc From 26669b9dbfacca22c0f45056797b12d923d79623 Mon Sep 17 00:00:00 2001 From: kparasch Date: Wed, 30 Sep 2026 15:04:57 +0200 Subject: [PATCH 04/10] Catch RecursionError in case it is caused by excessive recursive template usage. --- pyaml/configuration/fileloader.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index 9e7b1965..cd8b417c 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -297,7 +297,7 @@ def resolve_file(value: str, context: LoadContext | None = None) -> Any: @resolver("template") -def resolve_template(value: str, _context: LoadContext | None = None) -> Any: +def resolve_template(value: str, context: LoadContext | None = None) -> Any: """ Resolve a configuration by template. @@ -324,8 +324,14 @@ def resolve_template(value: str, _context: LoadContext | None = None) -> Any: name, args = value.split(",", maxsplit=1) arguments = args.split(",") - generated = TemplateManager.generate(name, *arguments) - return _context.expand(generated) + try: + generated = TemplateManager.generate(name, *arguments) + return context.expand(generated) + except RecursionError as exc: + raise PyAMLException( + f"Recursion limit reached while expanding template {name!r}. " + "Check for circular template references or excessive nesting." + ) from exc except KeyError as exc: raise PyAMLException(f"Invalid template resolver call {value}.") from exc From 0f7c1805e55bd25a8c51032404074d054feeb89d Mon Sep 17 00:00:00 2001 From: kparasch Date: Wed, 30 Sep 2026 15:31:01 +0200 Subject: [PATCH 05/10] Change to instance-based template managers that are attached to the configuration manager. --- pyaml/accelerator.py | 4 ---- pyaml/configuration/fileloader.py | 38 +++++++++++++++++++++---------- pyaml/configuration/manager.py | 10 +++++--- pyaml/configuration/template.py | 38 ++++++++++++++----------------- 4 files changed, 50 insertions(+), 40 deletions(-) diff --git a/pyaml/accelerator.py b/pyaml/accelerator.py index 58e4da5b..0bc27301 100644 --- a/pyaml/accelerator.py +++ b/pyaml/accelerator.py @@ -392,10 +392,6 @@ def load(filename: str, include_locations: bool = False, ignore_external=False, """ manager = ConfigurationManager() - # ensure TemplateManager is clean before loading a new accelerator config - from .configuration.template import TemplateManager - - TemplateManager.clear() if not validate and include_locations: warnings.warn( diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index cd8b417c..a79a9415 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -15,7 +15,7 @@ from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Optional, Union +from typing import Any, Union import yaml from yaml import CLoader @@ -139,6 +139,8 @@ class LoadContext: Preserve source locations in loaded mappings. include_stack : list[pathlib.Path], optional Active include chain. Usually left empty for a new session. + templates : TemplateManager, optional + Template registry shared by files in this session. Defaults to a fresh registry. Methods ------- @@ -148,8 +150,9 @@ class LoadContext: include_locations: bool = False include_stack: list[Path] = field(default_factory=list) - expand: Optional[Callable] = None + expand: Callable[[Any], Any] | None = None # Is populated with ConfigLoader's expand function for use in template resolver + templates: TemplateManager = field(default_factory=TemplateManager) @contextmanager def loading(self, path: Path): @@ -306,8 +309,8 @@ def resolve_template(value: str, context: LoadContext | None = None) -> Any: value : str Name and arguments of the template. Must be in the format: NAME,ARG1,ARG2,... - _context : LoadContext or None, optional - Unused loading context retained for resolver compatibility. + context : LoadContext or None, optional + Active loading context providing the template registry and expansion callback. Returns ------- @@ -317,15 +320,16 @@ def resolve_template(value: str, context: LoadContext | None = None) -> Any: Raises ------ PyAMLException - If the environment variable is not set. + If the template call is invalid or expansion exceeds the recursion limit. """ - try: - from .template import TemplateManager + if context is None or context.expand is None: + raise PyAMLException("Template resolver requires an active loading context.") + try: name, args = value.split(",", maxsplit=1) arguments = args.split(",") try: - generated = TemplateManager.generate(name, *arguments) + generated = context.templates.generate(name, *arguments) return context.expand(generated) except RecursionError as exc: raise PyAMLException( @@ -336,16 +340,22 @@ def resolve_template(value: str, context: LoadContext | None = None) -> Any: raise PyAMLException(f"Invalid template resolver call {value}.") from exc -def load(filename: str, include_locations: bool = False) -> Union[dict, list]: +def load(filename: str, include_locations: bool = False, *, templates: TemplateManager | None = None) -> Union[dict, list]: """ Load a configuration file. When include_locations is False, uses the faster C-based YAML loader and skips including source location metadata. + + ``templates`` may be supplied to share definitions across configuration + fragments. When omitted, this load gets its own registry. """ # Create a new context - context = LoadContext(include_locations=include_locations) + context = LoadContext( + include_locations=include_locations, + templates=templates if templates is not None else TemplateManager(), + ) return _load(filename, context) @@ -363,8 +373,12 @@ def _load(filename: str, context: LoadContext) -> Union[dict, list]: else: raise PyAMLException(f"{filename} File format not supported (only .yaml .yml or .json)") + previous_expand = context.expand context.expand = loader.expand - return loader.load() + try: + return loader.load() + finally: + context.expand = previous_expand def _is_supported_file(value: Any) -> bool: @@ -564,7 +578,7 @@ def register_templates(self, config): return for template in config.pop("templates", []): - TemplateManager.add( + self.context.templates.add( name=template["name"], parameters=template["parameters"], config=template["config"], diff --git a/pyaml/configuration/manager.py b/pyaml/configuration/manager.py index 68baf41c..b7626033 100644 --- a/pyaml/configuration/manager.py +++ b/pyaml/configuration/manager.py @@ -24,6 +24,7 @@ from ..common.exception import PyAMLConfigException from .fileloader import ROOT, load from .restfetcher import REMOTE_BASE_URL_KEY, SourceRoot, fetch_remote_config, is_remote_url, resolve_reference +from .template import TemplateManager _INTERNAL_METADATA_KEYS = {"__location__", "__fieldlocations__", REMOTE_BASE_URL_KEY} _VALID_QUERY_KEY_RE = re.compile(r"^[A-Z0-9_]+$") @@ -59,7 +60,7 @@ class ConfigurationManager: replace(category, element) Replace an existing named entry in an aggregated category. clear(category=None) - Clear the aggregated state, or a single root field/category. + Clear the aggregated state and templates, or a single root field/category. categories() Return categories that currently contain entries. keys(category=None) @@ -156,7 +157,8 @@ def __init__(self): Initialize an empty configuration manager. The manager starts with the default accelerator type and empty named - categories. Source tracking is enabled as fragments are added. + categories. Source tracking is enabled as fragments are added. Local + file fragments share this manager's private template registry. """ self._state: dict[str, Any] = {"class_path": self.DEFAULT_CLASS_PATH} self._items_by_category: dict[str, dict[str, dict[str, Any]]] = {category: {} for category in self.NAMED_CATEGORIES} @@ -164,6 +166,7 @@ def __init__(self): self._field_sources: dict[str, str] = {} self._build_root: SourceRoot = ROOT.get() self._build_root_locked = False + self._templates = TemplateManager() def add(self, payload, **kwargs) -> None: r""" @@ -306,6 +309,7 @@ def clear(self, category: str | None = None) -> None: >>> manager.clear() """ if category is None: + self._templates.clear() self._state = {"class_path": self.DEFAULT_CLASS_PATH} self._field_sources.clear() for name in self.NAMED_CATEGORIES: @@ -698,7 +702,7 @@ def _load_payload( source_root = resolved_path.parent try: ROOT.set(source_root) - fragment = load(resolved_path.name, include_locations) + fragment = load(resolved_path.name, include_locations, templates=self._templates) finally: ROOT.set(previous_root) diff --git a/pyaml/configuration/template.py b/pyaml/configuration/template.py index fc98093f..e5e01813 100644 --- a/pyaml/configuration/template.py +++ b/pyaml/configuration/template.py @@ -2,7 +2,6 @@ import logging import re -from ..common.element import Element from ..common.exception import PyAMLConfigException logger = logging.getLogger(__name__) @@ -21,22 +20,21 @@ def substitute(obj, arguments): class TemplateManager: - template_names = [] - template_parameters = {} - template_codes = {} + """Store template definitions for one configuration manager or loading session.""" - @classmethod - def add(cls, name: str, parameters: list[str], config: dict): - if name in cls.template_names: - raise PyAMLConfigException("Template '{name}' has already been registered.") + def __init__(self): + self.template_parameters: dict[str, list[str]] = {} + self.template_codes: dict[str, dict] = {} - cls.template_names.append(name) - cls.template_parameters[name] = parameters - cls.template_codes[name] = config + def add(self, name: str, parameters: list[str], config: dict): + if name in self.template_codes: + raise PyAMLConfigException(f"Template '{name}' has already been registered.") - @classmethod - def generate(cls, name: str, *args): - number_of_parameters = len(cls.template_parameters[name]) + self.template_parameters[name] = list(parameters) + self.template_codes[name] = copy.deepcopy(config) + + def generate(self, name: str, *args): + number_of_parameters = len(self.template_parameters[name]) number_of_arguments = len(args) if number_of_parameters != number_of_arguments: @@ -47,7 +45,7 @@ def generate(cls, name: str, *args): # name the arguments by position arguments_dict = {} - for arg_name, arg_value in zip(cls.template_parameters[name], args, strict=True): + for arg_name, arg_value in zip(self.template_parameters[name], args, strict=True): # check if {...} is included in any of the arguments, and a raise a warning if so. if re.search(r"\{[^{}]+\}", str(arg_value)): logger.warning( @@ -56,12 +54,10 @@ def generate(cls, name: str, *args): ) arguments_dict[arg_name] = arg_value - config = substitute(copy.deepcopy(cls.template_codes[name]), arguments_dict) + config = substitute(copy.deepcopy(self.template_codes[name]), arguments_dict) return config - @classmethod - def clear(cls): - cls.template_names = [] - cls.template_parameters = {} - cls.template_codes = {} + def clear(self): + self.template_parameters.clear() + self.template_codes.clear() From 957d2c2ffa6665773f1a24042d318d3dcb663b8b Mon Sep 17 00:00:00 2001 From: kparasch Date: Wed, 30 Sep 2026 15:41:49 +0200 Subject: [PATCH 06/10] add tests for templates --- tests/configuration/test_templates.py | 211 ++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 tests/configuration/test_templates.py diff --git a/tests/configuration/test_templates.py b/tests/configuration/test_templates.py new file mode 100644 index 00000000..acbf4714 --- /dev/null +++ b/tests/configuration/test_templates.py @@ -0,0 +1,211 @@ +import json + +import pytest +import yaml + +from pyaml import PyAMLException +from pyaml.accelerator import Accelerator +from pyaml.common.exception import PyAMLConfigException +from pyaml.configuration import ConfigurationManager +from pyaml.configuration.fileloader import FIELD_LOCATIONS_KEY, ROOT, load +from pyaml.configuration.template import TemplateManager + + +def _template_definition(prefix=""): + return { + "name": "device", + "parameters": ["name"], + "config": {"class": "pyaml.magnet.hcorrector.HCorrector", "name": prefix + "{name}"}, + } + + +def _write_config(path, config): + path.write_text(json.dumps(config) if path.suffix == ".json" else yaml.safe_dump(config, sort_keys=False)) + return path + + +@pytest.fixture +def template_config_root(tmp_path): + previous_root = ROOT.get() + ROOT.set(tmp_path) + try: + yield tmp_path + finally: + ROOT.set(previous_root) + + +def test_template_definitions_are_independent_of_inputs_and_results(): + registry = TemplateManager() + parameters = ["name"] + config = {"name": "{name}", "nested": [1]} + registry.add("T", parameters, config) + + parameters.clear() + config["nested"].append(2) + + generated = registry.generate("T", "Q1") + assert generated == {"name": "Q1", "nested": [1]} + + generated["nested"].append(3) + + assert registry.generate("T", "Q2") == {"name": "Q2", "nested": [1]} + + +def test_template_registry_rejects_duplicate_names(): + registry = TemplateManager() + registry.add("T", ["name"], {"name": "{name}"}) + + with pytest.raises(PyAMLConfigException, match="Template 'T'.*registered"): + registry.add("T", [], {}) + + +@pytest.mark.parametrize("suffix", [".yaml", ".json"]) +def test_standalone_loads_have_fresh_template_registries(tmp_path, suffix): + path = _write_config( + tmp_path / ("config" + suffix), + {"templates": [_template_definition()], "devices": ["${template:device,Q1}"]}, + ) + + first = load(str(path)) + second = load(str(path)) + + assert first["devices"][0]["name"] == "Q1" + assert second["devices"][0]["name"] == "Q1" + + use_only = _write_config(tmp_path / "use.yaml", {"devices": ["${template:device,Q1}"]}) + with pytest.raises(PyAMLException, match="Invalid template resolver call"): + load(str(use_only)) + + +def test_configuration_managers_have_independent_templates(tmp_path): + first_definitions = _write_config(tmp_path / "first.yaml", {"templates": [_template_definition("first-")]}) + second_definitions = _write_config(tmp_path / "second.json", {"templates": [_template_definition("second-")]}) + usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) + first = ConfigurationManager() + second = ConfigurationManager() + + first.add(first_definitions) + second.add(second_definitions) + first.add(usage) + second.add(usage) + + assert first.keys("devices") == ["first-Q1"] + assert second.keys("devices") == ["second-Q1"] + + first.clear() + second.clear("devices") + second.add(usage) + + assert second.keys("devices") == ["second-Q1"] + + +def test_manager_category_clear_preserves_templates(tmp_path): + definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) + usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) + manager = ConfigurationManager() + manager.add(definitions) + manager.add(usage) + + manager.clear("devices") + manager.add(usage) + + assert manager.keys("devices") == ["Q1"] + + +def test_manager_clear_resets_template_registration(tmp_path): + definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) + usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) + manager = ConfigurationManager() + manager.add(definitions) + + manager.clear() + + with pytest.raises(PyAMLException, match="Invalid template resolver call"): + manager.add(usage) + + manager.add(definitions) + manager.add(usage) + + assert manager.keys("devices") == ["Q1"] + + +def test_included_files_and_nested_templates_share_registry(template_config_root, monkeypatch): + tmp_path = template_config_root + monkeypatch.setenv("PYAML_TEST_HOST", "localhost") + _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) + _write_config(tmp_path / "Q1.json", {"factor": 1.5}) + path = _write_config( + tmp_path / "parent.yaml", + { + "templates": [ + { + "name": "wrapper", + "parameters": ["name"], + "config": { + "device": "${template:device,{name}}", + "model": "{name}.json", + "host": "${env:PYAML_TEST_HOST}", + }, + } + ], + "definitions": "definitions.yaml", + "result": "${template:wrapper,Q1}", + }, + ) + + loaded = load(str(path), include_locations=True) + result = loaded["result"] + + assert result["device"]["name"] == "Q1" + assert result["model"] == {"factor": 1.5} + assert result["host"] == "localhost" + assert "templates" in loaded[FIELD_LOCATIONS_KEY] + assert "name" in result["device"][FIELD_LOCATIONS_KEY] + + +def test_explicit_registry_can_be_shared_between_standalone_loads(tmp_path): + registry = TemplateManager() + definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) + usage = _write_config(tmp_path / "usage.json", {"devices": ["${template:device,Q1}"]}) + + load(str(definitions), templates=registry) + result = load(str(usage), templates=registry) + + assert result["devices"][0]["name"] == "Q1" + + +def test_recursive_template_still_reports_configuration_error(tmp_path): + path = _write_config( + tmp_path / "recursive.yaml", + { + "templates": [{"name": "T", "parameters": ["name"], "config": {"child": "${template:T,{name}}"}}], + "result": "${template:T,Q1}", + }, + ) + + with pytest.raises(PyAMLException, match="Recursion limit reached while expanding template"): + load(str(path)) + + +def test_accelerator_load_does_not_clear_an_existing_managers_templates(template_config_root): + tmp_path = template_config_root + definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) + manager = ConfigurationManager() + manager.add(definitions) + accelerator = _write_config( + tmp_path / "accelerator.yaml", + { + "class": "pyaml.accelerator.Accelerator", + "facility": "test", + "machine": "sr", + "energy": 3e9, + "templates": [_template_definition("other-")], + }, + ) + usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) + + for _ in range(2): + Accelerator.load(str(accelerator)) + manager.add(usage) + + assert manager.keys("devices") == ["Q1"] From da311724ec615c3c9ed224a2635aa7bc3f7daaec Mon Sep 17 00:00:00 2001 From: kparasch Date: Wed, 30 Sep 2026 16:15:03 +0200 Subject: [PATCH 07/10] Allow templates with zero arguments. --- pyaml/configuration/fileloader.py | 6 +++--- tests/configuration/test_templates.py | 29 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index a79a9415..ff96b43e 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -308,7 +308,7 @@ def resolve_template(value: str, context: LoadContext | None = None) -> Any: ---------- value : str Name and arguments of the template. - Must be in the format: NAME,ARG1,ARG2,... + Must be in the format: NAME,ARG1,ARG2,... or NAME for no arguments. context : LoadContext or None, optional Active loading context providing the template registry and expansion callback. @@ -326,8 +326,8 @@ def resolve_template(value: str, context: LoadContext | None = None) -> Any: raise PyAMLException("Template resolver requires an active loading context.") try: - name, args = value.split(",", maxsplit=1) - arguments = args.split(",") + name, separator, argument_text = value.partition(",") + arguments = argument_text.split(",") if separator else [] try: generated = context.templates.generate(name, *arguments) return context.expand(generated) diff --git a/tests/configuration/test_templates.py b/tests/configuration/test_templates.py index acbf4714..712040b7 100644 --- a/tests/configuration/test_templates.py +++ b/tests/configuration/test_templates.py @@ -174,6 +174,35 @@ def test_explicit_registry_can_be_shared_between_standalone_loads(tmp_path): assert result["devices"][0]["name"] == "Q1" +def test_template_calls_distinguish_no_arguments_from_an_empty_argument(tmp_path): + path = _write_config( + tmp_path / "arguments.yaml", + { + "templates": [ + {"name": "constant", "parameters": [], "config": {"name": "Q1"}}, + _template_definition(), + ], + "constant": "${template:constant}", + "empty": "${template:device,}", + }, + ) + + result = load(str(path)) + + assert result["constant"]["name"] == "Q1" + assert result["empty"]["name"] == "" + + +def test_template_call_without_required_arguments_reports_count_error(tmp_path): + path = _write_config( + tmp_path / "missing_argument.yaml", + {"templates": [_template_definition()], "result": "${template:device}"}, + ) + + with pytest.raises(PyAMLConfigException, match=r"Invalid number of arguments.*Expected 1"): + load(str(path)) + + def test_recursive_template_still_reports_configuration_error(tmp_path): path = _write_config( tmp_path / "recursive.yaml", From d4721f7fa29eac4ef2c094c3916f50d958b1f34d Mon Sep 17 00:00:00 2001 From: kparasch Date: Wed, 30 Sep 2026 16:46:18 +0200 Subject: [PATCH 08/10] docstring improvement --- examples/template/build_template_example.py | 2 + pyaml/configuration/fileloader.py | 78 +++++++++++++---- pyaml/configuration/manager.py | 6 +- pyaml/configuration/template.py | 93 ++++++++++++++++++++- 4 files changed, 160 insertions(+), 19 deletions(-) diff --git a/examples/template/build_template_example.py b/examples/template/build_template_example.py index b863f500..7572cb83 100644 --- a/examples/template/build_template_example.py +++ b/examples/template/build_template_example.py @@ -1,3 +1,5 @@ +"""Load and print templated devices; run from this example's directory.""" + from pyaml.accelerator import Accelerator sr = Accelerator.load("templated_config.yaml") diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index ff96b43e..75602e77 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -1,8 +1,9 @@ """ Load PyAML configuration files and expand nested references. -The loader supports YAML and JSON files, environment and path resolvers, -recursive file includes, and optional source-location metadata for diagnostics. +The loader supports YAML and JSON files, parameterized templates, environment +and path resolvers, recursive file includes, and optional source-location +metadata for diagnostics. """ import io @@ -139,6 +140,9 @@ class LoadContext: Preserve source locations in loaded mappings. include_stack : list[pathlib.Path], optional Active include chain. Usually left empty for a new session. + expand : callable or None, optional + Callback used to expand generated configurations. Installed temporarily + by the active file loader and restored when that loader finishes. templates : TemplateManager, optional Template registry shared by files in this session. Defaults to a fresh registry. @@ -302,15 +306,17 @@ def resolve_file(value: str, context: LoadContext | None = None) -> Any: @resolver("template") def resolve_template(value: str, context: LoadContext | None = None) -> Any: """ - Resolve a configuration by template. + Instantiate a template and recursively expand its configuration. Parameters ---------- value : str - Name and arguments of the template. - Must be in the format: NAME,ARG1,ARG2,... or NAME for no arguments. + Name and comma-separated string arguments in the format + ``NAME,ARG1,ARG2,...``, or ``NAME`` for no arguments. A trailing comma + supplies an empty-string argument. Commas in arguments are not escaped. context : LoadContext or None, optional - Active loading context providing the template registry and expansion callback. + Active loading context providing the template registry and expansion + callback. Required for resolution despite the compatibility default. Returns ------- @@ -320,7 +326,10 @@ def resolve_template(value: str, context: LoadContext | None = None) -> Any: Raises ------ PyAMLException - If the template call is invalid or expansion exceeds the recursion limit. + If the active context is missing, the template name is unknown, or + expansion exceeds the recursion limit. Expansion errors also propagate. + PyAMLConfigException + If the number of arguments does not match the template parameters. """ if context is None or context.expand is None: raise PyAMLException("Template resolver requires an active loading context.") @@ -342,13 +351,31 @@ def resolve_template(value: str, context: LoadContext | None = None) -> Any: def load(filename: str, include_locations: bool = False, *, templates: TemplateManager | None = None) -> Union[dict, list]: """ - Load a configuration file. + Load a configuration file, register templates, and expand references. - When include_locations is False, uses the faster C-based YAML loader - and skips including source location metadata. + Parameters + ---------- + filename : str + YAML or JSON filename. Relative paths are resolved against ``ROOT``. + include_locations : bool, optional + Preserve source-location metadata in YAML mappings. Defaults to False, + which uses the faster C-based YAML loader without location metadata. + templates : TemplateManager or None, optional + Registry to share across configuration fragments. If omitted, a fresh + registry is created. Included files share the same registry. - ``templates`` may be supplied to share definitions across configuration - fragments. When omitted, this load gets its own registry. + Returns + ------- + dict or list + Expanded configuration with root-level template definitions removed. + + Raises + ------ + PyAMLException + If the file format is unsupported, parsing fails, or reference + expansion fails. + PyAMLConfigException + If template registration or argument-count validation fails. """ # Create a new context @@ -401,6 +428,8 @@ class ConfigLoader(ABC): ------- expand(obj) Recursively expand configuration values. + register_templates(config) + Register and remove root-level template definitions before expansion. load() Load and parse the current configuration file. """ @@ -574,6 +603,23 @@ def _expand_list(self, items: list) -> list: return expanded def register_templates(self, config): + """ + Register root-level templates in the current loading context. + + Parameters + ---------- + config : object + Parsed configuration. For dictionaries, the ``templates`` section + is removed in place and its definitions are registered. Other + values are left unchanged. + + Raises + ------ + PyAMLConfigException + If a template name is already registered in this context's registry. + KeyError + If a definition lacks ``name``, ``parameters``, or ``config``. + """ if not isinstance(config, dict): return @@ -604,7 +650,7 @@ class YAMLLoader(ConfigLoader): Methods ------- load() - Parse the YAML file and expand nested configuration references. + Parse YAML, register root-level templates, and expand references. """ def __init__(self, path: Path, context: LoadContext): @@ -614,7 +660,7 @@ def __init__(self, path: Path, context: LoadContext): self._loader = SafeLineLoader if context.include_locations else CLoader def load(self) -> Union[dict, list]: - """Parse the YAML file and expand nested configuration references.""" + """Parse YAML, register root-level templates, and expand configuration references.""" logger.log(logging.DEBUG, f"Loading YAML file '{self.path}'") with open(self.path) as file: @@ -641,7 +687,7 @@ class JSONLoader(ConfigLoader): Methods ------- load() - Parse the JSON file and expand nested configuration references. + Parse JSON, register root-level templates, and expand references. """ def __init__(self, path: Path, context: LoadContext): @@ -650,7 +696,7 @@ def __init__(self, path: Path, context: LoadContext): super().__init__(path, context) def load(self) -> Union[dict, list]: - """Parse the JSON file and expand nested configuration references.""" + """Parse JSON, register root-level templates, and expand configuration references.""" logger.log(logging.DEBUG, f"Loading JSON file '{self.path}'") with open(self.path) as file: diff --git a/pyaml/configuration/manager.py b/pyaml/configuration/manager.py index b7626033..94998b94 100644 --- a/pyaml/configuration/manager.py +++ b/pyaml/configuration/manager.py @@ -290,12 +290,14 @@ def replace(self, category: str, element: dict) -> None: def clear(self, category: str | None = None) -> None: r""" - Clear the aggregated state, or a single root field/category. + Clear all configuration state and templates, or one root field/category. Parameters ---------- category : str, optional - If provided, only that category or root field is cleared. + If provided, only that category or root field is cleared and + template definitions are retained. If omitted, all aggregated + state and this manager's template registry are cleared. Examples -------- diff --git a/pyaml/configuration/template.py b/pyaml/configuration/template.py index e5e01813..d1bc6440 100644 --- a/pyaml/configuration/template.py +++ b/pyaml/configuration/template.py @@ -1,3 +1,11 @@ +""" +Store and instantiate parameterized configuration templates. + +Each registry owns its definitions. Generation substitutes positional arguments +into a copy of the template; the configuration loader expands any remaining +file, environment, or template references. +""" + import copy import logging import re @@ -8,6 +16,27 @@ def substitute(obj, arguments): + """ + Recursively substitute named placeholders in configuration values. + + Parameters + ---------- + obj : object + Configuration value to process. Dictionaries and lists are traversed; + dictionary keys and non-string scalar values are left unchanged. + arguments : dict[str, object] + Parameter names mapped to replacement values, converted to strings. + + Returns + ------- + object + Configuration with substitutions applied and new dictionaries and lists. + + Notes + ----- + Replacements use ``{name}`` placeholders and follow argument insertion order. + Text inserted by one replacement can be modified by a later replacement. + """ if isinstance(obj, dict): return {key: substitute(value, arguments) for key, value in obj.items()} if isinstance(obj, list): @@ -20,13 +49,45 @@ def substitute(obj, arguments): class TemplateManager: - """Store template definitions for one configuration manager or loading session.""" + """ + Store template definitions for one configuration manager or loading session. + + Definitions are isolated between instances and copied on registration and + generation so callers can modify their configurations independently. + + Methods + ------- + add(name, parameters, config) + Register a named template with ordered parameters. + generate(name, *args) + Substitute positional arguments into a copy of a template. + clear() + Remove all definitions from this registry. + """ def __init__(self): + """Initialize an empty template registry.""" self.template_parameters: dict[str, list[str]] = {} self.template_codes: dict[str, dict] = {} def add(self, name: str, parameters: list[str], config: dict): + """ + Register a template, copying its parameters and configuration. + + Parameters + ---------- + name : str + Template name, unique within this registry. + parameters : list[str] + Parameter names in the order expected by :meth:`generate`. + config : dict + Configuration body containing ``{parameter}`` placeholders. + + Raises + ------ + PyAMLConfigException + If the name is already registered. + """ if name in self.template_codes: raise PyAMLConfigException(f"Template '{name}' has already been registered.") @@ -34,6 +95,35 @@ def add(self, name: str, parameters: list[str], config: dict): self.template_codes[name] = copy.deepcopy(config) def generate(self, name: str, *args): + """ + Generate a configuration by substituting positional arguments. + + Parameters + ---------- + name : str + Name of a registered template. + *args : object + Values corresponding to the template's ordered parameters. + Values are converted to strings during substitution. + + Returns + ------- + dict + Independent configuration with placeholders replaced. Resolver + expressions and file references are left for the loader to expand. + + Raises + ------ + KeyError + If the template name is not registered. + PyAMLConfigException + If the number of arguments does not match the parameters. + + Notes + ----- + Arguments containing brace-delimited placeholders produce a logging + warning because sequential substitution may modify their contents. + """ number_of_parameters = len(self.template_parameters[name]) number_of_arguments = len(args) @@ -59,5 +149,6 @@ def generate(self, name: str, *args): return config def clear(self): + """Remove all definitions from this registry without affecting other instances.""" self.template_parameters.clear() self.template_codes.clear() From 702bda349634cf8a5bd863f9e57392579261d629 Mon Sep 17 00:00:00 2001 From: kparasch Date: Wed, 30 Sep 2026 16:47:25 +0200 Subject: [PATCH 09/10] fix typehints --- pyaml/configuration/template.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pyaml/configuration/template.py b/pyaml/configuration/template.py index d1bc6440..90ff3850 100644 --- a/pyaml/configuration/template.py +++ b/pyaml/configuration/template.py @@ -9,13 +9,15 @@ import copy import logging import re +from collections.abc import Mapping +from typing import Any from ..common.exception import PyAMLConfigException logger = logging.getLogger(__name__) -def substitute(obj, arguments): +def substitute(obj: Any, arguments: Mapping[str, object]) -> Any: """ Recursively substitute named placeholders in configuration values. @@ -24,7 +26,7 @@ def substitute(obj, arguments): obj : object Configuration value to process. Dictionaries and lists are traversed; dictionary keys and non-string scalar values are left unchanged. - arguments : dict[str, object] + arguments : Mapping[str, object] Parameter names mapped to replacement values, converted to strings. Returns @@ -65,12 +67,12 @@ class TemplateManager: Remove all definitions from this registry. """ - def __init__(self): + def __init__(self) -> None: """Initialize an empty template registry.""" self.template_parameters: dict[str, list[str]] = {} - self.template_codes: dict[str, dict] = {} + self.template_codes: dict[str, dict[str, Any]] = {} - def add(self, name: str, parameters: list[str], config: dict): + def add(self, name: str, parameters: list[str], config: dict[str, Any]) -> None: """ Register a template, copying its parameters and configuration. @@ -80,7 +82,7 @@ def add(self, name: str, parameters: list[str], config: dict): Template name, unique within this registry. parameters : list[str] Parameter names in the order expected by :meth:`generate`. - config : dict + config : dict[str, Any] Configuration body containing ``{parameter}`` placeholders. Raises @@ -94,7 +96,7 @@ def add(self, name: str, parameters: list[str], config: dict): self.template_parameters[name] = list(parameters) self.template_codes[name] = copy.deepcopy(config) - def generate(self, name: str, *args): + def generate(self, name: str, *args: object) -> dict[str, Any]: """ Generate a configuration by substituting positional arguments. @@ -108,7 +110,7 @@ def generate(self, name: str, *args): Returns ------- - dict + dict[str, Any] Independent configuration with placeholders replaced. Resolver expressions and file references are left for the loader to expand. @@ -134,7 +136,7 @@ def generate(self, name: str, *args): ) # name the arguments by position - arguments_dict = {} + arguments_dict: dict[str, object] = {} for arg_name, arg_value in zip(self.template_parameters[name], args, strict=True): # check if {...} is included in any of the arguments, and a raise a warning if so. if re.search(r"\{[^{}]+\}", str(arg_value)): @@ -148,7 +150,7 @@ def generate(self, name: str, *args): return config - def clear(self): + def clear(self) -> None: """Remove all definitions from this registry without affecting other instances.""" self.template_parameters.clear() self.template_codes.clear() From 4ef3b7876dab32129df8f3e66db6c57c8ca5992c Mon Sep 17 00:00:00 2001 From: kparasch Date: Wed, 30 Sep 2026 17:09:31 +0200 Subject: [PATCH 10/10] rename template_codes to template_configs --- pyaml/configuration/template.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyaml/configuration/template.py b/pyaml/configuration/template.py index 90ff3850..7cf5d2a9 100644 --- a/pyaml/configuration/template.py +++ b/pyaml/configuration/template.py @@ -70,7 +70,7 @@ class TemplateManager: def __init__(self) -> None: """Initialize an empty template registry.""" self.template_parameters: dict[str, list[str]] = {} - self.template_codes: dict[str, dict[str, Any]] = {} + self.template_configs: dict[str, dict[str, Any]] = {} def add(self, name: str, parameters: list[str], config: dict[str, Any]) -> None: """ @@ -90,11 +90,11 @@ def add(self, name: str, parameters: list[str], config: dict[str, Any]) -> None: PyAMLConfigException If the name is already registered. """ - if name in self.template_codes: + if name in self.template_configs: raise PyAMLConfigException(f"Template '{name}' has already been registered.") self.template_parameters[name] = list(parameters) - self.template_codes[name] = copy.deepcopy(config) + self.template_configs[name] = copy.deepcopy(config) def generate(self, name: str, *args: object) -> dict[str, Any]: """ @@ -146,11 +146,11 @@ def generate(self, name: str, *args: object) -> dict[str, Any]: ) arguments_dict[arg_name] = arg_value - config = substitute(copy.deepcopy(self.template_codes[name]), arguments_dict) + config = substitute(copy.deepcopy(self.template_configs[name]), arguments_dict) return config def clear(self) -> None: """Remove all definitions from this registry without affecting other instances.""" self.template_parameters.clear() - self.template_codes.clear() + self.template_configs.clear()