diff --git a/examples/template/build_template_example.py b/examples/template/build_template_example.py new file mode 100644 index 00000000..7572cb83 --- /dev/null +++ b/examples/template/build_template_example.py @@ -0,0 +1,7 @@ +"""Load and print templated devices; run from this example's directory.""" + +from pyaml.accelerator import Accelerator + +sr = Accelerator.load("templated_config.yaml") +for dev in sr._devices: + print(dev) diff --git a/examples/template/templated_config.yaml b/examples/template/templated_config.yaml new file mode 100644 index 00000000..71be2cc3 --- /dev/null +++ b/examples/template/templated_config.yaml @@ -0,0 +1,29 @@ +class: pyaml.accelerator.Accelerator +machine: sr +facility: PETRAIII +energy: 6e9 +controls: + - class: tango.pyaml.controlsystem.TangoControlSystem + tango_host: ebs-simu-3:10000 + name: live + catalog: + class: tango.pyaml.tango_catalog.TangoCatalog + disconnected: True +devices: +- ${template:SHI,SHI,01} +- ${template:SHI,SHI,02} +- ${template:SHI,SHI,03} +templates: + - name: SHI + parameters: + - sh_name + - ii + config: + class: pyaml.magnet.hcorrector.HCorrector + name: "{sh_name}-{ii}" + model: + class: pyaml.magnet.linear_model.LinearMagnetModel + unit: rad + hardware_unit: str + calibration_factor: 1.0 + powerconverter: tango/MAGNET/{sh_name}-{ii}/B1L diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index ef5f9f0e..75602e77 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -1,8 +1,9 @@ """ Load PyAML configuration files and expand nested references. -The loader supports YAML and JSON files, environment and path resolvers, -recursive file includes, and optional source-location metadata for diagnostics. +The loader supports YAML and JSON files, parameterized templates, environment +and path resolvers, recursive file includes, and optional source-location +metadata for diagnostics. """ import io @@ -23,6 +24,7 @@ from yaml.loader import SafeLoader from .. import PyAMLException +from .template import TemplateManager logger = logging.getLogger(__name__) @@ -138,6 +140,11 @@ class LoadContext: Preserve source locations in loaded mappings. include_stack : list[pathlib.Path], optional Active include chain. Usually left empty for a new session. + expand : callable or None, optional + Callback used to expand generated configurations. Installed temporarily + by the active file loader and restored when that loader finishes. + templates : TemplateManager, optional + Template registry shared by files in this session. Defaults to a fresh registry. Methods ------- @@ -147,6 +154,9 @@ class LoadContext: include_locations: bool = False include_stack: list[Path] = field(default_factory=list) + expand: Callable[[Any], Any] | None = None + # Is populated with ConfigLoader's expand function for use in template resolver + templates: TemplateManager = field(default_factory=TemplateManager) @contextmanager def loading(self, path: Path): @@ -293,16 +303,86 @@ def resolve_file(value: str, context: LoadContext | None = None) -> Any: return _load(value, context) -def load(filename: str, include_locations: bool = False) -> Union[dict, list]: +@resolver("template") +def resolve_template(value: str, context: LoadContext | None = None) -> Any: """ - Load a configuration file. + Instantiate a template and recursively expand its configuration. - When include_locations is False, uses the faster C-based YAML loader - and skips including source location metadata. + Parameters + ---------- + value : str + Name and comma-separated string arguments in the format + ``NAME,ARG1,ARG2,...``, or ``NAME`` for no arguments. A trailing comma + supplies an empty-string argument. Commas in arguments are not escaped. + context : LoadContext or None, optional + Active loading context providing the template registry and expansion + callback. Required for resolution despite the compatibility default. + + Returns + ------- + object + Parsed and expanded configuration data. + + Raises + ------ + PyAMLException + If the active context is missing, the template name is unknown, or + expansion exceeds the recursion limit. Expansion errors also propagate. + PyAMLConfigException + If the number of arguments does not match the template parameters. + """ + if context is None or context.expand is None: + raise PyAMLException("Template resolver requires an active loading context.") + + try: + name, separator, argument_text = value.partition(",") + arguments = argument_text.split(",") if separator else [] + try: + generated = context.templates.generate(name, *arguments) + return context.expand(generated) + except RecursionError as exc: + raise PyAMLException( + f"Recursion limit reached while expanding template {name!r}. " + "Check for circular template references or excessive nesting." + ) from exc + except KeyError as exc: + raise PyAMLException(f"Invalid template resolver call {value}.") from exc + + +def load(filename: str, include_locations: bool = False, *, templates: TemplateManager | None = None) -> Union[dict, list]: + """ + Load a configuration file, register templates, and expand references. + + Parameters + ---------- + filename : str + YAML or JSON filename. Relative paths are resolved against ``ROOT``. + include_locations : bool, optional + Preserve source-location metadata in YAML mappings. Defaults to False, + which uses the faster C-based YAML loader without location metadata. + templates : TemplateManager or None, optional + Registry to share across configuration fragments. If omitted, a fresh + registry is created. Included files share the same registry. + + Returns + ------- + dict or list + Expanded configuration with root-level template definitions removed. + + Raises + ------ + PyAMLException + If the file format is unsupported, parsing fails, or reference + expansion fails. + PyAMLConfigException + If template registration or argument-count validation fails. """ # Create a new context - context = LoadContext(include_locations=include_locations) + context = LoadContext( + include_locations=include_locations, + templates=templates if templates is not None else TemplateManager(), + ) return _load(filename, context) @@ -320,7 +400,12 @@ def _load(filename: str, context: LoadContext) -> Union[dict, list]: else: raise PyAMLException(f"{filename} File format not supported (only .yaml .yml or .json)") - return loader.load() + previous_expand = context.expand + context.expand = loader.expand + try: + return loader.load() + finally: + context.expand = previous_expand def _is_supported_file(value: Any) -> bool: @@ -343,6 +428,8 @@ class ConfigLoader(ABC): ------- expand(obj) Recursively expand configuration values. + register_templates(config) + Register and remove root-level template definitions before expansion. load() Load and parse the current configuration file. """ @@ -515,6 +602,34 @@ def _expand_list(self, items: list) -> list: return expanded + def register_templates(self, config): + """ + Register root-level templates in the current loading context. + + Parameters + ---------- + config : object + Parsed configuration. For dictionaries, the ``templates`` section + is removed in place and its definitions are registered. Other + values are left unchanged. + + Raises + ------ + PyAMLConfigException + If a template name is already registered in this context's registry. + KeyError + If a definition lacks ``name``, ``parameters``, or ``config``. + """ + if not isinstance(config, dict): + return + + for template in config.pop("templates", []): + self.context.templates.add( + name=template["name"], + parameters=template["parameters"], + config=template["config"], + ) + @abstractmethod def load(self) -> Union[dict, list]: """Load and parse the current configuration file.""" @@ -535,7 +650,7 @@ class YAMLLoader(ConfigLoader): Methods ------- load() - Parse the YAML file and expand nested configuration references. + Parse YAML, register root-level templates, and expand references. """ def __init__(self, path: Path, context: LoadContext): @@ -545,12 +660,15 @@ def __init__(self, path: Path, context: LoadContext): self._loader = SafeLineLoader if context.include_locations else CLoader def load(self) -> Union[dict, list]: - """Parse the YAML file and expand nested configuration references.""" + """Parse YAML, register root-level templates, and expand configuration references.""" logger.log(logging.DEBUG, f"Loading YAML file '{self.path}'") with open(self.path) as file: try: - return self.expand(yaml.load(file, Loader=self._loader)) + parsed_config = yaml.load(file, Loader=self._loader) + # "templates" is popped out here if it exists + self.register_templates(parsed_config) + return self.expand(parsed_config) except yaml.YAMLError as exc: raise PyAMLException(f"{self.path}: {exc}") from exc @@ -569,7 +687,7 @@ class JSONLoader(ConfigLoader): Methods ------- load() - Parse the JSON file and expand nested configuration references. + Parse JSON, register root-level templates, and expand references. """ def __init__(self, path: Path, context: LoadContext): @@ -578,12 +696,15 @@ def __init__(self, path: Path, context: LoadContext): super().__init__(path, context) def load(self) -> Union[dict, list]: - """Parse the JSON file and expand nested configuration references.""" + """Parse JSON, register root-level templates, and expand configuration references.""" logger.log(logging.DEBUG, f"Loading JSON file '{self.path}'") with open(self.path) as file: try: - return self.expand(json.load(file)) + parsed_config = json.load(file) + # "templates" is popped out here if it exists + self.register_templates(parsed_config) + return self.expand(parsed_config) except json.JSONDecodeError as exc: raise PyAMLException(f"{self.path}: {exc}") from exc diff --git a/pyaml/configuration/manager.py b/pyaml/configuration/manager.py index 68baf41c..94998b94 100644 --- a/pyaml/configuration/manager.py +++ b/pyaml/configuration/manager.py @@ -24,6 +24,7 @@ from ..common.exception import PyAMLConfigException from .fileloader import ROOT, load from .restfetcher import REMOTE_BASE_URL_KEY, SourceRoot, fetch_remote_config, is_remote_url, resolve_reference +from .template import TemplateManager _INTERNAL_METADATA_KEYS = {"__location__", "__fieldlocations__", REMOTE_BASE_URL_KEY} _VALID_QUERY_KEY_RE = re.compile(r"^[A-Z0-9_]+$") @@ -59,7 +60,7 @@ class ConfigurationManager: replace(category, element) Replace an existing named entry in an aggregated category. clear(category=None) - Clear the aggregated state, or a single root field/category. + Clear the aggregated state and templates, or a single root field/category. categories() Return categories that currently contain entries. keys(category=None) @@ -156,7 +157,8 @@ def __init__(self): Initialize an empty configuration manager. The manager starts with the default accelerator type and empty named - categories. Source tracking is enabled as fragments are added. + categories. Source tracking is enabled as fragments are added. Local + file fragments share this manager's private template registry. """ self._state: dict[str, Any] = {"class_path": self.DEFAULT_CLASS_PATH} self._items_by_category: dict[str, dict[str, dict[str, Any]]] = {category: {} for category in self.NAMED_CATEGORIES} @@ -164,6 +166,7 @@ def __init__(self): self._field_sources: dict[str, str] = {} self._build_root: SourceRoot = ROOT.get() self._build_root_locked = False + self._templates = TemplateManager() def add(self, payload, **kwargs) -> None: r""" @@ -287,12 +290,14 @@ def replace(self, category: str, element: dict) -> None: def clear(self, category: str | None = None) -> None: r""" - Clear the aggregated state, or a single root field/category. + Clear all configuration state and templates, or one root field/category. Parameters ---------- category : str, optional - If provided, only that category or root field is cleared. + If provided, only that category or root field is cleared and + template definitions are retained. If omitted, all aggregated + state and this manager's template registry are cleared. Examples -------- @@ -306,6 +311,7 @@ def clear(self, category: str | None = None) -> None: >>> manager.clear() """ if category is None: + self._templates.clear() self._state = {"class_path": self.DEFAULT_CLASS_PATH} self._field_sources.clear() for name in self.NAMED_CATEGORIES: @@ -698,7 +704,7 @@ def _load_payload( source_root = resolved_path.parent try: ROOT.set(source_root) - fragment = load(resolved_path.name, include_locations) + fragment = load(resolved_path.name, include_locations, templates=self._templates) finally: ROOT.set(previous_root) diff --git a/pyaml/configuration/template.py b/pyaml/configuration/template.py new file mode 100644 index 00000000..7cf5d2a9 --- /dev/null +++ b/pyaml/configuration/template.py @@ -0,0 +1,156 @@ +""" +Store and instantiate parameterized configuration templates. + +Each registry owns its definitions. Generation substitutes positional arguments +into a copy of the template; the configuration loader expands any remaining +file, environment, or template references. +""" + +import copy +import logging +import re +from collections.abc import Mapping +from typing import Any + +from ..common.exception import PyAMLConfigException + +logger = logging.getLogger(__name__) + + +def substitute(obj: Any, arguments: Mapping[str, object]) -> Any: + """ + Recursively substitute named placeholders in configuration values. + + Parameters + ---------- + obj : object + Configuration value to process. Dictionaries and lists are traversed; + dictionary keys and non-string scalar values are left unchanged. + arguments : Mapping[str, object] + Parameter names mapped to replacement values, converted to strings. + + Returns + ------- + object + Configuration with substitutions applied and new dictionaries and lists. + + Notes + ----- + Replacements use ``{name}`` placeholders and follow argument insertion order. + Text inserted by one replacement can be modified by a later replacement. + """ + if isinstance(obj, dict): + return {key: substitute(value, arguments) for key, value in obj.items()} + if isinstance(obj, list): + return [substitute(value, arguments) for value in obj] + if isinstance(obj, str): + for name, value in arguments.items(): + obj = obj.replace("{" + name + "}", str(value)) + return obj + return obj + + +class TemplateManager: + """ + Store template definitions for one configuration manager or loading session. + + Definitions are isolated between instances and copied on registration and + generation so callers can modify their configurations independently. + + Methods + ------- + add(name, parameters, config) + Register a named template with ordered parameters. + generate(name, *args) + Substitute positional arguments into a copy of a template. + clear() + Remove all definitions from this registry. + """ + + def __init__(self) -> None: + """Initialize an empty template registry.""" + self.template_parameters: dict[str, list[str]] = {} + self.template_configs: dict[str, dict[str, Any]] = {} + + def add(self, name: str, parameters: list[str], config: dict[str, Any]) -> None: + """ + Register a template, copying its parameters and configuration. + + Parameters + ---------- + name : str + Template name, unique within this registry. + parameters : list[str] + Parameter names in the order expected by :meth:`generate`. + config : dict[str, Any] + Configuration body containing ``{parameter}`` placeholders. + + Raises + ------ + PyAMLConfigException + If the name is already registered. + """ + if name in self.template_configs: + raise PyAMLConfigException(f"Template '{name}' has already been registered.") + + self.template_parameters[name] = list(parameters) + self.template_configs[name] = copy.deepcopy(config) + + def generate(self, name: str, *args: object) -> dict[str, Any]: + """ + Generate a configuration by substituting positional arguments. + + Parameters + ---------- + name : str + Name of a registered template. + *args : object + Values corresponding to the template's ordered parameters. + Values are converted to strings during substitution. + + Returns + ------- + dict[str, Any] + Independent configuration with placeholders replaced. Resolver + expressions and file references are left for the loader to expand. + + Raises + ------ + KeyError + If the template name is not registered. + PyAMLConfigException + If the number of arguments does not match the parameters. + + Notes + ----- + Arguments containing brace-delimited placeholders produce a logging + warning because sequential substitution may modify their contents. + """ + number_of_parameters = len(self.template_parameters[name]) + number_of_arguments = len(args) + + if number_of_parameters != number_of_arguments: + raise PyAMLConfigException( + f"Invalid number of arguments ({args}: {number_of_arguments}) passed to template {name}." + f" Expected {number_of_parameters}." + ) + + # name the arguments by position + arguments_dict: dict[str, object] = {} + for arg_name, arg_value in zip(self.template_parameters[name], args, strict=True): + # check if {...} is included in any of the arguments, and a raise a warning if so. + if re.search(r"\{[^{}]+\}", str(arg_value)): + logger.warning( + f"Argument {arg_name!r} for template {name!r} contains a placeholder: {arg_value!r}. " + "Sequential replacement may substitute placeholders inside this argument.", + ) + arguments_dict[arg_name] = arg_value + + config = substitute(copy.deepcopy(self.template_configs[name]), arguments_dict) + + return config + + def clear(self) -> None: + """Remove all definitions from this registry without affecting other instances.""" + self.template_parameters.clear() + self.template_configs.clear() diff --git a/tests/configuration/test_templates.py b/tests/configuration/test_templates.py new file mode 100644 index 00000000..712040b7 --- /dev/null +++ b/tests/configuration/test_templates.py @@ -0,0 +1,240 @@ +import json + +import pytest +import yaml + +from pyaml import PyAMLException +from pyaml.accelerator import Accelerator +from pyaml.common.exception import PyAMLConfigException +from pyaml.configuration import ConfigurationManager +from pyaml.configuration.fileloader import FIELD_LOCATIONS_KEY, ROOT, load +from pyaml.configuration.template import TemplateManager + + +def _template_definition(prefix=""): + return { + "name": "device", + "parameters": ["name"], + "config": {"class": "pyaml.magnet.hcorrector.HCorrector", "name": prefix + "{name}"}, + } + + +def _write_config(path, config): + path.write_text(json.dumps(config) if path.suffix == ".json" else yaml.safe_dump(config, sort_keys=False)) + return path + + +@pytest.fixture +def template_config_root(tmp_path): + previous_root = ROOT.get() + ROOT.set(tmp_path) + try: + yield tmp_path + finally: + ROOT.set(previous_root) + + +def test_template_definitions_are_independent_of_inputs_and_results(): + registry = TemplateManager() + parameters = ["name"] + config = {"name": "{name}", "nested": [1]} + registry.add("T", parameters, config) + + parameters.clear() + config["nested"].append(2) + + generated = registry.generate("T", "Q1") + assert generated == {"name": "Q1", "nested": [1]} + + generated["nested"].append(3) + + assert registry.generate("T", "Q2") == {"name": "Q2", "nested": [1]} + + +def test_template_registry_rejects_duplicate_names(): + registry = TemplateManager() + registry.add("T", ["name"], {"name": "{name}"}) + + with pytest.raises(PyAMLConfigException, match="Template 'T'.*registered"): + registry.add("T", [], {}) + + +@pytest.mark.parametrize("suffix", [".yaml", ".json"]) +def test_standalone_loads_have_fresh_template_registries(tmp_path, suffix): + path = _write_config( + tmp_path / ("config" + suffix), + {"templates": [_template_definition()], "devices": ["${template:device,Q1}"]}, + ) + + first = load(str(path)) + second = load(str(path)) + + assert first["devices"][0]["name"] == "Q1" + assert second["devices"][0]["name"] == "Q1" + + use_only = _write_config(tmp_path / "use.yaml", {"devices": ["${template:device,Q1}"]}) + with pytest.raises(PyAMLException, match="Invalid template resolver call"): + load(str(use_only)) + + +def test_configuration_managers_have_independent_templates(tmp_path): + first_definitions = _write_config(tmp_path / "first.yaml", {"templates": [_template_definition("first-")]}) + second_definitions = _write_config(tmp_path / "second.json", {"templates": [_template_definition("second-")]}) + usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) + first = ConfigurationManager() + second = ConfigurationManager() + + first.add(first_definitions) + second.add(second_definitions) + first.add(usage) + second.add(usage) + + assert first.keys("devices") == ["first-Q1"] + assert second.keys("devices") == ["second-Q1"] + + first.clear() + second.clear("devices") + second.add(usage) + + assert second.keys("devices") == ["second-Q1"] + + +def test_manager_category_clear_preserves_templates(tmp_path): + definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) + usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) + manager = ConfigurationManager() + manager.add(definitions) + manager.add(usage) + + manager.clear("devices") + manager.add(usage) + + assert manager.keys("devices") == ["Q1"] + + +def test_manager_clear_resets_template_registration(tmp_path): + definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) + usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) + manager = ConfigurationManager() + manager.add(definitions) + + manager.clear() + + with pytest.raises(PyAMLException, match="Invalid template resolver call"): + manager.add(usage) + + manager.add(definitions) + manager.add(usage) + + assert manager.keys("devices") == ["Q1"] + + +def test_included_files_and_nested_templates_share_registry(template_config_root, monkeypatch): + tmp_path = template_config_root + monkeypatch.setenv("PYAML_TEST_HOST", "localhost") + _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) + _write_config(tmp_path / "Q1.json", {"factor": 1.5}) + path = _write_config( + tmp_path / "parent.yaml", + { + "templates": [ + { + "name": "wrapper", + "parameters": ["name"], + "config": { + "device": "${template:device,{name}}", + "model": "{name}.json", + "host": "${env:PYAML_TEST_HOST}", + }, + } + ], + "definitions": "definitions.yaml", + "result": "${template:wrapper,Q1}", + }, + ) + + loaded = load(str(path), include_locations=True) + result = loaded["result"] + + assert result["device"]["name"] == "Q1" + assert result["model"] == {"factor": 1.5} + assert result["host"] == "localhost" + assert "templates" in loaded[FIELD_LOCATIONS_KEY] + assert "name" in result["device"][FIELD_LOCATIONS_KEY] + + +def test_explicit_registry_can_be_shared_between_standalone_loads(tmp_path): + registry = TemplateManager() + definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) + usage = _write_config(tmp_path / "usage.json", {"devices": ["${template:device,Q1}"]}) + + load(str(definitions), templates=registry) + result = load(str(usage), templates=registry) + + assert result["devices"][0]["name"] == "Q1" + + +def test_template_calls_distinguish_no_arguments_from_an_empty_argument(tmp_path): + path = _write_config( + tmp_path / "arguments.yaml", + { + "templates": [ + {"name": "constant", "parameters": [], "config": {"name": "Q1"}}, + _template_definition(), + ], + "constant": "${template:constant}", + "empty": "${template:device,}", + }, + ) + + result = load(str(path)) + + assert result["constant"]["name"] == "Q1" + assert result["empty"]["name"] == "" + + +def test_template_call_without_required_arguments_reports_count_error(tmp_path): + path = _write_config( + tmp_path / "missing_argument.yaml", + {"templates": [_template_definition()], "result": "${template:device}"}, + ) + + with pytest.raises(PyAMLConfigException, match=r"Invalid number of arguments.*Expected 1"): + load(str(path)) + + +def test_recursive_template_still_reports_configuration_error(tmp_path): + path = _write_config( + tmp_path / "recursive.yaml", + { + "templates": [{"name": "T", "parameters": ["name"], "config": {"child": "${template:T,{name}}"}}], + "result": "${template:T,Q1}", + }, + ) + + with pytest.raises(PyAMLException, match="Recursion limit reached while expanding template"): + load(str(path)) + + +def test_accelerator_load_does_not_clear_an_existing_managers_templates(template_config_root): + tmp_path = template_config_root + definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) + manager = ConfigurationManager() + manager.add(definitions) + accelerator = _write_config( + tmp_path / "accelerator.yaml", + { + "class": "pyaml.accelerator.Accelerator", + "facility": "test", + "machine": "sr", + "energy": 3e9, + "templates": [_template_definition("other-")], + }, + ) + usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) + + for _ in range(2): + Accelerator.load(str(accelerator)) + manager.add(usage) + + assert manager.keys("devices") == ["Q1"]