From cbf2d151a8436164012f6b801e2a7d971dbecfda Mon Sep 17 00:00:00 2001 From: Thomas Harvey Date: Sun, 1 Nov 2026 22:01:07 +0000 Subject: [PATCH 1/5] Update examples for current Synthesizer API - Import Greybody/Blackbody from synthesizer.emission_models (the emission_models.dust.emission module was removed). - Rename GalaxyBasis(build_grid=...) to build_library=... - Use emission_model.label as the emission model key where TotalEmission without dust emission relabels its root model. - Fix stale names (calculate_mwa, grid_folder, out_flux_unit='ABmag'). --- .../scripts/batch_library_generation.py | 4 ++-- .../scripts/final_library_generation.py | 4 ++-- .../scripts/final_library_generation_multinode.py | 14 +++++++------- .../library_generation/scripts/fixed_params.py | 12 +++++++----- .../scripts/generate_agn_library.py | 5 +++-- .../scripts/generate_library_basic.py | 3 +-- .../scripts/generate_library_full.py | 8 ++++---- .../scripts/generate_spectral_library.py | 10 +++++----- examples/library_generation/scripts/min_example.py | 4 ++-- examples/online/notebooks/online_learning.ipynb | 12 ++++++------ examples/online/scripts/online_learning.py | 4 ++-- 11 files changed, 41 insertions(+), 39 deletions(-) diff --git a/examples/library_generation/scripts/batch_library_generation.py b/examples/library_generation/scripts/batch_library_generation.py index 540a2a2..e88a3eb 100644 --- a/examples/library_generation/scripts/batch_library_generation.py +++ b/examples/library_generation/scripts/batch_library_generation.py @@ -10,7 +10,7 @@ ) # noqa mwn18 = MWN18() -from synthesizer.emission_models.dust.emission import Greybody, IR_templates # noqa +from synthesizer.emission_models import Greybody # noqa from synthesizer.emission_models.stellar.pacman_model import ( BimodalPacmanEmission, ) # noqa @@ -489,7 +489,7 @@ def db_sf_convert(param, param_dict, Nparam_SFH=3): alt_parametrizations=alt_parametrizations, redshift_dependent_sfh=True, params_to_ignore=sfh_params.get("params_to_ignore", []), - build_grid=False, + build_library=False, ) combined_basis = CombinedBasis( diff --git a/examples/library_generation/scripts/final_library_generation.py b/examples/library_generation/scripts/final_library_generation.py index 8aa374f..5fdbda3 100644 --- a/examples/library_generation/scripts/final_library_generation.py +++ b/examples/library_generation/scripts/final_library_generation.py @@ -5,10 +5,10 @@ import numpy as np from astropy.cosmology import Planck18 +from synthesizer.emission_models import Greybody # noqa from synthesizer.emission_models.attenuation import ( Calzetti2000, ) # noqa -from synthesizer.emission_models.dust.emission import Greybody, IR_templates # noqa from synthesizer.emission_models.stellar.pacman_model import ( PacmanEmission, ) # noqa @@ -499,7 +499,7 @@ def db_sf_convert(param, param_dict, Nparam_SFH=3): alt_parametrizations=alt_parametrizations, redshift_dependent_sfh=True, params_to_ignore=sfh_params.get("params_to_ignore", []), - build_grid=False, + build_library=False, log_stellar_masses=all_param_dict["log_masses"], ) diff --git a/examples/library_generation/scripts/final_library_generation_multinode.py b/examples/library_generation/scripts/final_library_generation_multinode.py index cc38c92..90c2d2c 100644 --- a/examples/library_generation/scripts/final_library_generation_multinode.py +++ b/examples/library_generation/scripts/final_library_generation_multinode.py @@ -7,10 +7,10 @@ import numpy as np from astropy.cosmology import Planck18 +from synthesizer.emission_models import Greybody # noqa from synthesizer.emission_models.attenuation import ( Calzetti2000, ) # noqa -from synthesizer.emission_models.dust.emission import Greybody, IR_templates # noqa from synthesizer.emission_models.stellar.pacman_model import ( PacmanEmission, ) # noqa @@ -33,21 +33,21 @@ from synference import ( GalaxyBasis, calculate_beta, + calculate_burstiness, calculate_colour, calculate_d4000, + calculate_line_ew, + calculate_line_flux, calculate_mass_weighted_age, calculate_muv, + calculate_Ndot_ion, calculate_sfh_quantile, calculate_surviving_mass, + calculate_xi_ion0, draw_from_hypercube, generate_constant_R, generate_random_DB_sfh, generate_sfh_basis, - calculate_line_ew, - calculate_line_flux, - calculate_xi_ion0, - calculate_burstiness, - calculate_Ndot_ion, ) # Filters @@ -584,7 +584,7 @@ def db_sf_convert(param, param_dict, Nparam_SFH=3): alt_parametrizations=alt_parametrizations, redshift_dependent_sfh=True, params_to_ignore=sfh_params.get("params_to_ignore", []), - build_grid=False, + build_library=False, log_stellar_masses=all_param_dict["log_masses"], ) diff --git a/examples/library_generation/scripts/fixed_params.py b/examples/library_generation/scripts/fixed_params.py index 83a7ed3..fa59514 100644 --- a/examples/library_generation/scripts/fixed_params.py +++ b/examples/library_generation/scripts/fixed_params.py @@ -15,8 +15,8 @@ from synference import ( GalaxyBasis, + calculate_mass_weighted_age, calculate_muv, - calculate_mwa, draw_from_hypercube, generate_constant_R, generate_sfh_basis, @@ -169,6 +169,8 @@ # Emission parameters +# With no dust emission model the root of this model is not labelled "total" +# (it is "attenuated" for fesc=0), so below we refer to it via emission_model.label. emission_model = TotalEmission( grid=grid, dust_curve=Calzetti2000(), @@ -206,7 +208,7 @@ def z_to_max_age(params, max_redshift=20): """Convert redshift to maximum age of the SFH at that redshift.""" z = params["redshift"] - from cosmology import Planck18 as cosmo + from astropy.cosmology import Planck18 as cosmo age = cosmo.age(z) - cosmo.age(max_redshift) age = age.to_value("Myr") * Myr @@ -222,7 +224,7 @@ def z_to_max_age(params, max_redshift=20): basis.plot_galaxy( i, log_stellar_mass=all_param_dict["log_masses"][i], - emission_model_keys=["total"], + emission_model_keys=[emission_model.label], out_dir=out_dir, ) @@ -234,11 +236,11 @@ def z_to_max_age(params, max_redshift=20): verbose=False, batch_size=50_000, mUV=(calculate_muv, cosmo), # Calculate mUV for the mock catalogue. - mwa=calculate_mwa, # Calculate MWA for the mock catalogue. + mwa=calculate_mass_weighted_age, # Calculate MWA for the mock catalogue. parameter_transforms_to_save={ "max_age": z_to_max_age }, # Save function to calculate the maximum age of the SFH at that redshift. - emission_model_key="total", + emission_model_key=emission_model.label, ) """ diff --git a/examples/library_generation/scripts/generate_agn_library.py b/examples/library_generation/scripts/generate_agn_library.py index bfdd954..983d031 100644 --- a/examples/library_generation/scripts/generate_agn_library.py +++ b/examples/library_generation/scripts/generate_agn_library.py @@ -5,13 +5,14 @@ from synthesizer import Grid from synthesizer.emission_models import ( AttenuatedEmission, + Blackbody, DustEmission, EmissionModel, + Greybody, PacmanEmission, UnifiedAGN, ) from synthesizer.emission_models.attenuation import PowerLaw -from synthesizer.emission_models.dust.emission import Blackbody, Greybody from synthesizer.instruments import FilterCollection, Instrument from synthesizer.parametric import SFH, ZDist from tqdm import tqdm @@ -264,7 +265,7 @@ def torus_vdisp_from_mbh(mbh): galaxy_params=galaxy_params, alt_parametrizations={}, redshift_dependent_sfh=True, - build_grid=False, + build_library=False, log_stellar_masses=all_param_dict["log_masses"], params_to_ignore=["max_age"], ) diff --git a/examples/library_generation/scripts/generate_library_basic.py b/examples/library_generation/scripts/generate_library_basic.py index 610d2ca..9836db5 100644 --- a/examples/library_generation/scripts/generate_library_basic.py +++ b/examples/library_generation/scripts/generate_library_basic.py @@ -4,11 +4,10 @@ import numpy as np from astropy.cosmology import Planck18 -from synthesizer.emission_models import PacmanEmission +from synthesizer.emission_models import Blackbody, PacmanEmission from synthesizer.emission_models.attenuation import ( Calzetti2000, ) # noqa -from synthesizer.emission_models.dust.emission import Blackbody from synthesizer.grid import Grid from synthesizer.instruments import FilterCollection, Instrument from synthesizer.parametric import SFH, ZDist diff --git a/examples/library_generation/scripts/generate_library_full.py b/examples/library_generation/scripts/generate_library_full.py index f4358d8..1740b07 100644 --- a/examples/library_generation/scripts/generate_library_full.py +++ b/examples/library_generation/scripts/generate_library_full.py @@ -4,10 +4,10 @@ import numpy as np from astropy.cosmology import Planck18 +from synthesizer.emission_models import Greybody # noqa from synthesizer.emission_models.attenuation import ( Calzetti2000, ) # noqa -from synthesizer.emission_models.dust.emission import Greybody, IR_templates # noqa from synthesizer.emission_models.stellar.pacman_model import ( BimodalPacmanEmission, ) # noqa @@ -20,8 +20,8 @@ from synference import ( CombinedBasis, GalaxyBasis, + calculate_mass_weighted_age, calculate_muv, - calculate_mwa, draw_from_hypercube, generate_constant_R, generate_random_DB_sfh, @@ -311,7 +311,7 @@ def db_sf_convert(param, param_dict): alt_parametrizations=alt_parametrizations, redshift_dependent_sfh=True, params_to_ignore=[], - build_grid=False, + build_library=False, ) @@ -339,7 +339,7 @@ def db_sf_convert(param, param_dict): combined_basis.process_bases( overwrite=True, mUV=(calculate_muv, cosmo), - mwa=calculate_mwa, + mwa=calculate_mass_weighted_age, n_proc=n_proc, verbose=False, batch_size=batch_size, diff --git a/examples/library_generation/scripts/generate_spectral_library.py b/examples/library_generation/scripts/generate_spectral_library.py index 9ea043e..197e58a 100644 --- a/examples/library_generation/scripts/generate_spectral_library.py +++ b/examples/library_generation/scripts/generate_spectral_library.py @@ -5,10 +5,10 @@ import numpy as np from astropy.cosmology import Planck18 +from synthesizer.emission_models import Greybody # noqa from synthesizer.emission_models.attenuation import ( Calzetti2000, ) # noqa -from synthesizer.emission_models.dust.emission import Greybody, IR_templates # noqa from synthesizer.emission_models.stellar.pacman_model import ( BimodalPacmanEmission, ) # noqa @@ -96,9 +96,9 @@ # path for this file dir_path = os.path.dirname(os.path.abspath(__file__)) -from synference import grid_folder +from synference import library_folder -out_dir = grid_folder +out_dir = library_folder def continuity_agebins( @@ -223,7 +223,7 @@ def continuity_agebins( sfh_name = str(sfh_type).split(".")[-1].split("'")[0] - name = f"BPASS_Chab_{sfh_name}_SFH_{redshift[0]}_z_{redshift[1]}_logN_{np.log10(Nmodels):.1f}_CF00_v2" # noqa: E501 + name = f"BPASS_Chab_{sfh_name}_SFH_z_{redshift}_logN_{np.log10(Nmodels):.1f}_CF00_v2" # noqa: E501 if os.path.exists(f"{out_dir}/v2/spectral_library_{name}.hdf5") and not overwrite: print(f"Grid {name} already exists, skipping.") continue @@ -414,7 +414,7 @@ def db_sf_convert(param, param_dict, Nparam_SFH=3): alt_parametrizations=alt_parametrizations, redshift_dependent_sfh=True, params_to_ignore=sfh_params.get("params_to_ignore", []), - build_grid=False, + build_library=False, ) combined_basis = CombinedBasis( diff --git a/examples/library_generation/scripts/min_example.py b/examples/library_generation/scripts/min_example.py index 2de85e6..b23f5c9 100644 --- a/examples/library_generation/scripts/min_example.py +++ b/examples/library_generation/scripts/min_example.py @@ -1,8 +1,8 @@ import os from astropy.cosmology import Planck18 +from synthesizer.emission_models import Greybody # noqa from synthesizer.emission_models.attenuation import Calzetti2000 # noqa -from synthesizer.emission_models.dust.emission import Greybody, IR_templates # noqa from synthesizer.emission_models.stellar.pacman_model import PacmanEmission # noqa from synthesizer.grid import Grid from synthesizer.instruments import FilterCollection, Instrument @@ -142,7 +142,7 @@ galaxy_params=galaxy_params, redshift_dependent_sfh=True, params_to_ignore=[], - build_grid=False, + build_library=False, ) # Create our mock training catalogue! diff --git a/examples/online/notebooks/online_learning.ipynb b/examples/online/notebooks/online_learning.ipynb index 99c192c..28a0808 100644 --- a/examples/online/notebooks/online_learning.ipynb +++ b/examples/online/notebooks/online_learning.ipynb @@ -151,12 +151,12 @@ " grid=grid,\n", " instrument=instrument,\n", " emission_model=emission_model,\n", - " emission_model_key=\"total\",\n", + " emission_model_key=emission_model.label,\n", " emitter_params=emitter_params,\n", " param_units={\"peak_age\": Myr, \"max_age\": Myr},\n", " normalize_method=None, # calculate_muv,\n", " output_type=\"photo_fnu\",\n", - " out_flux_unit=\"ABmag\",\n", + " out_flux_unit=\"AB\",\n", ")" ] }, @@ -429,12 +429,12 @@ " grid=grid,\n", " instrument=instrument,\n", " emission_model=emission_model,\n", - " emission_model_key=\"total\",\n", + " emission_model_key=emission_model.label,\n", " emitter_params=emitter_params,\n", " param_units={\"peak_age\": Myr, \"max_age\": Myr},\n", " normalize_method=None, # calculate_muv,\n", " output_type=\"photo_fnu\",\n", - " out_flux_unit=\"ABmag\",\n", + " out_flux_unit=\"AB\",\n", " fixed_params=fixed_params,\n", ")\n", "\n", @@ -530,12 +530,12 @@ " grid=grid,\n", " instrument=instrument,\n", " emission_model=emission_model,\n", - " emission_model_key=\"total\",\n", + " emission_model_key=emission_model.label,\n", " emitter_params=emitter_params,\n", " param_units={\"peak_age\": Myr, \"max_age\": Myr},\n", " normalize_method=None, # calculate_muv,\n", " output_type=\"photo_fnu\",\n", - " out_flux_unit=\"ABmag\",\n", + " out_flux_unit=\"AB\",\n", " fixed_params=fixed_params,\n", ")\n", "\n", diff --git a/examples/online/scripts/online_learning.py b/examples/online/scripts/online_learning.py index 799d23e..f4df35f 100644 --- a/examples/online/scripts/online_learning.py +++ b/examples/online/scripts/online_learning.py @@ -85,12 +85,12 @@ grid=grid, instrument=instrument, emission_model=emission_model, - emission_model_key="total", + emission_model_key=emission_model.label, # "emergent": no dust emission emitter_params=emitter_params, param_units={"peak_age": Myr, "max_age": Myr}, normalize_method=calculate_muv, output_type="photo_fnu", - out_flux_unit="ABmag", + out_flux_unit="AB", ) inputs = [ From fd6d1380a94ce7a8b4d77d97b2bcf57f74214922 Mon Sep 17 00:00:00 2001 From: Thomas Harvey Date: Sun, 1 Nov 2026 22:01:07 +0000 Subject: [PATCH 2/5] Add vectorised SFZH construction and tabulated IGM transmission fast_sfzh: make_parametric_stars builds parametric Stars from an SFH and ZDist using one vectorised SFR evaluation per galaxy instead of a scipy.quad call per age bin (~0.9 ms vs 12-112 ms per galaxy). SFZH agrees with the quad path to <=1e-3 L1 (median <=1e-8); unsupported inputs fall back to Synthesizer's own path. fast_igm: TabulatedIGM is a drop-in, picklable IGM that interpolates a precomputed transmission table in redshift (Inoue14 band errors <=1.3e-4 mag at dz=0.02), ~40-70x faster than recomputing per Sed. --- src/synference/fast_igm.py | 492 +++++++++++++++++++++++++ src/synference/fast_sfzh.py | 695 ++++++++++++++++++++++++++++++++++++ tests/test_fast_igm.py | 332 +++++++++++++++++ tests/test_fast_sfzh.py | 508 ++++++++++++++++++++++++++ 4 files changed, 2027 insertions(+) create mode 100644 src/synference/fast_igm.py create mode 100644 src/synference/fast_sfzh.py create mode 100644 tests/test_fast_igm.py create mode 100644 tests/test_fast_sfzh.py diff --git a/src/synference/fast_igm.py b/src/synference/fast_igm.py new file mode 100644 index 0000000..627c1c2 --- /dev/null +++ b/src/synference/fast_igm.py @@ -0,0 +1,492 @@ +"""Tabulated (pre-computed) IGM transmission for fast observed-frame SEDs. + +Synthesizer recomputes the IGM transmission (e.g. ``Inoue14``) from scratch +every time ``Sed.get_fnu`` / ``Line.get_flux`` is called. For Inoue14 on a +~4000 point wavelength grid this costs ~9-10 ms per call, which dominates the +cost of producing observed-frame spectra and photometry for large libraries. + +The transmission depends only on (redshift, wavelength), and for Inoue14 and +Madau96 it is exactly 1 redward of rest-frame Lyman-alpha (Inoue14: every +optical depth term is gated by ``lam_obs < lam_j (1 + z)`` with +``max(lam_j) = 1215.67 A``; Madau96: ``wl > 1216 (1 + z)`` returns 1). When a +spectrum with rest-frame wavelengths ``lam`` is observed at redshift ``z``, +``get_fnu`` passes ``obslam = lam * (1 + z)``, so ``obslam / (1 + z)`` recovers +the same rest-frame array for every redshift. :class:`TabulatedIGM` therefore +tabulates ``log(tau)`` on that rest-frame array (only the points blueward of +Lyman-alpha) on a redshift grid, and interpolates linearly in +``log(1 + z)``. At fixed rest wavelength every Inoue14 term is a power law +in ``(1 + z)``, so the interpolation is essentially exact except near the +kinks where a term switches power law (z = 1.2, 4.7 for the LAF, 2.0 for DLA, +in ``lam_obs / lam_j``); those set the error budget. + +Accuracy for Inoue14 on a 4000 point (50 A - 1e6 A, log) grid, z in [0, 20], +young BPASS SED, 25 JWST/NIRCam (wide+medium), HST/ACS, Subaru/HSC g-y and +CFHT u bands (bands with < 5 mag of IGM suppression): + + dz build [s] max|dT| max|dmag| + 0.01 6.4 1.5e-4 5e-5 + 0.02 3.2 3.9e-4 1.3e-4 (default) + 0.05 1.3 9.6e-4 6e-4 + 0.10 0.65 2.4e-3 2.3e-3 + +Evaluation costs ~0.07-0.13 ms vs ~5-10 ms for exact Inoue14. + +Usage (drop-in for the ``igm=`` argument): + + from synthesizer.emission_models.attenuation import Inoue14 + from synference.fast_igm import TabulatedIGM + + igm = TabulatedIGM(Inoue14, z_max=20, dz=0.02) + sed.get_fnu(cosmo, z, igm=igm) # drop-in replacement + pipeline.get_observed_spectra(cosmo, igm=igm) + pipeline.get_photometry_fluxes(instrument, igm=igm) + +Tables are built lazily the second time a given rest-frame wavelength array +is seen (the first call is computed exactly), or eagerly via ``lam=`` / +:meth:`TabulatedIGM.prepare`. Anything outside the table (redshift outside +``[z_min, z_max]``, unseen wavelength arrays, peculiar-velocity shifted +spectra) falls back to the exact base model, so results are always correct +to within the interpolation tolerance. +""" + +import hashlib +import os +import tempfile +import threading + +import numpy as np +from synthesizer.emission_models.transformers.igm import IGMBase, Inoue14, Madau96 +from unyt import angstrom + +__all__ = ["TabulatedIGM"] + +_TABLE_VERSION = 1 + + +class _Table: + """A transmission table for one rest-frame wavelength array.""" + + __slots__ = ("lam_rest", "blue", "n_blue", "logtau", "_atol") + + def __init__(self, lam_rest, blue, logtau): + self.lam_rest = lam_rest # full rest-frame wavelength array [A] + self.blue = blue # slice or index array of tabulated points + self.n_blue = logtau.shape[1] + self.logtau = logtau # (n_z, n_blue) log(tau) + self._atol = None # per-element match tolerance (set lazily) + + def matches(self, lam_rest, rtol): + """Check ``lam_rest`` equals this table's array within ``rtol``.""" + ref = self.lam_rest + if lam_rest.shape != ref.shape: + return False + if self._atol is None: + self._atol = rtol * np.abs(ref) + atol = self._atol + if abs(lam_rest[0] - ref[0]) > atol[0] or abs(lam_rest[-1] - ref[-1]) > atol[-1]: + return False + diff = np.subtract(lam_rest, ref) + np.abs(diff, out=diff) + return bool(np.all(diff <= atol)) + + def __getstate__(self): + return {s: getattr(self, s) for s in self.__slots__ if s != "_atol"} + + def __setstate__(self, state): + self._atol = None + for k, v in state.items(): + setattr(self, k, v) + + +def _to_angstrom(lam): + """Return a float64 ndarray in Angstrom (unitless input is assumed A).""" + units = getattr(lam, "units", None) + if units is None: + return np.asarray(lam, dtype=np.float64) + if units is angstrom or units == angstrom: # fast path, no copy + return np.asarray(lam.ndview, dtype=np.float64) + return np.asarray(lam.to_value(angstrom), dtype=np.float64) + + +class TabulatedIGM(IGMBase): + """Drop-in, tabulated replacement for a Synthesizer IGM model. + + Instances are *not* callable (``Sed.get_fnu`` would otherwise call + ``igm()``), expose ``get_transmission(redshift, lam_obs)`` with the same + semantics as the Synthesizer IGM classes (``lam_obs`` a unyt array or an + array in Angstrom; returns a dimensionless ndarray of the same shape), + and inherit the Transformer interface from ``IGMBase`` so they can also + be used as an ``igm=`` transformer in emission models. + + Args: + base (IGMBase class or instance): The exact IGM model to tabulate. + Defaults to ``Inoue14``. A class is instantiated with no args. + z_min (float): Lower edge of the redshift table. + z_max (float): Upper edge of the redshift table. + dz (float): Redshift spacing of the table. For Inoue14 on a 4000 + point grid (z in [0, 20]): dz=0.02 builds in ~3 s with max + |dT| ~4e-4 and band errors <~1.3e-4 mag; dz=0.05 builds in + ~1.3 s with max |dT| ~1e-3 and band errors <~6e-4 mag. + lam_rest_max (float): Rest-frame wavelength [A] redward of which the + transmission is assumed to be exactly 1 (checked at build time). + Use ``np.inf`` to tabulate every wavelength (e.g. for Asada25 + with a CGM damping wing). + lam (unyt_array/array, optional): Rest-frame wavelength array(s) to + tabulate immediately (e.g. ``grid.lam``). A list of arrays is + also accepted. + min_hits (int): A table is built for a new rest-frame wavelength + array the ``min_hits``-th time it is seen (earlier calls are + computed exactly). ``1`` builds on first sight. + max_tables (int): Maximum number of tables kept in memory (oldest + evicted first). + cache_dir (str, optional): If given, tables are saved to / loaded + from ``.npz`` files in this directory (written atomically), so + many processes / MPI ranks can share one build. + pickle_tables (bool): Whether pickling includes the built tables + (a few MB). If False, unpickled copies rebuild (or load from + ``cache_dir``) lazily. + dtype (np.dtype): Storage dtype of the ``log(tau)`` table. + rtol (float): Relative tolerance used to match a rest-frame + wavelength array against a stored table. + """ + + def __init__( + self, + base=Inoue14, + z_min=0.0, + z_max=20.0, + dz=0.02, + lam_rest_max=1216.0, + lam=None, + min_hits=2, + max_tables=8, + cache_dir=None, + pickle_tables=True, + dtype=np.float32, + rtol=1e-8, + ): + """Initialise the tabulated IGM (see class docstring).""" + if isinstance(base, type): + base = base() + if isinstance(base, TabulatedIGM): + base = base.base + if not hasattr(base, "get_transmission"): + raise TypeError("base must be an IGM model with get_transmission") + if z_max <= z_min or dz <= 0: + raise ValueError("Need z_max > z_min and dz > 0") + + IGMBase.__init__(self, f"Tabulated{base.name}") + + self.base = base + n_z = int(np.ceil((z_max - z_min) / dz - 1e-9)) + 1 + self.z_min = float(z_min) + self.dz = float(dz) + self.z_grid = self.z_min + self.dz * np.arange(n_z) + self.z_max = float(self.z_grid[-1]) + self._u_grid = np.log1p(self.z_grid) + self.lam_rest_max = float(lam_rest_max) + self.min_hits = int(min_hits) + self.max_tables = int(max_tables) + self.cache_dir = cache_dir + self.pickle_tables = bool(pickle_tables) + self.dtype = np.dtype(dtype) + self.rtol = float(rtol) + + self._tables = {} # n -> list[_Table] + self._order = [] # insertion order of tables (for eviction) + self._pending = [] # [lam_rest, hits] for not yet tabulated arrays + self._lock = threading.RLock() + + # Statistics (handy for checking the table is actually used) + self.n_table_calls = 0 + self.n_exact_calls = 0 + + if lam is not None: + lams = lam if isinstance(lam, (list, tuple)) else [lam] + for this_lam in lams: + self.prepare(this_lam) + + def __repr__(self): + """Return a string representation.""" + return ( + f"TabulatedIGM(base={self.base!r}, z_min={self.z_min}, " + f"z_max={self.z_max}, dz={self.dz}, n_tables={len(self._order)})" + ) + + # ------------------------------------------------------------------ + # Pickling + # ------------------------------------------------------------------ + def __getstate__(self): + """Drop the lock (and optionally the tables) when pickling.""" + state = self.__dict__.copy() + state.pop("_lock", None) + state["_pending"] = [] + if not self.pickle_tables: + state["_tables"] = {} + state["_order"] = [] + return state + + def __setstate__(self, state): + """Restore state and recreate the lock.""" + self.__dict__.update(state) + self._lock = threading.RLock() + + # ------------------------------------------------------------------ + # Table management + # ------------------------------------------------------------------ + @property + def tables(self): + """List of the rest-frame wavelength arrays currently tabulated.""" + return [t.lam_rest for t in self._order] + + def clear(self): + """Remove all in-memory tables.""" + with self._lock: + self._tables = {} + self._order = [] + self._pending = [] + + def _matches(self, a, b): + """Check two rest-frame arrays agree within ``rtol``.""" + if a.shape != b.shape: + return False + tol = self.rtol + if abs(a[0] - b[0]) > tol * abs(b[0]) or abs(a[-1] - b[-1]) > tol * abs(b[-1]): + return False + return bool(np.all(np.abs(a - b) <= tol * np.abs(b))) + + def _find(self, lam_rest): + for tab in self._tables.get(lam_rest.shape[0], ()): + if tab.matches(lam_rest, self.rtol): + return tab + return None + + def _cache_path(self, lam_rest): + h = hashlib.sha1() + h.update( + repr( + ( + _TABLE_VERSION, + type(self.base).__module__, + type(self.base).__qualname__, + repr(self.base), + getattr(self.base, "scale_tau", None), + self.z_min, + self.dz, + len(self.z_grid), + self.lam_rest_max, + self.dtype.str, + ) + ).encode() + ) + h.update(np.ascontiguousarray(lam_rest, dtype=np.float64).tobytes()) + return os.path.join(self.cache_dir, f"tabigm_{h.hexdigest()[:20]}.npz") + + def _insert(self, tab): + with self._lock: + if self._find(tab.lam_rest) is not None: + return + self._tables.setdefault(tab.lam_rest.shape[0], []).append(tab) + self._order.append(tab) + while len(self._order) > self.max_tables: + old = self._order.pop(0) + self._tables[old.lam_rest.shape[0]].remove(old) + + def prepare(self, lam_rest): + """Build (or load from ``cache_dir``) the table for a wavelength array. + + Args: + lam_rest (unyt_array/array): Rest-frame wavelengths (Angstrom if + unitless), e.g. ``grid.lam`` or ``sed.lam``. + + Returns: + TabulatedIGM: self (for chaining). + """ + lam_rest = np.array(_to_angstrom(lam_rest), dtype=np.float64).ravel() + if self._find(lam_rest) is None: + self._insert(self._load_or_build(lam_rest)) + return self + + def _load_or_build(self, lam_rest): + path = None + if self.cache_dir is not None: + path = self._cache_path(lam_rest) + if os.path.exists(path): + try: + with np.load(path) as f: + stored = f["lam_rest"] + if self._matches(lam_rest, stored): + blue = self._blue_index(lam_rest) + logtau = f["logtau"].astype(self.dtype) + if logtau.shape == (len(self.z_grid), self._count(blue)): + return _Table(lam_rest, blue, logtau) + except Exception: # corrupt / partially written file: rebuild + pass + tab = self._build(lam_rest) + if path is not None: + os.makedirs(self.cache_dir, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=self.cache_dir, suffix=".npz.tmp") + try: + with os.fdopen(fd, "wb") as fh: + np.savez(fh, lam_rest=lam_rest, logtau=tab.logtau) + os.replace(tmp, path) + except Exception: + if os.path.exists(tmp): + os.remove(tmp) + raise + return tab + + def _blue_index(self, lam_rest): + mask = lam_rest < self.lam_rest_max + idx = np.flatnonzero(mask) + if idx.size == 0: + return slice(0, 0) + if idx[-1] - idx[0] + 1 == idx.size: # contiguous (e.g. sorted grid) + return slice(int(idx[0]), int(idx[-1]) + 1) + return idx + + @staticmethod + def _count(blue): + if isinstance(blue, slice): + return blue.stop - blue.start + return blue.size + + def _build(self, lam_rest): + """Tabulate log(tau) for the blue part of ``lam_rest``.""" + blue = self._blue_index(lam_rest) + lam_blue = lam_rest[blue] + n_z = len(self.z_grid) + logtau = np.empty((n_z, lam_blue.size), dtype=self.dtype) + + # Check the "transmission == 1 redward of the cut" assumption + red = np.ones(lam_rest.size, dtype=bool) + red[blue] = False + if red.any(): + lam_red = lam_rest[red] + for z in np.unique([self.z_grid[0], self.z_grid[n_z // 2], self.z_grid[-1]]): + if z <= 0: + z = self.z_grid[1] + t = np.asarray(self.base.get_transmission(z, lam_red * (1.0 + z) * angstrom)) + if np.any(np.abs(t - 1.0) > 1e-12): + raise ValueError( + f"{self.base!r} transmission is not 1 redward of " + f"lam_rest_max={self.lam_rest_max} A (z={z}); " + "increase lam_rest_max (np.inf tabulates everything)." + ) + + if lam_blue.size: + for i, z in enumerate(self.z_grid): + t = np.asarray( + self.base.get_transmission(float(z), lam_blue * (1.0 + z) * angstrom), + dtype=np.float64, + ) + tau = -np.log(np.clip(t, 1e-300, 1.0)) + logtau[i] = np.log(np.maximum(tau, 1e-30)) + if isinstance(self.base, Madau96): + self._continue_madau_cut(lam_blue, logtau) + return _Table(lam_rest, blue, logtau) + + def _continue_madau_cut(self, lam_blue, logtau): + """Smoothly continue log(tau) into Madau96's hard-zero region. + + Madau96 (as implemented in Synthesizer) sets T = 0 for + ``lam_obs <= 950 (1 + z) - 1500`` even where the analytic + transmission is ~0.1-0.6, so T is discontinuous in z at fixed rest + wavelength. Interpolating across that edge would pull T to ~0 just + above it; instead the table is linearly extrapolated (in log tau vs + log(1 + z)) past the edge, and the exact cut is reapplied when + evaluating. + """ + u = self._u_grid + cut = lam_blue[None, :] <= (self.base.lams[-1] - 1500.0 / (1.0 + self.z_grid[:, None])) + n_z = len(u) + first = np.where(cut.any(axis=0), cut.argmax(axis=0), n_z) + for j in np.flatnonzero(first < n_z): + k = first[j] + if k >= 2: + slope = (logtau[k - 1, j] - logtau[k - 2, j]) / (u[k - 1] - u[k - 2]) + logtau[k:, j] = logtau[k - 1, j] + slope * (u[k:] - u[k - 1]) + + # ------------------------------------------------------------------ + # Transmission + # ------------------------------------------------------------------ + def _exact(self, redshift, lam_obs): + self.n_exact_calls += 1 + if not hasattr(lam_obs, "units"): + lam_obs = np.asarray(lam_obs, dtype=np.float64) * angstrom + shape = lam_obs.shape + out = self.base.get_transmission(redshift, lam_obs.reshape(-1)) + return np.asarray(out).reshape(shape) + + def _lookup(self, lam_rest): + """Find the table for ``lam_rest``, building it on the min_hits-th hit.""" + tab = self._find(lam_rest) + if tab is not None: + return tab + with self._lock: + tab = self._find(lam_rest) + if tab is not None: + return tab + for entry in self._pending: + if self._matches(lam_rest, entry[0]): + entry[1] += 1 + break + else: + entry = [lam_rest.copy(), 1] + self._pending.append(entry) + if len(self._pending) > 4 * self.max_tables: + self._pending.pop(0) + if entry[1] < self.min_hits: + return None + self._pending.remove(entry) + tab = self._load_or_build(entry[0]) + self._insert(tab) + return tab + + def get_transmission(self, redshift, lam_obs): + """Compute the IGM transmission. + + Args: + redshift (float): Redshift at which to evaluate the IGM. + lam_obs (unyt_array/array): Observed-frame wavelengths (Angstrom + if unitless), any shape. + + Returns: + np.ndarray: The transmission (same shape as ``lam_obs``). + """ + z = float(redshift) + if not (self.z_min <= z <= self.z_max): + return self._exact(redshift, lam_obs) + + lam = _to_angstrom(lam_obs) + shape = lam.shape + lam = lam.ravel() + if lam.size == 0: + return np.ones(shape) + + lam_rest = lam / (1.0 + z) + tab = self._lookup(lam_rest) + if tab is None: + return self._exact(redshift, lam_obs) + + self.n_table_calls += 1 + out = np.ones(lam.size, dtype=np.float64) + if tab.n_blue: + # Linear interpolation of log(tau) in log(1 + z) + i = int((z - self.z_min) / self.dz) + i = min(max(i, 0), len(self.z_grid) - 2) + u = np.log1p(z) + w = (u - self._u_grid[i]) / (self._u_grid[i + 1] - self._u_grid[i]) + lt = tab.logtau[i].astype(np.float64) + lt *= 1.0 - w + lt += w * tab.logtau[i + 1] + np.exp(lt, out=lt) + np.negative(lt, out=lt) + np.exp(lt, out=lt) + out[tab.blue] = lt + if isinstance(self.base, Madau96): + # Madau96 has a hard, z-dependent cut to zero transmission + # (lam_obs <= 950 (1 + z) - 1500 A) that interpolation in z + # would smear; reapply it exactly. + out[lam <= self.base.lams[-1] * (1.0 + z) - 1500.0] = 0.0 + return out.reshape(shape) diff --git a/src/synference/fast_sfzh.py b/src/synference/fast_sfzh.py new file mode 100644 index 0000000..6d12750 --- /dev/null +++ b/src/synference/fast_sfzh.py @@ -0,0 +1,695 @@ +"""Fast, vectorised construction of parametric SFZH grids. + +Building a :class:`synthesizer.parametric.Stars` object from an SFH function +and a metallicity distribution is dominated by ``Stars._get_sfzh``, which calls +:func:`scipy.integrate.quad` once per age bin (and once per metallicity bin for +non-delta metallicity distributions). For the 1e5-1e7 galaxies Synference +builds this is the bottleneck. + +This module reproduces exactly the same binning as Synthesizer but computes the +mass in each age bin with a single vectorised SFR evaluation: + +* The integration domain ``[0, ages[-1]]`` (shifted by ``age_offset``) is split + at every Synthesizer bin edge, at a cached set of log-spaced sub-nodes, and at + SFH specific nodes (``min_age``/``max_age`` truncations, internal step + breakpoints of bin based SFHs, interpolation knots of tabulated SFHs and + geometrically refined nodes around sharp features such as the peak of a + narrow log-normal or a short e-folding time close to ``max_age``). +* Every sub-interval is integrated with an ``n``-point Gauss-Legendre rule. + Gauss-Legendre nodes are strictly interior so step discontinuities placed at + interval boundaries are integrated exactly, piecewise constant / linear SFHs + are integrated exactly by the 1-point (midpoint) rule and smooth SFHs to + ~1e-8 relative accuracy. +* Sub-interval integrals are summed per age bin with ``np.add.reduceat``. + +Metallicity weights reproduce Synthesizer bit-for-bit for ``DeltaConstant`` (and +float metallicities) - the same cloud-in-cell assignment as the C extension - +and use the analytic error function integral for ``ZDist.Normal`` (agreeing with +Synthesizer's ``quad`` to ~1e-10). Unknown ZDist classes use the same ``quad`` +loop as Synthesizer. + +Anything not explicitly supported (instantaneous float SFHs, array SFHs, +unknown/user defined SFH classes, SFH subclasses not registered here, divergent +integrals, non-finite results) automatically falls back to Synthesizer's own +``Stars(sf_hist=..., metal_dist=...)`` path, so :func:`make_parametric_stars` +can be used unconditionally as a drop-in replacement. + +Example: + >>> stars = make_parametric_stars( + ... SFH.LogNormal( + ... tau=0.3, + ... peak_age=200 + ... * Myr, + ... max_age=1 + ... * Gyr, + ... ), + ... ZDist.DeltaConstant( + ... log10metallicity=-2.0 + ... ), + ... grid.log10ages, + ... grid.metallicities, + ... initial_mass=1e9 + ... * Msun, + ... ) +""" + +from __future__ import annotations + +from collections import OrderedDict +from typing import Callable, Optional, Tuple + +import numpy as np +from scipy import integrate +from scipy.special import erf, hyp2f1 +from synthesizer.parametric import SFH, ZDist +from synthesizer.parametric import Stars as ParametricStars +from unyt import unyt_array, unyt_quantity + +__all__ = [ + "sfh_bin_masses", + "zdist_weights", + "fast_sfzh", + "make_parametric_stars", + "is_supported_sfh", + "is_supported_zdist", + "FastParametricStars", + "age_bin_edges", +] + +# Gauss-Legendre order used for smooth SFHs. 6 points integrate polynomials of +# degree 11 exactly; with the node placement below this gives ~1e-9 relative +# accuracy per bin for every smooth parametrisation tested. +GL_ORDER = 6 + +# Number of log-spaced sub-intervals per Synthesizer age bin in the cached base +# node set (on top of SFH specific feature nodes). +SUB_PER_BIN = 3 + +# Number of geometric sub-nodes used to resolve the first age bin [0, e_1], +# spanning N_HEAD_DECADES decades below e_1. +N_HEAD_DECADES = 8 +N_HEAD_PER_DECADE = 3 + +_GL_CACHE: dict = {} +_NODE_CACHE: "OrderedDict" = OrderedDict() +_NODE_CACHE_MAX = 32 + + +class _Unsupported(Exception): + """Raised internally when the fast path cannot handle an input.""" + + +# -------------------------------------------------------------------------- +# Helpers +# -------------------------------------------------------------------------- +def _gl(order: int) -> Tuple[np.ndarray, np.ndarray]: + """Return (cached) Gauss-Legendre nodes and weights on [-1, 1].""" + if order not in _GL_CACHE: + _GL_CACHE[order] = np.polynomial.legendre.leggauss(order) + return _GL_CACHE[order] + + +def _to_yr(value) -> float: + """Convert a float (assumed yr) or unyt quantity to a float in yr.""" + if value is None: + return 0.0 + if isinstance(value, (unyt_quantity, unyt_array)): + return float(value.to("yr").value) + return float(value) + + +def age_bin_edges(log10ages) -> np.ndarray: + """Return the age bin edges (yr) Synthesizer integrates the SFH over. + + Matches ``synthesizer.parametric.Stars._get_sfzh``: ``[0, midpoints of + 10**log10ages (linear), ages[-1]]``. + + Args: + log10ages (np.ndarray): log10(age / yr) of the grid. + + Returns: + np.ndarray: ``len(log10ages) + 1`` bin edges in yr. + """ + ages = 10 ** np.asarray(log10ages, dtype=np.float64) + edges = np.empty(ages.size + 1) + edges[0] = 0.0 + if ages.size > 1: + # np.mean([a, b]) == (a + b) / 2 == 0.5 * (a + b) bit-for-bit + edges[1:-1] = 0.5 * (ages[1:] + ages[:-1]) + edges[-1] = ages[-1] + return edges + + +def _base_nodes(log10ages) -> Tuple[np.ndarray, np.ndarray]: + """Cached (edges, base_nodes) for an age grid (relative, no offset).""" + log10ages = np.asarray(log10ages, dtype=np.float64) + key = log10ages.tobytes() + hit = _NODE_CACHE.get(key) + if hit is not None: + _NODE_CACHE.move_to_end(key) + return hit + + edges = age_bin_edges(log10ages) + pieces = [edges] + # Log-spaced sub-nodes inside each bin except the first + lo, hi = edges[1:-1], edges[2:] + if lo.size and SUB_PER_BIN > 1: + frac = np.arange(1, SUB_PER_BIN) / SUB_PER_BIN + good = (lo > 0) & (hi > lo) + sub = np.exp(np.log(lo[good, None]) + frac[None, :] * np.log(hi[good] / lo[good])[:, None]) + pieces.append(sub.ravel()) + # Geometric refinement of the first bin [0, e_1] down towards 0 + if edges.size > 1 and edges[1] > 0: + n = N_HEAD_DECADES * N_HEAD_PER_DECADE + pieces.append(edges[1] * np.logspace(-N_HEAD_DECADES, 0, n + 1)) + nodes = np.unique(np.concatenate(pieces)) + nodes = nodes[(nodes >= edges[0]) & (nodes <= edges[-1])] + + _NODE_CACHE[key] = (edges, nodes) + if len(_NODE_CACHE) > _NODE_CACHE_MAX: + _NODE_CACHE.popitem(last=False) + return edges, nodes + + +# Precomputed dimensionless node patterns +_GEOM = np.concatenate([[0.0], np.geomspace(1e-4, 60.0, 25)]) # in units of tau +_U_LOGNORMAL = np.linspace(-12.0, 12.0, 49) # in units of tau (ln t) +_U_GAUSS = np.linspace(-10.0, 10.0, 41) # in units of sigma +_DPL_LOG = np.logspace(-6.0, 6.0, 97) # in units of peak_age + + +def _geom(scale: float) -> np.ndarray: + """Geometric offsets ``|scale| * [0, geomspace(1e-4, 60)]``.""" + s = abs(scale) + if not np.isfinite(s) or s == 0: + return _GEOM[:1] + return s * _GEOM + + +# -------------------------------------------------------------------------- +# SFH handlers: each returns (extra_nodes, vectorised_sfr, gl_order, +# head_integral) where head_integral is an optional callable giving the exact +# integral over [0, x] for SFHs singular at age 0. +# -------------------------------------------------------------------------- +def _h_constant(s, span): + return np.array([s.min_age, s.max_age]), s._sfrs, 1, None + + +def _h_gaussian(s, span): + nodes = np.concatenate([[s.min_age, s.max_age], s.peak_age + s.sigma * _U_GAUSS]) + return nodes, s._sfrs, GL_ORDER, None + + +def _h_exponential(s, span): + g = _geom(s.tau) + nodes = np.concatenate([[s.min_age, s.max_age], s.max_age - g, s.min_age + g]) + return nodes, s._sfrs, GL_ORDER, None + + +def _h_delayed(s, span): + g = _geom(s.tau) + nodes = np.concatenate([[s.min_age, s.max_age], s.max_age - g, s.min_age + g]) + return nodes, s._sfrs, GL_ORDER, None + + +def _h_lognormal(s, span): + # In t = max_age - age the SFR is a Gaussian in ln(t) of width tau + # centred on t_0 (it is the d ln t density); resolve +-12 sigma. + if not (np.isfinite(s.t_0) and s.tau > 0): + raise _Unsupported("degenerate LogNormal") + t = np.exp(s.t_0 + s.tau * _U_LOGNORMAL) + nodes = np.concatenate([[s.min_age, s.max_age], s.max_age - t]) + return nodes, s._sfrs, GL_ORDER, None + + +def _h_doublepowerlaw(s, span): + if not (s.peak_age > 0 and np.isfinite(s.alpha) and np.isfinite(s.beta)): + raise _Unsupported("degenerate DoublePowerLaw") + m = min(s.alpha, s.beta) + head = None + if s.min_age <= 0: + if m >= 1: + # Non-integrable singularity at age = 0: quad's answer is + # ill-defined, so defer to Synthesizer for identical behaviour. + raise _Unsupported("DoublePowerLaw diverges at age 0") + if m > 0: + p, d = s.peak_age, abs(s.alpha - s.beta) + + def head(x, p=p, m=m, d=d): + # Exact int_0^x da / ((a/p)^alpha + (a/p)^beta) + # = p int_0^Y y^-m / (1 + y^d) dy, Y = x / p + y = x / p + if d == 0: + return 0.5 * p * y ** (1 - m) / (1 - m) + c = (1 - m) / d + return p * y ** (1 - m) / (1 - m) * hyp2f1(1.0, c, 1.0 + c, -(y**d)) + + turn = _GEOM / max(abs(s.alpha), abs(s.beta), 1.0) + turn = turn[turn < 1.0] # beyond this the log-spaced nodes suffice + nodes = np.concatenate( + [ + [s.min_age, s.max_age, s.peak_age], + s.peak_age * _DPL_LOG, + # resolve the turnover at the peak, whose width in ln(age) is + # ~1/max(alpha, beta) (sharp for steep power laws) + s.peak_age * np.exp(turn), + s.peak_age * np.exp(-turn), + ] + ) + return nodes, s._sfrs, GL_ORDER, head + + +def _step_eval_lower(s): + """Vectorised Continuity/ContinuityFlex/Dirichlet ``_sfr``.""" + upper = np.asarray(s.bin_edges[1:], dtype=np.float64) + rates = np.asarray(s.sfrs, dtype=np.float64) + n = rates.size + + def f(age): + idx = np.searchsorted(upper, age) + out = rates[np.minimum(idx, n - 1)] + bad = (idx >= n) | (age < s.min_age) | (age > s.max_age) + return np.where(bad, 0.0, out) + + return f + + +def _h_continuity(s, span): + nodes = np.concatenate([[s.min_age, s.max_age], np.asarray(s.bin_edges, dtype=np.float64)]) + return nodes, _step_eval_lower(s), 1, None + + +def _h_continuity_psb(s, span): + edges = np.asarray(s.bin_edges, dtype=np.float64) + rates = np.asarray(s.sfrs, dtype=np.float64) + n = rates.size + + def f(age): + idx = np.searchsorted(edges, age, side="right") - 1 + out = rates[np.clip(idx, 0, n - 1)] + bad = (idx < 0) | (idx >= n) | (age < s.min_age) | (age >= s.max_age) + return np.where(bad, 0.0, out) + + return np.concatenate([[s.min_age, s.max_age], edges]), f, 1, None + + +def _h_interp(s, span): + # DenseBasis / Stochastic: np.interp on (finegrid, intsfh) -> piecewise + # linear, integrated exactly by the midpoint rule with knots as nodes. + return np.asarray(s.finegrid, dtype=np.float64), s._sfrs, 1, None + + +def _h_combined(s, span): + parts = [_handler_for(m)(m, span) for m in s.sfhs] + if any(p[3] is not None for p in parts): + raise _Unsupported("CombinedSFH with singular component") + weights = np.asarray(s.weights, dtype=np.float64) + funcs = [p[1] for p in parts] + + def f(age): + out = np.zeros_like(age, dtype=np.float64) + for w, fn in zip(weights, funcs): + out = out + fn(age) * w + return out + + nodes = np.concatenate([p[0] for p in parts]) + return nodes, f, max(p[2] for p in parts), None + + +_HANDLERS = { + SFH.Constant: _h_constant, + SFH.Gaussian: _h_gaussian, + SFH.Exponential: _h_exponential, + SFH.TruncatedExponential: _h_exponential, + SFH.DecliningExponential: _h_exponential, + SFH.DelayedExponential: _h_delayed, + SFH.LogNormal: _h_lognormal, + SFH.DoublePowerLaw: _h_doublepowerlaw, + SFH.DenseBasis: _h_interp, +} +for _name, _h in ( + ("Continuity", _h_continuity), + ("ContinuityFlex", _h_continuity), + ("Dirichlet", _h_continuity), + ("ContinuityPSB", _h_continuity_psb), + ("CombinedSFH", _h_combined), + ("Stochastic", _h_interp), +): + if hasattr(SFH, _name): + _HANDLERS[getattr(SFH, _name)] = _h + + +def _handler_for(sfh) -> Callable: + # Exact type match only: subclasses may override _sfr. + h = _HANDLERS.get(type(sfh)) + if h is None: + raise _Unsupported(f"unsupported SFH type {type(sfh).__name__}") + return h + + +def is_supported_sfh(sfh) -> bool: + """Return True if ``sfh`` has a vectorised fast path.""" + combined = getattr(SFH, "CombinedSFH", None) + if combined is not None and type(sfh) is combined: + return all(is_supported_sfh(m) for m in sfh.sfhs) + return type(sfh) in _HANDLERS + + +def is_supported_zdist(metal_dist) -> bool: + """Return True if ``metal_dist`` has an exact/vectorised fast path.""" + return isinstance(metal_dist, (ZDist.DeltaConstant, ZDist.Normal, float, np.floating)) or ( + isinstance(metal_dist, unyt_quantity) and metal_dist.size == 1 + ) + + +# -------------------------------------------------------------------------- +# Age axis +# -------------------------------------------------------------------------- +def _quad_bin_masses(sfh, edges, offset): + """Synthesizer's own per-bin quad loop (reference / fallback).""" + out = np.zeros(edges.size - 1) + for i in range(out.size): + out[i] = integrate.quad(sfh.get_sfr, edges[i] + offset, edges[i + 1] + offset)[0] + return out + + +def _fast_bin_masses(sfh, log10ages, offset) -> np.ndarray: + edges, base = _base_nodes(log10ages) + span = edges[-1] - edges[0] + extra, func, order, head = _handler_for(sfh)(sfh, span) + + if offset != 0.0: + edges = edges + offset + base = base + offset + lo, hi = edges[0], edges[-1] + extra = np.asarray(extra, dtype=np.float64) + extra = extra[np.isfinite(extra) & (extra > lo) & (extra < hi)] + nodes = np.concatenate([base, extra]) + nodes.sort() + keep = np.empty(nodes.size, dtype=bool) + keep[0] = True + np.greater(nodes[1:], nodes[:-1], out=keep[1:]) + nodes = nodes[keep] + + a = nodes[:-1] + h = 0.5 * (nodes[1:] - a) + m = a + h + if order == 1: + seg = 2.0 * h * func(m) + else: + x, w = _gl(order) + pts = (m[:, None] + h[:, None] * x[None, :]).ravel() + seg = h * (func(pts).reshape(-1, order) @ w) + + if head is not None and lo == 0.0: + # Replace the first (singular) interval by its analytic integral + seg[0] = head(nodes[1]) + + idx = np.searchsorted(nodes, edges) + return np.add.reduceat(seg, idx[:-1]) + + +def sfh_bin_masses(sfh, log10ages, age_offset=0.0, fallback: bool = True) -> np.ndarray: + """Mass formed per Synthesizer age bin for an SFH function object. + + Matches ``Stars._get_sfzh``: bin ``i`` spans + ``[edges[i] + age_offset, edges[i + 1] + age_offset]`` with + ``edges = [0, midpoints of 10**log10ages, 10**log10ages[-1]]`` (yr), and + the result is the integral of ``sfh.get_sfr`` over each bin (un-normalised). + + Args: + sfh (SFH.Common): The SFH function object. + log10ages (np.ndarray): log10(age / yr) of the age grid. + age_offset (float/unyt_quantity): Offset (yr if float) added to edges. + fallback (bool): If True, unsupported SFHs (or non-finite fast + results) are integrated with Synthesizer's quad loop instead of + raising. + + Returns: + np.ndarray: Mass per age bin (same units as ``sfh.get_sfr`` * yr). + + Raises: + TypeError: If ``fallback=False`` and the SFH is unsupported. + """ + offset = _to_yr(age_offset) + try: + with np.errstate(all="ignore"): + out = _fast_bin_masses(sfh, log10ages, offset) + if not np.all(np.isfinite(out)): + raise _Unsupported("non-finite result") + return out + except _Unsupported as e: + if not fallback: + raise TypeError(str(e)) from None + return _quad_bin_masses(sfh, age_bin_edges(log10ages), offset) + + +# -------------------------------------------------------------------------- +# Metallicity axis +# -------------------------------------------------------------------------- +_UNIT_MASS = None + + +def _unit_mass() -> float: + """Internal value of the 1 Msun dummy particle mass Synthesizer uses. + + Synthesizer converts ``1.0 * Msun`` to its internal (galactic) unit + system, giving ``1.0000000000000002``; reproducing it keeps the + metallicity weights bit-for-bit identical. + """ + global _UNIT_MASS + if _UNIT_MASS is None: + from synthesizer.particle import Stars as ParticleStars + from unyt import Msun, yr + + p = ParticleStars( + initial_masses=np.array([1.0]) * Msun, + ages=np.array([0.0]) * yr, + metallicities=np.array([0.01]), + ) + _UNIT_MASS = float(np.asarray(p._initial_masses, dtype=np.float64)[0]) + return _UNIT_MASS + + +def _cic_weights(z: float, metallicities: np.ndarray) -> np.ndarray: + """Replicates the C cloud-in-cell (ParticleStars.get_metal_dist, mass 1).""" + grid = metallicities + n = grid.size + m = _unit_mass() + w = np.zeros(n) + if n == 1: + w[0] = m + return w + if z <= grid[0]: + w[0] = 1.0 * m + elif z >= grid[-1]: + w[-1] = 1.0 * m + else: + up = int(np.searchsorted(grid, z, side="right")) + lo = up - 1 + frac = (z - grid[lo]) / (grid[up] - grid[lo]) + w[lo] = (1.0 - frac) * m + w[up] = frac * m + return w + + +def _particle_metal_weights(z: float, metallicities) -> np.ndarray: + """Synthesizer's own dummy particle path (used for non-float64 grids).""" + from synthesizer.particle import Stars as ParticleStars + from unyt import Msun, yr + + p = ParticleStars( + initial_masses=np.array([1.0]) * Msun, + ages=np.array([0.0]) * yr, + metallicities=np.array([z]), + ) + return p.get_metal_dist(metallicities) + + +def _metal_edges(metallicities: np.ndarray) -> np.ndarray: + z = metallicities + edges = np.empty(z.size + 1) + edges[0] = 0.0 + edges[1:-1] = 0.5 * (z[1:] + z[:-1]) + edges[-1] = z[-1] + return edges + + +def zdist_weights(metal_dist, metallicities) -> np.ndarray: + """Weights per metallicity bin, matching ``Stars._get_sfzh``. + + * ``ZDist.DeltaConstant`` / float metallicity: cloud-in-cell assignment in + linear Z, bit-for-bit identical to Synthesizer (clamped to grid ends). + * ``ZDist.Normal``: analytic integral of the Gaussian over Synthesizer's + metallicity bins ``[0, midpoints, Z_max]`` (matches quad to ~1e-10). + * Any other ``ZDist`` object: Synthesizer's quad loop. + + Args: + metal_dist: ZDist object or a float metallicity. + metallicities (np.ndarray): grid metallicities (linear). + + Returns: + np.ndarray: weight per metallicity bin (un-normalised). + """ + metallicities = np.asarray(metallicities) + if isinstance(metal_dist, ZDist.DeltaConstant) or isinstance( + metal_dist, (float, np.floating, unyt_quantity) + ): + if isinstance(metal_dist, ZDist.DeltaConstant): + z = metal_dist.get_metallicity() + else: + z = metal_dist + z = float(np.asarray(z)) + if metallicities.dtype != np.float64 or metallicities.ndim != 1: + return _particle_metal_weights(z, metallicities) + return _cic_weights(z, metallicities) + + if type(metal_dist) is ZDist.Normal: + mu, sig = float(metal_dist.mean), float(metal_dist.sigma) + if sig > 0 and np.isfinite(mu) and np.isfinite(sig): + e = _metal_edges(metallicities.astype(np.float64)) + c = 0.5 * erf((e - mu) / (sig * np.sqrt(2.0))) + out = np.diff(c) + # erf differences lose precision deep in the same tail; fall + # back to quad only in that (rare) case. + if np.all(np.isfinite(out)): + return out + + if hasattr(metal_dist, "get_dist_weight"): + e = _metal_edges(metallicities.astype(np.float64)) + return np.array( + [ + integrate.quad(metal_dist.get_dist_weight, e[i], e[i + 1])[0] + for i in range(e.size - 1) + ] + ) + raise TypeError(f"Unsupported metallicity distribution {type(metal_dist)}") + + +# -------------------------------------------------------------------------- +# SFZH and Stars +# -------------------------------------------------------------------------- +def fast_sfzh(sfh, metal_dist, log10ages, metallicities, age_offset=0.0) -> Optional[np.ndarray]: + """Un-normalised SFZH ``sfh_bins[:, None] * z_weights[None, :]``. + + Returns None if either input has no fast path (callers should then use + Synthesizer's own machinery). + """ + if not is_supported_sfh(sfh): + return None + if not (is_supported_zdist(metal_dist) or isinstance(metal_dist, ZDist.Common)): + return None + try: + with np.errstate(all="ignore"): + sf = _fast_bin_masses(sfh, log10ages, _to_yr(age_offset)) + except _Unsupported: + return None + if not np.all(np.isfinite(sf)): + return None + zw = zdist_weights(metal_dist, metallicities) + return sf[:, np.newaxis] * zw + + +def make_parametric_stars( + sfh, + metal_dist, + log10ages, + metallicities, + initial_mass=None, + subclass: bool = False, + **kwargs, +) -> ParametricStars: + """Drop-in fast replacement for ``Stars(sf_hist=sfh, metal_dist=...)``. + + The returned object is constructed through Synthesizer's normal + ``Stars.__init__`` with ``sf_hist``/``metal_dist`` passed as usual (so + ``sf_hist_func``, ``metal_dist_func``, ``_instant_sf`` and + ``_instant_metallicity`` are set exactly as before) plus a precomputed + ``sfzh=``, which Synthesizer uses instead of its quad integration. + Normalisation (``initial_mass`` or ``surviving_mass`` + ``grid``), + ``sfzh_normalisation``, ``sf_hist``/``metal_dist`` projections and all + other attributes are therefore produced by Synthesizer itself. + + Falls back to the original quad path transparently whenever the SFH or + ZDist is not supported. + + Args: + sfh: SFH function object (or anything Stars accepts as ``sf_hist``). + metal_dist: ZDist object or float (or anything Stars accepts). + log10ages (np.ndarray): Grid log10 ages. + metallicities (np.ndarray): Grid metallicities. + initial_mass (unyt_quantity): Total initial stellar mass. + subclass (bool): If True return a :class:`FastParametricStars`, whose + ``_get_sfzh`` (used by ``get_at_earlier_time`` and + ``calculate_*_mass_at_age``) also uses the fast path. Default + False returns a plain ``synthesizer.parametric.Stars``. + **kwargs: Passed to ``Stars`` (e.g. ``surviving_mass``, ``grid``, + ``fesc``, emitter parameters). + + Returns: + synthesizer.parametric.Stars + """ + cls = FastParametricStars if subclass else ParametricStars + if subclass: + # The subclass computes its SFZH lazily in _get_sfzh. + return cls( + log10ages=log10ages, + metallicities=metallicities, + sf_hist=sfh, + metal_dist=metal_dist, + initial_mass=initial_mass, + **kwargs, + ) + + sfzh = kwargs.pop("sfzh", None) + if sfzh is None: + sfzh = fast_sfzh(sfh, metal_dist, log10ages, metallicities) + + stars = ParametricStars( + log10ages=log10ages, + metallicities=metallicities, + sf_hist=sfh, + metal_dist=metal_dist, + initial_mass=initial_mass, + sfzh=sfzh, + **kwargs, + ) + # Synthesizer's _get_sfzh sets this side effect for DeltaConstant + if sfzh is not None and isinstance(metal_dist, ZDist.Common): + if metal_dist.name == "DeltaConstant": + stars._instant_metallicity = metal_dist.get_metallicity() + return stars + + +class FastParametricStars(ParametricStars): + """``synthesizer.parametric.Stars`` with a vectorised ``_get_sfzh``. + + Identical to the parent except that SFZH integration (at construction and + in ``get_at_earlier_time``/``calculate_*_mass_at_age``) uses + :func:`fast_sfzh` when possible, falling back to the parent otherwise. + """ + + def _get_sfzh(self, age_offset=None): + """Compute the SFZH, using the fast path when supported.""" + sfh = self.sf_hist_func + zd = self.metal_dist_func + if sfh is not None and self._instant_sf is None: + if zd is not None: + metal = zd + elif self._instant_metallicity is not None: + metal = self._instant_metallicity + else: + metal = None + if metal is not None: + sfzh = fast_sfzh( + sfh, + metal, + self.log10ages, + self.metallicities, + age_offset=0.0 if age_offset is None else age_offset, + ) + if sfzh is not None: + if zd is not None and zd.name == "DeltaConstant": + self._instant_metallicity = zd.get_metallicity() + return sfzh + return super()._get_sfzh(age_offset=age_offset) diff --git a/tests/test_fast_igm.py b/tests/test_fast_igm.py new file mode 100644 index 0000000..af24903 --- /dev/null +++ b/tests/test_fast_igm.py @@ -0,0 +1,332 @@ +"""Tests for the tabulated IGM transmission (synference.fast_igm).""" + +import os +import pickle +import sys + +import numpy as np +import pytest +from astropy.cosmology import Planck18 +from synthesizer.emission_models import PacmanEmission +from synthesizer.emission_models.attenuation import Calzetti2000, Inoue14, Madau96 +from synthesizer.emissions import Sed +from synthesizer.instruments import FilterCollection, Instrument +from synthesizer.parametric import SFH, Galaxy, Stars, ZDist +from unyt import Angstrom, Hz, Msun, Myr, erg, s + +try: + from synference.fast_igm import TabulatedIGM +except ImportError: # pragma: no cover - e.g. optional deps of the package missing + import importlib.util + + _path = os.path.join(os.path.dirname(__file__), "..", "src", "synference", "fast_igm.py") + _spec = importlib.util.spec_from_file_location("synference_fast_igm", _path) + _mod = importlib.util.module_from_spec(_spec) + sys.modules["synference_fast_igm"] = _mod # so instances can be pickled + _spec.loader.exec_module(_mod) + TabulatedIGM = _mod.TabulatedIGM + + +# A modest log-spaced rest-frame grid (fast to tabulate) +LAM = np.logspace(np.log10(80.0), np.log10(5e5), 1500) +Z_MAX = 15.0 +DZ = 0.05 + + +@pytest.fixture(scope="module") +def tab_igm(): + """Tabulated Inoue14 prepared for LAM.""" + return TabulatedIGM(Inoue14, z_max=Z_MAX, dz=DZ, lam=LAM * Angstrom) + + +@pytest.fixture(scope="module") +def sf_sed(): + """A young, star-forming like SED: blue power law with a Lyman break.""" + lnu = 1e28 * (LAM / 1500.0) ** 0.5 # beta ~ -2.5 in f_lambda + lnu[LAM < 912.0] *= 0.05 # partial Lyman-continuum escape + lnu *= 1 + 5 * np.exp(-0.5 * ((LAM - 1215.67) / 2.0) ** 2) # Lya + return Sed(LAM * Angstrom, lnu * erg / s / Hz) + + +@pytest.fixture(scope="module") +def tophat_filters(): + """Broad and medium top-hat bands spanning u to 5 micron.""" + tophat = {} + for cen, frac in [ + (3550, 0.18), + (4800, 0.30), + (6200, 0.25), + (7700, 0.20), + (8900, 0.12), + (9800, 0.08), + (11500, 0.22), + (14000, 0.10), + (15000, 0.21), + (16200, 0.10), + (20000, 0.23), + (27700, 0.25), + (35600, 0.22), + (41000, 0.05), + (44400, 0.24), + ]: + w = cen * frac + tophat[f"th{cen}_{int(100 * frac)}"] = { + "lam_min": (cen - w / 2) * Angstrom, + "lam_max": (cen + w / 2) * Angstrom, + } + return FilterCollection(tophat_dict=tophat, new_lam=np.logspace(3, 5, 6000) * Angstrom) + + +@pytest.fixture(scope="module") +def small_grid(): + """The synthesizer test grid (downloaded by conftest's fixture if needed).""" + from synthesizer import get_grids_dir + from synthesizer.grid import Grid + + grid_dir = get_grids_dir() + if not os.path.exists(os.path.join(grid_dir, "test_grid.hdf5")): + pytest.skip("synthesizer test grid not available") + new_lam = np.logspace(np.log10(80.0), np.log10(5e5), 2000) * Angstrom + return Grid(grid_name="test_grid", grid_dir=grid_dir, new_lam=new_lam) + + +def _galaxies(grid, zs): + gals = [] + for i, z in enumerate(zs): + stars = Stars( + log10ages=grid.log10ages, + metallicities=grid.metallicities, + sf_hist=SFH.Constant(max_age=(50 + 30 * i) * Myr), + metal_dist=ZDist.DeltaConstant(log10metallicity=-2.5), + initial_mass=1e9 * Msun, + ) + gals.append(Galaxy(stars=stars, redshift=z)) + return gals + + +def _model(grid): + return PacmanEmission(grid=grid, tau_v=0.2, dust_curve=Calzetti2000(), fesc=0.1) + + +# ---------------------------------------------------------------------------- +# Basic behaviour +# ---------------------------------------------------------------------------- +def test_not_callable_and_interface(tab_igm): + """Instances must not be callable (get_fnu would call igm()).""" + assert not callable(tab_igm) + assert hasattr(tab_igm, "get_transmission") + assert tab_igm.name == "TabulatedInoue14" + # Class or instance bases are both accepted + assert isinstance(TabulatedIGM(Inoue14(scale_tau=1.0), dz=0.5).base, Inoue14) + + +def test_red_side_is_exactly_one(): + """Inoue14 and Madau96 are exactly transparent redward of Lya.""" + red = LAM[LAM >= 1216.0] + for base in (Inoue14(), Madau96()): + for z in (0.1, 3.0, 7.5, 14.0): + t = base.get_transmission(z, red * (1 + z) * Angstrom) + assert np.all(t == 1.0) + + +@pytest.mark.parametrize("base", [Inoue14, Madau96]) +def test_transmission_accuracy(base): + """Per-wavelength transmission vs the exact model at random redshifts.""" + lam = LAM[::3] + tab = TabulatedIGM(base, z_max=Z_MAX, dz=DZ, lam=lam) + exact = base() + rng = np.random.default_rng(42) + max_err = 0.0 + for z in np.concatenate([rng.uniform(0.01, Z_MAX, 25), [1.2, 4.7, 6.0]]): + lam_obs = lam * (1 + z) * Angstrom + t_tab = tab.get_transmission(z, lam_obs) + t_ex = exact.get_transmission(z, lam_obs) + assert t_tab.shape == t_ex.shape + assert np.all((t_tab >= 0) & (t_tab <= 1)) + np.testing.assert_array_equal(t_tab[lam >= 1216.0], 1.0) + max_err = max(max_err, np.abs(t_tab - t_ex).max()) + assert tab.n_exact_calls == 0 + assert max_err < 2.5e-3 + + +def test_photometric_accuracy(tab_igm, sf_sed, tophat_filters): + """Band fluxes vs exact Inoue14 to < 1e-3 mag (bands not dropped out).""" + exact = Inoue14() + rng = np.random.default_rng(1) + worst = 0.0 + for z in np.concatenate([rng.uniform(0.05, Z_MAX, 40), [1.2, 4.7]]): + sed = Sed(sf_sed.lam, sf_sed.lnu) + sed.get_fnu(Planck18, z, igm=None) + fnu0 = np.asarray(sed._fnu, dtype=np.float64) + obslam = sed.obslam + arr = np.vstack( + [ + fnu0, + fnu0 * exact.get_transmission(z, obslam), + fnu0 * tab_igm.get_transmission(z, obslam), + ] + ) + f = tophat_filters.apply_filters( + np.ascontiguousarray(arr), nu=np.asarray(sed._obsnu, dtype=np.float64) + ) + f0, fe, ft = f[:, 0], f[:, 1], f[:, 2] + ok = fe > 1e-2 * f0 # band not (almost) completely dropped out + dmag = np.abs(2.5 * np.log10(ft[ok] / fe[ok])) + if dmag.size: + worst = max(worst, dmag.max()) + # Absolute flux error relative to the unattenuated flux everywhere + assert np.all(np.abs(ft - fe) <= 1e-3 * f0 + 1e-30) + assert worst < 1e-3 + + +# ---------------------------------------------------------------------------- +# Fallbacks, lazy building, caching and pickling +# ---------------------------------------------------------------------------- +def test_out_of_table_fallback(tab_igm): + """Redshifts outside the table use the exact base model.""" + exact = Inoue14() + z = Z_MAX + 2.0 + lam_obs = LAM * (1 + z) * Angstrom + n0 = tab_igm.n_exact_calls + np.testing.assert_array_equal( + tab_igm.get_transmission(z, lam_obs), exact.get_transmission(z, lam_obs) + ) + assert tab_igm.n_exact_calls == n0 + 1 + + +def test_lazy_build_and_arbitrary_arrays(): + """Unseen arrays are exact on first sight and tabulated on the second.""" + tab = TabulatedIGM(Inoue14, z_max=8.0, dz=0.1) + exact = Inoue14() + lam = np.linspace(700.0, 3000.0, 200) + t1 = tab.get_transmission(3.0, lam * 4.0 * Angstrom) + assert tab.n_exact_calls == 1 and len(tab.tables) == 0 + np.testing.assert_array_equal(t1, exact.get_transmission(3.0, lam * 4.0 * Angstrom)) + # Second sight (different redshift, same rest-frame array) builds + t2 = tab.get_transmission(5.0, lam * 6.0 * Angstrom) + assert len(tab.tables) == 1 and tab.n_table_calls == 1 + np.testing.assert_allclose(t2, exact.get_transmission(5.0, lam * 6.0 * Angstrom), atol=5e-3) + # Unitless input (Angstrom) and other length units work too + t3 = tab.get_transmission(5.0, lam * 6.0) + t4 = tab.get_transmission(5.0, (lam * 6.0 * Angstrom).to("um")) + np.testing.assert_allclose(t3, t2) + np.testing.assert_allclose(t4, t2) + # Scalar-like / 2D shapes are preserved + assert tab.get_transmission(5.0, np.array([5000.0]) * Angstrom).shape == (1,) + + +def test_pickle_round_trip(tab_igm): + """Pickled copies give identical results and don't rebuild.""" + z = 7.3 + lam_obs = LAM * (1 + z) * Angstrom + ref = tab_igm.get_transmission(z, lam_obs) + blob = pickle.dumps(tab_igm) + clone = pickle.loads(blob) + n_exact = clone.n_exact_calls + np.testing.assert_array_equal(clone.get_transmission(z, lam_obs), ref) + assert clone.n_exact_calls == n_exact + assert not callable(clone) + # Without tables the pickle is tiny and the clone rebuilds lazily + light = TabulatedIGM(Inoue14, z_max=Z_MAX, dz=DZ, pickle_tables=False) + light._tables, light._order = tab_igm._tables, tab_igm._order + small = pickle.dumps(light) + assert len(small) < len(blob) / 10 + assert pickle.loads(small).tables == [] + + +def test_disk_cache(tmp_path): + """Tables written to cache_dir are reused by new instances.""" + lam = LAM[::4] + a = TabulatedIGM(Inoue14, z_max=6.0, dz=0.1, cache_dir=str(tmp_path), lam=lam) + files = list(tmp_path.glob("tabigm_*.npz")) + assert len(files) == 1 + b = TabulatedIGM(Inoue14, z_max=6.0, dz=0.1, cache_dir=str(tmp_path)) + + def no_build(*args, **kwargs): + raise AssertionError("should have loaded the table from cache_dir") + + b._build = no_build + b.prepare(lam) + lam_obs = lam * 4.5 * Angstrom + np.testing.assert_array_equal( + a.get_transmission(3.5, lam_obs), b.get_transmission(3.5, lam_obs) + ) + + +def test_bad_red_cut_raises(): + """A cut redward of which T != 1 is detected at build time.""" + with pytest.raises(ValueError): + TabulatedIGM(Inoue14, z_max=5.0, dz=0.5, lam_rest_max=1000.0, lam=LAM) + + +# ---------------------------------------------------------------------------- +# Integration with Synthesizer +# ---------------------------------------------------------------------------- +def test_sed_get_fnu_and_photo(tab_igm, sf_sed, tophat_filters): + """Works as igm= in Sed.get_fnu and the subsequent get_photo_fnu.""" + for z in (2.5, 6.8, 11.0): + s_ex = Sed(sf_sed.lam, sf_sed.lnu) + s_tab = Sed(sf_sed.lam, sf_sed.lnu) + f_ex = s_ex.get_fnu(Planck18, z, igm=Inoue14) + n_tab = tab_igm.n_table_calls + f_tab = s_tab.get_fnu(Planck18, z, igm=tab_igm) + assert tab_igm.n_table_calls == n_tab + 1 + assert f_tab.units == f_ex.units + scale = f_ex.max() + assert np.abs(f_tab - f_ex).max() <= 3e-3 * scale + p_ex = s_ex.get_photo_fnu(tophat_filters) + p_tab = s_tab.get_photo_fnu(tophat_filters) + for f in tophat_filters.filter_codes: + assert abs(p_tab[f] - p_ex[f]) <= 1e-3 * abs(p_ex[f]) + 1e-6 * scale + + +def test_galaxy_observed_spectra_and_lines(small_grid): + """Works as igm= in Galaxy.get_observed_spectra / get_observed_lines.""" + tab = TabulatedIGM(Inoue14, z_max=Z_MAX, dz=0.1, min_hits=1) + model = _model(small_grid) + line_ids = [lid for lid in small_grid.available_lines if "1215" not in lid][:4] + for igm in (Inoue14, tab): + gal = _galaxies(small_grid, [6.5])[0] + gal.stars.get_spectra(model) + gal.get_observed_spectra(Planck18, igm=igm) + gal.stars.get_lines(line_ids, model) + gal.get_observed_lines(Planck18, igm=igm) + if igm is Inoue14: + ref_fnu = gal.stars.spectra["emergent"].fnu.copy() + ref_flux = gal.stars.lines["emergent"].flux.copy() + else: + fnu = gal.stars.spectra["emergent"].fnu + flux = gal.stars.lines["emergent"].flux + assert tab.n_table_calls > 0 + np.testing.assert_allclose(fnu, ref_fnu, rtol=5e-3, atol=1e-3 * ref_fnu.max().value) + np.testing.assert_allclose(flux, ref_flux, rtol=5e-3, atol=1e-6 * ref_flux.max().value) + + +def test_pipeline(small_grid, tophat_filters): + """Works as igm= in a small Pipeline (nthreads=1) incl. photometry & lines.""" + from synthesizer.pipeline import Pipeline + + tab = TabulatedIGM(Inoue14, z_max=Z_MAX, dz=0.1, lam=small_grid.lam) + inst = Instrument("TH", filters=tophat_filters) + line_ids = [lid for lid in small_grid.available_lines if "1215" not in lid][:3] + zs = [3.0, 6.2, 9.4] + out = {} + for name, igm in (("exact", Inoue14), ("tab", tab)): + gals = _galaxies(small_grid, zs) + pipe = Pipeline(emission_model=_model(small_grid), nthreads=1, verbose=0) + pipe.get_observed_spectra(Planck18, igm=igm) + pipe.get_photometry_fluxes(inst, igm=igm) + pipe.get_observed_lines(Planck18, igm=igm, line_ids=line_ids) + pipe.add_galaxies(gals) + pipe.run() + out[name] = gals + # Only the first sighting of the line wavelength array is computed exactly + assert tab.n_table_calls > 0 and tab.n_exact_calls <= 1 + for g_ex, g_tab in zip(out["exact"], out["tab"]): + ph_ex = g_ex.stars.photo_fnu["emergent"] + ph_tab = g_tab.stars.photo_fnu["emergent"] + for f in tophat_filters.filter_codes: + assert abs(ph_tab[f] - ph_ex[f]) <= 1e-3 * abs(ph_ex[f]) + 1e-30 * ph_ex[f].units + np.testing.assert_allclose( + g_tab.stars.lines["emergent"].flux, g_ex.stars.lines["emergent"].flux, rtol=5e-3 + ) diff --git a/tests/test_fast_sfzh.py b/tests/test_fast_sfzh.py new file mode 100644 index 0000000..c37bae4 --- /dev/null +++ b/tests/test_fast_sfzh.py @@ -0,0 +1,508 @@ +"""Tests for synference.fast_sfzh (vectorised parametric SFZH construction). + +The fast path is compared against Synthesizer's own ``Stars(sf_hist=..., +metal_dist=...)`` quad integration. Synthesizer's ``quad`` is itself +inaccurate for sharply peaked SFHs / steps inside wide age bins (we have seen +errors up to ~1e-2), so each case is additionally compared against a refined +reference (quad split at the SFH's natural breakpoints, tight tolerances): the +fast result must match Synthesizer to < 1e-3 OR, where Synthesizer's own error +vs the refined reference exceeds 5e-4, match the refined reference to < 1e-5. + +Run with ``pytest -s tests/test_fast_sfzh.py`` to see the error distributions. +""" + +import os +import warnings + +import numpy as np +import pytest +from scipy import integrate +from synthesizer.grid import Grid +from synthesizer.parametric import SFH, Galaxy, Stars, ZDist +from unyt import Msun, Myr, angstrom, unyt_array, yr + +from synference import fast_sfzh as F + +GRID_NAME = "bpass-2.2.1-bin_chabrier03-0.1,300.0_cloudy-c23.01-sps.hdf5" +AGE_MAX = 13.5e3 # Myr +TOL_SYNTH = 1e-3 +TOL_REF = 1e-5 +SUMMARY = {} + + +def _grid_dir(): + d = os.environ.get("SYNTHESIZER_GRID_DIR") + if d is None: + try: + from synthesizer import get_grids_dir + + d = str(get_grids_dir()) + except Exception: # pragma: no cover + return None + return d + + +@pytest.fixture(scope="module") +def grid(): + """BPASS grid (falls back to the synthesizer test grid).""" + d = _grid_dir() + if d is None: + pytest.skip("No SYNTHESIZER_GRID_DIR") + for name in (GRID_NAME, "test_grid.hdf5"): + if os.path.exists(os.path.join(d, name)): + return Grid(name, grid_dir=d) + pytest.skip("No suitable grid found") + + +# --------------------------------------------------------------------------- +# Random SFH generators (incl. edge cases) +# --------------------------------------------------------------------------- +def _maxa(rng, lo=1.0): + return 10 ** rng.uniform(np.log10(lo), np.log10(AGE_MAX)) * Myr + + +def _gen(name, rng, i): + maxa = _maxa(rng) + if name == "LogNormal": + edge = [ + dict(tau=0.05, peak_age=0.999 * maxa, max_age=maxa), # sharp, at max_age + dict(tau=0.05, peak_age=0.5 * Myr, max_age=1 * Myr), # very young + dict(tau=2.5, peak_age=0.001 * maxa, max_age=maxa), + ] + if i < len(edge): + return SFH.LogNormal(**edge[i]) + return SFH.LogNormal( + tau=rng.uniform(0.05, 2.5), peak_age=rng.uniform(0.001, 0.999) * maxa, max_age=maxa + ) + if name == "DelayedExponential": + edge = [dict(tau=10 * Myr, max_age=13000 * Myr), dict(tau=1 * Myr, max_age=2 * Myr)] + if i < len(edge): + return SFH.DelayedExponential(**edge[i]) + return SFH.DelayedExponential(tau=10 ** rng.uniform(0, 4.3) * Myr, max_age=maxa) + if name == "DecliningExponential": + return SFH.DecliningExponential(tau=10 ** rng.uniform(0, 4.3) * Myr, max_age=maxa) + if name == "Exponential": + tau = 10 ** rng.uniform(0, 4) * Myr + if rng.uniform() < 0.5: # rising; keep exp(max_age / |tau|) finite + tau = -max(tau, maxa / 300) + return SFH.Exponential(tau=tau, max_age=maxa, min_age=rng.uniform(0, 0.5) * maxa) + if name == "TruncatedExponential": + return SFH.TruncatedExponential( + tau=10 ** rng.uniform(0, 4) * Myr, max_age=maxa, min_age=rng.uniform(0, 0.9) * maxa + ) + if name == "Constant": + edge = [dict(max_age=1 * Myr), dict(max_age=1.05 * Myr, min_age=1.0 * Myr)] + if i < len(edge): + return SFH.Constant(**edge[i]) + return SFH.Constant(max_age=maxa, min_age=rng.uniform(0, 0.9) * maxa * rng.choice([0, 1])) + if name == "Gaussian": + return SFH.Gaussian( + peak_age=rng.uniform(0, 1) * maxa, sigma=10 ** rng.uniform(-2, 0) * maxa, max_age=maxa + ) + if name == "DoublePowerLaw": + # Integrable at age 0 only for min(alpha, beta) < 1 (otherwise falls back) + a, b = 10 ** rng.uniform(-1, np.log10(0.99)), 10 ** rng.uniform(-1, 3) + if rng.uniform() < 0.5: + a, b = b, a + return SFH.DoublePowerLaw(peak_age=rng.uniform(0.01, 0.99) * AGE_MAX * Myr, alpha=a, beta=b) + if name == "Continuity": + n = int(rng.integers(3, 9)) + top = max(maxa.to("yr").value, 2e7) + e = np.concatenate([[1.0], np.logspace(6.5, np.log10(top), n)]) + agebins = unyt_array(np.array([e[:-1], e[1:]]).T, "yr") + return SFH.Continuity.init_from_prior(agebins=agebins) + if name == "ContinuityFlex": + return SFH.ContinuityFlex.init_from_prior( + N_flex_bins=int(rng.integers(2, 5)), + fixed_young_bin=[0.0, 7.5], + fixed_old_bin=[9.7, 10.136], + ) + if name == "Dirichlet": + e = np.concatenate([[1.0], np.logspace(7, 10.1, 5)]) + agebins = unyt_array(np.array([e[:-1], e[1:]]).T, "yr") + return SFH.Dirichlet.init_from_prior(agebins=agebins) + if name == "ContinuityPSB": + return SFH.ContinuityPSB.init_from_prior() + if name == "DenseBasis": + return SFH.DenseBasis.init_from_prior( + N_tx=3, log_mass=10, log_sfr=rng.uniform(-2, 2), redshift=rng.uniform(0.1, 10) + ) + if name == "CombinedSFH": + return SFH.CombinedSFH( + [ + SFH.LogNormal(tau=rng.uniform(0.1, 1), peak_age=0.5 * maxa, max_age=maxa), + SFH.Constant(max_age=rng.uniform(0.01, 1) * maxa), + ], + weights=[1.0, 1e-9], + ) + raise ValueError(name) + + +# (class name, number of random draws). DenseBasis quad is ~0.1 s per galaxy. +CASES = [ + ("LogNormal", 60), + ("DelayedExponential", 50), + ("DecliningExponential", 40), + ("Exponential", 30), + ("TruncatedExponential", 20), + ("Constant", 40), + ("Gaussian", 25), + ("DoublePowerLaw", 40), + ("Continuity", 25), + ("ContinuityFlex", 10), + ("Dirichlet", 10), + ("ContinuityPSB", 10), + ("DenseBasis", 8), + ("CombinedSFH", 10), +] + + +def _breakpoints(sfh): + """Natural breakpoints of an SFH (for the refined quad reference).""" + pts = [] + for attr in ("min_age", "max_age", "peak_age"): + v = getattr(sfh, attr, None) + if v is not None and np.isfinite(v): + pts.append(float(v)) + if hasattr(sfh, "tau") and hasattr(sfh, "max_age") and not isinstance(sfh, SFH.LogNormal): + pts += list(sfh.max_age - abs(sfh.tau) * np.array([0.01, 0.1, 1, 3, 10, 30])) + pts += list(sfh.min_age + abs(sfh.tau) * np.array([0.01, 0.1, 1, 3, 10, 30])) + if isinstance(sfh, SFH.LogNormal): + pts += list(sfh.max_age - np.exp(sfh.t_0 + sfh.tau * np.linspace(-6, 6, 13))) + if isinstance(sfh, SFH.Gaussian): + pts += list(sfh.peak_age + sfh.sigma * np.linspace(-6, 6, 13)) + if hasattr(sfh, "bin_edges"): + pts += list(np.asarray(sfh.bin_edges, dtype=float)) + if hasattr(sfh, "sfhs"): + for m in sfh.sfhs: + pts += _breakpoints(m) + return pts + + +def _refined_reference(sfh, edges): + pts = np.array(_breakpoints(sfh)) + out = np.zeros(edges.size - 1) + for i in range(out.size): + a, b = edges[i], edges[i + 1] + inner = np.unique(pts[(pts > a) & (pts < b)]) + knots = np.concatenate([[a], inner, [b]]) + if hasattr(sfh, "finegrid"): # interpolated SFHs: split at knots + fg = np.asarray(sfh.finegrid) + knots = np.unique(np.concatenate([knots, fg[(fg > a) & (fg < b)]])) + out[i] = sum( + integrate.quad(sfh.get_sfr, x0, x1, limit=200, epsabs=0, epsrel=1e-11)[0] + for x0, x1 in zip(knots[:-1], knots[1:]) + ) + return out + + +def _l1(a, b): + return np.abs(a - b).sum() / np.abs(b).sum() + + +# --------------------------------------------------------------------------- +# Metallicity weights +# --------------------------------------------------------------------------- +def test_delta_constant_bitwise(grid): + """DeltaConstant / float weights are identical to Synthesizer's CIC.""" + from synthesizer.particle import Stars as ParticleStars + + zs = list(grid.metallicities) + list(10 ** np.random.default_rng(0).uniform(-6, -1, 40)) + zs += [0.5 * grid.metallicities[0], 2 * grid.metallicities[-1]] + for z in zs: + p = ParticleStars( + initial_masses=np.array([1.0]) * Msun, + ages=np.array([0.0]) * yr, + metallicities=np.array([z]), + ) + ref = p.get_metal_dist(grid.metallicities) + got = F.zdist_weights(ZDist.DeltaConstant(metallicity=z), grid.metallicities) + np.testing.assert_array_equal(got, ref) + np.testing.assert_array_equal(F.zdist_weights(float(z), grid.metallicities), ref) + + +def test_normal_zdist_matches_quad(grid): + """ZDist.Normal analytic weights match Synthesizer's quad loop.""" + rng = np.random.default_rng(1) + Z = grid.metallicities + e = np.concatenate([[0.0], 0.5 * (Z[1:] + Z[:-1]), [Z[-1]]]) + for _ in range(20): + zd = ZDist.Normal(mean=10 ** rng.uniform(-4, -1.4), sigma=10 ** rng.uniform(-4, -2)) + ref = np.array( + [integrate.quad(zd.get_dist_weight, e[i], e[i + 1])[0] for i in range(Z.size)] + ) + got = F.zdist_weights(zd, Z) + np.testing.assert_allclose(got, ref, rtol=1e-7, atol=1e-9 * ref.sum()) + + +# --------------------------------------------------------------------------- +# SFH accuracy +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("name,n", CASES, ids=[c[0] for c in CASES]) +def test_sfh_accuracy(grid, name, n): + """Fast SFZH agrees with Synthesizer (or a refined quad where it is off).""" + if not hasattr(SFH, name): + pytest.skip(f"SFH.{name} not available") + if name == "DenseBasis": + pytest.importorskip("dense_basis") + warnings.filterwarnings("ignore") + rng = np.random.default_rng(sum(map(ord, name))) + edges = F.age_bin_edges(grid.log10ages) + errs, ref_errs, synth_bad, n_fallback = [], [], 0, 0 + zd = ZDist.DeltaConstant(log10metallicity=-2.3) + for i in range(n): + sfh = _gen(name, rng, i) + new = F.make_parametric_stars( + sfh, zd, grid.log10ages, grid.metallicities, initial_mass=1e9 * Msun + ) + old = Stars( + log10ages=grid.log10ages, + metallicities=grid.metallicities, + sf_hist=sfh, + metal_dist=zd, + initial_mass=1e9 * Msun, + ) + assert type(new) is Stars + if not F.is_supported_sfh(sfh): + n_fallback += 1 + try: + F.sfh_bin_masses(sfh, grid.log10ages, fallback=False) + except TypeError: + n_fallback += 1 + np.testing.assert_array_equal(new.sfzh, old.sfzh) + continue + err = _l1(new.sfzh, old.sfzh) + errs.append(err) + if err >= TOL_SYNTH or name in ("Constant", "DecliningExponential"): + ref = _refined_reference(sfh, edges) + ref_sfzh = ref[:, None] * F.zdist_weights(zd, grid.metallicities)[None, :] + ref_sfzh *= 1e9 / ref_sfzh.sum() + synth_err = _l1(old.sfzh, ref_sfzh) + new_err = _l1(new.sfzh, ref_sfzh) + ref_errs.append(new_err) + if err >= TOL_SYNTH: + assert synth_err > 5e-4, (name, sfh.parameters, err, synth_err) + assert new_err < TOL_REF, (name, sfh.parameters, new_err) + synth_bad += 1 + else: + assert err < TOL_SYNTH + errs = np.array(errs) + SUMMARY[name] = (len(errs), n_fallback, np.median(errs), errs.max(), synth_bad, ref_errs) + print( + f"\n{name}: n={len(errs)} fallback={n_fallback} L1 vs synthesizer quad: " + f"median={np.median(errs):.1e} p90={np.percentile(errs, 90):.1e} " + f"max={errs.max():.1e}; synthesizer-quad failures={synth_bad}" + + (f"; max err vs refined ref={max(ref_errs):.1e}" if ref_errs else "") + ) + + +def test_photometry(grid): + """Photometry of fast vs quad Stars agrees to < 1e-3 mag.""" + from synthesizer.emission_models import IncidentEmission, PacmanEmission + from synthesizer.emission_models.attenuation import Calzetti2000 + from synthesizer.instruments import FilterCollection + + filters = FilterCollection( + tophat_dict={ + "UV": {"lam_eff": 1500 * angstrom, "lam_fwhm": 300 * angstrom}, + "U": {"lam_eff": 3650 * angstrom, "lam_fwhm": 660 * angstrom}, + "V": {"lam_eff": 5510 * angstrom, "lam_fwhm": 880 * angstrom}, + "K": {"lam_eff": 22000 * angstrom, "lam_fwhm": 4000 * angstrom}, + }, + new_lam=grid.lam, + ) + models = { + "incident": IncidentEmission(grid), + "attenuated": PacmanEmission(grid, tau_v="tau_v", dust_curve=Calzetti2000(), fesc=0.0), + } + rng = np.random.default_rng(7) + worst = 0.0 + for name in ( + "LogNormal", + "DelayedExponential", + "DecliningExponential", + "Constant", + "Continuity", + ): + for i in range(4): + sfh = _gen(name, rng, i + 10) + zd = ZDist.DeltaConstant(log10metallicity=rng.uniform(-4, -1.4)) + kw = dict(initial_mass=1e9 * Msun, tau_v=rng.uniform(0, 1), fesc_ly_alpha=1.0) + new = F.make_parametric_stars(sfh, zd, grid.log10ages, grid.metallicities, **kw) + old = Stars( + log10ages=grid.log10ages, + metallicities=grid.metallicities, + sf_hist=sfh, + metal_dist=zd, + **kw, + ) + for label, model in models.items(): + mags = [] + for st in (new, old): + gal = Galaxy(stars=st, redshift=0.0) + gal.stars.get_spectra(model) + lnu = gal.stars.spectra[label].get_photo_lnu(filters).photo_lnu.value + mags.append(-2.5 * np.log10(lnu)) + d = np.nanmax(np.abs(mags[0] - mags[1])) + worst = max(worst, d) + assert d < 1e-3, (name, label, sfh.parameters, d) + print(f"\nmax photometric difference: {worst:.2e} mag") + + +# --------------------------------------------------------------------------- +# Attribute parity, normalisation, fallbacks +# --------------------------------------------------------------------------- +def test_attribute_parity(grid): + """Downstream attributes and methods agree with the quad-built Stars.""" + sfh = SFH.LogNormal(tau=0.4, peak_age=300 * Myr, max_age=2000 * Myr) + zd = ZDist.DeltaConstant(log10metallicity=-2.1) + kw = dict(initial_mass=3e9 * Msun, tau_v=0.3, fesc=0.1) + new = F.make_parametric_stars(sfh, zd, grid.log10ages, grid.metallicities, **kw) + old = Stars( + log10ages=grid.log10ages, + metallicities=grid.metallicities, + sf_hist=sfh, + metal_dist=zd, + **kw, + ) + assert new.sf_hist_func is sfh and new.metal_dist_func is zd + assert new._instant_sf is None and old._instant_sf is None + assert new._instant_metallicity == old._instant_metallicity + assert new.tau_v == old.tau_v and new.fesc == old.fesc + assert new.metallicity_grid_type == old.metallicity_grid_type + assert set(vars(new)) == set(vars(old)) + np.testing.assert_allclose(new.initial_mass, old.initial_mass, rtol=1e-12) + np.testing.assert_allclose(new.sfzh.sum(), 3e9, rtol=1e-12) + np.testing.assert_allclose(new.sfzh_normalisation, old.sfzh_normalisation, rtol=1e-6) + np.testing.assert_allclose(new.sf_hist, old.sf_hist, rtol=1e-5, atol=1e-8 * 3e9) + np.testing.assert_allclose(new.metal_dist, old.metal_dist, rtol=1e-6) + np.testing.assert_allclose( + new.calculate_average_sfr((0, 1e8)), old.calculate_average_sfr((0, 1e8)), rtol=1e-5 + ) + np.testing.assert_allclose(new.calculate_mean_age(), old.calculate_mean_age(), rtol=1e-6) + np.testing.assert_allclose( + new.calculate_mean_metallicity(), old.calculate_mean_metallicity(), rtol=1e-10 + ) + np.testing.assert_allclose( + new.calculate_surviving_mass(grid), old.calculate_surviving_mass(grid), rtol=1e-6 + ) + early_new = new.get_at_earlier_time(100 * Myr) + early_old = old.get_at_earlier_time(100 * Myr) + np.testing.assert_allclose(early_new.sfzh, early_old.sfzh, rtol=1e-5, atol=1e-8 * 3e9) + + # Surviving mass normalisation path + new = F.make_parametric_stars( + sfh, zd, grid.log10ages, grid.metallicities, surviving_mass=1e9 * Msun, grid=grid + ) + old = Stars( + log10ages=grid.log10ages, + metallicities=grid.metallicities, + sf_hist=sfh, + metal_dist=zd, + surviving_mass=1e9 * Msun, + grid=grid, + ) + np.testing.assert_allclose(new.initial_mass, old.initial_mass, rtol=1e-6) + np.testing.assert_allclose((new.sfzh * grid.stellar_fraction).sum(), 1e9, rtol=1e-10) + + # No mass given: initial_mass is the raw integral + new = F.make_parametric_stars(sfh, zd, grid.log10ages, grid.metallicities) + old = Stars( + log10ages=grid.log10ages, metallicities=grid.metallicities, sf_hist=sfh, metal_dist=zd + ) + np.testing.assert_allclose(new.initial_mass, old.initial_mass, rtol=1e-6) + + +def test_normal_zdist_stars(grid): + """End-to-end with a Normal metallicity distribution.""" + sfh = SFH.DelayedExponential(tau=500 * Myr, max_age=3000 * Myr) + zd = ZDist.Normal(mean=0.01, sigma=0.005) + new = F.make_parametric_stars( + sfh, zd, grid.log10ages, grid.metallicities, initial_mass=1e9 * Msun + ) + old = Stars( + log10ages=grid.log10ages, + metallicities=grid.metallicities, + sf_hist=sfh, + metal_dist=zd, + initial_mass=1e9 * Msun, + ) + assert _l1(new.sfzh, old.sfzh) < 1e-6 + + +def test_age_offset(grid): + """age_offset reproduces Synthesizer's shifted integration.""" + edges = F.age_bin_edges(grid.log10ages) + for sfh in ( + SFH.LogNormal(tau=0.3, peak_age=300 * Myr, max_age=2000 * Myr), + SFH.Constant(max_age=500 * Myr, min_age=50 * Myr), + ): + for off in (1e6, 3e7, 3e8): + ref = _refined_reference(sfh, edges + off) + got = F.sfh_bin_masses(sfh, grid.log10ages, age_offset=off * yr) + assert _l1(got, ref) < 1e-6 + sfh = SFH.LogNormal(tau=0.3, peak_age=300 * Myr, max_age=2000 * Myr) + zd = ZDist.DeltaConstant(log10metallicity=-2.0) + fast = F.make_parametric_stars( + sfh, zd, grid.log10ages, grid.metallicities, initial_mass=1e9 * Msun, subclass=True + ) + assert isinstance(fast, F.FastParametricStars) and isinstance(fast, Stars) + old = Stars( + log10ages=grid.log10ages, + metallicities=grid.metallicities, + sf_hist=sfh, + metal_dist=zd, + initial_mass=1e9 * Msun, + ) + assert _l1(fast.sfzh, old.sfzh) < 1e-6 + assert ( + _l1(fast.get_at_earlier_time(200 * Myr).sfzh, old.get_at_earlier_time(200 * Myr).sfzh) + < 1e-6 + ) + np.testing.assert_allclose( + fast.calculate_initial_mass_at_age(200 * Myr), + old.calculate_initial_mass_at_age(200 * Myr), + rtol=1e-6, + ) + + +class _MyLogNormal(SFH.LogNormal): + """User subclass: must fall back (it may override _sfr).""" + + +def test_fallbacks(grid): + """Unsupported inputs reproduce Synthesizer's result exactly.""" + warnings.filterwarnings("ignore") + zd = ZDist.DeltaConstant(log10metallicity=-2.0) + cases = [ + (_MyLogNormal(tau=0.3, peak_age=100 * Myr, max_age=1000 * Myr), zd), + (SFH.DoublePowerLaw(peak_age=3000 * Myr, alpha=2.0, beta=30.0), zd), # diverges at 0 + (30 * Myr, zd), # instantaneous burst + (30 * Myr, 0.01), # instantaneous burst + float Z + (SFH.Constant(max_age=100 * Myr), 0.004), # float Z (fast path) + ] + for sfh, z in cases: + new = F.make_parametric_stars( + sfh, z, grid.log10ages, grid.metallicities, initial_mass=1e9 * Msun + ) + old = Stars( + log10ages=grid.log10ages, + metallicities=grid.metallicities, + sf_hist=sfh, + metal_dist=z, + initial_mass=1e9 * Msun, + ) + if F.is_supported_sfh(sfh) and not isinstance(sfh, SFH.DoublePowerLaw): + assert _l1(new.sfzh, old.sfzh) < 1e-6 + else: + np.testing.assert_array_equal(new.sfzh, old.sfzh) + assert not F.is_supported_sfh(_MyLogNormal(tau=0.3, peak_age=100 * Myr, max_age=1000 * Myr)) + assert F.fast_sfzh(30 * Myr, zd, grid.log10ages, grid.metallicities) is None + # sfh_bin_masses always returns an answer (quad loop fallback) + s = _MyLogNormal(tau=0.3, peak_age=100 * Myr, max_age=1000 * Myr) + np.testing.assert_allclose( + F.sfh_bin_masses(s, grid.log10ages), + F._quad_bin_masses(s, F.age_bin_edges(grid.log10ages), 0.0), + ) + with pytest.raises(TypeError): + F.sfh_bin_masses(s, grid.log10ages, fallback=False) From a378157c8fdad3f892f2ee86a623072e5516ea83 Mon Sep 17 00:00:00 2001 From: Thomas Harvey Date: Sun, 1 Nov 2026 22:01:07 +0000 Subject: [PATCH 3/5] Speed up library generation and fix Synthesizer v1.3 incompatibilities Speed: - Use make_parametric_stars and TabulatedIGM in GalaxyBasis and GalaxySimulator (opt out with fast_sfzh=False / tabulate_igm=False). - Replace within-galaxy threading with process parallelism across galaxies: workers build their chunk lazily, run a single-threaded Pipeline and write chunk files that are merged into the usual layout. n_proc now means worker processes (threads per rank under MPI). - Save only the root spectrum by default when no keys are given. - GalaxySimulator computes observed fluxes only for the requested key. Fixes: - Draw hypercube samples in float64 (float32 is rejected by Synthesizer's precision checks). - Keep spectra required by dust-emission line generation when lines are requested. - Return line luminosities in erg/s after Synthesizer's Lsun change. - Fix dust emission save/load round trip and unit loss when saving dust curve parameters. - Validate emission model keys with a clear error; warn when galaxy parameters shadow dust-curve parameters. - Fix serial runs with multiple batches; remove obsolete workarounds. --- src/synference/__init__.py | 10 + src/synference/library.py | 1421 +++++++++++++++++++++++------- src/synference/utils.py | 251 +++++- tests/test_library_fast_paths.py | 495 +++++++++++ 4 files changed, 1873 insertions(+), 304 deletions(-) create mode 100644 tests/test_library_fast_paths.py diff --git a/src/synference/__init__.py b/src/synference/__init__.py index d7398cd..3fb16f8 100644 --- a/src/synference/__init__.py +++ b/src/synference/__init__.py @@ -28,6 +28,14 @@ +try: + # Fast drop-in replacements for Synthesizer's parametric SFZH integration + # and IGM transmission (used by default in library generation). + from .fast_sfzh import make_parametric_stars + from .fast_igm import TabulatedIGM +except ImportError: # synthesizer not installed + pass + from .noise_models import ( EmpiricalUncertaintyModel, create_uncertainty_models_from_EPOCHS_cat, DepthUncertaintyModel, UncertaintyModel, AsinhEmpiricalUncertaintyModel, @@ -114,4 +122,6 @@ "create_database_universal", "download_test_data", "test_data_dir", + "make_parametric_stars", + "TabulatedIGM", ] \ No newline at end of file diff --git a/src/synference/library.py b/src/synference/library.py index f1d7588..d08511f 100644 --- a/src/synference/library.py +++ b/src/synference/library.py @@ -3,6 +3,7 @@ import copy import inspect import os +import sys import threading from collections import defaultdict from datetime import datetime @@ -16,7 +17,6 @@ import torch from astropy.cosmology import Cosmology, Planck18, z_at_value from dill.source import getsource -from joblib import Parallel, delayed, parallel_config from matplotlib.ticker import FuncFormatter, ScalarFormatter from scipy.linalg import inv from scipy.stats import qmc @@ -35,6 +35,9 @@ from synthesizer.particle.stars import sample_sfzh from synthesizer.pipeline import Pipeline + from .fast_igm import TabulatedIGM + from .fast_sfzh import make_parametric_stars + synthesizer_available = True except ImportError as e: print(e) @@ -65,7 +68,13 @@ from .noise_models import ( EmpiricalUncertaintyModel, ) -from .utils import check_log_scaling, check_scaling, list_parameters, save_emission_model +from .utils import ( + check_log_scaling, + check_scaling, + list_parameters, + load_dust_emission_params, + save_emission_model, +) file_path = os.path.dirname(os.path.realpath(__file__)) library_folder = os.path.join(os.path.dirname(os.path.dirname(file_path)), "libraries") @@ -458,7 +467,9 @@ def calculate_line_luminosity( emission_model_key: The key for the emission model to use (default is 'total'). Returns: - A dictionary with line names as keys and their luminosities as values. + The line luminosity as a unyt_quantity in erg/s. (Synthesizer's + internal luminosity unit is now Lsun; erg/s is kept for continuity + with libraries created with older versions.) """ from synthesizer.emissions.utils import aliases @@ -466,7 +477,7 @@ def calculate_line_luminosity( galaxy.stars.get_lines(([line]), emission_model) - return galaxy.stars.lines[emission_model_key].luminosity[0] + return galaxy.stars.lines[emission_model_key].luminosity[0].to("erg/s") def calculate_sfh_quantile(galaxy, quantile=0.5, norm=False, cosmo=Planck18): @@ -547,7 +558,7 @@ def calculate_xi_ion0(galaxy, emission_model, emission_model_key="total"): galaxy.stars.get_lines(([Ha]), emission_model) - ha_luminosity = galaxy.stars.lines[emission_model_key].luminosity[0] + ha_luminosity = galaxy.stars.lines[emission_model_key].luminosity[0].to("erg/s") MUV = calculate_MUV(galaxy, cosmo=Planck18)[emission_model_key].to("erg/s/Hz") @@ -630,8 +641,10 @@ def calculate_agn_fraction( wav = galaxy.spectra[total_em_key].lam - total_lum = galaxy.spectra[total_em_key].luminosity # erg/s - agn_lum = galaxy.black_holes.spectra[agn_em_key].luminosity # erg/s + # nu * L_nu in Synthesizer's internal luminosity unit (Lsun); the units + # cancel in the ratio below. + total_lum = galaxy.spectra[total_em_key].luminosity + agn_lum = galaxy.black_holes.spectra[agn_em_key].luminosity mask = (wav >= min_wav_rest) & (wav <= max_wav_rest) agn_fraction = np.trapezoid(agn_lum[mask], x=wav[mask]) / np.trapezoid( @@ -1099,7 +1112,9 @@ def draw_from_hypercube( all_param_dict = {} for i, key in enumerate(param_ranges.keys()): - samples = scaled_samples[:, i].astype(np.float32) + # Keep float64: Synthesizer's precision checks reject mixing float32 + # parameters (e.g. ZDist metallicities) with float64 masses/grids. + samples = scaled_samples[:, i].astype(np.float64) if key in unlog_keys: samples = 10**samples @@ -1135,7 +1150,8 @@ def load_hypercube_from_npy(file_path: str): # Load the hypercube data from the .npy file hypercube = np.load(file_path) - return hypercube.astype(np.float32) + # float64 to match Synthesizer's default precision (see draw_from_hypercube) + return hypercube.astype(np.float64) def generate_sfh_basis( @@ -1338,7 +1354,65 @@ def generate_sfh_basis( return np.array(sfhs), redshifts -# from joblib import delayed, Parallel, wrap_non_picklable_objects, parallel_config +# ------------------------------------------ +# Galaxy construction +# ------------------------------------------ + +# Environment variables controlling the size of native thread pools. Worker +# processes used for process-based parallelism set these to 1 so that N +# processes do not each spin up N threads (oversubscription). +_THREAD_ENV_VARS = ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", + "NUMEXPR_NUM_THREADS", +) + +# Minimum number of galaxies per worker for process-based parallelism to be +# worthwhile (smaller jobs are run serially). +_MIN_GALAXIES_PER_WORKER = 25 +# Target number of chunks per worker (for load balancing) and maximum chunk +# size (bounds per-worker memory and the size of the temporary chunk files). +_CHUNKS_PER_WORKER = 4 +_MIN_CHUNK_SIZE = 100 +_MAX_CHUNK_SIZE = 2000 +# Number of galaxies above which tabulate_igm="auto" switches on the tabulated +# IGM (building the table costs ~1-3 s, saving ~5-10 ms per observed Sed). +_TABULATE_IGM_MIN_GALAXIES = 500 + + +def _as_float64_scalar(value): + """Cast reduced-precision numpy floating scalars to Python floats. + + Synthesizer refuses to mix float32 and float64 arrays, so e.g. a float32 + tau_v drawn from a hypercube/torch tensor can break spectra generation + when everything else is float64. float64 values are returned unchanged. + """ + if isinstance(value, np.floating) and value.dtype != np.float64: + return float(value) + if isinstance(value, unyt_quantity) and value.dtype.kind == "f" and value.dtype != np.float64: + return value.astype(np.float64) + return value + + +def _float64_metal_dist(metal_dist): + """Return a DeltaConstant ZDist whose metallicity is float64. + + A ``ZDist.DeltaConstant`` built from a float32 metallicity would make + Synthesizer's quad SFZH path fail its precision checks. Returns a shallow + copy (the user's object is not modified) or the input unchanged. + """ + if ( + isinstance(metal_dist, ZDist.Common) + and getattr(metal_dist, "name", None) == "DeltaConstant" + and isinstance(getattr(metal_dist, "metallicity_", None), np.floating) + and metal_dist.metallicity_.dtype != np.float64 + ): + metal_dist = copy.copy(metal_dist) + metal_dist.metallicity_ = float(metal_dist.metallicity_) + metal_dist.log10metallicity_ = float(metal_dist.log10metallicity_) + return metal_dist def create_galaxy( @@ -1347,11 +1421,35 @@ def create_galaxy( metal_dist: Type[ZDist.Common], grid: Grid, log_stellar_masses: Union[float, list] = 9, + fast_sfzh: bool = True, **galaxy_kwargs, ) -> Type[Galaxy]: - """Create a new galaxy with the specified parameters.""" - # Initialise the parametric Stars object + """Create a new galaxy with the specified parameters. + Parameters + ---------- + sfh : SFH.Common + Star formation history. + redshift : float + Redshift of the galaxy. + metal_dist : ZDist.Common + Metallicity distribution. + grid : Grid + SPS grid (sets the age/metallicity binning of the SFZH). + log_stellar_masses : float or list of float + log10 initial stellar mass in Msun. A list creates a particle galaxy + by sampling the SFZH. + fast_sfzh : bool + If True (default) the SFZH is computed with the vectorised + :func:`synference.fast_sfzh.make_parametric_stars` (~1 ms instead of + 10-100 ms; agrees with Synthesizer's quad integration to <=1e-3 + relative per bin, usually ~1e-8). It falls back to Synthesizer's own + path for unsupported SFH/ZDist classes. Set False to use + ``synthesizer.parametric.Stars`` directly (the original behaviour). + galaxy_kwargs : dict + Emitter parameters set on the Stars object; keys prefixed ``bh_`` / + ``gas_`` create a parametric BlackHole / Gas component. + """ assert not isinstance(log_stellar_masses, (unyt_array, unyt_quantity)), ( "log_stellar_masses must be a float or list of floats, not a unyt array" ) @@ -1360,12 +1458,13 @@ def create_galaxy( log_stellar_masses[0] if isinstance(log_stellar_masses, (list)) else log_stellar_masses ) - single_mass = 10**single_mass * Msun + single_mass = 10 ** float(single_mass) * Msun # If there are any black hole or gas kwargs, pop them out of galaxy_kwargs bh_kwargs = {} gas_kwargs = {} for key in list(galaxy_kwargs.keys()): + galaxy_kwargs[key] = _as_float64_scalar(galaxy_kwargs[key]) if key.startswith("bh_"): new_key = key.replace("bh_", "") bh_kwargs[new_key] = galaxy_kwargs.pop(key) @@ -1373,14 +1472,26 @@ def create_galaxy( new_key = key.replace("gas_", "") gas_kwargs[new_key] = galaxy_kwargs.pop(key) - param_stars = Stars( - log10ages=grid.log10ages, - metallicities=grid.metallicities, - sf_hist=sfh, - metal_dist=metal_dist, - initial_mass=single_mass, - **galaxy_kwargs, # most parameters want to be on the emitter - ) + metal_dist = _float64_metal_dist(metal_dist) + + if fast_sfzh: + param_stars = make_parametric_stars( + sfh, + metal_dist, + np.asarray(grid.log10ages), + grid.metallicities, + initial_mass=single_mass, + **galaxy_kwargs, # most parameters want to be on the emitter + ) + else: + param_stars = Stars( + log10ages=grid.log10ages, + metallicities=grid.metallicities, + sf_hist=sfh, + metal_dist=metal_dist, + initial_mass=single_mass, + **galaxy_kwargs, # most parameters want to be on the emitter + ) # Define the number of stellar particles n = len(log_stellar_masses) if isinstance(log_stellar_masses, list) else 1 @@ -1410,8 +1521,6 @@ def create_galaxy( from synthesizer.parametric import BlackHole bh = BlackHole(**bh_kwargs) - # Dumb fix - bh.accretion_rates_eddington = bh.accretion_rate_eddington else: bh = None @@ -1428,74 +1537,468 @@ def create_galaxy( return galaxy +def _galaxy_save_params(galaxy_data, fixed_params, alt_parametrizations): + """Return the parameter dictionary stored for a matched galaxy. + + This is the dictionary attached to the galaxy as ``gal.all_params`` and + written to the pipeline output (one dataset per key). It only depends on + the (lightweight) galaxy description, so it can be computed without + building the galaxy. + """ + params = fixed_params.copy() + params.update(galaxy_data["varying_params"]) + save_params = copy.deepcopy(params) + + save_params["redshift"] = galaxy_data["redshift"] + try: + save_params.update(galaxy_data["sfh"].parameters) + except AttributeError: + # This occurs if SFH is just a burst e.g. a single value + save_params["sfh"] = galaxy_data["sfh"] + + try: + save_params.update(galaxy_data["metal_dist"].parameters) + except AttributeError: + save_params["metal_dist"] = galaxy_data["metal_dist"] + + # Apply alternative parametrizations + if len(alt_parametrizations) > 0: + to_remove = set() + for key, (new_key, func) in alt_parametrizations.items(): + if key in save_params: + if isinstance(new_key, str): + save_params[new_key] = func(save_params) + to_remove.add(key) + elif isinstance(new_key, (list, tuple)): + for k in new_key: + save_params[k] = func(k, save_params) + to_remove.add(key) + + for key in to_remove: + save_params.pop(key, None) + + return save_params + + +def _build_matched_galaxy(galaxy_data, fixed_params, grid, save_params=None, fast_sfzh=True): + """Build one galaxy from its lightweight description.""" + params = fixed_params.copy() + params.update(galaxy_data["varying_params"]) + gal = create_galaxy( + sfh=galaxy_data["sfh"], + redshift=galaxy_data["redshift"], + metal_dist=galaxy_data["metal_dist"], + log_stellar_masses=galaxy_data["log_stellar_mass"], + grid=grid, + fast_sfzh=fast_sfzh, + **params, + ) + gal.all_params = save_params + return gal + + +class _GalaxyRecipe: + """Lightweight, picklable description of a set of galaxies. + + Holds, for every galaxy, the small objects needed to build it (SFH, + ZDist, redshift, mass and varying emitter parameters) plus the parameter + dictionary to attach as ``gal.all_params``. Galaxies are only built when + needed (:meth:`build`), so the parent process of a parallel run never + holds every galaxy, and worker processes only build their own chunk. + + Slicing returns a new recipe for the sub-range. + """ + + def __init__(self, galaxy_data, save_params, fixed_params, fast_sfzh=True): + """Store the galaxy descriptions (see class docstring).""" + assert len(galaxy_data) == len(save_params) + self.galaxy_data = galaxy_data + self.save_params = save_params + self.fixed_params = fixed_params + self.fast_sfzh = fast_sfzh + + def __len__(self): + """Number of galaxies described.""" + return len(self.galaxy_data) + + def __getitem__(self, item): + """Return a sub-recipe for a slice.""" + if not isinstance(item, slice): + raise TypeError("_GalaxyRecipe only supports slicing; use build().") + return _GalaxyRecipe( + self.galaxy_data[item], self.save_params[item], self.fixed_params, self.fast_sfzh + ) + + @property + def redshifts(self): + """Redshifts of the described galaxies.""" + return np.array([float(d["redshift"]) for d in self.galaxy_data]) + + def build(self, grid) -> List[Type[Galaxy]]: + """Build all described galaxies.""" + return [ + _build_matched_galaxy(d, self.fixed_params, grid, p, fast_sfzh=self.fast_sfzh) + for d, p in zip(self.galaxy_data, self.save_params) + ] + + def _init_worker(grid, alt_parametrizations, fixed_params): - """Initialize worker process with shared data.""" + """Initialize worker with shared data (kept for backwards compatibility).""" _thread_local.grid = grid _thread_local.alt_parametrizations = alt_parametrizations _thread_local.fixed_params = fixed_params -def _process_galaxy_batch(galaxy_indices_and_data): - """Process a batch of galaxies in a single worker.""" - # Access shared data from thread-local storage +def _process_galaxy_batch(galaxy_indices_and_data, fast_sfzh=True): + """Build a batch of matched galaxies (kept for backwards compatibility).""" grid = _thread_local.grid alt_parametrizations = _thread_local.alt_parametrizations base_params = _thread_local.fixed_params galaxies = [] - for galaxy_idx, galaxy_data in galaxy_indices_and_data: - # Reconstruct minimal parameters for this galaxy - params = base_params.copy() # Only copy once per batch item - params.update(galaxy_data["varying_params"]) - - save_params = copy.deepcopy(params) - - # Create galaxy - gal = create_galaxy( - sfh=galaxy_data["sfh"], - redshift=galaxy_data["redshift"], - metal_dist=galaxy_data["metal_dist"], - log_stellar_masses=galaxy_data["log_stellar_mass"], - grid=grid, # Use shared reference - **params, + save_params = _galaxy_save_params(galaxy_data, base_params, alt_parametrizations) + gal = _build_matched_galaxy( + galaxy_data, base_params, grid, save_params, fast_sfzh=fast_sfzh ) + galaxies.append((galaxy_idx, gal)) - # Process parameters (reuse logic from original) - save_params["redshift"] = galaxy_data["redshift"] - try: - save_params.update(galaxy_data["sfh"].parameters) - except AttributeError: - # This occurs if SFH is just a burst e.g. a single value - save_params["sfh"] = galaxy_data["sfh"] - pass + return galaxies + + +# ------------------------------------------ +# Process-based parallelism for the pipeline +# ------------------------------------------ + +# State shared with worker processes. With the fork start method this is +# inherited copy-on-write (nothing is pickled); otherwise it is pickled once +# per worker by the executor initializer. +_PARALLEL_STATE: Dict[str, Any] = {} + + +def _limit_native_threads(): + """Limit OpenMP/BLAS/torch thread pools of this process to one thread.""" + for var in _THREAD_ENV_VARS: + os.environ[var] = "1" + try: + from threadpoolctl import threadpool_limits + + threadpool_limits(1) + except Exception: # pragma: no cover - threadpoolctl is optional + pass + try: + torch.set_num_threads(1) + except Exception: # pragma: no cover + pass + + +def _parallel_worker_init(state): + """Initializer for pipeline worker processes.""" + global _PARALLEL_STATE + _PARALLEL_STATE = state + _limit_native_threads() + + +def _parallel_worker_run(task): + """Build a chunk of galaxies, run a serial Pipeline on it and write it. + + Args: + task (tuple): (chunk_id, start, end, items, out_path). ``items`` is + None when the galaxy source was inherited from the parent (fork), + otherwise the chunk's galaxies / recipe. + + Returns: + tuple: (chunk_id, number of galaxies, elapsed seconds) + """ + chunk_id, start, end, items, out_path = task + state = _PARALLEL_STATE + basis = state["basis"] + if items is None: + items = state["source"][start:end] + + t0 = datetime.now() + if isinstance(items, _GalaxyRecipe): + galaxies = items.build(basis.grid) + else: + galaxies = list(items) + + pipeline = basis._setup_pipeline( + nthreads=1, + comm=None, + verbose=0, + igm=state["igm"], + em_lines_to_save=state["em_lines_to_save"], + extra_analysis_functions=state["extra_analysis_functions"], + ) + # NOTE: Pipeline.run pops galaxies from the list it was given + n = len(galaxies) + pipeline.add_galaxies(galaxies) + pipeline.run() + pipeline.write(out_path, verbose=0) + del pipeline, galaxies + return chunk_id, n, (datetime.now() - t0).total_seconds() + + +class _PipelinePool: + """Worker processes running the Synthesizer pipeline on chunks of galaxies. + + The galaxies (a list of Galaxy objects or a lazy ``_GalaxyRecipe``) are + split into contiguous chunks. Each worker process builds its chunk's + galaxies, runs a serial Synthesizer Pipeline (nthreads=1, no MPI) on + them and writes a temporary HDF5 file; the chunk files are then + concatenated in order (:func:`synference.utils.merge_pipeline_files`) + into a file with exactly the layout a serial ``Pipeline.write`` produces. + + On Linux the ``fork`` start method is used: the GalaxyBasis (grid, + emission model, instrument), the IGM and the galaxy descriptions are + inherited copy-on-write and only chunk indices are sent to the workers. + Elsewhere (or with ``start_method="loky"``) a loky executor is used: the + shared state is pickled once per worker and each chunk is sent with its + task. The pool is created once and reused for every batch, since + starting worker processes is not free (~0.1 s per fork for a typical + Synference process). + """ + def __init__( + self, + basis, + galaxies, + n_proc, + igm=None, + em_lines_to_save=None, + extra_analysis_functions=None, + start_method=None, + ): + """Start the worker processes (see class docstring).""" + import multiprocessing as mp + + self.basis = basis + self.galaxies = galaxies + self.n_workers = max(1, int(n_proc)) + + if start_method is None: + start_method = ( + "fork" + if sys.platform.startswith("linux") and "fork" in mp.get_all_start_methods() + else "loky" + ) + self.start_method = start_method + + state = { + "basis": basis, + "source": galaxies if start_method == "fork" else None, + "igm": igm, + "em_lines_to_save": em_lines_to_save, + "extra_analysis_functions": extra_analysis_functions or {}, + } + + if start_method == "loky": + from joblib.externals.loky import get_reusable_executor + + self.executor = get_reusable_executor( + max_workers=self.n_workers, + initializer=_parallel_worker_init, + initargs=(state,), + env={var: "1" for var in _THREAD_ENV_VARS}, + reuse=False, + ) + else: + from concurrent.futures import ProcessPoolExecutor + + self.executor = ProcessPoolExecutor( + max_workers=self.n_workers, + mp_context=mp.get_context(start_method), + initializer=_parallel_worker_init, + initargs=(state,), + ) + + def __enter__(self): + """Use as a context manager (shuts the workers down on exit).""" + return self + + def __exit__(self, *exc): + """Shut the workers down.""" + self.shutdown() + return False + + def shutdown(self): + """Shut the worker processes down.""" + if self.executor is not None: + if self.start_method == "loky": + self.executor.shutdown(wait=True) + else: + self.executor.shutdown(wait=True, cancel_futures=True) + self.executor = None + + def run(self, start: int, end: int, out_path: str) -> None: + """Process galaxies[start:end] and write them to out_path.""" + import shutil + import tempfile + import warnings + from concurrent.futures import as_completed + + from .utils import merge_pipeline_files + + n_gal = end - start + n_workers = max(1, min(self.n_workers, n_gal)) + # A few chunks per worker for load balancing, but not too small + # (each chunk pays a fixed pipeline set-up + file write cost). + chunk_size = max( + int(np.ceil(n_gal / (n_workers * _CHUNKS_PER_WORKER))), + min(_MIN_CHUNK_SIZE, int(np.ceil(n_gal / n_workers))), + ) + chunk_size = max(1, min(chunk_size, _MAX_CHUNK_SIZE)) + # Use a multiple of the number of workers so they finish together. + n_chunks = int(np.ceil(n_gal / chunk_size)) + n_chunks = int(np.ceil(n_chunks / n_workers)) * n_workers + chunk_size = max(1, int(np.ceil(n_gal / n_chunks))) + ranges = [(s, min(s + chunk_size, end)) for s in range(start, end, chunk_size)] + + out_dir = os.path.dirname(os.path.abspath(out_path)) + base = os.path.splitext(os.path.basename(out_path))[0] + tmp_dir = tempfile.mkdtemp(prefix=f".{base}_chunks_", dir=out_dir) + chunk_paths = [os.path.join(tmp_dir, f"chunk_{i:06d}.hdf5") for i in range(len(ranges))] + + logger.info( + f"Running pipeline for {n_gal} galaxies on {n_workers} worker processes " + f"({len(ranges)} chunks of <= {chunk_size}, start method '{self.start_method}')." + ) + + futures = [] + t_start = datetime.now() try: - save_params.update(galaxy_data["metal_dist"].parameters) - except AttributeError: - save_params["metal_dist"] = galaxy_data["metal_dist"] - pass + with warnings.catch_warnings(): + # Python >= 3.12 warns about fork() in multi-threaded processes; + # the workers only run single-threaded serial code. + warnings.filterwarnings("ignore", category=DeprecationWarning) + for i, (s, e) in enumerate(ranges): + items = None if self.start_method == "fork" else self.galaxies[s:e] + futures.append( + self.executor.submit(_parallel_worker_run, (i, s, e, items, chunk_paths[i])) + ) + done = 0 + for fut in tqdm( + as_completed(futures), + total=len(futures), + desc=f"Processing {self.basis.model_name} galaxies", + disable=(rank != 0), + ): + _, n_done, _ = fut.result() + done += n_done + if done != n_gal: + raise RuntimeError(f"Expected {n_gal} galaxies to be processed, got {done}.") + + t_merge = datetime.now() + # Uncompressed: gzip saves only a few per cent on float data but + # would make the (serial) merge several times slower. + merge_pipeline_files(chunk_paths, out_path, compression=None) + logger.info( + f"Workers took {t_merge - t_start}; merging {len(chunk_paths)} chunk files " + f"took {datetime.now() - t_merge}." + ) + except BaseException: + for fut in futures: + fut.cancel() + self.shutdown() + raise + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + +def _validate_emission_model_keys(emission_model, keys, what="emission_model_key"): + """Check that each key (label or glob pattern) exists in an emission model. + + Raises: + ValueError: with the available labels and the model's root label, + since e.g. Synthesizer's TotalEmission without dust emission + returns a model whose root label is 'attenuated' or 'emergent' + rather than the requested 'total'. + """ + import fnmatch + + if keys is None: + return + if isinstance(keys, str): + keys = [keys] + labels = list(emission_model._models.keys()) + for key in keys: + if not isinstance(key, str): + raise TypeError(f"{what} must be a string label, got {type(key)}.") + if key in labels or any(fnmatch.fnmatchcase(label, key) for label in labels): + continue + raise ValueError( + f"{what}='{key}' does not match any model in the emission model " + f"{type(emission_model).__name__}. Its root label is " + f"'{emission_model.label}' (use emission_model.label), available labels: " + f"{sorted(labels)}. Note that some Synthesizer models ignore label= " + "(e.g. TotalEmission without dust emission returns a model labelled " + "'attenuated' or 'emergent')." + ) - # Apply alternative parametrizations - if len(alt_parametrizations) > 0: - to_remove = set() - for key, (new_key, func) in alt_parametrizations.items(): - if key in save_params: - if isinstance(new_key, str): - save_params[new_key] = func(save_params) - to_remove.add(key) - elif isinstance(new_key, (list, tuple)): - for k in new_key: - save_params[k] = func(k, save_params) - to_remove.add(key) - - for key in to_remove: - save_params.pop(key, None) - - gal.all_params = save_params - galaxies.append((galaxy_idx, gal)) - return galaxies +def _line_dependency_labels(emission_model) -> List[str]: + """Labels whose spectra must be kept for emission-line generation. + + Energy-balance / scaled generators (e.g. dust emission: Greybody, + Blackbody, Casey12, DraineLi07) generate their lines from the spectra of + the generator model itself and of the models they are scaled by + (intrinsic/attenuated/scaler). Since Synthesizer only keeps saved spectra + (and skips unsaved branches), these must be saved when lines are wanted. + """ + labels = [] + for label, model in emission_model._models.items(): + if not getattr(model, "_is_generating", False): + continue + gen = model.generator + deps = [] + for attr in ("_intrinsic", "_attenuated", "_scaler"): + dep = getattr(gen, attr, None) + if dep is None: + continue + deps.append(dep if isinstance(dep, str) else getattr(dep, "label", None)) + deps = [d for d in deps if d is not None and d in emission_model._models] + if len(deps) > 0: + for lab in [label] + deps: + if lab not in labels: + labels.append(lab) + return labels + + +def _dust_curve_param_clashes(emission_model, param_names) -> Dict[str, List[str]]: + """Find emitter parameters with the same name as a dust curve parameter. + + Synthesizer lets an emitter attribute override a transformer parameter + of the same name, so e.g. a galaxy parameter called ``slope`` would + silently replace a fixed ``PowerLaw(slope=-1)``. A curve parameter set to + a string deliberately aliases an emitter attribute and is not reported. + + Returns: + dict: curve class name -> list of clashing parameter names. + """ + try: + from synthesizer.emission_models.transformers.dust_attenuation import ( + AttenuationLaw, + ) + except ImportError: # pragma: no cover + return {} + + clashes = {} + seen = set() + for model in emission_model._models.values(): + curve = getattr(model, "_transformer", None) + if not isinstance(curve, AttenuationLaw) or id(curve) in seen: + continue + seen.add(id(curve)) + try: + curve_params = set(inspect.signature(type(curve).__init__).parameters) - {"self"} + except (TypeError, ValueError): # pragma: no cover + continue + for name in param_names: + if name in curve_params and hasattr(curve, name): + if isinstance(getattr(curve, name), str): + continue + clashes.setdefault(type(curve).__name__, []).append(name) + return clashes class GalaxyBasis: @@ -1532,6 +2035,8 @@ def __init__( redshift_dependent_sfh: bool = False, params_to_ignore: List[str] = None, build_library: bool = False, + fast_sfzh: bool = True, + tabulate_igm: Union[bool, str] = "auto", ) -> None: """Initialize the GalaxyBasis object with SFHs, redshifts, and other parameters. @@ -1576,6 +2081,18 @@ def __init__( If False, assume all dimensions of parameters are the same size and build the library from the parameters. I.e don't generate combinations of parameters, just use the parameters as they are. + fast_sfzh : bool, optional + If True (default), build the parametric SFZH of each galaxy with the + vectorised :mod:`synference.fast_sfzh` (10-100x faster than + Synthesizer's per-bin ``scipy.quad``; per-bin SFZH agrees to <=1e-3 + relative, photometry to <~1e-6 mag). Set False to reproduce the + original Synthesizer ``Stars`` path exactly. + tabulate_igm : bool or "auto", optional + Whether :meth:`process_galaxies` replaces the IGM model by a + :class:`synference.fast_igm.TabulatedIGM` (pre-computed transmission + table, ~50x faster; band errors <~1.3e-4 mag). "auto" (default) + tabulates when at least 500 galaxies are processed (building the + table takes ~1-3 s). False reproduces the original exact IGM. """ if galaxy_params is None: galaxy_params = {} @@ -1601,6 +2118,9 @@ def __init__( self.log_stellar_masses = log_stellar_masses self.params_to_ignore = params_to_ignore self.build_library = build_library + self.fast_sfzh = fast_sfzh + self.tabulate_igm = tabulate_igm + self._igm_cache = {} self.galaxies = [] @@ -1629,6 +2149,21 @@ def __init__( # If the value is a dictionary, process it as a prior self.galaxy_params[key] = self.process_priors(value) + # Synthesizer lets emitter attributes override dust curve parameters of + # the same name, which can silently change a fixed curve parameter. + if isinstance(emission_model, EmissionModel): + emitter_param_names = set(self.galaxy_params.keys()) + clashes = _dust_curve_param_clashes(emission_model, emitter_param_names) + for curve_name, names in clashes.items(): + logger.warning( + f"galaxy_params {names} have the same name as parameters of the " + f"dust curve {curve_name} in the emission model. Synthesizer uses " + "the emitter (galaxy) value in preference to the value fixed on " + "the curve, so the curve's own value will be overridden. Rename the " + "galaxy parameter, or set the curve parameter to the galaxy " + "parameter's name (a string) if this is intended." + ) + if not build_library: logger.info("Generating library directly from provided parameter samples.") elif self.redshift_dependent_sfh: @@ -1773,6 +2308,7 @@ def _create_galaxies( metal_dist=Z_dist, log_stellar_masses=log_base_masses, grid=self.grid, + fast_sfzh=self.fast_sfzh, **params, ) @@ -1961,6 +2497,14 @@ def _check_model_simplicity(self, parameter_transforms_to_save=None, verbose=Tru ] ) + try: + if isinstance(self.emission_model, EmissionModel): + save_emission_model(self.emission_model) + except ValueError as e: + if verbose: + logger.warning(f"{e} Cannot store model.") + accept = False + if type(self.emission_model).__name__ not in PREMADE_MODELS: if verbose: logger.warning( @@ -2166,6 +2710,7 @@ def create_galaxy( Type[Galaxy] Galaxy object created with the specified parameters. """ + galaxy_kwargs.setdefault("fast_sfzh", getattr(self, "fast_sfzh", True)) return create_galaxy( sfh=sfh, redshift=redshift, @@ -2175,37 +2720,27 @@ def create_galaxy( **galaxy_kwargs, ) - def create_galaxies_optimized( + def _matched_galaxy_recipe( self, fixed_params, varying_param_names, log_base_masses, galaxies_mask=None, - n_proc=28, - batch_size=None, - ): - """Optimized version with reduced serialization overhead.""" - # Determine optimal batch size if not provided - if batch_size is None: - total_galaxies = len(self.sfhs) - if galaxies_mask is not None and len(galaxies_mask) > 0: - total_galaxies = np.sum(galaxies_mask) - batch_size = max(1, total_galaxies // (n_proc)) # 1 batch per thread - logger.info(total_galaxies) - - # Prepare lightweight job inputs (grouped into batches) - job_batches = [] - current_batch = [] - - for i in tqdm(range(len(self.sfhs)), desc="Preparing galaxy batches", disable=(rank != 0)): + ) -> "_GalaxyRecipe": + """Describe the matched galaxies without building them. + + Returns a :class:`_GalaxyRecipe` holding, for each (unmasked) index, + the SFH, ZDist, redshift, mass and varying parameters, plus the + ``all_params`` dictionary stored for that galaxy. + """ + galaxy_data = [] + save_params = [] + for i in tqdm(range(len(self.sfhs)), desc="Preparing galaxies", disable=(rank != 0)): # Skip this galaxy if the mask is provided and is False if galaxies_mask is not None and len(galaxies_mask) > 0 and not galaxies_mask[i]: continue - # Create minimal data structure (no large objects) - varying_params = {} - for key in varying_param_names: - varying_params[key] = self.galaxy_params[key][i] + varying_params = {key: self.galaxy_params[key][i] for key in varying_param_names} # Determine the stellar mass for this galaxy try: @@ -2213,69 +2748,53 @@ def create_galaxies_optimized( except (IndexError, TypeError): mass = log_base_masses - galaxy_data = { + data = { "sfh": self.sfhs[i], "redshift": self.redshifts[i], "metal_dist": self.metal_dists[i], "log_stellar_mass": mass, "varying_params": varying_params, # Only varying parameters } + galaxy_data.append(data) + save_params.append(_galaxy_save_params(data, fixed_params, self.alt_parametrizations)) + + return _GalaxyRecipe( + galaxy_data, + save_params, + fixed_params, + fast_sfzh=getattr(self, "fast_sfzh", True), + ) - current_batch.append((i, galaxy_data)) - - # Create batch when it reaches desired size - if len(current_batch) >= batch_size: - job_batches.append(current_batch) - current_batch = [] - - # Add remaining galaxies to final batch - if current_batch: - job_batches.append(current_batch) - - if n_proc > 1: - logger.info(f"Creating {len(job_batches)} batches across {n_proc} processes.") - logger.info( - f"Average batch size: {sum(len(batch) for batch in job_batches) / len(job_batches):.1f}" # noqa: E501 - ) - - # Create a wrapper function that includes the shared data - def process_batch_with_shared_data(batch): - # Initialize thread-local data for this worker - _init_worker(self.grid, self.alt_parametrizations, fixed_params) - return _process_galaxy_batch(batch) - - # Use threading backend - with parallel_config("threading"): - tasks = [delayed(process_batch_with_shared_data)(batch) for batch in job_batches] - - # Process batches - batch_results = Parallel(n_jobs=n_proc)( - tqdm(tasks, desc="Creating galaxy batches", disable=(rank != 0)) - ) - - # Flatten results and sort by original index - all_results = [] - for batch_result in batch_results: - all_results.extend(batch_result) - - # Sort by original galaxy index to maintain order - all_results.sort(key=lambda x: x[0]) - self.galaxies = [galaxy for _, galaxy in all_results] + def create_galaxies_optimized( + self, + fixed_params, + varying_param_names, + log_base_masses, + galaxies_mask=None, + n_proc=28, + batch_size=None, + ): + """Build all matched galaxies (serially) and store them in self.galaxies. - else: - # Sequential processing - initialize once for the main thread - _init_worker(self.grid, self.alt_parametrizations, fixed_params) - self.galaxies = [] - for batch in tqdm(job_batches, desc="Creating galaxies", disable=(rank != 0)): - batch_galaxies = _process_galaxy_batch(batch) - self.galaxies.extend([galaxy for _, galaxy in batch_galaxies]) + With the fast SFZH galaxy creation costs ~1 ms, so it is no longer + parallelised here; ``n_proc`` and ``batch_size`` are accepted for + backwards compatibility and ignored. Parallelism across galaxies is + done by :meth:`process_galaxies` (worker processes build *and* process + their own galaxies). + """ + recipe = self._matched_galaxy_recipe( + fixed_params, varying_param_names, log_base_masses, galaxies_mask + ) + self.galaxies = recipe.build(self.grid) + return self.galaxies def _create_matched_galaxies( self, log_base_masses: Union[float, np.ndarray] = 9, galaxies_mask: Optional[np.ndarray] = None, n_proc: int = 1, - ) -> List[Type[Galaxy]]: + lazy: bool = False, + ) -> Union[List[Type[Galaxy]], "_GalaxyRecipe"]: """Creates galaxies where all parameters have been sampled. A galaxy is created for each SFH, redshift, and metallicity distribution @@ -2285,13 +2804,21 @@ def _create_matched_galaxies( ---------- log_base_masses : Union[float, np.ndarray], optional Base mass (or array of base masses) to use for the galaxies. - n_procs : int, optional - Number of processes to use for parallel processing, by default 1. + galaxies_mask : np.ndarray, optional + Boolean mask selecting which galaxies to create. + n_proc : int, optional + Unused (kept for backwards compatibility). See process_galaxies. + lazy : bool, optional + If True, do not build the galaxies; return a lightweight + ``_GalaxyRecipe`` which :meth:`process_galaxies` builds batch by + batch (or inside worker processes). self.galaxies is left empty. + Parameter bookkeeping (varying/fixed parameter names etc.) is + identical in both cases. Returns: ------- - List[Type[Galaxy]] - List of Galaxy objects created with the specified parameters. + List[Type[Galaxy]] or _GalaxyRecipe + List of Galaxy objects (or the recipe if lazy=True). """ if len(self.metal_dists) == 1: # Just reference the first one @@ -2341,47 +2868,15 @@ def _create_matched_galaxies( ): self.per_particle = True - """ - job_inputs: List[Dict[str, Any]] = [] - for i in tqdm(range(len(self.sfhs)), desc="Batching galaxy inputs", disable=(rank != 0)): - # Skip this galaxy if the mask is provided and is False - if galaxies_mask is not None and len(galaxies_mask) > 0 and not galaxies_mask[i]: - continue - - # Assemble the parameters for this specific galaxy - params = fixed_params.copy() - for key in varying_param_names: - params[key] = self.galaxy_params[key][i] - - # Determine the stellar mass for this galaxy - try: - mass = log_base_masses[i] - except (IndexError, TypeError): - mass = log_base_masses - - job_inputs.append( - { - "sfh": self.sfhs[i], - "redshift": self.redshifts[i], - "metal_dist": self.metal_dists[i], - "log_stellar_mass": mass, - "params": params, - "grid": self.grid, - "alt_parametrizations": self.alt_parametrizations, - } - ) - """ - - # Use the optimized version - self.create_galaxies_optimized( - galaxies_mask=galaxies_mask, - varying_param_names=varying_param_names, - log_base_masses=log_base_masses, - fixed_params=fixed_params, - n_proc=1, + recipe = self._matched_galaxy_recipe( + fixed_params, varying_param_names, log_base_masses, galaxies_mask ) - - logger.info(f"Created {len(self.galaxies)} galaxies.") + if lazy: + self.galaxies = [] + logger.info(f"Prepared {len(recipe)} galaxies (built on demand).") + else: + self.galaxies = recipe.build(self.grid) + logger.info(f"Created {len(self.galaxies)} galaxies.") # Use sets instead of lists for faster lookups and unique values self.all_parameters = defaultdict(set) @@ -2389,12 +2884,12 @@ def _create_matched_galaxies( param_units = {} # Process all galaxies in one pass - for i, gal in enumerate(self.galaxies): + for i, all_params in enumerate(recipe.save_params): # Store the galaxy parameters directly - self.all_params[i] = gal.all_params + self.all_params[i] = all_params # Add unique values to sets using update operation - for key, value in gal.all_params.items(): + for key, value in all_params.items(): if key not in self.params_to_ignore: if isinstance(value, (unyt_quantity, unyt_array)): unit = str(value.units) @@ -2452,11 +2947,189 @@ def _create_matched_galaxies( logger.info("Finished creating galaxies.") - return self.galaxies + return recipe if lazy else self.galaxies + + def _configure_saved_spectra( + self, + emission_model_keys=None, + spectra_to_save=None, + em_lines_to_save=None, + ) -> Optional[List[str]]: + """Set which spectra the emission model keeps (and therefore computes). + + Synthesizer only generates the models needed for saved spectra, and + the pipeline computes observed spectra/photometry for every saved + spectrum, so saving only what is needed matters a lot for speed. + + - ``emission_model_keys`` given: save those (+ ``spectra_to_save``). + - ``emission_model_keys`` None and the emission model still has its + default "save everything" flags: save only the root model + (``emission_model.label``) + ``spectra_to_save``, with a warning. + (Before v0.2 every spectrum was saved, ~4x slower for Pacman models; + pass ``emission_model_keys="*"`` to get that.) + - ``emission_model_keys`` None and the user configured the saved + spectra already (``emission_model.save_spectra``): respected. + + If emission lines are requested, the spectra that energy-balance dust + emission generators need to generate lines are also saved. + + Returns: + The list of labels that were set to be saved (None if unchanged). + """ + model = self.emission_model + spectra_to_save = list(spectra_to_save) if spectra_to_save is not None else [] + + if emission_model_keys is None: + default_flags = all( + m.save for label, m in model._models.items() if not label.startswith("_") + ) + if default_flags: + keys = [model.label] + logger.warning( + "emission_model_keys not given: only saving the root spectra " + f"'{model.label}' (+ spectra_to_save={spectra_to_save}). Previous " + f"versions saved all {len(model._models)} spectra of the emission " + "model, which is several times slower; pass emission_model_keys='*' " + "(or a list of labels) to save more." + ) + else: + keys = list(model.saved_labels) + elif isinstance(emission_model_keys, str): + keys = [emission_model_keys] + else: + keys = list(emission_model_keys) + + _validate_emission_model_keys(model, keys, "emission_model_keys") + _validate_emission_model_keys(model, spectra_to_save, "spectra_to_save") + + keys = keys + [k for k in spectra_to_save if k not in keys] + if em_lines_to_save is not None and len(em_lines_to_save) > 0: + extra = [k for k in _line_dependency_labels(model) if k not in keys] + if len(extra) > 0: + logger.info( + f"Also saving spectra {extra} (needed to generate emission lines " + "for the dust emission model)." + ) + keys = keys + extra + + model.save_spectra(*keys) + return keys + + def _resolve_igm(self, igm, redshifts, tabulate_igm=None): + """Return the IGM object to use (possibly a cached TabulatedIGM). + + Args: + igm: IGM class/instance requested by the user (or None). + redshifts (array): Redshifts of all galaxies to be processed. + tabulate_igm (bool/str/None): Override of self.tabulate_igm. + """ + if igm is None or isinstance(igm, TabulatedIGM): + return igm + + mode = getattr(self, "tabulate_igm", "auto") if tabulate_igm is None else tabulate_igm + redshifts = np.asarray(redshifts, dtype=float) + if mode == "auto": + mode = len(redshifts) >= _TABULATE_IGM_MIN_GALAXIES + if not mode or len(redshifts) == 0: + return igm + + # Table up to just above the maximum redshift (anything outside the + # table falls back to the exact model anyway). + z_max = max(0.5, float(np.ceil((np.max(redshifts) + 0.05) * 2.0) / 2.0)) + base_key = igm if isinstance(igm, type) else id(igm) + lam = np.asarray(self.grid.lam.to(Angstrom).value, dtype=np.float64) + key = (base_key, z_max, lam.shape[0], float(lam[0]), float(lam[-1])) + + if not hasattr(self, "_igm_cache"): + self._igm_cache = {} + if key not in self._igm_cache: + start = datetime.now() + self._igm_cache[key] = TabulatedIGM(base=igm, z_min=0.0, z_max=z_max, lam=lam) + logger.info( + f"Built tabulated IGM ({self._igm_cache[key]}) in {datetime.now() - start}." + ) + return self._igm_cache[key] + + def _setup_pipeline( + self, + nthreads=1, + comm=None, + verbose=0, + igm=Inoue14, + em_lines_to_save=None, + extra_analysis_functions=None, + ) -> Pipeline: + """Create a Pipeline with the analysis functions and outputs Synference needs.""" + if extra_analysis_functions is None: + extra_analysis_functions = {} + + pipeline = Pipeline( + emission_model=self.emission_model, + nthreads=nthreads, + verbose=verbose, + comm=comm, + ) + + for key in self.all_parameters.keys(): + pipeline.add_analysis_func( + lambda gal, key=key: gal.all_params[key], + result_key=key, + ) + + pipeline.add_analysis_func(lambda gal: gal.stars.initial_mass, result_key="mass") + + # Add any extra analysis functions requested by the user. + for key, params in extra_analysis_functions.items(): + if callable(params): + func = copy.deepcopy(params) + params = [] + else: + func = params[0] + params = params[1:] + + pipeline.add_analysis_func(func, f"supp_{key}", *params) + + # pipeline.get_spectra() # Switch off so they aren't saved + pipeline.get_observed_spectra(self.cosmo, igm=igm) + + if self.instrument.can_do_photometry: + pipeline.get_photometry_fluxes(self.instrument, igm=igm) + + if em_lines_to_save is not None and len(em_lines_to_save) > 0: + # ["H 1 6562.80A", "O 3 5006.84A", "H 1 4861.32A"] + pipeline.get_lines(line_ids=em_lines_to_save) + pipeline.get_observed_lines(self.cosmo, igm=igm) + + return pipeline + + def _run_pipeline_parallel( + self, + galaxies, + out_path: str, + n_proc: int, + igm=None, + em_lines_to_save=None, + extra_analysis_functions=None, + start_method: Optional[str] = None, + ) -> None: + """Run the pipeline over galaxies with worker processes and write out_path. + + Convenience wrapper around :class:`_PipelinePool` for a single batch. + """ + with _PipelinePool( + self, + galaxies, + n_proc, + igm=igm, + em_lines_to_save=em_lines_to_save, + extra_analysis_functions=extra_analysis_functions, + start_method=start_method, + ) as pool: + pool.run(0, len(galaxies), out_path) def process_galaxies( self, - galaxies: List[Type[Galaxy]], + galaxies: Union[List[Type[Galaxy]], "_GalaxyRecipe"], out_name: str = "auto", out_dir: str = "internal", n_proc: int = 4, @@ -2470,14 +3143,18 @@ def process_galaxies( spectra_to_save: list = None, em_lines_to_save: list = None, igm=Inoue14, + tabulate_igm: Union[bool, str, None] = None, + start_method: Optional[str] = None, **extra_analysis_functions, - ) -> Pipeline: + ) -> None: """Processes galaxies through Synthesizer pipeline. Parameters ---------- - galaxies : List[Type[Galaxy]] - List of Galaxy objects to process. + galaxies : List[Type[Galaxy]] or _GalaxyRecipe + List of Galaxy objects to process, or the lazy recipe returned by + ``_create_matched_galaxies(lazy=True)`` (galaxies are then built + batch by batch, or inside the worker processes). out_name : str, optional Name of the output file to save the pipeline results, by default "auto". If "auto", uses the model_name. @@ -2485,50 +3162,82 @@ def process_galaxies( Directory to save the output file, by default "internal". If "internal", saves to the Synthesizer grids directory. n_proc : int, optional - Number of processes to use for parallel processing, by default 4. + Number of worker *processes* (not threads) used to process + galaxies in parallel on this node, by default 4. Each process + builds and processes a chunk of galaxies with a single-threaded + Synthesizer pipeline (OpenMP/BLAS limited to 1 thread), and the + chunk outputs are merged into the same file a serial run writes. + n_proc=1 runs serially in this process. Small jobs (< 25 + galaxies per process) use fewer processes. With ``multi_node=True`` + n_proc keeps its old meaning: OpenMP threads of the per-rank + Synthesizer pipeline (MPI ranks provide the parallelism). verbose : int, optional Verbosity level for the pipeline, by default 1. save : bool, optional If True, saves the pipeline results to disk, by default True. - emission_model_keys : List[str], optional - List of emission model keys to save spectra for, by default None. - If None, saves all spectra. + (Parallel processing requires save=True.) + emission_model_keys : str or List[str], optional + Emission model label(s) to save spectra for, by default None. If + None and the saved spectra of the emission model have not been + configured, only the root model (``emission_model.label``) is saved + (behaviour change: previously all spectra were saved, which is + several times slower; use "*" for all). See + :meth:`_configure_saved_spectra`. batch_galaxies : bool, optional If True, processes galaxies in batches, by default True. If False, processes all galaxies in a single batch. batch_size : int, optional Size of each batch of galaxies to process, by default 40,000. + Each batch is written to its own file (``_.hdf5``) when + there is more than one batch. spectra_to_save : list, optional - List of spectra types to save, by default None. If None, saves only the root spectra. + Additional spectra labels to save, by default None. em_lines_to_save : list, optional List of emission lines to save, by default None. + igm : IGM class or instance, optional + The IGM model, by default Inoue14. None disables IGM attenuation. + tabulate_igm : bool or "auto", optional + Override GalaxyBasis.tabulate_igm for this call (tabulated IGM + transmission, see :class:`synference.fast_igm.TabulatedIGM`). + start_method : str, optional + Multiprocessing start method for n_proc > 1: None (default: "fork" + on Linux, else "loky"), "fork", "forkserver", "spawn" or "loky". extra_analysis_functions : dict, optional Additional analysis functions to add to the pipeline. Should be a dictionary where keys are function names and values are tuples of the function and its parameters. - Returns: - ------- - Pipeline - The pipeline object after processing the galaxies. - """ self.emission_model.set_per_particle(self.per_particle) - if spectra_to_save is None: - spectra_to_save = [] - - if emission_model_keys is not None: - self.emission_model.save_spectra(emission_model_keys, *spectra_to_save) + self._configure_saved_spectra(emission_model_keys, spectra_to_save, em_lines_to_save) logger.info("Creating pipeline.") - if not batch_galaxies: - batch_size = len(galaxies) + if multi_node: + from synthesizer.extensions.openmp_check import check_openmp + + if not check_openmp() and n_proc > 1: + logger.warning( + "Synthesizer not installed with OpenMP support. " + "Pipeline will run on a single core" + ) + n_proc = 1 n_gal = len(galaxies) + if not batch_galaxies: + batch_size = max(n_gal, 1) + n_batches = int(np.ceil(n_gal / batch_size)) + # Choose the IGM once for all batches (a tabulated IGM is built once). + if isinstance(galaxies, _GalaxyRecipe): + all_redshifts = galaxies.redshifts + else: + all_redshifts = [gal.redshift for gal in galaxies] + igm = self._resolve_igm(igm, all_redshifts, tabulate_igm) + + galaxies_all = galaxies if n_batches > 1: logger.info(f"Splitting galaxies into {n_batches} batches of size {batch_size}.") galaxies = [galaxies[i * batch_size : (i + 1) * batch_size] for i in range(n_batches)] @@ -2536,6 +3245,67 @@ def process_galaxies( logger.info("Processing all galaxies in a single batch.") galaxies = [galaxies] + # Worker processes (single node, save=True) are started once and + # reused for every batch. The pool shares the full (unbatched) galaxy + # source, so batches are addressed by global index ranges. + pool = None + if not multi_node and save and int(n_proc) > 1: + max_batch = min(batch_size, n_gal) + n_workers = min(int(n_proc), max(1, max_batch // _MIN_GALAXIES_PER_WORKER)) + if n_workers > 1: + pool = _PipelinePool( + self, + galaxies_all, + n_workers, + igm=igm, + em_lines_to_save=em_lines_to_save, + extra_analysis_functions=extra_analysis_functions, + start_method=start_method, + ) + elif not multi_node and not save and int(n_proc) > 1: + logger.warning("Parallel processing requires save=True; running serially.") + + try: + self._process_batches( + galaxies, + pool=pool, + batch_size=batch_size, + n_batches=n_batches, + n_gal=n_gal, + out_name=out_name, + out_dir=out_dir, + n_proc=n_proc, + verbose=verbose, + save=save, + overwrite=overwrite, + multi_node=multi_node, + igm=igm, + em_lines_to_save=em_lines_to_save, + extra_analysis_functions=extra_analysis_functions, + ) + finally: + if pool is not None: + pool.shutdown() + + def _process_batches( + self, + galaxies, + pool, + batch_size, + n_batches, + n_gal, + out_name, + out_dir, + n_proc, + verbose, + save, + overwrite, + multi_node, + igm, + em_lines_to_save, + extra_analysis_functions, + ): + """Run (and write) each batch of galaxies; see process_galaxies.""" for batch_i, batch_gals in enumerate(galaxies): skip = False if save: @@ -2563,76 +3333,54 @@ def process_galaxies( skip = True if not skip: + ngal = len(batch_gals) if multi_node: logger.info("Running pipeline in multi-node mode with MPI.") - # logger.debug(f"SIZE: {size}, RANK: {rank}") + n_workers = 1 else: logger.info("Running in single-node mode.") + n_workers = min(int(n_proc), max(1, ngal // _MIN_GALAXIES_PER_WORKER)) + if pool is None: + n_workers = 1 - from synthesizer.extensions.openmp_check import check_openmp + # Path written by this batch. In single-node mode with several + # batches each batch goes straight to its own _.hdf5. + write_path = fullpath if (save and (multi_node or n_batches == 1)) else None + if save and write_path is None: + write_path = final_fullpath - if not check_openmp() and n_proc > 1: - logger.warning( - "Synthesizer not installed with OpenMP support. " - "Pipeline will run on a single core" - ) - n_proc = 1 - - pipeline = Pipeline( - emission_model=self.emission_model, - nthreads=n_proc, - verbose=verbose, - comm=comm, - ) + start = datetime.now() + logger.info(f"Running pipeline at {start} for {ngal} galaxies") - for key in self.all_parameters.keys(): - pipeline.add_analysis_func( - lambda gal, key=key: gal.all_params[key], - result_key=key, + if n_workers > 1: + batch_start = batch_i * batch_size + pool.run(batch_start, batch_start + ngal, write_path) + pipeline = None + else: + if isinstance(batch_gals, _GalaxyRecipe): + batch_gals = batch_gals.build(self.grid) + + pipeline = self._setup_pipeline( + nthreads=n_proc if multi_node else 1, + comm=comm if multi_node else None, + verbose=verbose, + igm=igm, + em_lines_to_save=em_lines_to_save, + extra_analysis_functions=extra_analysis_functions, ) + logger.info("Added analysis functions to pipeline.") + if multi_node: + logger.info(f"Pipeline MPI: {pipeline.using_mpi}") - pipeline.add_analysis_func(lambda gal: gal.stars.initial_mass, result_key="mass") - - logger.info("Added analysis functions to pipeline.") - - if multi_node: - logger.info(f"Pipeline MPI: {pipeline.using_mpi}") - - # Add any extra analysis functions requested by the user. - - for key, params in extra_analysis_functions.items(): - if callable(params): - func = copy.deepcopy(params) - params = [] - else: - func = params[0] - params = params[1:] - - pipeline.add_analysis_func(func, f"supp_{key}", *params) - - # pipeline.get_spectra() # Switch off so they aren't saved - pipeline.get_observed_spectra(self.cosmo, igm=igm) - - if self.instrument.can_do_photometry: - pipeline.get_photometry_fluxes(self.instrument, igm=igm) - - if em_lines_to_save is not None and len(em_lines_to_save) > 0: - # ["H 1 6562.80A", "O 3 5006.84A", "H 1 4861.32A"] - pipeline.get_lines(line_ids=em_lines_to_save) - pipeline.get_observed_lines(self.cosmo, igm=igm) - - pipeline.add_galaxies(batch_gals) + pipeline.add_galaxies(batch_gals) + pipeline.run() - ngal = len(batch_gals) - start = datetime.now() - logger.info(f"Running pipeline at {start} for {ngal} galaxies") - pipeline.run() elapsed = datetime.now() - start logger.info(f"Finished running pipeline at {datetime.now()} for {ngal} galaxies") logger.info(f"Pipeline took {elapsed} to run.") - if save: + if save and pipeline is not None: # Save the pipeline to a file - pipeline.write(fullpath, verbose=0) + pipeline.write(write_path, verbose=0) if multi_node: logger.info("Combining HDF5 files across nodes.") @@ -2736,6 +3484,7 @@ def plot_galaxy( metal_dist=self.metal_dists[idx], log_stellar_masses=log_stellar_mass, grid=self.grid, + fast_sfzh=getattr(self, "fast_sfzh", True), **galaxy_params, ) else: @@ -3069,7 +3818,10 @@ def create_mock_library( Directory to save the output file, by default "library_folder". If "library_folder", saves to the Synthesizer grids directory. n_proc : int, optional - Number of processes to use for the pipeline, by default 6. + Number of worker processes (not threads) used to build and process + galaxies in parallel on this node, by default 6. 1 runs serially. + See GalaxyBasis.process_galaxies. With multi_node=True this is the + number of OpenMP threads per MPI rank (as before). overwrite : Union[bool, List[bool]], optional If True, overwrites the output file if it exists, by default False. If a list, must be the same length as bases. @@ -3113,6 +3865,10 @@ def create_mock_library( """ # make a CombinedBasis object with the current GalaxyBasis + # Fail early (before building any galaxy) if the key is not a label + # of the emission model. + _validate_emission_model_keys(self.emission_model, emission_model_key) + if log_stellar_masses is None: assert self.log_stellar_masses is not None, ( "log_stellar_masses must be provided or set in the GalaxyBasis" @@ -3315,7 +4071,10 @@ def process_bases( Parameters ---------- n_proc : int - Number of processes to use for the pipeline. + Number of worker processes (not threads) used to build and process + galaxies in parallel on this node; 1 runs serially. See + GalaxyBasis.process_galaxies. With multi_node=True this is the + number of OpenMP threads per MPI rank (as before). overwrite : bool or list of bools If True, overwrite the existing files. If False, skip the files that already exist. @@ -3340,6 +4099,11 @@ def process_bases( list of booleans with the same length as bases""" ) + for i, base in enumerate(self.bases): + _validate_emission_model_keys( + base.emission_model, self.base_emission_model_keys[i], "base_emission_model_keys" + ) + for i, base in enumerate(self.bases): full_out_path = f"{self.out_dir}/{base.model_name}.hdf5" ngalaxies = len(self.log_stellar_masses) @@ -3370,8 +4134,13 @@ def process_bases( "galaxies_mask is not implemented for draw_parameter_combinations=False." ) else: + # Lazy: galaxies are built batch by batch (or inside the + # worker processes) by process_galaxies. galaxies = base._create_matched_galaxies( - log_base_masses=self.log_base_masses, galaxies_mask=galaxies_mask, n_proc=n_proc + log_base_masses=self.log_base_masses, + galaxies_mask=galaxies_mask, + n_proc=n_proc, + lazy=True, ) logger.info(f"Created {len(galaxies)} galaxies for base {base.model_name}") @@ -4970,6 +5739,9 @@ def __init__( ignore_scatter: bool = False, return_type="array", device: str = "cpu", + fast_sfzh: bool = True, + igm=Inoue14, + tabulate_igm: bool = True, ) -> None: """Parameters @@ -5055,6 +5827,19 @@ def __init__( device : str Device to use for tensor calculations. Default is 'cpu'. Can be 'cpu' or 'cuda'. Only used if return_type is 'tensor'. + fast_sfzh : bool + If True (default), build the SFZH with the vectorised + :func:`synference.fast_sfzh.make_parametric_stars` (10-100x faster + than Synthesizer's quad integration; photometry agrees to <~1e-6 + mag). False reproduces the original ``Stars`` path. + igm : IGM class or instance + IGM model applied to observed-frame outputs. Default Inoue14 (as + before). None disables IGM attenuation. + tabulate_igm : bool + If True (default), wrap ``igm`` in a + :class:`synference.fast_igm.TabulatedIGM` (z in [0, 20], built on + first use, ~1-3 s; afterwards ~50x faster than exact Inoue14, + band errors <~1.3e-4 mag). False reproduces the exact IGM. """ if fixed_params is None: @@ -5103,6 +5888,11 @@ def __init__( self.ignore_scatter = ignore_scatter self.return_type = return_type self.device = device + self.fast_sfzh = fast_sfzh + self.igm = igm + self.tabulate_igm = tabulate_igm + self._tabulated_igm = None + self._log10ages = np.asarray(self.grid.log10ages) self.unused_params = [] self.reported_unused = False @@ -5179,6 +5969,9 @@ def __init__( self.emitter_params = emitter_params self.num_emitter_params = np.sum([len(emitter_params[key]) for key in emitter_params]) + _validate_emission_model_keys(self.emission_model, emission_model_key, "emission_model_key") + # Only the requested spectra are kept, so Synthesizer skips every + # model branch the output does not depend on. self.emission_model.save_spectra(emission_model_key) self.total_possible_keys = ( @@ -5419,6 +6212,19 @@ def from_library( else: dust_model_params[key] = value + # Libraries written by older versions stored dimensionful + # curve parameters (e.g. Calzetti2000 cent_lam) without + # units: assume the units of the constructor default. + dust_sig = inspect.signature(dust_model.__init__).parameters + for key, value in dust_model_params.items(): + default = dust_sig[key].default if key in dust_sig else None + if ( + hasattr(default, "units") + and not hasattr(value, "units") + and isinstance(value, (int, float, np.number)) + ): + dust_model_params[key] = value * default.units + dust_model = dust_model(**dust_model_params) else: dust_model = None @@ -5433,23 +6239,12 @@ def from_library( " in synthesizer.emission_models. Cannot create from_library." ) - dust_emission_model_params = {} - dust_emission_param_keys = em_group.attrs["dust_emission_keys"] - dust_emission_param_values = em_group.attrs["dust_emission_values"] - dust_emission_param_units = em_group.attrs["dust_emission_units"] - for key, value, unit in zip( - dust_emission_param_keys, - dust_emission_param_values, - dust_emission_param_units, - ): - if unit != "": - dust_emission_model_params[key] = unyt_array(value, unit) - else: - dust_emission_model_params[key] = value - cmb = dust_emission_model_params.pop("cmb_factor", None) - dust_emission_model_params.pop("temperature_z", None) - # if cmb is not None: - # dust_emission_model_params["cmb_heating"] = cmb != 1 + dust_emission_model_params = load_dust_emission_params( + dust_emission_model, + em_group.attrs["dust_emission_keys"], + em_group.attrs["dust_emission_values"], + em_group.attrs["dust_emission_units"], + ) dust_emission_model = dust_emission_model(**dust_emission_model_params) else: dust_emission_model = None @@ -5600,6 +6395,10 @@ def simulate(self, params): params.update(self.fixed_params) + # float32 scalars (e.g. from torch) would trip Synthesizer's precision + # checks when mixed with the float64 grid. + params = {key: _as_float64_scalar(value) for key, value in params.items()} + for key in self.required_keys: if key not in params: raise ValueError(f"Missing required parameter {key}. Cannot create photometry.") @@ -5722,14 +6521,24 @@ def simulate(self, params): for key in found_params: stellar_keys[key] = params[key] - stars = Stars( - log10ages=self.grid.log10ages, - metallicities=self.grid.metallicities, - sf_hist=sfh, - metal_dist=zdist, - initial_mass=mass, - **stellar_keys, - ) + if getattr(self, "fast_sfzh", True): + stars = make_parametric_stars( + sfh, + zdist, + self._log10ages, + self.grid.metallicities, + initial_mass=mass, + **stellar_keys, + ) + else: + stars = Stars( + log10ages=self.grid.log10ages, + metallicities=self.grid.metallicities, + sf_hist=sfh, + metal_dist=zdist, + initial_mass=mass, + **stellar_keys, + ) galaxy_keys = {} if "galaxy" in self.emitter_params: @@ -5775,8 +6584,10 @@ def simulate(self, params): outputs["photo_lnu"] = copy.deepcopy(fluxes) if "fnu" in self.output_type or "photo_fnu" in self.output_type: - # Apply IGM and distance - galaxy.get_observed_spectra(self.cosmo) + # Apply IGM and distance, only for the spectra we return + galaxy.stars.spectra[self.emission_model_key].get_fnu( + self.cosmo, galaxy.redshift, igm=self._get_igm() + ) if "photo_fnu" in self.output_type: fluxes = galaxy.stars.spectra[self.emission_model_key].get_photo_fnu( @@ -5877,6 +6688,18 @@ def inspect_func(func, locals): return fluxes + def _get_igm(self): + """Return the IGM used for observed-frame outputs (tabulated if requested).""" + igm = getattr(self, "igm", Inoue14) + if igm is None or not getattr(self, "tabulate_igm", False): + return igm + if isinstance(igm, TabulatedIGM): + return igm + if getattr(self, "_tabulated_igm", None) is None: + # Tables are built for each rest-frame wavelength array on first use. + self._tabulated_igm = TabulatedIGM(base=igm, z_min=0.0, z_max=20.0, min_hits=1) + return self._tabulated_igm + def _normalize(self, fluxes, method=None, norm_unit="AB", add_norm_pos=-1): if method is None: return fluxes diff --git a/src/synference/utils.py b/src/synference/utils.py index b5dd45b..7adf056 100644 --- a/src/synference/utils.py +++ b/src/synference/utils.py @@ -807,6 +807,97 @@ def asinh_err_to_f_jy( return unyt_array(f_jy_err, units=Jy) +# Constructor arguments of dust emission generators that are wired up by the +# parent emission model (not user parameters) and so are never stored. +_DUST_EMISSION_NON_PARAMS = ("self", "intrinsic", "attenuated", "scaler", "verbose") + + +def _dust_emission_init_params(generator) -> dict: + """Return the constructor parameters of a dust emission generator. + + Uses ``inspect.signature(type(generator).__init__)`` intersected with the + generator's attributes, so derived attributes (e.g. Casey12's ``lam_c`` + and ``n_pl``) are not stored and the generator can be re-created with + ``type(generator)(**params)``. ``do_cmb_heating`` is included. + + Raises: + ValueError: if a parameter cannot be stored in an HDF5 attribute + (e.g. the dust grid of DraineLi07). + """ + import inspect + + sig = inspect.signature(type(generator).__init__).parameters + params = {} + for name, par in sig.items(): + if name in _DUST_EMISSION_NON_PARAMS or par.kind in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ): + continue + if not hasattr(generator, name): + continue + value = getattr(generator, name) + if isinstance(value, np.ndarray) and not hasattr(value, "units") and value.ndim == 0: + value = value.item() + if isinstance(value, np.generic): + value = value.item() + storable = value is None or isinstance(value, (bool, int, float, str)) + storable = storable or (hasattr(value, "units") and np.ndim(value) == 0) + if not storable: + raise ValueError( + f"Cannot store dust emission model {type(generator).__name__}: parameter " + f"'{name}' is a {type(value).__name__}, which cannot be saved to the " + "library file. Libraries using this dust emission model cannot be " + "re-created with GalaxySimulator.from_library; pass the emission model " + "explicitly via override_emission_model instead." + ) + if value is None: + # None cannot be stored in an HDF5 attribute list; the + # constructor default is used on reload. + continue + params[name] = value + return params + + +def load_dust_emission_params(generator_class, keys, values, units) -> dict: + """Decode dust emission parameters stored by :func:`save_emission_model`. + + Args: + generator_class: The dust emission generator class. + keys: The stored ``dust_emission_keys`` attribute. + values: The stored ``dust_emission_values`` attribute. + units: The stored ``dust_emission_units`` attribute. + + Returns: + dict: keyword arguments for ``generator_class(**kwargs)``. + """ + import ast + import inspect + + sig = inspect.signature(generator_class.__init__).parameters + params = {} + for key, value, unit in zip(keys, values, units): + key = key.decode("utf-8") if isinstance(key, bytes) else str(key) + unit = unit.decode("utf-8") if isinstance(unit, bytes) else str(unit) + if isinstance(value, bytes): + value = value.decode("utf-8") + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, str): + try: + parsed = ast.literal_eval(value) + if isinstance(parsed, (bool, int, float)): + value = parsed + except (ValueError, SyntaxError): + pass # a genuine string, e.g. an emitter attribute name + if unit != "" and not isinstance(value, str): + value = unyt_quantity(float(value), unit) + elif key in sig and isinstance(sig[key].default, bool) and not isinstance(value, str): + value = bool(value) + params[key] = value + return params + + def save_emission_model(model): """Save the fixed parameters of the emission model. @@ -837,7 +928,11 @@ def save_emission_model(model): dust_attenuation_keys.pop("description", None) # drop any hidden keys (starting with _) for key in list(dust_attenuation_keys.keys()): - if isinstance(dust_attenuation_keys[key], np.ndarray): + # (unyt quantities are ndarrays too: keep them, their units are + # stored separately below) + if isinstance(dust_attenuation_keys[key], np.ndarray) and not hasattr( + dust_attenuation_keys[key], "units" + ): dust_attenuation_keys[key] = dust_attenuation_keys[key].tolist() # force numpy types to native python types if "numpy" in str(type(dust_attenuation_keys[key])) and not hasattr( @@ -856,10 +951,7 @@ def save_emission_model(model): if "dust_emission" in model._models: dust_em = model._models["dust_emission"].generator - settable_params = dust_em.__annotations__ - dust_emission_keys.update( - {param: getattr(dust_em, param) for param in settable_params.keys()} - ) + dust_emission_keys.update(_dust_emission_init_params(dust_em)) dust_emission_model = type(dust_em).__name__ else: dust_emission_model = None @@ -900,6 +992,10 @@ def save_emission_model(model): dust_emission_units.append("") dust_emission_values = list(dust_emission_keys.values()) + # HDF5 attributes cannot hold mixed str/number lists: store everything as + # strings in that case (decoded again by GalaxySimulator.from_library). + if any(isinstance(v, str) for v in dust_emission_values): + dust_emission_values = [str(v) for v in dust_emission_values] dust_emission_keys = list(dust_emission_keys.keys()) return { @@ -2215,6 +2311,151 @@ def make_serializable(obj: Any, allowed_types=None) -> Any: return f"" +def merge_pipeline_files(paths, out_path, galaxies_group="Galaxies", compression="same"): + """Concatenate Synthesizer pipeline HDF5 files (in order) into one file. + + Used to merge the per-chunk files written by worker processes in + :meth:`synference.library.GalaxyBasis.process_galaxies`. The result has + the layout a single serial ``Pipeline.write`` would produce: + + * root attributes and every top-level group other than ``Galaxies`` + (``Instruments``, ``EmissionModel``, ``Lines`` ...) are copied from the + first file; + * every dataset under ``Galaxies`` is concatenated along the first + (galaxy) axis in the order of ``paths``, keeping its attributes + (e.g. ``Units``), dtype and compression; group attributes are kept. + + Args: + paths (list of str): Pipeline files to merge, in galaxy order. + out_path (str): Output file path (overwritten). + galaxies_group (str): Name of the per-galaxy group. + compression (str/None): Compression of the merged per-galaxy datasets. + "same" (default) keeps the filter of the input datasets (gzip for + Synthesizer files); None writes them uncompressed, which is ~20x + faster to write (gzip only saves a few per cent on float spectra + and photometry) and is transparent to readers. + + Raises: + ValueError: If the files do not contain the same per-galaxy datasets, + or the datasets of a file disagree on the number of galaxies. + """ + if len(paths) == 0: + raise ValueError("No files to merge.") + + def _collect(group, prefix, dsets, groups): + for key, item in group.items(): + path = f"{prefix}/{key}" if prefix else key + if isinstance(item, h5py.Group): + groups[path] = dict(item.attrs) + _collect(item, path, dsets, groups) + else: + dsets[path] = item + + # Pass 1: work out the merged dataset shapes and dtypes. + specs = {} + counts = [] + for i, path in enumerate(paths): + with h5py.File(path, "r") as f: + dsets, groups = {}, {} + if galaxies_group in f: + _collect(f[galaxies_group], "", dsets, groups) + if i == 0: + group_attrs = groups + for name, d in dsets.items(): + specs[name] = { + "tail": d.shape[1:], + "dtype": d.dtype, + "ndim": d.ndim, + "attrs": dict(d.attrs), + "compression": d.compression, + "compression_opts": d.compression_opts, + } + elif set(dsets) != set(specs): + missing = sorted(set(specs) ^ set(dsets)) + raise ValueError( + f"Cannot merge {path}: per-galaxy datasets differ from {paths[0]} " + f"({missing[:10]})." + ) + n = None + for name, d in dsets.items(): + spec = specs[name] + if d.ndim == 0 or spec["ndim"] == 0: + continue + if d.shape[1:] != spec["tail"]: + raise ValueError( + f"Cannot merge {path}: dataset {name} has shape {d.shape}, " + f"expected (N, *{spec['tail']})." + ) + if n is None: + n = d.shape[0] + elif d.shape[0] != n: + raise ValueError( + f"Cannot merge {path}: datasets disagree on the number of galaxies." + ) + if d.dtype != spec["dtype"]: + if d.dtype.kind == "S" and spec["dtype"].kind == "S": + if d.dtype.itemsize > spec["dtype"].itemsize: + spec["dtype"] = d.dtype + else: + spec["dtype"] = np.result_type(spec["dtype"], d.dtype) + counts.append(0 if n is None else n) + + n_total = int(np.sum(counts)) + + # Pass 2: write the merged file. + with h5py.File(out_path, "w") as out: + with h5py.File(paths[0], "r") as first: + for attr, value in first.attrs.items(): + out.attrs[attr] = value + for key in first.keys(): + if key != galaxies_group: + first.copy(first[key], out, name=key) + if galaxies_group in first: + gal = out.create_group(galaxies_group) + for attr, value in first[galaxies_group].attrs.items(): + gal.attrs[attr] = value + for gpath, attrs in group_attrs.items(): + grp = gal.require_group(gpath) + for attr, value in attrs.items(): + grp.attrs[attr] = value + for name, spec in specs.items(): + if spec["ndim"] == 0: + first.copy(first[galaxies_group][name], gal, name=name) + continue + kwargs = {} + if compression != "same": + if compression is not None: + kwargs["compression"] = compression + elif spec["compression"] is not None and n_total > 0: + kwargs["compression"] = spec["compression"] + kwargs["compression_opts"] = spec["compression_opts"] + dset = gal.create_dataset( + name, shape=(n_total, *spec["tail"]), dtype=spec["dtype"], **kwargs + ) + for attr, value in spec["attrs"].items(): + dset.attrs[attr] = value + + if galaxies_group in out: + gal = out[galaxies_group] + files = [h5py.File(path, "r") for path in paths] + try: + # One dataset at a time: read it from every file, concatenate + # and write it in a single call (writing many small slices + # into a compressed, chunked dataset is very slow). + for name, spec in specs.items(): + if spec["ndim"] == 0: + continue + parts = [f[galaxies_group][name][()] for f, n in zip(files, counts) if n > 0] + if len(parts) > 0: + gal[name][...] = np.concatenate(parts, axis=0).astype( + spec["dtype"], copy=False + ) + del parts + finally: + for f in files: + f.close() + + def combine_rank_files(size, filepath, num_galaxies, starts, ends): """Combine the rank files into a single file. diff --git a/tests/test_library_fast_paths.py b/tests/test_library_fast_paths.py new file mode 100644 index 0000000..9b0baa1 --- /dev/null +++ b/tests/test_library_fast_paths.py @@ -0,0 +1,495 @@ +"""Tests for the fast/parallel library generation paths and audit bug fixes.""" + +import h5py +import numpy as np +import pytest +from astropy.cosmology import Planck18 +from synthesizer.emission_models import PacmanEmission, TotalEmission +from synthesizer.emission_models.attenuation import Calzetti2000, PowerLaw +from synthesizer.emission_models.generators.dust.blackbody import Blackbody +from synthesizer.emission_models.generators.dust.casey12 import Casey12 +from synthesizer.emission_models.generators.dust.greybody import Greybody +from synthesizer.parametric import SFH, Galaxy, Stars, ZDist +from unyt import K, Msun, Myr, um + +from synference import GalaxyBasis, GalaxySimulator, draw_from_hypercube +from synference.fast_igm import TabulatedIGM +from synference.library import ( + _dust_curve_param_clashes, + _line_dependency_labels, + calculate_line_luminosity, + create_galaxy, +) +from synference.utils import ( + _dust_emission_init_params, + load_dust_emission_params, + save_emission_model, +) + +# Synthesizer serialises the child labels of each emission model from a set, +# so their order depends on the (per process) string hash seed. +_UNORDERED_SUFFIX = "/Children" +_RUN_ATTRS = ("date_created", "pipeline_time", "CreationDT", "timestamp") + + +def _make_inputs(n, seed=0): + rng = np.random.default_rng(seed) + z = rng.uniform(0.5, 12.0, n) + age = (Planck18.age(z) - Planck18.age(20)).to("Myr").value + sfhs = [ + SFH.LogNormal( + tau=rng.uniform(0.1, 1.5), + peak_age=rng.uniform(0.05, 0.95) * age[i] * Myr, + max_age=age[i] * Myr, + ) + for i in range(n) + ] + zds = [ZDist.DeltaConstant(log10metallicity=rng.uniform(-3.5, -1.5)) for i in range(n)] + return z, sfhs, zds, rng.uniform(7, 11, n), rng.uniform(0, 2, n) + + +def _make_basis(grid, model, instrument, n, name="fast", seed=0, **kwargs): + z, sfhs, zds, logm, tau_v = _make_inputs(n, seed) + basis = GalaxyBasis( + model_name=name, + redshifts=z, + grid=grid, + emission_model=model, + sfhs=sfhs, + metal_dists=zds, + log_stellar_masses=logm, + galaxy_params={"tau_v": tau_v}, + instrument=instrument, + build_library=False, + redshift_dependent_sfh=True, + **kwargs, + ) + return basis, logm + + +def _dust_model(grid, **kwargs): + return PacmanEmission( + grid=grid, + tau_v="tau_v", + dust_curve=Calzetti2000(), + dust_emission=Greybody(temperature=40 * K, emissivity=1.5), + fesc=0.0, + fesc_ly_alpha=0.0, + **kwargs, + ) + + +def _read_all(path): + out = {} + + def visit(name, obj): + if isinstance(obj, h5py.Dataset): + out[name] = (obj[()], dict(obj.attrs), obj.dtype, obj.shape) + else: + out[name] = (None, dict(obj.attrs), None, None) + + with h5py.File(path, "r") as f: + f.visititems(visit) + root = dict(f.attrs) + return root, out + + +def _assert_same_file(path_a, path_b, phot_atol_mag=None): + root_a, a = _read_all(path_a) + root_b, b = _read_all(path_b) + assert set(root_a) == set(root_b) + for key in root_a: + if key not in _RUN_ATTRS: + assert np.array_equal(np.asarray(root_a[key]), np.asarray(root_b[key])), key + assert set(a) == set(b) + for name in a: + va, attrs_a, dt_a, sh_a = a[name] + vb, attrs_b, dt_b, sh_b = b[name] + assert set(attrs_a) == set(attrs_b), name + assert dt_a == dt_b and sh_a == sh_b, name + if va is None: + continue + if name.endswith(_UNORDERED_SUFFIX): + assert sorted(va.tolist()) == sorted(vb.tolist()), name + continue + if phot_atol_mag is not None and va.dtype.kind == "f" and "Photometry" in name: + pos = (va > 0) & (vb > 0) + assert np.array_equal(va > 0, vb > 0), name + dmag = np.abs(2.5 * np.log10(va[pos] / vb[pos])) + assert dmag.size == 0 or dmag.max() < phot_atol_mag, (name, dmag.max()) + elif phot_atol_mag is None: + assert np.array_equal(va, vb, equal_nan=va.dtype.kind == "f"), name + + +# --------------------------------------------------------------------------- +# Parallel vs serial +# --------------------------------------------------------------------------- +def test_parallel_matches_serial(test_grid, mock_instrument, tmp_path): + """n_proc=2 (worker processes + merge) gives the same files as n_proc=1.""" + n = 60 # >= 2 * 25 galaxies so two workers are used + paths = {} + for n_proc in (1, 2): + model = _dust_model(test_grid) + basis, logm = _make_basis(test_grid, model, mock_instrument, n) + out = tmp_path / f"np{n_proc}" + basis.create_mock_library( + "lib", + log_stellar_masses=logm, + emission_model_key="total", + out_dir=str(out), + n_proc=n_proc, + overwrite=True, + ) + paths[n_proc] = out + # no temporary chunk files left behind + assert sorted(p.name for p in out.iterdir()) == ["fast.hdf5", "lib.hdf5"] + + _assert_same_file(paths[1] / "fast.hdf5", paths[2] / "fast.hdf5") + _assert_same_file(paths[1] / "lib.hdf5", paths[2] / "lib.hdf5") + + +def test_parallel_prebuilt_galaxies_and_batches(test_grid, mock_instrument, tmp_path): + """Prebuilt galaxy lists and several batches also work in parallel.""" + model = _dust_model(test_grid) + basis, _ = _make_basis(test_grid, model, mock_instrument, 60) + galaxies = basis._create_matched_galaxies(log_base_masses=9) + assert len(galaxies) == 60 + basis.process_galaxies( + galaxies, + "pre.hdf5", + out_dir=str(tmp_path), + n_proc=2, + verbose=0, + emission_model_keys="total", + batch_size=30, + ) + # two batches -> two files, each with 30 galaxies and the metadata attrs + for i in (1, 2): + with h5py.File(tmp_path / f"pre_{i}.hdf5", "r") as f: + assert f["Galaxies/redshift"].shape == (30,) + assert "varying_param_names" in f.attrs + assert "Wavelengths" in f + + +def test_parallel_loky_and_errors(test_grid, mock_instrument, tmp_path): + """The non-fork (loky) executor gives the same result; worker errors propagate.""" + files = {} + for method in ("fork", "loky"): + model = _dust_model(test_grid) + basis, _ = _make_basis(test_grid, model, mock_instrument, 50) + recipe = basis._create_matched_galaxies(log_base_masses=9, lazy=True) + assert basis.galaxies == [] and len(recipe) == 50 + basis.process_galaxies( + recipe, + f"{method}.hdf5", + out_dir=str(tmp_path), + n_proc=2, + verbose=0, + emission_model_keys="total", + start_method=method, + ) + files[method] = tmp_path / f"{method}.hdf5" + _assert_same_file(files["fork"], files["loky"]) + + def broken(gal): + raise RuntimeError("boom") + + model = _dust_model(test_grid) + basis, _ = _make_basis(test_grid, model, mock_instrument, 50) + recipe = basis._create_matched_galaxies(log_base_masses=9, lazy=True) + with pytest.raises(RuntimeError, match="boom"): + basis.process_galaxies( + recipe, + "err.hdf5", + out_dir=str(tmp_path / "err"), + n_proc=2, + verbose=0, + emission_model_keys="total", + broken=broken, + ) + assert list((tmp_path / "err").iterdir()) == [] # chunk files cleaned up + + +# --------------------------------------------------------------------------- +# Fast path opt-out +# --------------------------------------------------------------------------- +def test_create_galaxy_fast_sfzh_opt_out(test_grid, simple_sfh, simple_zdist): + """fast_sfzh=False reproduces Stars(...) exactly; the fast path is close.""" + ref = Stars( + log10ages=test_grid.log10ages, + metallicities=test_grid.metallicities, + sf_hist=simple_sfh, + metal_dist=simple_zdist, + initial_mass=1e9 * Msun, + tau_v=0.3, + ) + slow = create_galaxy( + simple_sfh, 7.0, simple_zdist, test_grid, log_stellar_masses=9, fast_sfzh=False, tau_v=0.3 + ) + fast = create_galaxy( + simple_sfh, 7.0, simple_zdist, test_grid, log_stellar_masses=9, fast_sfzh=True, tau_v=0.3 + ) + assert np.array_equal(slow.stars.sfzh, ref.sfzh) + np.testing.assert_allclose(fast.stars.sfzh, ref.sfzh, rtol=0, atol=1e-6 * ref.sfzh.max()) + assert fast.stars.tau_v == 0.3 + + +def test_fast_paths_vs_opt_out(test_grid, mock_instrument, tmp_path): + """Default fast paths agree with the original paths to < 1e-3 mag.""" + paths = {} + for label, kwargs in { + "slow": dict(fast_sfzh=False, tabulate_igm=False), + "fast": dict(fast_sfzh=True, tabulate_igm=True), + }.items(): + model = _dust_model(test_grid) + basis, logm = _make_basis(test_grid, model, mock_instrument, 8, **kwargs) + basis.create_mock_library( + "lib", + log_stellar_masses=logm, + emission_model_key="total", + out_dir=str(tmp_path / label), + n_proc=1, + overwrite=True, + ) + paths[label] = tmp_path / label + _assert_same_file(paths["slow"] / "fast.hdf5", paths["fast"] / "fast.hdf5", phot_atol_mag=1e-3) + + +def test_resolve_igm(test_grid, mock_instrument): + """tabulate_igm controls whether a (cached) TabulatedIGM is used.""" + from synthesizer.emission_models.attenuation import Inoue14 + + model = _dust_model(test_grid) + basis, _ = _make_basis(test_grid, model, mock_instrument, 4) + assert basis._resolve_igm(Inoue14, [1.0, 2.0]) is Inoue14 # auto, few galaxies + assert basis._resolve_igm(None, [1.0], tabulate_igm=True) is None + tab = basis._resolve_igm(Inoue14, [1.0, 7.3], tabulate_igm=True) + assert isinstance(tab, TabulatedIGM) and tab.z_max >= 7.3 + assert basis._resolve_igm(Inoue14, [3.0, 7.3], tabulate_igm=True) is tab # cached + assert basis._resolve_igm(Inoue14, [1.0], tabulate_igm=False) is Inoue14 + + +def test_default_saved_spectra(test_grid, mock_instrument): + """emission_model_keys=None saves only the root model unless configured.""" + model = _dust_model(test_grid) + basis, _ = _make_basis(test_grid, model, mock_instrument, 4) + assert basis._configure_saved_spectra(None) == ["total"] + assert model.saved_labels == ["total"] + # A user-configured selection is respected + model.save_spectra("total", "attenuated") + basis._configure_saved_spectra(None) + assert sorted(model.saved_labels) == ["attenuated", "total"] + + +# --------------------------------------------------------------------------- +# Audit fixes +# --------------------------------------------------------------------------- +def test_s1_float64_hypercube_and_float32_inputs(test_grid, simple_sfh): + """Hypercube draws are float64; float32 inputs no longer break Synthesizer.""" + draws = draw_from_hypercube({"a": (0, 1), "b": (-3, -1)}, N=10, rng=1) + assert all(v.dtype == np.float64 for v in draws.values()) + + zd = ZDist.DeltaConstant(log10metallicity=np.float32(-2.0)) + for fast in (True, False): + gal = create_galaxy( + simple_sfh, + 6.0, + zd, + test_grid, + log_stellar_masses=np.float32(9.0), + fast_sfzh=fast, + tau_v=np.float32(0.4), + ) + assert gal.stars.sfzh.dtype == np.float64 + assert isinstance(gal.stars.tau_v, float) + # the user's ZDist is not modified + assert isinstance(zd.metallicity_, np.float32) + + +def test_s2_lines_with_greybody(test_grid, mock_instrument, tmp_path): + """Emission lines can be saved with an energy-balance dust emission model.""" + model = _dust_model(test_grid) + assert set(_line_dependency_labels(model)) == {"dust_emission", "reprocessed", "attenuated"} + basis, logm = _make_basis(test_grid, model, mock_instrument, 3) + basis.create_mock_library( + "lib", + log_stellar_masses=logm, + emission_model_key="total", + out_dir=str(tmp_path), + n_proc=1, + overwrite=True, + em_lines_to_save=["H 1 6562.80A"], + ) + with h5py.File(tmp_path / "fast.hdf5", "r") as f: + lum = f["Galaxies/Stars/Lines/Luminosity/total"] + assert lum.shape[0] == 3 and np.all(lum[()] > 0) + + +def test_s5_line_luminosity_erg_s(test_parametric_galaxy, mock_emission_model): + """calculate_line_luminosity returns erg/s.""" + lum = calculate_line_luminosity(test_parametric_galaxy, mock_emission_model, "Ha", "emergent") + assert str(lum.units) == "erg/s" + assert lum.value > 1e38 + + +@pytest.mark.parametrize( + "gen", + [ + Greybody(temperature=45 * K, emissivity=1.7, lam_0=80 * um, do_cmb_heating=True), + Blackbody(temperature=55 * K), + Casey12(temperature=42 * K, emissivity=1.6, alpha=2.2), + ], + ids=["Greybody", "Blackbody", "Casey12"], +) +def test_s6_dust_emission_round_trip(test_grid, gen, tmp_path): + """Dust emission parameters survive save -> HDF5 attrs -> reload.""" + model = PacmanEmission( + grid=test_grid, + tau_v=0.2, + dust_curve=Calzetti2000(), + dust_emission=gen, + fesc=0.0, + fesc_ly_alpha=0.0, + ) + saved = save_emission_model(model) + # Calzetti2000's cent_lam/gamma keep their units (needed by from_library) + units = dict(zip(saved["dust_attenuation_keys"], saved["dust_attenuation_units"])) + assert units["cent_lam"] != "" and units["gamma"] != "" + assert "lam_c" not in saved["dust_emission_keys"] + assert "do_cmb_heating" in saved["dust_emission_keys"] + with h5py.File(tmp_path / "m.hdf5", "w") as f: + for key in ("dust_emission_keys", "dust_emission_values", "dust_emission_units"): + f.attrs[key] = saved[key] + keys, values, units = ( + f.attrs["dust_emission_keys"], + f.attrs["dust_emission_values"], + f.attrs["dust_emission_units"], + ) + params = load_dust_emission_params(type(gen), keys, values, units) + new = type(gen)(**params) + assert _dust_emission_init_params(new).keys() == _dust_emission_init_params(gen).keys() + for key, value in _dust_emission_init_params(gen).items(): + other = getattr(new, key) + if hasattr(value, "units"): + assert np.isclose(other.to(value.units).value, value.value), key + else: + assert other == value and type(other) is type(value), key + + +def test_s6_unstorable_dust_emission_raises(): + """Generators with parameters that cannot be stored give a clear error.""" + + class FakeGrid: + pass + + class FakeGen: + def __init__(self, grid, qpah=1.0): + self.grid = grid + self.qpah = qpah + + with pytest.raises(ValueError, match="cannot be saved"): + _dust_emission_init_params(FakeGen(FakeGrid())) + + +def test_s9_invalid_emission_model_key(test_grid, mock_instrument, tmp_path): + """A key that is not a label of the model gives a clear error up front.""" + model = TotalEmission(test_grid, Calzetti2000(), tau_v=0.1, fesc=0.0) + assert model.label != "total" # Synthesizer ignores label= here + basis, logm = _make_basis(test_grid, model, mock_instrument, 3) + with pytest.raises(ValueError, match="emission_model.label"): + basis.create_mock_library( + "lib", + log_stellar_masses=logm, + emission_model_key="total", + out_dir=str(tmp_path), + n_proc=1, + ) + with pytest.raises(ValueError, match="root label"): + GalaxySimulator( + SFH.LogNormal, + ZDist.DeltaConstant, + test_grid, + mock_instrument, + model, + emission_model_key="total", + ) + + +def test_dust_curve_param_clash(test_grid): + """Galaxy params named like a fixed dust curve parameter are reported.""" + model = PacmanEmission( + grid=test_grid, tau_v="tau_v", dust_curve=PowerLaw(slope=-1.0), dust_emission=None + ) + assert _dust_curve_param_clashes(model, {"tau_v", "slope"}) == {"PowerLaw": ["slope"]} + aliased = PacmanEmission( + grid=test_grid, tau_v="tau_v", dust_curve=PowerLaw(slope="slope"), dust_emission=None + ) + assert _dust_curve_param_clashes(aliased, {"slope"}) == {} + + +# --------------------------------------------------------------------------- +# Simulator +# --------------------------------------------------------------------------- +def test_simulator_fast_vs_original(test_grid, mock_instrument): + """GalaxySimulator fast paths agree with the original ones (< 1e-3 mag).""" + out = {} + for label, kw in { + "slow": dict(fast_sfzh=False, tabulate_igm=False), + "fast": dict(fast_sfzh=True, tabulate_igm=True), + }.items(): + model = _dust_model(test_grid) + sim = GalaxySimulator( + sfh_model=SFH.LogNormal, + zdist_model=ZDist.DeltaConstant, + grid=test_grid, + instrument=mock_instrument, + emission_model=model, + emission_model_key="total", + emitter_params={"stellar": ["tau_v"]}, + param_order=[ + "redshift", + "log_mass", + "tau", + "peak_age", + "max_age", + "log10metallicity", + "tau_v", + ], + param_units={"peak_age": Myr, "max_age": Myr}, + out_flux_unit="AB", + **kw, + ) + rng = np.random.default_rng(3) + res = [] + for _ in range(5): + z = rng.uniform(1, 10) + age = (Planck18.age(z) - Planck18.age(20)).to("Myr").value + p = [ + z, + rng.uniform(8, 10), + rng.uniform(0.2, 1), + rng.uniform(0.1, 0.9) * age, + age, + rng.uniform(-3, -1.5), + np.float32(rng.uniform(0, 1)), + ] + res.append(sim.simulate(p)) + out[label] = np.array(res) + assert np.all(np.isfinite(out["slow"])) + np.testing.assert_allclose(out["fast"], out["slow"], rtol=0, atol=1e-3) + + +def test_parametric_galaxy_unchanged_by_fast_stars(test_grid, simple_sfh, simple_zdist): + """Spectra from fast-SFZH stars match the original Stars.""" + model = _dust_model(test_grid) + model.save_spectra("total") + gals = [ + create_galaxy(simple_sfh, 5.0, simple_zdist, test_grid, 9, fast_sfzh=f, tau_v=0.5) + for f in (False, True) + ] + for gal in gals: + assert isinstance(gal, Galaxy) + gal.stars.get_spectra(model) + np.testing.assert_allclose( + gals[1].stars.spectra["total"].lnu, gals[0].stars.spectra["total"].lnu, rtol=1e-5 + ) From 19b02059349df5e2052c1476b0d71a778e78c39b Mon Sep 17 00:00:00 2001 From: Thomas Harvey Date: Sun, 1 Nov 2026 22:17:14 +0000 Subject: [PATCH 4/5] Surface synference-download errors and fix stale notebook instruction - download_test_data now raises a RuntimeError containing the command's own output when synference-download fails (previously the output was captured and discarded), and a clear error when the command, which is provided by Synthesizer, is not installed. - The diffusion noise model notebook told readers to fetch a pre-trained model with synference-download, but that command does not provide it; the notebook reloads the model it saved a few cells earlier. Reword to match. --- .../diffusion_noise_models.ipynb | 2 +- src/synference/utils.py | 25 ++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/source/noise_modelling/diffusion_noise_models.ipynb b/docs/source/noise_modelling/diffusion_noise_models.ipynb index 838f6da..8d7d493 100644 --- a/docs/source/noise_modelling/diffusion_noise_models.ipynb +++ b/docs/source/noise_modelling/diffusion_noise_models.ipynb @@ -117,7 +117,7 @@ "source": [ "### Loading the Model\n", "\n", - "We can load in the saved model from disk, just to prove it works! Here we will switch to our pre-trained model which you can get by running `synference-download`, but feel free to load in your own trained model if you have one.\n" + "We can load the model we just saved back in from disk, just to prove it works! You can load any other trained noise model the same way.\n" ] }, { diff --git a/src/synference/utils.py b/src/synference/utils.py index 7adf056..13bb43f 100644 --- a/src/synference/utils.py +++ b/src/synference/utils.py @@ -3032,19 +3032,38 @@ def interval_sharpness( def download_test_data(): - """Downloads test data for Synference using the synference-download CLI tool.""" + """Downloads test data for Synference using the synference-download CLI tool. + + The ``synference-download`` command is provided by Synthesizer. + + Raises: + RuntimeError: If ``synference-download`` is not installed, or if the + download fails. The error includes the command's own output. + """ + import shutil import subprocess data_dir = get_data_dir() / "synference" data_dir.mkdir(parents=True, exist_ok=True) + if shutil.which("synference-download") is None: + raise RuntimeError( + "The 'synference-download' command was not found. It is installed " + "with Synthesizer (cosmos-synthesizer); check Synthesizer is " + "installed in the active environment." + ) + # Download the test data - subprocess.run( + result = subprocess.run( ["synference-download", "--test", "--destination", f"{str(data_dir)}"], - check=True, capture_output=True, text=True, ) + if result.returncode != 0: + output = "\n".join(s.strip() for s in (result.stdout, result.stderr) if s.strip()) + raise RuntimeError( + f"synference-download failed with exit code {result.returncode}:\n{output}" + ) print(f"Test data downloaded to {data_dir}") From ab83cf2ac2295141b83bf688ba897f3cd7a3f60d Mon Sep 17 00:00:00 2001 From: Thomas Harvey Date: Sun, 1 Nov 2026 22:39:56 +0000 Subject: [PATCH 5/5] Address review: IGM pending removal, streaming merge, spectral overwrite - TabulatedIGM: remove pending entries by identity. list.remove compared the stored ndarrays with == and raised when a different array was pending ahead of the one being built (default min_hits=2). Add a regression test with two arrays seen alternately. - merge_pipeline_files: open one chunk file at a time and write its rows at the running offset, instead of holding every file open and concatenating whole datasets in memory. Output is unchanged. - generate_spectral_library.py: check for an existing library at the path create_spectral_grid writes to, and pass the configured overwrite value instead of always overwriting. --- .../scripts/generate_spectral_library.py | 4 +-- src/synference/fast_igm.py | 4 ++- src/synference/utils.py | 31 ++++++++++--------- tests/test_fast_igm.py | 21 +++++++++++++ 4 files changed, 42 insertions(+), 18 deletions(-) diff --git a/examples/library_generation/scripts/generate_spectral_library.py b/examples/library_generation/scripts/generate_spectral_library.py index 197e58a..14dd6f2 100644 --- a/examples/library_generation/scripts/generate_spectral_library.py +++ b/examples/library_generation/scripts/generate_spectral_library.py @@ -224,7 +224,7 @@ def continuity_agebins( sfh_name = str(sfh_type).split(".")[-1].split("'")[0] name = f"BPASS_Chab_{sfh_name}_SFH_z_{redshift}_logN_{np.log10(Nmodels):.1f}_CF00_v2" # noqa: E501 - if os.path.exists(f"{out_dir}/v2/spectral_library_{name}.hdf5") and not overwrite: + if os.path.exists(f"{out_dir}/spectral_library_{name}.hdf5") and not overwrite: print(f"Grid {name} already exists, skipping.") continue @@ -464,5 +464,5 @@ def db_sf_convert(param, param_dict, Nparam_SFH=3): # Create grid - kinda overkill for a single case, but it does work. combined_basis.create_spectral_grid( - overwrite=True, overload_out_name=f"spectral_library_{name}" + overwrite=overwrite, overload_out_name=f"spectral_library_{name}" ) diff --git a/src/synference/fast_igm.py b/src/synference/fast_igm.py index 627c1c2..a8c28c4 100644 --- a/src/synference/fast_igm.py +++ b/src/synference/fast_igm.py @@ -438,7 +438,9 @@ def _lookup(self, lam_rest): self._pending.pop(0) if entry[1] < self.min_hits: return None - self._pending.remove(entry) + # Remove by identity: list.remove compares entries with ==, which + # would compare the stored ndarrays and raise. + self._pending = [e for e in self._pending if e is not entry] tab = self._load_or_build(entry[0]) self._insert(tab) return tab diff --git a/src/synference/utils.py b/src/synference/utils.py index 13bb43f..82bd9bc 100644 --- a/src/synference/utils.py +++ b/src/synference/utils.py @@ -2437,23 +2437,24 @@ def _collect(group, prefix, dsets, groups): if galaxies_group in out: gal = out[galaxies_group] - files = [h5py.File(path, "r") for path in paths] - try: - # One dataset at a time: read it from every file, concatenate - # and write it in a single call (writing many small slices - # into a compressed, chunked dataset is very slow). - for name, spec in specs.items(): - if spec["ndim"] == 0: - continue - parts = [f[galaxies_group][name][()] for f, n in zip(files, counts) if n > 0] - if len(parts) > 0: - gal[name][...] = np.concatenate(parts, axis=0).astype( + # Open one input file at a time and write its rows straight into + # the output at the running offset. This keeps a single file + # handle and a single chunk in memory, so the merge is bounded + # neither by the open-file limit nor by the size of the merged + # datasets. + offset = 0 + for path, n in zip(paths, counts): + if n == 0: + continue + with h5py.File(path, "r") as f: + src = f[galaxies_group] + for name, spec in specs.items(): + if spec["ndim"] == 0: + continue + gal[name][offset : offset + n] = src[name][()].astype( spec["dtype"], copy=False ) - del parts - finally: - for f in files: - f.close() + offset += n def combine_rank_files(size, filepath, num_galaxies, starts, ends): diff --git a/tests/test_fast_igm.py b/tests/test_fast_igm.py index af24903..a3e9cf2 100644 --- a/tests/test_fast_igm.py +++ b/tests/test_fast_igm.py @@ -215,6 +215,27 @@ def test_lazy_build_and_arbitrary_arrays(): assert tab.get_transmission(5.0, np.array([5000.0]) * Angstrom).shape == (1,) +@pytest.mark.parametrize("n_b", [200, 150], ids=["same_shape", "different_shape"]) +def test_alternating_unseen_arrays_build(n_b): + """Two pending arrays seen alternately both reach the build path. + + Regression test: removing a pending entry with ``list.remove`` compared + the stored ndarrays with ``==`` and raised when another array was pending + ahead of it. + """ + tab = TabulatedIGM(Inoue14, z_max=8.0, dz=0.1, min_hits=2) + exact = Inoue14() + lam_a = np.linspace(700.0, 3000.0, 200) + lam_b = np.linspace(650.0, 2500.0, n_b) + for z, lam in [(3.0, lam_a), (3.0, lam_b), (5.0, lam_b), (5.0, lam_a)]: + t = tab.get_transmission(z, lam * (1 + z) * Angstrom) + np.testing.assert_allclose( + t, exact.get_transmission(z, lam * (1 + z) * Angstrom), atol=5e-3 + ) + assert len(tab.tables) == 2 + assert tab.n_table_calls == 2 and tab.n_exact_calls == 2 + + def test_pickle_round_trip(tab_igm): """Pickled copies give identical results and don't rebuild.""" z = 7.3