diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ca808b5..546b66c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -43,6 +43,9 @@ jobs: pip install wheel sudo apt install pandoc pip install "cosmos-synthesizer@git+https://github.com/synthesizer-project/synthesizer.git" + # syntillate (score-based diffusion noise model) is a hard dependency but + # is not yet on PyPI, so install it before resolving synference itself. + pip install "syntillate@git+https://github.com/synthesizer-project/syntillate.git" pip install .[test,docs] pip install "dense_basis@git+https://github.com/kartheikiyer/dense_basis.git" pip install "ltu_ili@git+https://github.com/maho3/ltu-ili.git" @@ -50,10 +53,15 @@ jobs: pip install pytest-xdist # enable parallel pytest execution - name: Download test data run: | - # Download test grid data + # Download test grid data mkdir -p data/libraries/ - synthesizer-download --test-grids --dust-grid + synthesizer-download --test-grids --dust-grid --instruments JWSTNIRCamWide python -c "from synference.utils import download_test_data; download_test_data()" + - name: Download SVO filter cache + run: | + # Pre-fetch cached SVO filter responses so the notebooks never hit the + # live SVO service, which is intermittently unavailable. + synthesizer-download --svo-filter-cache - name: Sphinx Build run: | # Test sphinx build (runs all notebooks) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 964338d..c11d9d8 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -31,16 +31,25 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + # syntillate (score-based diffusion noise model) is a hard dependency but + # is not yet on PyPI, so install it before resolving synference itself. + pip install "syntillate@git+https://github.com/synthesizer-project/syntillate.git" pip install . pip install ruff pytest pip install "ltu_ili@git+https://github.com/maho3/ltu-ili.git" pip install "cosmos-synthesizer@git+https://github.com/synthesizer-project/synthesizer.git#egg=synthesizer" python -c "from synference.utils import download_test_data; download_test_data()" - + + - name: Download SVO filter cache + run: | + # Pre-fetch cached SVO filter responses so the tests never hit the live + # SVO service, which is intermittently unavailable. + synthesizer-download --svo-filter-cache + - name: Test import run: | - python -c "import synference" - + python -c "import synference" + - name: Test with pytest run: | pytest diff --git a/docs/source/library_gen/synthesizer_crash_course.ipynb b/docs/source/library_gen/synthesizer_crash_course.ipynb index d19ceb3..90e7ce1 100644 --- a/docs/source/library_gen/synthesizer_crash_course.ipynb +++ b/docs/source/library_gen/synthesizer_crash_course.ipynb @@ -512,7 +512,11 @@ ] } ], - "metadata": {}, + "metadata": { + "language_info": { + "name": "python" + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/docs/source/noise_modelling/creating_noise_model.rst b/docs/source/noise_modelling/creating_noise_model.rst index a6d5b12..33b8f58 100644 --- a/docs/source/noise_modelling/creating_noise_model.rst +++ b/docs/source/noise_modelling/creating_noise_model.rst @@ -7,4 +7,5 @@ Here we detail the various ways to create noise models in Synference. Noise mode :maxdepth: 1 noise_models + diffusion_noise_models diff --git a/docs/source/noise_modelling/diffusion_noise_models.ipynb b/docs/source/noise_modelling/diffusion_noise_models.ipynb new file mode 100644 index 0000000..838f6da --- /dev/null +++ b/docs/source/noise_modelling/diffusion_noise_models.ipynb @@ -0,0 +1,166 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "5413f0b4", + "metadata": {}, + "source": [ + "# Diffusion Noise Models\n", + "\n", + "This page describes the diffusion noise model available in `synference`. In contrast to the empirical noise models discussed in the previous sections, which simply model per-band uncertainties as a function of magnitude using binned interpolation, the diffusion noise model is a generative model that learns to the full latent distribution of noise in the data. This allows it to capture more complex noise properties, such as correlations between bands, and to generate more realistic noise realizations.\n", + "\n", + "This noise model is based on the diffusion probabilistic model framework, which has been successfully applied to a wide range of generative modeling tasks in machine learning. The basic idea is to model the noise as a stochastic process that gradually transforms a simple initial distribution (e.g., Gaussian noise) into the complex noise distribution observed in the data. By learning this transformation, the model can generate realistic noise samples that can be added to synthetic photometry to better match the properties of real observations.\n", + "\n", + "This is implemented as the `ScoreBasedUncertaintyModel`, which learns the conditional uncertainty distribution p(\u03c3|m), where \u03c3 is the uncertainty and m is the magnitude. It lives in the standalone [syntillate](https://github.com/synthesizer-project/syntillate) package, which Synference installs as a dependency and re-exports from `synference.noise_models`, so it can be used and serialized alongside Synference's other uncertainty models. The model is trained on a dataset of real photometric measurements, where the input is the magnitude and the target is the corresponding (log-scaled) uncertainty. Once trained, the model can be used to generate noise samples for synthetic photometry by sampling from the learned distribution.\n", + "\n", + "We will demonstate this on the standard COSMOSO2020 'Farmer' catalog (Weaver et al. 2020). If you want to train the noise model yourself, you will have to download the full catalog (~3 GB) from [here](https://cosmos2020.calet.org/).\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3fc591b0", + "metadata": {}, + "outputs": [], + "source": [ + "from syntillate import ScoreBasedUncertaintyModel\n", + "\n", + "?ScoreBasedUncertaintyModel" + ] + }, + { + "cell_type": "markdown", + "id": "ced7ee22", + "metadata": {}, + "source": [ + "### Training the Noise Model\n", + "\n", + "The first part is just loading and cleaning the COSMOS2020 catalog, which is fairly standard. We filter low signal to noise sources which are faint in IRAC Ch. 1. As we are interested in learning the full conditional distribution of noise across all the filters, we also remove sources with missing photometry in any of the filters." + ] + }, + { + "cell_type": "markdown", + "id": "cfad048c", + "metadata": {}, + "source": [ + "```python\n", + "\n", + "from astropy.table import Table\n", + "import numpy as np\n", + "\n", + "table = Table.read('COSMOS2020_FARMER_R1_v2.2_p3.fits', memmap=True)\n", + "\n", + "bands = ['CFHT_u', 'HSC_g', 'HSC_r', 'HSC_i', 'HSC_z',\n", + " 'HSC_y', 'UVISTA_Y', 'UVISTA_J', 'UVISTA_H', \n", + " 'UVISTA_Ks', 'IRAC_CH1', 'IRAC_CH2']\n", + "\n", + "snr_filter = table['IRAC_CH1_MAG'] < 26\n", + "table = table[snr_filter]\n", + "\n", + "for band in bands:\n", + " table = table[np.isfinite(table[band + '_MAG'])]\n" + ] + }, + { + "cell_type": "markdown", + "id": "f9388d26", + "metadata": {}, + "source": [ + "We then construct our training dataset, which consists of the magnitudes and uncertainties for each band. \n", + "\n", + "```python\n", + "\n", + "mag_array = np.array([table[band + '_MAG'] for band in bands]).T\n", + "flux_error_array = np.array([table[band + '_FLUXERR'] for band in bands]).T\n", + "\n", + "```\n", + "\n", + "Next, we can train the `ScoreBasedUncertaintyModel` on this dataset. This will learn the conditional distribution of uncertainties given magnitudes for each band. We will leave the model architecture as it's default, which is a simple MLP with 5 layers and a hidden dimension of 256. The training process can take some time, especially if you are using a CPU, so be patient!\n", + "\n", + "You can adjust the number of epochs and batch size as needed --> larger batch sizes will speed up training but require more memory, while more epochs will allow the model to learn better but will take longer.\n", + "\n", + "```python\n", + "\n", + "from syntillate import ScoreBasedUncertaintyModel\n", + "noise_model = ScoreBasedUncertaintyModel(filter_names=bands)\n", + "\n", + "# COSMOS2020 FLUXERR columns are in uJy; the model reports sigma back in\n", + "# the same units (see noise_model.sigma_units).\n", + "noise_model.fit(\n", + " mag_array,\n", + " flux_error_array,\n", + " flux_uncertainty_units='uJy',\n", + " n_epochs=200,\n", + " batch_size=1024,\n", + ")\n", + "\n", + "```\n", + "\n", + "### Saving the Model\n", + "\n", + "We can save the trained model to disk for later use - the model can be serialized directly into a custom HDF5 format, and re-loaded later without needing to re-train, or rely on unstable dependencies like `pickle` or `torch.save`.\n", + "\n", + "```python\n", + "\n", + "import h5py\n", + "with h5py.File('COSMOS_noise_model.h5', 'w') as f:\n", + " noise_model.serialize_to_hdf5(f.create_group('noise_model'))\n", + "\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "46d187b0", + "metadata": {}, + "source": [ + "### Loading the Model\n", + "\n", + "We can load in the saved model from disk, just to prove it works! Here we will switch to our pre-trained model which you can get by running `synference-download`, but feel free to load in your own trained model if you have one.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "833cc1b5", + "metadata": {}, + "outputs": [], + "source": [ + "import h5py\n", + "\n", + "with h5py.File(\"COSMOS_noise_model.h5\", \"r\") as f:\n", + " noise_model = ScoreBasedUncertaintyModel._from_hdf5_group(f[\"noise_model\"])" + ] + }, + { + "cell_type": "markdown", + "id": "d2e11370", + "metadata": {}, + "source": [ + "### Validating the Noise Model" + ] + }, + { + "cell_type": "markdown", + "id": "827471fa", + "metadata": {}, + "source": [ + "### Using the Noise Model in Synference" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "synference", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/sbi/scripts/train_model.py b/examples/sbi/scripts/train_model.py index f083503..8724681 100644 --- a/examples/sbi/scripts/train_model.py +++ b/examples/sbi/scripts/train_model.py @@ -13,6 +13,7 @@ from synference import ( AsinhEmpiricalUncertaintyModel, SBI_Fitter, + ScoreBasedUncertaintyModel, Simformer_Fitter, create_uncertainty_models_from_EPOCHS_cat, ) @@ -74,7 +75,7 @@ class Args: additional_model_args: tuple = () parameters_to_add: tuple = () parameters_to_remove: tuple = () - data_err_file: str = """/home/tharvey/Downloads/JADES-Deep-GS_MASTER_Sel-f277W+f356W+f444W_v9_loc_depth_masked_10pc_EAZY_matched_selection_ext_src_UV.fits""" # noqa + data_err_file: str = None # Path to FITS file with photometric errors for empirical noise model data_err_hdu: str = "OBJECTS" # The HDU name in the FITS file background: bool = False model_features: tuple = () @@ -87,6 +88,7 @@ class Args: noise_model_class: str = "general" # Type of noise model to use - general, depth or asinh. custom_config_yaml: str = None sql_db_path: str = None # Path to SQLite database for storing results + precomputed_noise_model_path: str = None # Path saved HDF5 file with noise parser.add_arguments(Args, dest="args") @@ -122,6 +124,15 @@ def main_task(args: Args) -> None: hst_bands = ["F435W", "F606W", "F775W", "F814W", "F850LP"] if args.scatter_fluxes > 0: + if (args.data_err_file is None and args.precomputed_noise_model_path is None) or ( + args.data_err_file is not None and args.precomputed_noise_model_path is not None + ): + raise ValueError( + "When scatter_fluxes > 0, exactly one of data_err_file or " + "precomputed_noise_model_path must be provided." + ) + + if args.scatter_fluxes > 0 and args.data_err_file: table = Table.read(args.data_err_file, format="fits", hdu=args.data_err_hdu) bands = [i.split("_")[-1] for i in table.colnames if i.startswith("loc_depth")] if len(phot_to_remove) > 0: @@ -148,6 +159,12 @@ def main_task(args: Args) -> None: min_flux_error=args.min_flux_error, model_class=args.noise_model_class, ) + elif args.scatter_fluxes > 0 and args.precomputed_noise_model_path: + from synference.noise_models import load_unc_model_from_hdf5 + + empirical_noise_models = load_unc_model_from_hdf5(args.precomputed_noise_model_path)[ + "noise_model" + ] else: empirical_noise_models = {} @@ -278,29 +295,43 @@ def log10_floor(x, floor=-6): unused_filters.append(band_name) print("Photometry in grid:", empirical_model_fitter.raw_observation_names, file=sys.stdout) + use_noise_models = args.scatter_fluxes > 0 and args.include_errors_in_feature_array + is_score_model = isinstance(empirical_noise_models, ScoreBasedUncertaintyModel) + for filt in empirical_model_fitter.raw_observation_names: - if ( - args.scatter_fluxes > 0 - and args.include_errors_in_feature_array - and filt not in empirical_noise_models - ): + if not use_noise_models: + continue + if is_score_model: + # The score model conditions on all of its bands at once, so any + # filter it was not trained on cannot be used. + check_names = [i.lower() for i in empirical_noise_models.filter_names] + if filt.lower() not in check_names: + print( + f"Filter {filt} not in the score-based noise model. Adding to unused filters.", + file=sys.stdout, + ) + unused_filters.append(filt) + elif filt not in empirical_noise_models: unused_filters.append(filt) print(f"Unused filters: {unused_filters}", file=sys.stdout) # flux_units is "AB" unless scatter_fluxes and include_errors_in_feature_array - # and all noise models are AsinhUncertaintyModels + # and the noise model(s) work in asinh magnitudes flux_units = "AB" - if args.scatter_fluxes > 0 and args.include_errors_in_feature_array: - if all( - isinstance(model, AsinhEmpiricalUncertaintyModel) - for model in empirical_noise_models.values() - ): + if use_noise_models: + if is_score_model: + asinh = empirical_noise_models.is_asinh_scaled + else: + asinh = all( + isinstance(model, AsinhEmpiricalUncertaintyModel) + for model in empirical_noise_models.values() + ) + if asinh: flux_units = "asinh" - print(empirical_noise_models) empirical_model_fitter.create_feature_array_from_raw_photometry( - extra_features=list(args.model_features), + extra_features=list(args.model_features[0].split(",")) if args.model_features else [], normalize_method=args.norm_method, include_errors_in_feature_array=args.include_errors_in_feature_array, scatter_fluxes=args.scatter_fluxes, @@ -441,3 +472,7 @@ def log10_floor(x, floor=-6): else: # Run in the foreground as normal main_task(args) + + +# Example invocation with a pre-trained score-based noise model: +# python synference/examples/sbi/scripts/train_model.py --scatter_fluxes=1 --include_errors_in_feature_array=True --photometry_to_remove=HST/ACS_WFC.F606W,HST/ACS_WFC.F775W,HST/ACS_WFC.F814W,HST/ACS_WFC.F850LP,JWST/NIRCam.F070,HST/WFC3_IR.F105W,HST/WFC3_IR.F110W,HST/WFC3_IR.F110W,HST/WFC3_IR.F125W,HST/WFC3_IR.F140W,HST/WFC3_IR.F160W,JWST/NIRCam.F070W,JWST/NIRCam.F090W,JWST/NIRCam.F115W,JWST/NIRCam.F140M,JWST/NIRCam.F150W,JWST/NIRCam.F162M,JWST/NIRCam.F182M,JWST/NIRCam.F200W,JWST/NIRCam.F210M,JWST/NIRCam.F250M,JWST/NIRCam.F277W,JWST/NIRCam.F300M,JWST/NIRCam.F335M,JWST/NIRCam.F356W,JWST/NIRCam.F360M,JWST/NIRCam.F410M,JWST/NIRCam.F430M,JWST/NIRCam.F444W,JWST/NIRCam.F460M,JWST/NIRCam.F480M,HST/ACS_WFC/F475W,JWST/MIRI.F560W,JWST/MIRI.F770W,JWST/MIRI.F1000W,JWST/MIRI.F1130W,JWST/MIRI.F1280W,JWST/MIRI.F1500W,JWST/MIRI.F1800W,JWST/MIRI.F2100W,JWST/MIRI.F2550W,Spitzer/IRAC.I3,Spitzer/IRAC.I4,CTIO/DECam.u,CTIO/DECam.g,CTIO/DECam.r,CTIO/DECam.i,CTIO/DECam.z,CTIO/DECam.Y,LSST/LSST.u,LSST/LSST.g,LSST/LSST.r,LSST/LSST.i,LSST/LSST.z,LSST/LSST.Y,PAN-STARRS/PS1.g,PAN-STARRS/PS1.r,PAN-STARRS/PS1.i,PAN-STARRS/PS1.w,PAN-STARRS/PS1.z,PAN-STARRS/PS1.y,Paranal/VISTA.Z,CFHT/MegaCam.g,CFHT/MegaCam.r,CFHT/MegaCam.i,CFHT/MegaCam.z,HST/ACS_WFC.F435W,HST/ACS_WFC.F475W,Euclid/NISP.Y,Euclid/NISP.J/Euclid/NISP.H --grid_path=/cosma/apps/dp276/dc-harv3/synference/libraries/grid_BPASS_Chab_DenseBasis_SFH_0.01_z_14_logN_5.0_Calzetti_v4_multinode.hdf5 --model_name=BPASS_DenseBasis_COSMOS2020 --name_append=nsf --train_test_fraction=0.90 --parameter_transformations="Av=log10,sfr_10=log10_floor,mass_weighted_age=log10" --parameters_to_add="mass_weighted_age,sfr_10,log_surviving_mass,beta" --model_types=nsf --custom_config_yaml=/cosma/apps/dp276/dc-harv3/synference/examples/sbi/configs/best_params_2101.yaml --precomputed_noise_model_path=/cosma/apps/dp276/dc-harv3/synference/priv/COSMOS2020_noise_model_asinh.h5 # noqa: E501 diff --git a/pyproject.toml b/pyproject.toml index 55e3da7..ccc90c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,8 @@ dependencies = [ "plotext", "matplotlib", "numba", + # Score-based diffusion noise model (ScoreBasedUncertaintyModel). + "syntillate", ] [project.optional-dependencies] diff --git a/src/synference/__init__.py b/src/synference/__init__.py index 6140915..d7398cd 100644 --- a/src/synference/__init__.py +++ b/src/synference/__init__.py @@ -31,7 +31,8 @@ from .noise_models import ( EmpiricalUncertaintyModel, create_uncertainty_models_from_EPOCHS_cat, DepthUncertaintyModel, UncertaintyModel, AsinhEmpiricalUncertaintyModel, - GeneralEmpiricalUncertaintyModel, save_unc_model_to_hdf5, load_unc_model_from_hdf5 + GeneralEmpiricalUncertaintyModel, save_unc_model_to_hdf5, load_unc_model_from_hdf5, + ScoreBasedUncertaintyModel ) from .custom_runner import SBICustomRunner @@ -103,6 +104,7 @@ "DepthUncertaintyModel", "UncertaintyModel", "GeneralEmpiricalUncertaintyModel", + "ScoreBasedUncertaintyModel", "save_unc_model_to_hdf5", "load_unc_model_from_hdf5", "optimize_sfh_xlimit", diff --git a/src/synference/library.py b/src/synference/library.py index 169c133..e686b0b 100644 --- a/src/synference/library.py +++ b/src/synference/library.py @@ -30,6 +30,7 @@ from synthesizer.emissions import plot_spectra from synthesizer.grid import Grid from synthesizer.instruments import UVJ, FilterCollection, Instrument + from synthesizer.instruments.instrument_base import InstrumentBase from synthesizer.parametric import SFH, Galaxy, Stars, ZDist from synthesizer.particle.stars import sample_sfzh from synthesizer.pipeline import Pipeline @@ -101,7 +102,10 @@ "MUV": {"lam_eff": 1500 * Angstrom, "lam_fwhm": 100 * Angstrom}, } -muv_filter = FilterCollection(tophat_dict=tophats, verbose=False) +if synthesizer_available: + muv_filter = FilterCollection(tophat_dict=tophats, verbose=False) +else: + muv_filter = None try: @@ -5073,7 +5077,9 @@ def __init__( assert isinstance(grid, Grid), f"Grid must be a subclass of Grid. Got {type(grid)} instead." self.grid = grid - assert isinstance(instrument, Instrument), f"""Instrument must be a subclass of Instrument. + assert isinstance( + instrument, InstrumentBase + ), f"""Instrument must be a subclass of InstrumentBase. Got {type(instrument)} instead.""" assert isinstance( emission_model, EmissionModel @@ -5774,12 +5780,7 @@ def simulate(self, params): fluxes = galaxy.stars.spectra[self.emission_model_key].get_photo_fnu( self.instrument.filters ) - print("check") - print(galaxy.redshift) - print(type(fluxes)) - print(fluxes) - print(fluxes.__dict__) - print(fluxes.__class__.__dict__["photo_fnu"].unit) + outputs["photo_fnu"] = fluxes.photo_fnu outputs["photo_wav"] = fluxes.filters.pivot_lams @@ -5999,8 +6000,8 @@ def _scatter( out_units=self.out_flux_unit, ) - scattered_fluxes[i] = scattered_flux - errors[i] = sigma + scattered_fluxes[i] = scattered_flux[0] + errors[i] = sigma[0] return scattered_fluxes, errors diff --git a/src/synference/noise_models.py b/src/synference/noise_models.py index 814338f..d0fe603 100644 --- a/src/synference/noise_models.py +++ b/src/synference/noise_models.py @@ -2,6 +2,11 @@ This module provides a robust and serializable framework for creating and applying various photometric noise models. + +The score-based diffusion noise model lives in the standalone `syntillate +`_ package; +``ScoreBasedUncertaintyModel`` is re-exported here so it can be used and +serialized alongside Synference's own uncertainty models. """ import warnings @@ -14,10 +19,25 @@ from astropy.table import Table from scipy import stats from scipy.interpolate import interp1d +from syntillate import ScoreBasedUncertaintyModel from unyt import Jy, Unit, unyt_array from .utils import f_jy_err_to_asinh, f_jy_to_asinh +__all__ = [ + "UncertaintyModel", + "DepthUncertaintyModel", + "SpectralUncertaintyModel", + "EmpiricalUncertaintyModel", + "AsinhEmpiricalUncertaintyModel", + "GeneralEmpiricalUncertaintyModel", + "ScoreBasedUncertaintyModel", + "MODEL_CLASS_REGISTRY", + "save_unc_model_to_hdf5", + "load_unc_model_from_hdf5", + "create_uncertainty_models_from_EPOCHS_cat", +] + # ============================================================================= # BASE CLASSES # ============================================================================= @@ -1107,6 +1127,7 @@ def apply_scalings( "DepthUncertaintyModel": DepthUncertaintyModel, "AsinhEmpiricalUncertaintyModel": AsinhEmpiricalUncertaintyModel, "GeneralEmpiricalUncertaintyModel": GeneralEmpiricalUncertaintyModel, + "ScoreBasedUncertaintyModel": ScoreBasedUncertaintyModel, } diff --git a/src/synference/sbi_runner.py b/src/synference/sbi_runner.py index 96e0c49..4646230 100644 --- a/src/synference/sbi_runner.py +++ b/src/synference/sbi_runner.py @@ -2,6 +2,7 @@ import copy import glob +import inspect import json import logging import os @@ -59,6 +60,7 @@ from .noise_models import ( AsinhEmpiricalUncertaintyModel, EmpiricalUncertaintyModel, + ScoreBasedUncertaintyModel, UncertaintyModel, load_unc_model_from_hdf5, save_unc_model_to_hdf5, @@ -739,10 +741,16 @@ def save_state( ): noise_models = param_dict["feature_array_flags"].pop("empirical_noise_models") noise_model_path = save_path.replace(".pkl", "_empirical_noise_models.h5") - param_dict["empirical_noise_models"] = {} - for key, model in noise_models.items(): - save_unc_model_to_hdf5(model, noise_model_path, key, overwrite=True) - param_dict["empirical_noise_models"][key] = noise_model_path + if isinstance(noise_models, ScoreBasedUncertaintyModel): + save_unc_model_to_hdf5( + noise_models, noise_model_path, "__score_based__", overwrite=True + ) + param_dict["empirical_noise_models"] = {"__score_based__": noise_model_path} + else: + param_dict["empirical_noise_models"] = {} + for key, model in noise_models.items(): + save_unc_model_to_hdf5(model, noise_model_path, key, overwrite=True) + param_dict["empirical_noise_models"][key] = noise_model_path if "stats" in param_dict: stats_path = f"{out_dir}/{self.name}{name_append}_summary.json" @@ -902,6 +910,125 @@ def apply_noise_model(filter_name): return scattered_photometry + def _apply_score_based_noise_model( + self, + photometry_array, + phot_names: List[str], + score_model: "ScoreBasedUncertaintyModel", + N_scatters: int = 5, + flux_units=None, + return_errors: bool = False, + normed_flux_units: str = "AB", + ): + """Apply a ScoreBasedUncertaintyModel to the full multi-band photometry array. + + Unlike the per-filter empirical path, the score model conditions on all + bands simultaneously and returns per-band uncertainties in a single call. + The returned arrays match the shape contract of + ``_apply_empirical_noise_models`` so the caller can treat both paths + identically (set ``converted=True`` after calling this method). + + Parameters: + photometry_array: Flux array of shape ``(n_filters, N)``, as a + ``unyt_array`` in any flux unit. + phot_names: List of filter names, length ``n_filters``. Must contain + the same filters as ``score_model.filter_names``; any ordering + is accepted and the output is returned in this order. + score_model: Trained :class:`ScoreBasedUncertaintyModel`. + N_scatters: Number of independent noise realisations per galaxy. + flux_units: Input flux units (used only if ``photometry_array`` is + not a unyt_array; ignored otherwise). + return_errors: If ``True`` returns ``(scattered, errors)``. + normed_flux_units: Target units for the returned scattered + photometry. ``"AB"`` returns AB magnitudes and AB-mag errors, + ``"asinh"`` returns asinh magnitudes using the score model's own + softening parameter (and requires an asinh-scaled model). Any + other string is treated as a physical flux unit (e.g. ``"nJy"``) + and the values are converted accordingly. + + Returns: + ``np.ndarray`` of shape ``(n_filters, N*N_scatters)`` when + ``return_errors=False``, or a tuple of two such arrays + ``(scattered_photometry, errors)`` when ``return_errors=True``. + + Raises: + ValueError: If the filter names do not match the model, or if + ``photometry_array`` carries no units and ``flux_units`` is not + given. + """ + if normed_flux_units == "asinh": + assert score_model.is_asinh_scaled, ( + "asinh-scaled output requested but score model is not asinh-scaled." + ) + + expected = [f.lower() for f in score_model.filter_names] + actual = [f.lower() for f in phot_names] + reset_order = False + if actual != expected: + # Reorder photometry_array to the model's filter order, reverting + # to the caller's order before returning. + if sorted(actual) != sorted(expected): + raise ValueError( + f"Filter names in phot_names do not match score model filter names. " + f"Expected: {sorted(set(expected))}, actual: {sorted(set(actual))}" + ) + + reset_order = True + new_indices = [actual.index(filt) for filt in expected] + photometry_array = photometry_array[new_indices, :] + resetting_indices = np.argsort(new_indices) + + # Ensure we have a unyt_array in Jy + if isinstance(photometry_array, unyt_array): + phot_jy = photometry_array.to("Jy").value # (n_filters, N) + elif flux_units is not None: + phot_jy = unyt_array(photometry_array, units=flux_units).to("Jy").value + else: + raise ValueError( + "flux_units must be provided when photometry_array is not a unyt_array." + ) + + # Repeat each galaxy N_scatters times: (n_filters, N*N_scatters) + phot_jy_rep = np.repeat(phot_jy, N_scatters, axis=1) + + # Sample per-band uncertainties: (N*N_scatters, n_filters). The model + # applies its own AB/asinh conversion, so pass fluxes with units. + sigma = score_model.sample_uncertainty(unyt_array(phot_jy_rep.T, units=Jy), n_samples=1) + sigma_jy = unyt_array(sigma, units=score_model.sigma_units).to_value(Jy) + + # Gaussian noise in Jy, then noisy flux: (N*N_scatters, n_filters) + noise = np.random.normal(0.0, sigma_jy) + noisy_jy = phot_jy_rep.T + noise # (N*N_scatters, n_filters) + + if normed_flux_units == "asinh": + # f_b is stored per filter; broadcast it over the object axis so the + # (N, n_filters) layout used here matches the utility's expectations. + f_b = unyt_array( + np.broadcast_to(score_model.asinh_b_factor.to_value(Jy), noisy_jy.shape).copy(), + units=Jy, + ) + errors = f_jy_err_to_asinh(noisy_jy * Jy, sigma_jy * Jy, f_b=f_b).T + scattered = f_jy_to_asinh(noisy_jy * Jy, f_b=f_b).T + elif normed_flux_units == "AB": + # Noisy AB magnitudes + safe_noisy = np.where(noisy_jy > 0, noisy_jy, np.finfo(np.float32).tiny) + scattered = (-2.5 * np.log10(safe_noisy / 3631.0)).T # (n_filters, N*N_scatters) + # AB magnitude error: σ_AB = (2.5/ln10) * σ_Jy / F_Jy + errors = ((2.5 / np.log(10)) * sigma_jy / safe_noisy).T + else: + scale = unyt_array(1.0, "Jy").to(normed_flux_units).value + scattered = (noisy_jy * scale).T + errors = (sigma_jy * scale).T + + if reset_order: + # revert to original filter order + scattered = scattered[resetting_indices] + errors = errors[resetting_indices] + + if return_errors: + return scattered, errors + return scattered + def detect_misspecification(self, x_obs, X_train=None, retrain=False): """Tests misspecification of the model using the MarginalTrainer from sbi. @@ -1434,7 +1561,9 @@ def create_feature_array_from_raw_photometry( normalization_unit: str = "AB", verbose: bool = True, scatter_fluxes: Union[int, bool] = False, - empirical_noise_models: Optional[Dict[str, EmpiricalUncertaintyModel]] = None, + empirical_noise_models: Optional[ + Union[Dict[str, EmpiricalUncertaintyModel], "ScoreBasedUncertaintyModel"] + ] = None, depths: Optional[unyt_array] = None, include_errors_in_feature_array: bool = False, min_flux_pc_error: float = 0.0, @@ -1595,6 +1724,8 @@ def create_feature_array_from_raw_photometry( for model in empirical_noise_models.values() ] ) + if isinstance(empirical_noise_models, ScoreBasedUncertaintyModel): + err_is_asinh = empirical_noise_models.is_asinh_scaled assert asinh_softening_parameters is not None or err_is_asinh, ( "asinh_softening_parameters must be provided for asinh normalization." ) @@ -1678,6 +1809,20 @@ def create_feature_array_from_raw_photometry( ) converted = False + elif isinstance(empirical_noise_models, ScoreBasedUncertaintyModel): + logger.info( + f"Using ScoreBasedUncertaintyModel with {scatter_fluxes} scatters per row." + ) + self.empirical_noise_models = empirical_noise_models + phot, phot_errors = self._apply_score_based_noise_model( + phot, + raw_observation_names, + empirical_noise_models, + N_scatters=scatter_fluxes, + return_errors=True, + normed_flux_units=normed_flux_units, + ) + converted = True elif empirical_noise_models is not None: logger.info(f"Using empirical noise models with {scatter_fluxes} scatters per row.") self.empirical_noise_models = empirical_noise_models @@ -2265,25 +2410,35 @@ def bin_noisy_testing_data( return X_test, y_test phot_units = self.feature_array_flags["normed_flux_units"] - if self.feature_array_flags.get("empirical_noise_models", None) is not None: - nm = self.feature_array_flags["empirical_noise_models"] - nm = all([isinstance(nm, AsinhEmpiricalUncertaintyModel) for nm in nm.values()]) - else: - nm = False + # `nm` records whether the features are in asinh magnitudes, in which case + # `asinh_b_factors` maps each feature to the softening parameter used. + nm = False + asinh_b_factors = {} + noise_models = self.feature_array_flags.get("empirical_noise_models", None) + if isinstance(noise_models, ScoreBasedUncertaintyModel): + nm = noise_models.is_asinh_scaled + if nm: + b_factor = noise_models.asinh_b_factor.to("Jy") + asinh_b_factors = { + name.lower(): b_factor[i] for i, name in enumerate(noise_models.filter_names) + } + elif isinstance(noise_models, dict) and noise_models: + nm = all(isinstance(m, AsinhEmpiricalUncertaintyModel) for m in noise_models.values()) + if nm: + asinh_b_factors = {name.lower(): m.b.to("Jy") for name, m in noise_models.items()} snrs = [] for feature_name in snr_feature_names or self.feature_names: if feature_name.startswith("unc_"): break - elif nm: + elif nm and feature_name.lower() in asinh_b_factors: feature_index = self.feature_names.index(feature_name) err_index = self.feature_names.index(f"unc_{feature_name}") # Need to know SNR from asinh magnitudes. - f_b = self.feature_array_flags["empirical_noise_models"][feature_name].b.to("Jy") snr = asinh_to_snr( X_test[:, feature_index], X_test[:, err_index], - f_b=f_b, + f_b=asinh_b_factors[feature_name.lower()], ) elif phot_units == "AB": @@ -2609,9 +2764,9 @@ def create_features_from_observations( "empirical_noise_models" in feature_array_flags and feature_array_flags["empirical_noise_models"] is not None ): - if all( - isinstance(model, AsinhEmpiricalUncertaintyModel) - for model in feature_array_flags["empirical_noise_models"] + nm_obs = feature_array_flags["empirical_noise_models"] + if isinstance(nm_obs, dict) and all( + isinstance(model, AsinhEmpiricalUncertaintyModel) for model in nm_obs.values() ): training_flux_units = "asinh" else: @@ -2766,7 +2921,9 @@ def create_features_from_observations( # Check if the flux units match the training data training_flux_units = feature_array_flags["normed_flux_units"] - if feature_array_flags["empirical_noise_models"] is not None: + if feature_array_flags["empirical_noise_models"] is not None and isinstance( + feature_array_flags["empirical_noise_models"], dict + ): for pos, (model_name, val) in enumerate( feature_array_flags["empirical_noise_models"].items() ): @@ -3431,7 +3588,7 @@ def split_dataset( if verbose: logger.info( - f"""Splitting dataset with {num_samples} samples into training""" + f"""Splitting dataset with {num_samples} samples into training """ f"""and testing sets with {train_fraction:.2f} train fraction.""" ) indices = np.arange(num_samples) @@ -5234,15 +5391,13 @@ def fit_observation_using_sampler( # convert to nJy if needed # Check asinh errors - if self.feature_array_flags.get("empirical_noise_models", False) and all( - [ - isinstance(nm, AsinhEmpiricalUncertaintyModel) - for nm in self.feature_array_flags["empirical_noise_models"].values() - ] - ): # noqa: E501 + _enm = self.feature_array_flags.get("empirical_noise_models", None) + if isinstance(_enm, dict) and all( + isinstance(nm, AsinhEmpiricalUncertaintyModel) for nm in _enm.values() + ): logger.info("Converting asinh photometry and errors to f_nu.") filters = self.feature_array_flags.get("raw_observation_names", []) - nms = self.feature_array_flags["empirical_noise_models"] + nms = _enm if phot_err is not None: phot_err = [ asinh_err_to_f_jy(phot_obs[i], phot_err[i], nms[filters[i]].b).to("nJy").value @@ -6066,7 +6221,7 @@ def recover_SED( unit = self.feature_array_flags.get("normed_flux_units", "AB") noise_models = self.feature_array_flags.get("empirical_noise_models", None) - if noise_models is not None: + if isinstance(noise_models, dict) and filter in noise_models: if isinstance(noise_models[filter], AsinhEmpiricalUncertaintyModel): unit = "asinh" if phot_unit != unit: @@ -6432,6 +6587,15 @@ def sample_posterior( # ) # shape = test_sample.shape + sampling_kwargs = {} + # check if sampler.sample accepts arbitrary keyword arguments, + # if so pass them the timeout_seconds_per_test + varkw = inspect.signature(sampler.sample).parameters + if "kwargs" in varkw: + if timeout_seconds_per_test is not None and sample_method == "direct": + print(f"{timeout_seconds_per_test}s per sample.") + sampling_kwargs["max_sampling_time"] = timeout_seconds_per_test + if log_times: times = [] # Draw samples from the posterior @@ -6441,7 +6605,9 @@ def sample_posterior( start_time = time.time() try: # print(X_test_array[i]) - samples[i] = sampler.sample(nsteps=num_samples, x=X_test_array[i], progress=False) + samples[i] = sampler.sample( + nsteps=num_samples, x=X_test_array[i], progress=False, **sampling_kwargs + ) """samples[i] = sample_with_timeout( sampler, num_samples, @@ -7580,6 +7746,20 @@ def find_class(self, module: str, name: str) -> Any: self.feature_array_flags["empirical_noise_models"] = params[ "empirical_noise_models" ] + # Restore ScoreBasedUncertaintyModel from its HDF5 sentinel key + _nm = self.feature_array_flags.get("empirical_noise_models") + if isinstance(_nm, dict) and set(_nm.keys()) == {"__score_based__"}: + _path = _nm["__score_based__"] + try: + self.feature_array_flags["empirical_noise_models"] = ( + load_unc_model_from_hdf5( + filepath=_path, group_name="__score_based__" + ) + ) + except (FileNotFoundError, KeyError) as e: + logger.warning( + f"Could not restore ScoreBasedUncertaintyModel from {_path}: {e}" + ) if learning_type == "offline": self.feature_names = params["feature_names"] @@ -7987,11 +8167,10 @@ def init_from_synference(cls, synference, **run_params): units = feature_array_flags.get("normed_flux_units", "AB") # Swap units to asinh if all uncertainty models are asinh - if uncertainty_models is not None: - if all( - isinstance(m, AsinhEmpiricalUncertaintyModel) for m in uncertainty_models.values() - ): - units = "asinh" + if isinstance(uncertainty_models, dict) and all( + isinstance(m, AsinhEmpiricalUncertaintyModel) for m in uncertainty_models.values() + ): + units = "asinh" # Need to figure out unit conversions here. return cls( diff --git a/src/synference/utils.py b/src/synference/utils.py index c0917bb..b5dd45b 100644 --- a/src/synference/utils.py +++ b/src/synference/utils.py @@ -26,7 +26,9 @@ try: from synthesizer import get_data_dir except ImportError: - get_data_dir = lambda: os.path.dirname(__file__) # noqa: E731 + from pathlib import Path as _Path + + get_data_dir = lambda: _Path(os.path.dirname(__file__)) # noqa: E731 try: import plotext as plo @@ -855,7 +857,9 @@ def save_emission_model(model): if "dust_emission" in model._models: dust_em = model._models["dust_emission"].generator settable_params = dust_em.__annotations__ - dust_emission_keys.update({param:getattr(dust_em, param) for param in settable_params.keys()}) + dust_emission_keys.update( + {param: getattr(dust_em, param) for param in settable_params.keys()} + ) dust_emission_model = type(dust_em).__name__ else: dust_emission_model = None diff --git a/tests/test_uncertainty_models.py b/tests/test_uncertainty_models.py index e4ceab9..976bc63 100644 --- a/tests/test_uncertainty_models.py +++ b/tests/test_uncertainty_models.py @@ -6,12 +6,13 @@ import numpy as np import pytest -from unyt import Jy, uJy +from unyt import Jy, uJy, unyt_array from synference import ( AsinhEmpiricalUncertaintyModel, DepthUncertaintyModel, GeneralEmpiricalUncertaintyModel, + ScoreBasedUncertaintyModel, UncertaintyModel, load_unc_model_from_hdf5, save_unc_model_to_hdf5, @@ -509,5 +510,229 @@ def test_upper_limit_error_behaviours(mock_data, err_behaviour, expected_error_f ) +# ============================================================================= +# Tests for ScoreBasedUncertaintyModel +# +# The model itself lives in the syntillate package and is tested there; these +# tests cover Synference's integration with it — the re-export, the shared +# HDF5 serialization registry, and the SBI_Fitter noise-application path. +# ============================================================================= + +# Minimal training kwargs — tiny network, very few epochs so the tests run in +# seconds while still exercising the full code path. +_SCORE_TRAIN_KWARGS = dict( + n_epochs=50, + batch_size=128, + learning_rate=1e-3, + min_epochs=10, + patience=15, + val_freq=5, + verbose=False, +) +_SCORE_SAMPLE_KWARGS = dict(n_steps=25) + + +@pytest.fixture(scope="module") +def score_model_data(): + """Minimal photometric data (AB mags + flux uncertainties in uJy) for two bands.""" + rng = np.random.default_rng(42) + N = 2000 + filter_names = ["F115W", "F200W"] + # AB mags: two bands with realistic scatter + mags = np.column_stack( + [ + rng.normal(25.0, 1.5, N), + rng.normal(24.5, 1.5, N), + ] + ).astype(np.float32) + # Flux uncertainties in uJy: faint-end power law + sigma = 10 ** (-0.4 * (mags - 8.9)) * 0.1 * rng.lognormal(0, 0.3, mags.shape) + return filter_names, mags, sigma.astype(np.float32) + + +@pytest.fixture(scope="module") +def trained_score_model(score_model_data): + """A ScoreBasedUncertaintyModel trained with minimal settings.""" + filter_names, mags, sigma = score_model_data + model = ScoreBasedUncertaintyModel( + filter_names=filter_names, + hidden_dim=32, + n_layers=2, + time_embed_dim=16, + device="cpu", + ) + model.fit(mags, sigma, **_SCORE_TRAIN_KWARGS) + return model + + +# --- Re-export --- + + +def test_score_model_reexported_from_syntillate(): + """Synference re-exports syntillate's ScoreBasedUncertaintyModel unchanged.""" + from syntillate import ScoreBasedUncertaintyModel as SyntillateModel + + assert ScoreBasedUncertaintyModel is SyntillateModel + + +# --- HDF5 serialization through synference's registry --- + + +def test_score_model_hdf5_round_trip(trained_score_model, temp_dir): + """A ScoreBasedUncertaintyModel round-trips through synference's HDF5 helpers.""" + path = os.path.join(temp_dir, "score_model.h5") + group = "score_test" + save_unc_model_to_hdf5(trained_score_model, path, group, overwrite=True) + loaded = load_unc_model_from_hdf5(path, group) + + assert isinstance(loaded, ScoreBasedUncertaintyModel) + assert loaded.filter_names == trained_score_model.filter_names + assert loaded.n_filters == trained_score_model.n_filters + assert loaded.sigma_units == trained_score_model.sigma_units + assert loaded._is_trained + + # Sampling should work and give finite values + mags = np.array([[25.0, 24.5]], dtype=np.float32) + sigma_orig = trained_score_model.sample_uncertainty(mags, **_SCORE_SAMPLE_KWARGS) + sigma_load = loaded.sample_uncertainty(mags, **_SCORE_SAMPLE_KWARGS) + assert np.all(np.isfinite(sigma_orig)) + assert np.all(np.isfinite(sigma_load)) + # exp(x) >= 0 always; underflow to 0 is possible with an undertrained model + assert np.all(sigma_orig >= 0) and np.all(sigma_load >= 0) + + +def test_score_model_hdf5_sentinel_key_round_trip(trained_score_model, temp_dir): + """ScoreBasedUncertaintyModel round-trips through HDF5 under the save_state sentinel key. + + The sentinel key ``__score_based__`` is what save_state uses when it writes a + ScoreBasedUncertaintyModel; load_model_from_pkl then reads it back by that key. + This test verifies that the HDF5 infrastructure supports it correctly. + """ + path = os.path.join(temp_dir, "score_sentinel.h5") + sentinel = "__score_based__" + + # Simulate what save_state does: write under the sentinel key + save_unc_model_to_hdf5(trained_score_model, path, sentinel, overwrite=True) + assert os.path.exists(path) + + # Simulate what load_model_from_pkl does: load back by sentinel key + loaded = load_unc_model_from_hdf5(path, sentinel) + + assert isinstance(loaded, ScoreBasedUncertaintyModel) + assert loaded.filter_names == trained_score_model.filter_names + assert loaded._is_trained + + mags = np.array([[25.0, 24.5]], dtype=np.float32) + sigma = loaded.sample_uncertainty(mags, **_SCORE_SAMPLE_KWARGS) + assert np.all(np.isfinite(sigma)) + assert np.all(sigma >= 0) + + +# --- sbi_runner integration --- + + +def _score_fitter(): + """A minimal SBI_Fitter stub exposing only _apply_score_based_noise_model.""" + from unittest.mock import MagicMock + + from synference.sbi_runner import SBI_Fitter + + fitter = MagicMock(spec=SBI_Fitter) + fitter._apply_score_based_noise_model = SBI_Fitter._apply_score_based_noise_model.__get__( + fitter, SBI_Fitter + ) + return fitter + + +def test_sbi_runner_apply_score_based_noise_model(trained_score_model): + """_apply_score_based_noise_model returns correctly shaped arrays.""" + fitter = _score_fitter() + + filter_names = trained_score_model.filter_names + n_filters = len(filter_names) + N = 20 + N_scatters = 3 + + rng = np.random.default_rng(0) + phot = unyt_array(rng.uniform(1e-7, 1e-5, (n_filters, N)), units="Jy") + + scattered, errors = fitter._apply_score_based_noise_model( + phot, + filter_names, + trained_score_model, + N_scatters=N_scatters, + return_errors=True, + normed_flux_units="AB", + ) + + assert scattered.shape == (n_filters, N * N_scatters) + assert errors.shape == (n_filters, N * N_scatters) + assert np.all(np.isfinite(scattered)) + assert np.all(np.isfinite(errors)) + # exp(x) >= 0 always; underflow to 0 is possible with an undertrained model + assert np.all(errors >= 0) + + +def test_sbi_runner_apply_score_reorders_filters(trained_score_model): + """Filters supplied out of order are reordered for the model and restored after.""" + fitter = _score_fitter() + + n_filters = trained_score_model.n_filters + N = 5 + phot = unyt_array(np.arange(1, n_filters * N + 1, dtype=float).reshape(n_filters, N) * 1e-7) + phot = unyt_array(phot.value, units="Jy") + reversed_names = list(reversed(trained_score_model.filter_names)) + + in_order = fitter._apply_score_based_noise_model( + phot[::-1], + trained_score_model.filter_names, + trained_score_model, + N_scatters=1, + normed_flux_units="nJy", + ) + out_of_order = fitter._apply_score_based_noise_model( + phot[::-1], + reversed_names, + trained_score_model, + N_scatters=1, + normed_flux_units="nJy", + ) + + assert out_of_order.shape == (n_filters, N) + # The reordering only permutes rows; the noise draw differs, but the input + # flux each row was built from must line up with the caller's ordering. + assert np.all(np.isfinite(out_of_order)) + assert in_order.shape == out_of_order.shape + + +def test_sbi_runner_apply_score_unknown_filters(trained_score_model): + """_apply_score_based_noise_model raises when the filter sets do not match.""" + fitter = _score_fitter() + + n_filters = trained_score_model.n_filters + phot = unyt_array(np.ones((n_filters, 5)), units="Jy") + + with pytest.raises(ValueError, match="do not match"): + fitter._apply_score_based_noise_model( + phot, ["not_a_filter"] * n_filters, trained_score_model, normed_flux_units="AB" + ) + + +def test_sbi_runner_apply_score_asinh_raises(trained_score_model): + """_apply_score_based_noise_model rejects asinh output from an AB-scaled model.""" + fitter = _score_fitter() + + n_filters = trained_score_model.n_filters + phot = unyt_array(np.ones((n_filters, 5)), units="Jy") + + with pytest.raises(AssertionError, match="asinh"): + fitter._apply_score_based_noise_model( + phot, + trained_score_model.filter_names, + trained_score_model, + normed_flux_units="asinh", + ) + + if __name__ == "__main__": pytest.main([__file__])