From 6742aea2e51ede70e7ba94273186ba78c4d66a9d Mon Sep 17 00:00:00 2001 From: mebiri Date: Thu, 17 Sep 2026 19:39:04 -0500 Subject: [PATCH 1/8] Added HyperPipe variants of existing RIFT code HyperPuffball: new version with flexible rotation/reflection/bounds system (better than RoboCode) ConstructHyperPosterior: ConstructEOSPosterior with flexible rotation system (better than RoboCode), but missing newer features added to CEP > April 2026 CIP_faster: a faster version of util_ConstructIntrinsicPosterior_GenericCoordinates, via optional imports and n_events > 1 functionality --- .../Code/bin/CIP_faster_v2.py | 3896 +++++++++++++++++ .../Code/bin/util_ConstructHyperPosterior.py | 977 +++++ .../Code/bin/util_HyperPuffball_new.py | 365 ++ 3 files changed, 5238 insertions(+) create mode 100644 MonteCarloMarginalizeCode/Code/bin/CIP_faster_v2.py create mode 100644 MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py create mode 100644 MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py diff --git a/MonteCarloMarginalizeCode/Code/bin/CIP_faster_v2.py b/MonteCarloMarginalizeCode/Code/bin/CIP_faster_v2.py new file mode 100644 index 000000000..1c925a50c --- /dev/null +++ b/MonteCarloMarginalizeCode/Code/bin/CIP_faster_v2.py @@ -0,0 +1,3896 @@ +#! /usr/bin/env python +# +# GOAL +# - load in lnL data +# - fit peak to quadratic (standard), GP, etc. +# - pass as input to mcsampler, to generate posterior samples +# +# FORMAT +# - pankow simplification of standard format +# +# COMPARE TO +# util_NRQuadraticFit.py +# postprocess_1d_cumulative +# util_QuadraticMassPosterior.py +# +feedback = True + +if feedback: + print("===Initializing CIP===") + import time + t1 = time.perf_counter() +import RIFT.interpolators.BayesianLeastSquares as BayesianLeastSquares +if feedback: print(" Imported: BayesianLeastSquares ({:.6f} s)".format(time.perf_counter()-t1)) + +import argparse +import sys +import numpy as np +import numpy.lib.recfunctions +import scipy +import scipy.stats +import scipy.special +import RIFT.lalsimutils as lalsimutils +import lalsimulation as lalsim +#import lalframe #unused import +import lal +import functools +#import itertools #unused except in one instance, reimported at that point + +from RIFT.misc.samples_utils import add_field + +import joblib # http://scikit-learn.org/stable/modules/model_persistence.html +if feedback: print(" Imported: standard imports and RIFT tools") + +# GPU acceleration: NOT YET, just do usual +xpy_default=numpy # just in case, to make replacement clear and to enable override +identity_convert = lambda x: x # trivial return itself +cupy_success=False + + +no_plots = True +internal_dtype = np.float32 # only use 32 bit storage! Factor of 2 memory savings for GP code in high dimensions + +C_CGS=2.997925*10**10 # Argh, Monica! + +#required imports every time +if feedback: print(" Importing: igwn_ligolw ..."); t1 = time.perf_counter() +from igwn_ligolw import lsctables, ligolw #, utils #unused import +lsctables.use_in(ligolw.LIGOLWContentHandler) +if feedback: print(" Imported: igwn_ligolw ({:.6f} s)\n Importing: mcsampler ...".format(time.perf_counter()-t1)); t1 = time.perf_counter() +#Ugh, slow: usual 1st output 'import vegas' is in here: +import RIFT.integrators.mcsampler as mcsampler +if feedback: print(" Imported: mcsampler ({:.6f} s)".format(time.perf_counter()-t1)) + +#============!!!CONDITIONAL IMPORTS BELOW PARSER ARGUMENTS!!!=================# + + +def render_coord(x): + if x in lalsimutils.tex_dictionary.keys(): + return lalsimutils.tex_dictionary[x] + if 'product(' in x: + a=x.replace(' ', '') # drop spaces + a = a[:len(a)-1] # drop last + a = a[8:] + terms = a.split(',') + exprs =list(map(render_coord, terms)) + exprs = list(map( lambda x: x.replace('$', ''), exprs)) + my_label = ' '.join(exprs) + return '$'+my_label+'$' + else: + return x + +def render_coordinates(coord_names): + return list(map(render_coord, coord_names)) + + +def extract_combination_from_LI(samples_LI, p): + """ + extract_combination_from_LI + - reads in known columns from posterior samples + - for selected known combinations not always available, it will compute them from standard quantities + """ + if p in samples_LI.dtype.names: # e.g., we have precomputed it + return samples_LI[p] + if p in remap_ILE_2_LI.keys(): + if remap_ILE_2_LI[p] in samples_LI.dtype.names: + return samples_LI[ remap_ILE_2_LI[p] ] + # Return cartesian components of spin1, spin2. NOTE: I may already populate these quantities in 'Add important quantities' + if p == 'chiz_plus': + print(" Transforming ") + if 'a1z' in samples_LI.dtype.names: + return (samples_LI['a1z']+ samples_LI['a2z'])/2. + if 'theta1' in samples_LI.dtype.names: + return (samples_LI['a1']*np.cos(samples_LI['theta1']) + samples_LI['a2']*np.cos(samples_LI['theta2']) )/2. +# return (samples_LI['a1']+ samples_LI['a2'])/2. + if p == 'chiz_minus': + print(" Transforming ") + if 'a1z' in samples_LI.dtype.names: + return (samples_LI['a1z']- samples_LI['a2z'])/2. + if 'theta1' in samples_LI.dtype.names: + return (samples_LI['a1']*np.cos(samples_LI['theta1']) - samples_LI['a2']*np.cos(samples_LI['theta2']) )/2. +# return (samples_LI['a1']- samples_LI['a2'])/2. + if 'theta1' in samples_LI.dtype.names: + if p == 's1x': + return samples_LI["a1"]*np.sin(samples_LI[ 'theta1']) * np.cos( samples_LI['phi1']) + if p == 's1y' : + return samples_LI["a1"]*np.sin(samples_LI[ 'theta1']) * np.sin( samples_LI['phi1']) + if p == 's2x': + return samples_LI["a2"]*np.sin(samples_LI[ 'theta2']) * np.cos( samples_LI['phi2']) + if p == 's2y': + return samples_LI["a2"]*np.sin(samples_LI[ 'theta2']) * np.sin( samples_LI['phi2']) + if p == 'chi1_perp' : + return samples_LI["a1"]*np.sin(samples_LI[ 'theta1']) + if p == 'chi2_perp': + return samples_LI["a2"]*np.sin(samples_LI[ 'theta2']) + if 'lambdat' in samples_LI.dtype.names: # LI does sampling in these tidal coordinates + lambda1, lambda2 = lalsimutils.tidal_lambda_from_tilde(samples_LI["m1"], samples_LI["m2"], samples_LI["lambdat"], samples_LI["dlambdat"]) + if p == "lambda1": + return lambda1 + if p == "lambda2": + return lambda2 + if p == 'delta' or p=='delta_mc': + return (samples_LI['m1'] - samples_LI['m2'])/((samples_LI['m1'] + samples_LI['m2'])) + # Return cartesian components of Lhat + if p == 'product(sin_beta,sin_phiJL)': + return np.sin(samples_LI[ remap_ILE_2_LI['beta'] ]) * np.sin( samples_LI['phi_jl']) + if p == 'product(sin_beta,cos_phiJL)': + return np.sin(samples_LI[ remap_ILE_2_LI['beta'] ]) * np.cos( samples_LI['phi_jl']) + + print(" No access for parameter ", p) + return np.zeros(len(samples_LI['m1'])) # to avoid causing a hard failure + + +parser = argparse.ArgumentParser() +parser.add_argument("--check-good-enough", action='store_true', help="If active, tests if a file 'cip_good_enough' in the current directory exists and has content of nonzero length. Terminates with 'success' if the file exists and has nonzero length ") +parser.add_argument("--fname",help="filename of *.dat file [standard ILE output]") +parser.add_argument("--input-tides",action='store_true',help="Use input format with tidal fields included.") +parser.add_argument("--input-eos-index",action='store_true',help="Use input format with eos index fields included") +parser.add_argument("--n-events-to-analyze",default=1,type=int,help="Number of EOS realizations to analyze. Currently only supports 1") +parser.add_argument("--input-distance",action='store_true',help="Use input format with distance fields (but not tidal fields?) enabled.") +parser.add_argument("--fname-lalinference",help="filename of posterior_samples.dat file [standard LI output], to overlay on corner plots") +parser.add_argument("--fname-output-samples",default="output-ILE-samples",help="output posterior samples (default output-ILE-samples -> output-ILE)") +parser.add_argument("--fname-output-integral",default="integral_result",help="output filename for integral result. Postfixes appended") +parser.add_argument("--no-save-samples",action='store_true') +parser.add_argument("--approx-output",default="SEOBNRv2", help="approximant to use when writing output XML files.") +parser.add_argument("--amplitude-order",default=-1,type=int,help="Set ampO for grid. Used in PN") +parser.add_argument("--phase-order",default=7,type=int,help="Set phaseO for grid. Used in PN") +parser.add_argument("--fref",default=20,type=float, help="Reference frequency used for spins in the ILE output. (Since I usually use SEOBNRv3, the best choice is 20Hz)") +parser.add_argument("--fmin",type=float,default=20) +parser.add_argument("--fname-rom-samples",default=None,help="*.rom_composite output. Treated identically to set of posterior samples produced by mcsampler after constructing fit.") +parser.add_argument("--n-output-samples",default=3000,type=int,help="output posterior samples (default 3000)") +parser.add_argument("--desc-lalinference",type=str,default='',help="String to adjoin to legends for LI") +parser.add_argument("--desc-ILE",type=str,default='',help="String to adjoin to legends for ILE") +parser.add_argument("--parameter", action='append', help="Parameters used as fitting parameters AND varied at a low level to make a posterior") +parser.add_argument("--parameter-implied", action='append', help="Parameter used in fit, but not independently varied for Monte Carlo") +parser.add_argument("--no-adapt-parameter",action='append',help="Disable adaptive sampling in a parameter. Useful in cases where a parameter is not well-constrainxed, and the a prior sampler is well-chosen.") +parser.add_argument("--mc-range",default=None,help="Chirp mass range [mc1,mc2]. Important if we have a low-mass object, to avoid wasting time sampling elsewhere.") +parser.add_argument("--eta-range",default=None,help="Eta range. Important if we have a BNS or other item that has a strong constraint.") +parser.add_argument("--mtot-range",default=None,help="Chirp mass range [mc1,mc2]. Important if we have a low-mass object, to avoid wasting time sampling elsewhere.") +parser.add_argument("--trust-sample-parameter-box",action='store_true', help="If used, sets the prior range to the SAMPLE range for any parameters. NOT IMPLEMENTED. This should be automatically done for mc!") +parser.add_argument("--plots-do-not-force-large-range",action='store_true', help = "If used, the plots do NOT automatically set the chieff range to [-1,1], the eta range to [0,1/4], etc") +parser.add_argument("--downselect-parameter",action='append', help='Name of parameter to be used to eliminate grid points ') +parser.add_argument("--downselect-parameter-range",action='append',type=str) +parser.add_argument("--no-downselect",action='store_true',help='Prevent using downselection on output points' ) +parser.add_argument("--no-downselect-grid",action='store_true',help='Prevent using downselection on input points. Applied only to mc range' ) +parser.add_argument("--downselect-enforce-kerr",action='store_true',help="Provides limits that enforce the kerr limit. Also imposed in coordinate transformations.") +parser.add_argument("--aligned-prior", default="uniform",help="Options are 'uniform', 'volumetric', and 'alignedspin-zprior'. Only influences s1z, s2z") +parser.add_argument("--transverse-prior", default="uniform",help="Options are (default), 'uniform-mag', 'taper-down', 'sqrt-prior', 'alignedspin-zprior', and 'Rbar-singular'. Only influences s1x,s1y,s2x,s2y,Rbar. Usually NOT intended for final work, except for Rbar-singular.") +parser.add_argument("--prior-in-integrand-correction",default=None,help="Implmement integrand = Lp/ps for p_s the default coordinate sampling prior, to allow using priors not naturally associated with coordinates. Intended for spin only at present. Options are 'uniform_over_rbar_singular' (convert rbar singular prior to uniform magnitude), 'uniform_over_volumetric' (convert from volumetric sampling to uniform) and 'volumetric_over_uniform'. Intent is to enable uniform-spin-magnitude sampling in other coordinate systems, etc. Note you MUST use the --transverse-prior Rbar_singular to use the uniform_over_rbar_singular prior ") +parser.add_argument("--spin-prior-chizplusminus-alternate-sampling",default='alignedspin_zprior',help="Use gaussian sampling when using chizplus, chizminus, to make reweighting more efficient.") +parser.add_argument("--import-prior-dictionary-file",default=None,type=str,help="File with dictionary stored_param_dict = 'name':func and stored_param_ranges = 'name':[left,right]. Use to overwrite priors with user-specified function") +parser.add_argument("--output-prior-dictionary-file",default=None,type=str,help="File with dictionary 'name':func. ") +parser.add_argument("--prior-gaussian-mass-ratio",action='store_true',help="Applies a gaussian mass ratio prior (mean=0.5, width=0.2 by default). Only viable in mtot, q coordinates. Not properly normalized, so will break bayes factors by about 2%%") +parser.add_argument("--prior-tapered-mass-ratio",action='store_true',help="Applies a tapered mass ratio prior (transition 0.8, kappa=20). Only viable in mtot, q coordinates. Not properly normalized, a tapering factor instread") +parser.add_argument("--prior-gaussian-spin1-magnitude",action='store_true',help="Applies a gaussian spin magnitude prior (mean=0.7, width=0.1 by default) for FIRST spin. Only viable in polar spin coordinates. Not properly normalized, so will break bayes factors by a small amount (depending on chi_max). Used for 2g+1g merger arguments") +parser.add_argument("--prior-tapered-spin1-magnitude",action='store_true',help="Applies a tapered prior to spin1 magnitude") +parser.add_argument("--prior-tapered-spin1z",action='store_true',help="Applies a tapered prior to spin1's z component") +parser.add_argument("--pseudo-uniform-magnitude-prior", action='store_true',help="Applies volumetric prior internally, and then reweights at end step to get uniform spin magnitude prior") +parser.add_argument("--pseudo-uniform-magnitude-prior-alternate-sampling", action='store_true',help="Changes the internal sampling to be gaussian, not volumetric") +parser.add_argument("--pseudo-gaussian-mass-prior",action='store_true', help="Applies a gaussian mass prior in postprocessing. Done via reweighting so we can use arbitrary mass sampling coordinates.") +parser.add_argument("--pseudo-gaussian-mass-prior-mean",default=1.33,type=float, help="Mean value for reweighting") +parser.add_argument("--pseudo-gaussian-mass-prior-std",default=0.09, type=float,help="Width for reweighting") +parser.add_argument("--prior-lambda-linear",action='store_true',help="Use p(lambda) ~ lambdamax -lambda. Intended for first few iterations, to insure strong coverage of the low-lambda_k corner") +parser.add_argument("--prior-lambda-power",type=float,default=1,help="Use p(lambda) ~ (lambdamax -lambda)^p. Intended for first few iterations, to insure strong coverage of the low-lambda_k corner") +parser.add_argument("--mirror-points",action='store_true',help="Use if you have many points very near equal mass (BNS). Doubles the number of points in the fit, each of which has a swapped m1,m2") +parser.add_argument("--cap-points",default=-1,type=int,help="Maximum number of points in the sample, if positive. Useful to cap the number of points ued for GP. See also lnLoffset. Note points are selected AT RANDOM") +parser.add_argument("--chi-max", default=1,type=float,help="Maximum range of 'a' allowed. Use when comparing to models that aren't calibrated to go to the Kerr limit.") +parser.add_argument("--chi-small-max", default=None,type=float,help="Maximum range of 'a' allowed on the smaller body. If not specified, defaults to chi_max") +parser.add_argument("--ecc-max", default=0.9,type=float,help="Maximum range of 'eccentricity' allowed.") +parser.add_argument("--ecc-min", default=0.0,type=float,help="Minimum range of 'eccentricity' allowed.") +parser.add_argument("--meanPerAno-max", default=2*np.pi,type=float,help="Maximum range of 'meanPerAno' allowed.") +parser.add_argument("--meanPerAno-min", default=0.0,type=float,help="Minimum range of 'meanPerAno' allowed.") +parser.add_argument("--chiz-plus-range", default=None,help="USE WITH CARE: If you are using chiz_minus, chiz_plus for a near-equal-mass system, then setting the chiz-plus-range can improve convergence (e.g., for aligned-spin systems), loosely by setting a chi_eff range that is allowed") +parser.add_argument("--lambda-min", default=0.01,type=float,help="Minimum value of 'Lambda' allowed. This is a very small number slightly different than zero by default, but can be required to be larger for targeted investigations") +parser.add_argument("--lambda-max", default=4000,type=float,help="Maximum value of 'Lambda' allowed. Minimum value is ZERO, not negative.") +parser.add_argument("--lambda-small-max", default=None,type=float,help="Maximum range of 'Lambda' allowed for smaller body. If provided and smaller than lambda_max, used ") +parser.add_argument("--lambda-plus-max", default=None,type=float,help="Maximum range of 'Lambda_plus' allowed. Used for sampling. Pick small values to accelerate sampling! Otherwise, use lambda-max.") +parser.add_argument("--parameter-nofit", action='append', help="Parameter used to initialize the implied parameters, and varied at a low level, but NOT the fitting parameters") +parser.add_argument("--use-precessing",action='store_true') +parser.add_argument("--lnL-downscale-factor",type=float,default=None,help="Multiply log likelihood by this number. Intended for early stages of iterative analyses. Broadens the posterior. Assumes lnL is usual scale. Applied by MULTIPLYING INPUT DATA BY THIS FACTOR, before anything else applied. Note also applied BEFORE MANUAL OFFSETS") +parser.add_argument("--lnL-shift-prevent-overflow",default=None,type=float,help="Define this quantity to be a large positive number to avoid overflows. Note that we do *not* define this dynamically based on sample values, to insure reproducibility and comparable integral results. BEWARE: If you shift the result to be below zero, because the GP relaxes to 0, you will get crazy answers.") +parser.add_argument("--lnL-protect-overflow",action='store_true',help="Before fitting, subtract lnLmax - 100. Add this quantity back at the end.") +parser.add_argument("--lnL-offset",type=float,default=np.inf,help="lnL offset. ONLY POINTS within lnLmax - lnLoffset are used in the calculation! VERY IMPORTANT - default value chosen to include all points, not viable for production with some fit techniques like gp") +parser.add_argument("--lnL-offset-n-random",type=int,default=0,help="Add this many random points past the threshold") +parser.add_argument("--lnL-cut",type=float,default=None,help="lnL cut [MANUAL]. Remove points below this likelihood value from consideration. Generally should not use") +parser.add_argument("--M-max-cut",type=float,default=1e5,help="Maximum mass to consider (e.g., if there is a cut on distance, this matters)") +parser.add_argument("--sigma-cut",type=float,default=0.6,help="Eliminate points with large error from the fit.") +parser.add_argument("--ignore-errors-in-data",action='store_true',help='Ignore reported error in lnL. Helpful for testing purposes (i.e., if the error is zero)') +parser.add_argument("--lnL-peak-insane-cut",type=float,default=np.inf,help="Throw away lnL greater than this value. Should not be necessary") +parser.add_argument("--verbose", action="store_true",default=False, help="Required to build post-frame-generating sanity-test plots") +parser.add_argument("--save-plots",default=False,action='store_true', help="Write plots to file (only useful for OSX, where interactive is default") +parser.add_argument("--inj-file", help="Name of injection file") +parser.add_argument("--event-num", type=int, default=0,help="Zero index of event in inj_file") +parser.add_argument("--report-best-point",action='store_true') +parser.add_argument("--force-no-adapt",action='store_true',help="Disable adaptation, both of the tempering exponent *and* the individual sampling prior(s)") +parser.add_argument("--fit-uses-reported-error",action='store_true') +parser.add_argument("--fit-uses-reported-error-factor",type=float,default=1,help="Factor to add to standard deviation of fit, before adding to lnL. Multiplies number fitting dimensions") +parser.add_argument("--n-max",default=3e8,type=float) +parser.add_argument("--n-eff",default=3e3,type=float) +parser.add_argument("--internal-bound-factor-if-n-eff-small",default=None,type=float,help="If n_eff < n_ouptut_samples, we truncate the output size based on n_eff*(factor)") +parser.add_argument("--n-chunk",default=1e5,type=int) +parser.add_argument("--contingency-unevolved-neff",default=None,help="Contingency planning for when n_eff produced by CIP is small, and user doesn't want to have hard failures. Note --fail-unless-n-eff will prevent this from happening. Options: quadpuff, ...") +parser.add_argument("--not-worker",action='store_true',help="Nonworker jobs, IF we have workers present, don't have the 'fail unless' statement active") +parser.add_argument("--fail-unless-n-eff",default=None,type=float,help="If nonzero, places a minimum requirement on n_eff. Code will exit if not achieved, with no sample generation") +parser.add_argument("--fit-method",default="rf",help="rf (default) : rf|gp|quadratic|polynomial|gp_hyper|gp_lazy|cov|kde. Note 'polynomial' with --fit-order 0 will fit a constant") +parser.add_argument("--fit-load-quadratic",default=None,help="Filename of hdf5 file to load quadratic fit from. ") +parser.add_argument("--fit-load-quadratic-path",default="GW190814/annealing_mc_source_eta_chieff",help="Path in hdf5 file to specific covariance matrix to be used") +parser.add_argument("--pool-size",default=3,type=int,help="Integer. Number of GPs to use (result is averaged)") +parser.add_argument("--fit-load-gp",default=None,type=str,help="Filename of GP fit to load. Overrides fitting process, but user MUST correctly specify coordinate system to interpret the fit with. Does not override loading and converting the data.") +parser.add_argument("--fit-save-gp",default=None,type=str,help="Filename of GP fit to save. ") +parser.add_argument("--fit-order",type=int,default=2,help="Fit order (polynomial case: degree)") +parser.add_argument("--fit-uncertainty-added",default=False, action='store_true', help="Reported likelihood is lnL+(fit error). Use for placement and use of systematic errors.") +parser.add_argument("--no-plots",action='store_true') +parser.add_argument("--tabular-eos-file",type=str,default=None,help="Tabular file of EOS to use. The default prior will be UNIFORM in this table!") +parser.add_argument("--tabular-eos-file-format",type=str,default=None,help="Format of tabular file of EOS to use. The default prior will be UNIFORM in this table!") +parser.add_argument("--tabular-eos-order-statistic",type=str,default=None,help="Order statistic to use. Options will include R1p4, LambdaTildeQ1, and ...}") +parser.add_argument("--using-eos", type=str, default=None, help="Name of EOS. Fit parameter list should physically use lambda1, lambda2 information (but need not). If starts with 'file:', uses a filename with EOS parameters ") +parser.add_argument("--using-eos-for-prior", action='store_true', default=None, help="Alternate (hacky) implementation, which overrides using-eos and using-eos-index, to handle loading in a hyperprior") +parser.add_argument("--using-eos-index", type=int, default=None, help="Index of EOS parameters in file.") +parser.add_argument("--no-use-lal-eos",action='store_true',help="Do not use LAL EOS interface. Used for spectral EOS. Do not use this.") +parser.add_argument("--no-matter1", action='store_true', help="Set the lambda parameters to zero (BBH) but return them") +parser.add_argument("--no-matter2", action='store_true', help="Set the lambda parameters to zero (BBH) but return them") +parser.add_argument("--protect-coordinate-conversions", action='store_true', help="Adds an extra layer to coordinate conversions with range tests. Slows code down, but adds layer of safety for out-of-range EOS parameters for example") +parser.add_argument("--source-redshift",default=0,type=float,help="Source redshift (used to convert from source-frame mass [integration limits] to arguments of fitting function. Note that if nonzero, integration done in SOURCE FRAME MASSES, but the fit is calculated using DETECTOR FRAME") +parser.add_argument("--eos-param", type=str, default=None, help="parameterization of equation of state") +parser.add_argument("--eos-param-values", default=None, help="Specific parameter list for EOS") +parser.add_argument("--sampler-method",default="adaptive_cartesian",help="adaptive_cartesian|GMM|adaptive_cartesian_gpu|portfolio") +parser.add_argument("--sampler-portfolio",default=None,action='append',type=str,help="comma-separated strings, matching sampler methods other than portfolio") +parser.add_argument("--sampler-portfolio-args",default=None, action='append', type=str, help='eval-able dictionary to be passed to that sampler_') +parser.add_argument("--sampler-portfolio-breakpoints",default=None, type=str, help='string representing list') +parser.add_argument("--sampler-oracle",default=None, action='append', type=str, help='names of oracles to be used') +parser.add_argument("--sampler-oracle-args",default=None, action='append', type=str, help='eval-able dictionary to be passed to that oracle') +parser.add_argument("--oracle-reference-sample-file",default=None, type=str, help='filename of reference sample file to be used as oracle for seeding sampler') +parser.add_argument("--oracle-reference-sample-params",default=None, type=str, help='parameters to be pulled from sample file (format comma-separated string)') +parser.add_argument("--internal-use-lnL",action='store_true',help="integrator internally manipulates lnL. ONLY VIABLE FOR GMM AT PRESENT") +parser.add_argument("--internal-temper-log",action='store_true',help="integrator internally uses lnL as sampling weights (only). Designed to reduce insane contrast and overfitting for high-amplitude cases") +parser.add_argument("--internal-correlate-parameters",default=None,type=str,help="comman-separated string indicating parameters that should be sampled allowing for correlations. Must be sampling parameters. Only implemented for gmm. If string is 'all', correlate *all* parameters") +parser.add_argument("--internal-n-comp",default=1,type=int,help="number of components to use for GMM sampling. Default is 1, because we expect a unimodal posterior in well-adapted coordinates. If you have crappy coordinates, use more") +parser.add_argument("--internal-gmm-memory-chisquared-factor",default=None,type=float,help="Multiple of the number of degrees of freedom to save. 5 is a part in 10^6, 4 is 10^{-4}, and None keeps all up to lnL_offset. Note that low-weight points can contribute notably to n_eff, and it can be dangerous to assume a simple chisquared likelihood! Provided in case we need very long runs") +parser.add_argument("--assume-eos-but-primary-bh",action='store_true',help="Special case of known EOS, but primary is a BH") +parser.add_argument("--use-eccentricity", action="store_true") +parser.add_argument("--use-meanPerAno", action="store_true") +parser.add_argument("--tripwire-fraction",default=0.05,type=float,help="Fraction of nmax of iterations after which n_eff needs to be greater than 1+epsilon for a small number epsilon") + +# FIXME hacky options added by me (Liz) to try to get my capstone project to work. +# I needed a way to fix the component masses and nothing else seemed to work. +parser.add_argument("--fixed-parameter", action="append") +parser.add_argument("--fixed-parameter-value", action="append") + +# Supplemental likelihood factors: convenient way to effectively change the mass/spin prior in arbitrary ways for example +# Note this supplemental factor is passed the *fitting* arguments, directly. Use with extreme caution, since we often change the dimension in a DAG +parser.add_argument("--supplementary-likelihood-factor-code", default=None,type=str,help="Import a module (in your pythonpath!) containing a supplementary factor for the likelihood. Used to impose supplementary external priors of arbitrary complexity and external dependence (e.g., external astro priors). EXPERTS-ONLY") +parser.add_argument("--supplementary-likelihood-factor-function", default=None,type=str,help="With above option, specifies the specific function used as an external likelihood. EXPERTS ONLY") +parser.add_argument("--supplementary-likelihood-factor-ini", default=None,type=str,help="With above option, specifies an ini file that is parsed (here) and passed to the preparation code, called when the module is first loaded, to configure the module. EXPERTS ONLY") +parser.add_argument("--supplementary-prior-code",default=None,type=str,help="Import external priors, assumed in scope as extra_prior.prior_dict_pdf, extra_prior.prior_range. Currentlyonly supports seperable external priors") + +#CIP faster args (can be ignored otherwise) +parser.add_argument("--save-hyperfile-only",action="store_true",help="Use to only save MARG-X-X+annotation.dat file, the only one used by HyperPipe (reduces number of output files)") +parser.add_argument("--chunk-save",action="store_true",help="Use to save single-line output files as one file for multiple EOS lines (reduces number of output files)") + +opts= parser.parse_args() + +# good enough file: terminate always with success if present, don't try any more work +if opts.check_good_enough: + fname = 'cip_good_enough' + import os + if os.path.isfile(fname): +# dat = np.loadtxt(fname,dtype=str) + if os.path.getsize(fname) > 0: + print(" Good enough file valid: terminating CIP") + sys.exit(0) + else: + print(" Good enough file ZERO LENGTH, continuing") + + +#================!!!CONDITIONAL IMPORTS BEGIN HERE!!!=========================# + +if not opts.no_plots: + try: #this is the slowest import, by a long shot + if feedback: print(" Importing: matplotlib ..."); t1 = time.perf_counter() + import matplotlib + matplotlib.use('agg') # prevent requests for DISPLAY + import matplotlib.pyplot as plt + #from mpl_toolkits.mplot3d import Axes3D #unused import + import matplotlib.lines as mlines + import corner + + no_plots=False + if feedback: print(" Imported: matplotlib ({:.6f} s)".format(time.perf_counter()-t1)) + except ImportError: + print(" - no matplotlib - ") #don't get this output +else: + if feedback: print(" NOTICE: matplotlib not imported") + +if feedback: print(" -- SAMPLER METHOD:",opts.sampler_method," -- ") +port_list = [] +sampler_check = 0 +if opts.sampler_method == "portfolio": + port_list = opts.sampler_portfolio + +#technically should define all other checks here, except they're never used +mcsampler_NF_ok=False +mcsampler_Portfolio_ok=False +internalGP_ok = False +gpytorch_ok = False + +#portfolio needs: AV, GMM, adaptive_cartesian_gpu, AC, NFlow, mcsamplerPortfolio +if opts.sampler_method == "GMM" or "GMM" in port_list: + try: + import RIFT.integrators.mcsamplerEnsemble as mcsamplerEnsemble #nothing in here + mcsampler_gmm_ok = True #never used + sampler_check += 1 + if feedback: print(" Imported sampler: GMM") + except: + print(" No mcsamplerEnsemble ") #don't get this output + mcsampler_gmm_ok = False +if opts.sampler_method == "adaptive_cartesian_gpu" or "AC" in port_list or "adaptive_cartesian_gpu" in port_list: + try: + import RIFT.integrators.mcsamplerGPU as mcsamplerGPU #no cupy (mcsamplerGPU) is in here + mcsampler_gpu_ok = True #never used + mcsamplerGPU.xpy_default =xpy_default # force consistent, in case GPU present + mcsamplerGPU.identity_convert = identity_convert + sampler_check += 1 + if feedback: print(" Imported sampler: adaptive_cartesian_gpu/AC") + except: + print( " No mcsamplerGPU ") #don't get this output + mcsampler_gpu_ok = False +if opts.sampler_method == "AV" or "AV" in port_list: + try: + import RIFT.integrators.mcsamplerAdaptiveVolume as mcsamplerAdaptiveVolume #no cupy (mcsamplerAV) is in here + mcsampler_AV_ok = True #never used + sampler_check += 1 + if feedback: print(" Imported sampler: AV") + except: + print(" No mcsamplerAV ") #don't get this output + mcsampler_AV_ok = False +#handled later: expensive import +mcsamplerNFlow = None #not really necessary +if opts.sampler_method == "NFlow" or "NFlow" in port_list: + sampler_check += 1 + if feedback: print(" Import requested: mcsamplerNFlow - will try later") +#Needed if sampler_method == portfolio, or if method is not any of the above (or those failed to import) +if opts.sampler_method == 'portfolio' or sampler_check == 0: + try: + import RIFT.integrators.mcsamplerPortfolio as mcsamplerPortfolio #'no cupy (mcsamplerPortfolio)' & 'Portfolio discovery: loading' are in here + mcsampler_Portfolio_ok = True + if feedback: print(" Imported sampler: mcsamplerPortfolio") + except: + print(" No mcsamplerPortfolio ") #---------------usual print - so usually False + +if feedback: print(" -- FIT METHOD:",opts.fit_method," -- ") +#from sklearn.preprocessing import PolynomialFeatures #unused import +if opts.fit_method == "polynomial": + try: #only used for opts.fit_method == "polynomial" + import RIFT.misc.ModifiedScikitFit as msf # altenative polynomialFeatures + if feedback: print(" Imported: RIFT ModifiedScikitFit") + except: + print(" - Failed ModifiedScikitFit : No polynomial fits - ") #don't get this output + from sklearn import linear_model #also only for fit polynomial + if feedback: print(" Imported: sklearn") +if opts.fit_method == 'nn' or opts.fit_method == 'nn_rfwrapper': + try: + import RIFT.interpolators.senni as senni #only used for opts.fit_method == 'nn_rfwrapper' & opts.fit_method == 'nn' + senni_ok = True #never used + if feedback: print(" Imported: senni") + except: + print( " No senni ") #don't get this output + senni_ok = False +if opts.fit_method == 'gp_sparse': + try: + import RIFT.interpolators.internal_GP #this is just a test - imported again later if ok + internalGP_ok = True #checked later to import again: only used if opts.fit_method == gp_sparse + if feedback: print(" Import check: internal_GP OK") + except: + print( " - no internal_GP - ") #don't get this output + internalGP_ok = False +if opts.fit_method == 'gp-torch': + try: + import RIFT.interpolators.gpytorch_wrapper as gpytorch_wrapper #only used if opts.fit_method == gp-torch + gpytorch_ok = True + if feedback: print(" Imported: gpytorch_wrapper") + except: + print( " No gpytorch_wrapper ") #-------- usual print + gpytorch_ok = False + +#=======================!!!END CONDITIONAL IMPORTS!!!=========================# + +if not(opts.no_adapt_parameter): + opts.no_adapt_parameter =[] # needs to default to empty list +ECC_MAX = opts.ecc_max +ECC_MIN = opts.ecc_min +MEANPERANO_MAX = 2*np.pi +MEANPERANO_MIN = 0 +no_plots = no_plots | opts.no_plots +if opts.n_events_to_analyze > 1: #force no plots if > 1 EOS realization + opts.no_plots = True + no_plots = True +lnL_shift = 0 +lnL_default_large_negative = -500 +if opts.lnL_shift_prevent_overflow: + lnL_shift = opts.lnL_shift_prevent_overflow +if not(opts.force_no_adapt): + opts.force_no_adapt=False # force explicit boolean false + +ok_lnL_methods = ['GMM', 'adaptive_cartesian', 'adaptive_cartesian_gpu', 'AV', 'NFlow', 'portfolio'] +bad_lnL_methods = ['default'] +if opts.internal_use_lnL and (opts.sampler_method in bad_lnL_methods ): + print(" OPTION MISMATCH : --internal-use-lnL not compatible with", opts.sampler_method, " can only use ", ok_lnL_methods) + sys.exit(99) + + +source_redshift=opts.source_redshift + +# require eos index input and +if opts.input_eos_index and not(opts.tabular_eos_file): + print(" warning: input EOS index, but not using it; presumably you are doing a model-free test ") +if not(opts.input_eos_index) and (opts.tabular_eos_file): + raise Exception(" Fail: must process EOS input to be able to use it ") +# ensure using_eos_index valid for eos file length +if opts.using_eos and opts.using_eos.startswith('file:') and not(opts.using_eos_index is None): + fname = opts.using_eos.replace('file:', '') + try: + dat = np.loadtxt(fname)[opts.using_eos_index] + except Exception as e: + print(" Fail: EOS index out of range:\n ",e) + sys.exit(0) + +#eos initialization moved to loop below + +with open('args.txt','w') as fp: + import sys + fp.write(' '.join(sys.argv)) + +if opts.fit_method == "quadratic": + opts.fit_order = 2 # overrride + + +### +### Comparison data (from LI) +### +remap_ILE_2_LI = { + "s1z":"a1z", "s2z":"a2z", + "s1x":"a1x", "s1y":"a1y", + "s2x":"a2x", "s2y":"a2y", + "chi1_perp":"chi1_perp", + "chi2_perp":"chi2_perp", + "chi1":'a1', + "chi2":'a2', + "cos_phiJL": 'cos_phiJL', + "sin_phiJL": 'sin_phiJL', + "cos_theta1":'costilt1', + "cos_theta2":'costilt2', + "theta1":"tilt1", + "theta2":"tilt2", + "xi":"chi_eff", + "chiMinus":"chi_minus", + "delta":"delta", + "delta_mc":"delta", + "mtot":'mtotal', "mc":"mc", "eta":"eta","m1":"m1","m2":"m2", + "cos_beta":"cosbeta", + "beta":"beta", + "LambdaTilde":"lambdat", + "DeltaLambdaTilde": "dlambdat"} + +if opts.fname_lalinference: + print(" Loading lalinference samples for direct comparison ", opts.fname_lalinference) + samples_LI = np.genfromtxt(opts.fname_lalinference,names=True) + + print(" Checking consistency between fref in samples and fref assumed here ") + try: + print(set(samples_LI['f_ref']), opts.fref) + except: + print(" No fref") + + print(" Checking LI samples have desired parameters ") + try: + for p in opts.parameter: + if p in remap_ILE_2_LI: + print(p , " -> ", remap_ILE_2_LI[p]) + else: + print(p, " NOT LISTED IN KEYS") + except: + print("remap check failed") + + ### + ### Add important quantities easily derived from the samples but not usually provided + ### + + delta_dat = (samples_LI["m1"] - samples_LI["m2"])/(samples_LI["m1"]+samples_LI["m2"]) + samples_LI = add_field(samples_LI, [('delta', float)]); samples_LI['delta'] = delta_dat + if "a1z" in samples_LI.dtype.names: + chi_minus = (samples_LI["a1z"]*samples_LI["m1"] - samples_LI["a2z"]*samples_LI["m2"])/(samples_LI["m1"]+samples_LI["m2"]) + samples_LI = add_field(samples_LI, [('chi_minus', float)]); samples_LI['chi_minus'] = chi_minus + + if "tilt1" in samples_LI.dtype.names: + a1x_dat = samples_LI["a1"]*np.sin(samples_LI["theta1"])*np.cos(samples_LI["phi1"]) + a1y_dat = samples_LI["a1"]*np.sin(samples_LI["theta1"])*np.sin(samples_LI["phi1"]) + chi1_perp = samples_LI["a1"]*np.sin(samples_LI["theta1"]) + + a2x_dat = samples_LI["a2"]*np.sin(samples_LI["theta2"])*np.cos(samples_LI["phi2"]) + a2y_dat = samples_LI["a2"]*np.sin(samples_LI["theta2"])*np.sin(samples_LI["phi2"]) + chi2_perp = samples_LI["a2"]*np.sin(samples_LI["theta2"]) + + + samples_LI = add_field(samples_LI, [('a1x', float)]); samples_LI['a1x'] = a1x_dat + samples_LI = add_field(samples_LI, [('a1y', float)]); samples_LI['a1y'] = a1y_dat + samples_LI = add_field(samples_LI, [('a2x', float)]); samples_LI['a2x'] = a2x_dat + samples_LI = add_field(samples_LI, [('a2y', float)]); samples_LI['a2y'] = a2y_dat + samples_LI = add_field(samples_LI, [('chi1_perp', float)]); samples_LI['chi1_perp'] = chi1_perp + samples_LI = add_field(samples_LI, [('chi2_perp', float)]); samples_LI['chi2_perp'] = chi2_perp + + samples_LI = add_field(samples_LI, [('cos_phiJL',float)]); samples_LI['cos_phiJL'] = np.cos(samples_LI['phi_jl']) + samples_LI = add_field(samples_LI, [('sin_phiJL',float)]); samples_LI['sin_phiJL'] = np.sin(samples_LI['phi_jl']) + + samples_LI = add_field(samples_LI, [('product(sin_beta,sin_phiJL)', float)]) + samples_LI['product(sin_beta,sin_phiJL)'] =np.sin(samples_LI[ remap_ILE_2_LI['beta'] ]) * np.sin( samples_LI['phi_jl']) + + + samples_LI = add_field(samples_LI, [('product(sin_beta,cos_phiJL)', float)]) + samples_LI['product(sin_beta,cos_phiJL)'] =np.sin(samples_LI[ remap_ILE_2_LI['beta'] ]) * np.cos( samples_LI['phi_jl']) + + # Add all keys in samples_LI to the remapper + for key in samples_LI.dtype.names: + if not (key in remap_ILE_2_LI.keys()): + remap_ILE_2_LI[key] = key # trivial remapping + +#setting up downselect ranges +downselect_dict = {} +dlist = [] +dlist_ranges=[] +if opts.downselect_parameter: + dlist = opts.downselect_parameter + dlist_ranges = list(map(eval,opts.downselect_parameter_range)) +else: + dlist = [] + dlist_ranges = [] +if len(dlist) != len(dlist_ranges): + print(" downselect parameters inconsistent", dlist, dlist_ranges) +for indx in np.arange(len(dlist_ranges)): + downselect_dict[dlist[indx]] = dlist_ranges[indx] + +chi_max = opts.chi_max +chi_small_max = chi_max +if not opts.chi_small_max is None: + chi_small_max = opts.chi_small_max +lambda_min=opts.lambda_min +lambda_max=opts.lambda_max +lambda_small_max = lambda_max +if not (opts.lambda_small_max is None): + lambda_small_max = opts.lambda_small_max +lambda_plus_max = opts.lambda_max +if opts.lambda_plus_max: + lambda_plus_max = opts.lambda_max +if not('chi1' in downselect_dict): + # dont override + downselect_dict['chi1'] = [0,chi_max] +if not('chi2' in downselect_dict): + downselect_dict['chi2'] = [0,chi_small_max] +if opts.input_tides: + # only insert these cuts if we are using a composite file with tides! + # do not downselect if lambda1 is not in ! allows NSBH + if not('lambda1' in downselect_dict) and 'lambda1' in opts.parameter: + downselect_dict['lambda1'] = [lambda_min,lambda_max] + if not('lambda2' in downselect_dict): + downselect_dict['lambda2'] = [lambda_min,lambda_small_max] +for param in ['s1z', 's2z', 's1x','s2x', 's1y', 's2y']: + downselect_dict[param] = [-chi_max,chi_max] +# Enforce definition of eta +downselect_dict['eta'] = [0,0.25] + +if opts.no_downselect: + downselect_dict={} + + +test_converged={} #sent to sampler as-is, apparently; i.e., sampler does not test convergence +#test_converged['neff'] = functools.partial(mcsampler.convergence_test_MostSignificantPoint,0.01) # most significant point less than 1/neff of probability. Exactly equivalent to usual neff threshold. +#test_converged["normal_integral"] = functools.partial(mcsampler.convergence_test_NormalSubIntegrals, 25, 0.01, 0.1) # 20 sub-integrals are gaussian distributed [weakly; mainly to rule out outliers] *and* relative error < 10%, based on sub-integrals . Should use # of intervals << neff target from above. Note this sets our target error tolerance on lnLmarg. Note the specific test requires >= 20 sub-intervals, which demands *very many* samples (each subintegral needs to be converged). + +### +### Parameters in use +### +coord_names = opts.parameter # Used in fit +if coord_names is None: + coord_names = [] +low_level_coord_names = list(coord_names) # Used for Monte Carlo. Use 'list' to force re-create/copy +if 'chi_pavg' in coord_names: + low_level_coord_names += ['chi_pavg'] +if opts.parameter_implied: + coord_names = coord_names+opts.parameter_implied +if opts.parameter_nofit: + if opts.parameter is None: + low_level_coord_names = opts.parameter_nofit # Used for Monte Carlo + else: + low_level_coord_names = opts.parameter+opts.parameter_nofit # Used for Monte Carlo +# SANITY COMPATIBILITY CHECK +if 'q' in low_level_coord_names and 'mc' in low_level_coord_names: + print(" Coordinate compatibility error: mc,eta or mc,delta_mc or M,q are compatible coordinates for masses. Do not mix!") + sys.exit(1) +error_factor = len(coord_names) +if error_factor ==0 : + raise Exception(" Coordinate list for fit empty; exiting ") +if opts.fit_uses_reported_error: + error_factor=len(coord_names)*opts.fit_uses_reported_error_factor +# TeX dictionary +tex_dictionary = lalsimutils.tex_dictionary +print(" Coordinate names for fit :, ", coord_names) +if not(opts.no_plots): + print(" Rendering coordinate names : ", render_coordinates(coord_names)) # map(lambda x: tex_dictionary[x], coord_names) +if opts.fit_method =="polynomial" or opts.fit_method == 'quadratic': + print(" Symmetry for these fitting coordinates :", lalsimutils.symmetry_sign_exchange(coord_names)) +print(" Coordinate names for Monte Carlo :, ", low_level_coord_names) +if not(opts.no_plots): + print(" Rendering coordinate names : ", list(map(lambda x: tex_dictionary[x], low_level_coord_names))) + + +### +### Prior functions : a dictionary - a whole TON of over-definition but whatever +### + +# mcmin, mcmax : to be defined later +def M_prior(x): # not normalized; see section II.C of https://arxiv.org/pdf/1701.01137.pdf + return 2*x/(mc_max**2-mc_min**2) +def q_prior(x,norm_factor=1.): + return norm_factor/(1+x)**2 # not normalized; see section II.C of https://arxiv.org/pdf/1701.01137.pdf +def m1_prior(x): + return 1./200 +def m2_prior(x): + return 1./200 +def s1z_prior(x): + return 1./(2*chi_max) +def s2z_prior(x): + return 1./(2*chi_max) +def mc_prior(x): + return 2*x/(mc_max**2-mc_min**2) +def unscaled_eta_prior_cdf(eta_min): + r""" + cumulative for integration of x^(-6/5)(1-4x)^(-1/2) from eta_min to 1/4. + Used to normalize the eta prior + Derivation in mathematica: + Integrate[ 1/\[Eta]^(6/5) 1/Sqrt[1 - 4 \[Eta]], {\[Eta], \[Eta]min, 1/4}] + """ + return 2**(2./5.) *np.sqrt(np.pi)*scipy.special.gamma(-0.2)/scipy.special.gamma(0.3) + 5*scipy.special.hyp2f1(-0.2,0.5,0.8, 4*eta_min)/(eta_min**(0.2)) +def eta_prior(x,norm_factor=1.44): + """ + eta_prior returns the eta prior. + Change norm_factor by the output + """ + return 1./np.power(x,6./5.)/np.power(1-4.*x, 0.5)/norm_factor +def delta_mc_prior(x,norm_factor=1.44): + """ + delta_mc = sqrt(1-4eta) <-> eta = 1/4(1-delta^2) + Transform the prior above + """ + eta_here = 0.25*(1 -x*x) + return 2./np.power(eta_here, 6./5.)/norm_factor + +def m_prior(x): + return 1/(1e3-1.) # uniform in mass, use a square. Should always be used as m1,m2 in pairs. Note this does NOT restrict m1>m2. + +def triangle_prior(x,R=chi_max): + return (np.ones(x.shape)-np.abs(x/R))/R # triangle from -R to R centered on zero +def xi_uniform_prior(x): + return np.ones(x.shape) +def s_component_uniform_prior(x,R=chi_max): # If all three are used, a volumetric prior + return np.ones(x.shape)/(2.*R) +def s_magnitude_uniform_prior(x,R=chi_max): + return np.ones(x.shape)/R +def s_component_sqrt_prior(x,R=chi_max): # If all three are used, a volumetric prior + return 1./(4.*R*np.sqrt(np.abs(x)/R)) # -R,R range +def s_component_gaussian_prior(x,R=chi_max/3.): + """ + (proportinal to) prior on range in one-dimensional components, in a cartesian domain. + Could be useful to sample densely near zero spin. + [Note: we should use 'truncnorm' instead...] + """ + xp = np.array(x,dtype=float) + val= scipy.stats.truncnorm(-chi_max/R,chi_max/R,scale=R).pdf(xp) # stupid casting problem : x is dtype 'object' + return val + +def s_component_zprior(x,R=chi_max): + # assume maximum spin =1. Should get from appropriate prior range + # Integrate[-1/2 Log[Abs[x]], {x, -1, 1}] == 1 + val = -1./(2*R) * np.log( (np.abs(x)/R+1e-7).astype(float)) + return val +def s_component_zprior_positive(x,R=chi_max): + # assume maximum spin =1. Should get from appropriate prior range + # Integrate[-1/2 Log[Abs[x]], {x, -1, 1}] == 1 + val = -1./(2*R) * np.log( (np.abs(x)/R+1e-7).astype(float)) + return val*2 + +def s_component_volumetricprior(x,R=1.): + # assume maximum spin =1. Should get from appropriate prior range + # for SPIN MAGNITUDE OF PRECESSING SPINS only + return (1./3.* np.power(x/R,2)) + +def s_component_aligned_volumetricprior(x,R=1.): + # assume maximum spin =1. Should get from appropriate prior range + # for SPIN COMPONENT ALIGNED (s1z,s2z) for aligned spins only + #This is a probability that is defined on x\in[-R,R], such that \int_a^b dx p(x) is the volume of a sphere between horizontal slices at height a,b: + #p(x)dx = pi R^2 (1- (x/R)^2)/ (4 pi R^3 /3) = 3/4 * (1 - (x/R)^2) /R + return (3./4.*(1- np.power(x/R,2))/R) + +def lambda_prior(x): + return np.ones(x.shape)/(lambda_max-lambda_min) # assume arbitrary +def lambda_small_prior(x): + return np.ones(x.shape)/(lambda_small_max -lambda_min) # assume arbitrary + + +# DO NOT USE UNLESS REQUIRED FOR COMPATIBILITY +def lambda_tilde_prior(x): + return np.ones(x.shape)/opts.lambda_max # 0,4000 +def delta_lambda_tilde_prior(x): + return np.ones(x.shape)/1000. # -500,500 + +def gaussian_mass_prior(x,mu=0.,sigma=1.): # actually viable for *any* prior. + y = np.array(x,dtype=np.float32) + return np.exp( - 0.5*(y-mu)**2/sigma**2)/np.sqrt(2*np.pi*sigma**2) + +def tapered_magnitude_prior(x,loc=0.65,kappa=19.): # + """ + tapered_magnitude_prior is 1 inside a region and tapers to 0 outside + The scale factor is designed so the taper is very strong and has no effect away from the region of significance + Equivalent to + (1 - 1/(1+f1)) / (1+f2) = f1/(1+f1)(1+f2) + """ + y = np.array(x,dtype=np.float32) # problem of object type data + f1 = np.exp( - (y-loc)*kappa) + f2 = np.exp( - (y+loc)*kappa) + + return f1/(1+f1)/(1+f2) + +def tapered_magnitude_prior_alt(x,loc=0.8,kappa=20.): # + """ + tapered_magnitude_prior is 1 above the scale factor and 0 below it + 1/ (1+f) = + """ + y = np.array(x,dtype=np.float32) # problem of object type data + f1 = np.exp( - (y-loc)*kappa) + + return 1/(1+f1) + +def eccentricity_prior(x): + return np.ones(x.shape) / (ECC_MAX-ECC_MIN) # uniform over the interval [0.0, ECC_MAX] + +def eccentricity_squared_prior(x): # note this is INCONSISTENT with the prior above -- we are designed to give a CDF = (e/emax)^2 for example here, or more generally (e^2 - emin^2)/(emax^2-emin^2) + return np.ones(x.shape) / (ECC_MAX**2-ECC_MIN**2) # uniform over the interval [ECC_MIN, ECC_MAX] + +def meanPerAno_prior(x): + return np.ones(x.shape) / (MEANPERANO_MAX-MEANPERANO_MIN) # uniform over the interval [MEANPERANO_MIN, MEANPERANO_MAX] + +def precession_prior(x): + return 0.5*np.ones(x.shape) # uniform over the interval [0.0, 2.0] + +def unnormalized_uniform_prior(x): + return np.ones(x.shape) +def unnormalized_log_prior(x): + return 1./x + +def normalized_Rbar_prior(x): + return 2*x +p_Rbar = lalsimutils.p_R +def normalized_Rbar_singular_prior(x): + return np.power(x, p_Rbar-1.)*p_Rbar +def normalized_zbar_prior(z): + return 3.*(1.-z**2)/4. + +prior_map = { "mtot": M_prior, "q":q_prior, "s1z":s_component_uniform_prior, "s2z":functools.partial(s_component_uniform_prior, R=chi_small_max), "mc":mc_prior, "eta":eta_prior, 'delta_mc':delta_mc_prior, 'xi':xi_uniform_prior,'chi_eff':xi_uniform_prior,'delta': (lambda x: 1./2), + 's1x':s_component_uniform_prior, + 's2x':functools.partial(s_component_uniform_prior, R=chi_small_max), + 's1y':s_component_uniform_prior, + 's2y': functools.partial(s_component_uniform_prior, R=chi_small_max), + 'chiz_plus':s_component_uniform_prior, + 'chiz_minus':s_component_uniform_prior, + 'm1':m_prior, + 'm2':m_prior, + 'lambda1':lambda_prior, + 'lambda2':lambda_small_prior, + 'lambda_plus': lambda_prior, + 'lambda_minus': lambda_prior, + 'LambdaTilde':lambda_tilde_prior, + 'DeltaLambdaTilde':delta_lambda_tilde_prior, + # Polar spin components (uniform magnitude by default) + 'chi1':s_magnitude_uniform_prior, + 'chi2':functools.partial(s_magnitude_uniform_prior, R=chi_small_max), + 'theta1': mcsampler.uniform_samp_theta, + 'theta2': mcsampler.uniform_samp_theta, + 'cos_theta1': mcsampler.uniform_samp_cos_theta, + 'cos_theta2': mcsampler.uniform_samp_cos_theta, + 'phi1':mcsampler.uniform_samp_phase, + 'phi2':mcsampler.uniform_samp_phase, + # Pseudo-cylindrical : note this is a VOLUMETRIC prior + 'chi1_perp_bar':normalized_Rbar_prior, + 'chi1_perp_u':unnormalized_uniform_prior, + 'chi2_perp_bar':normalized_Rbar_prior, + 'chi2_perp_u':unnormalized_uniform_prior, + 's1z_bar':normalized_zbar_prior, + 's2z_bar':normalized_zbar_prior, + # Other priors + 'eccentricity':eccentricity_prior, + 'eccentricity_squared':eccentricity_squared_prior, + 'meanPerAno':meanPerAno_prior, + 'chi_pavg':precession_prior, + 'mu1': unnormalized_log_prior, + 'mu2': unnormalized_uniform_prior +} +prior_range_map = {"mtot": [1, 300], "q":[0.01,1], "s1z":[-0.999*chi_max,0.999*chi_max], "s2z":[-0.999*chi_small_max,0.999*chi_small_max], "mc":[0.9,250], "eta":[0.01,0.2499999],'delta_mc':[0,0.9], 'xi':[-chi_max,chi_max],'chi_eff':[-chi_max,chi_max],'delta':[-1,1], + 's1x':[-chi_max,chi_max], + 's2x':[-chi_small_max,chi_small_max], + 's1y':[-chi_max,chi_max], + 's2y':[-chi_small_max,chi_small_max], + 'chiz_plus':[-chi_max,chi_max], # BEWARE BOUNDARIES + 'chiz_minus':[-chi_max,chi_max], + 'm1':[0.9,1e3], + 'm2':[0.9,1e3], + 'lambda1':[lambda_min,lambda_max], + 'lambda2':[lambda_min,lambda_small_max], + 'lambda_plus':[lambda_min,lambda_plus_max], + 'lambda_minus':[-lambda_max,lambda_max], # will include the true region always...lots of overcoverage for small lambda, but adaptation will save us. + 'eccentricity':[ECC_MIN, ECC_MAX], + 'eccentricity_squared':[ECC_MIN**2, ECC_MAX**2], + 'meanPerAno':[MEANPERANO_MIN, MEANPERANO_MAX], + 'chi_pavg':[0.0,2.0], + # strongly recommend you do NOT use these as parameters! Only to insure backward compatibility with LI results + 'LambdaTilde':[0.01,5000], + 'DeltaLambdaTilde':[-500,500], + 'chi1':[0,chi_max], + 'chi2':[0,chi_small_max], + 'theta1':[0,np.pi], + 'theta2':[0,np.pi], + 'cos_theta1':[-1,1], + 'cos_theta2':[-1,1], + 'phi1':[0,2*np.pi], + 'phi2':[0,2*np.pi], + 'chi1_perp_bar':[0,1], + 'chi2_perp_bar':[0,1], + 'chi1_perp_u':[0,1], + 'chi2_perp_u':[0,1], + 's1z_bar':[-1,1], + 's2z_bar':[-1,1], + 'mu1':[0.0001,1e3], # suboptimal, but something + 'mu2':[-300,1e3] +} +if not (opts.chiz_plus_range is None): + print(" Warning: Overriding default chiz_plus range. USE WITH CARE", opts.chiz_plus_range) + prior_range_map['chiz_plus']=eval(opts.chiz_plus_range) + +if not (opts.eta_range is None): + print(f" Warning: Overriding default eta range to {eval(opts.eta_range)}. USE WITH CARE") + eta_range=prior_range_map['eta'] = eval(opts.eta_range) # really only useful if eta is a coordinate. USE WITH CARE + prior_range_map['delta_mc'] = np.sqrt(1-4*np.array(prior_range_map['eta']))[::-1] # reverse + + # change eta range normalization factors to match prior range on eta + norm_factor = unscaled_eta_prior_cdf(eta_range[0]) - unscaled_eta_prior_cdf(eta_range[1]) + prior_map['eta'] = functools.partial(eta_prior, norm_factor=norm_factor) + prior_map['delta_mc'] = functools.partial(delta_mc_prior, norm_factor=norm_factor) + + # if q in parameters, also correctly normalize the prior for q, in case user is using (M,q) coordinates + # Note CDF of 1/(1+q)^2 is -1/(1+q), so the normalization is analytic + if 'q' in low_level_coord_names: + delta_range = np.sqrt(1 - 4*np.array(eta_range)) + q_range = (1-delta_range)/(1+delta_range) + norm_factor_q = 1./(1+q_range[0]) - 1./(1+q_range[1]) + prior_map['q'] = functools.partial(q_prior, norm_factor=norm_factor_q) + prior_range_map['q'] = q_range + + +### +### Modify priors, as needed +### +# https://bugs.ligo.org/redmine/issues/5020 +# https://github.com/lscsoft/lalsuite/blob/master/lalinference/src/LALInferencePrior.c +if opts.aligned_prior == 'alignedspin-zprior': + # prior on s1z constructed to produce the standard distribution + prior_map["s1z"] = s_component_zprior + prior_map["s2z"] = functools.partial(s_component_zprior,R=chi_small_max) + prior_map["s1z_bar"] = s_component_zprior + prior_map["s2z_bar"] = functools.partial(s_component_zprior,R=chi_small_max) + if 'chiz_plus' in low_level_coord_names: + if opts.spin_prior_chizplusminus_alternate_sampling == 'alignedspin_zprior': + # just a trick to make reweighting more efficient. + prior_map['chiz_plus'] = s_component_zprior + prior_map['chiz_minus'] = s_component_zprior + else: + prior_map['chiz_plus'] = s_component_gaussian_prior + prior_map['chiz_minus'] = s_component_gaussian_prior +elif opts.aligned_prior == 'alignedspin-zprior-positive': + # prior on s1z constructed to produce the standard distribution + prior_map["s1z"] = s_component_zprior_positive + prior_map["s2z"] = functools.partial(s_component_zprior_positive,R=chi_small_max) + prior_map["s1z_bar"] = s_component_zprior_positive + prior_map["s2z_bar"] = functools.partial(s_component_zprior_positive,R=chi_small_max) + prior_range_map['s1z'] = [0,chi_max] + prior_range_map['s2z'] = [0,chi_small_max] + prior_range_map['s1z_bar'] = [0,chi_max] + prior_range_map['s2z_bar'] = [0,chi_small_max] + +if opts.transverse_prior == 'uniform': + # Don't do anything: let the default uniform priros for s1x, s1y ... OR chi1_perp-bar, etc used be used + # Note these have different physical meaning! + print(" Transverse prior: Uniform in PROPOSED COORDINATES (defaults). Note physical meaning depends on coordinate choice!") + True +elif opts.transverse_prior == 'uniform-mag': # intent: only for these chi1_perp_bar, chi2_perp_bar coordinates + # allow for better transverse spin prior, + prior_map['chi1_perp_bar'] = unnormalized_uniform_prior + prior_map['chi2_perp_bar'] = unnormalized_uniform_prior +elif opts.transverse_prior == "Rbar-singular": + prior_map["chi1_perp_bar"] = normalized_Rbar_singular_prior + prior_map["chi2_perp_bar"] = normalized_Rbar_singular_prior +elif opts.transverse_prior == 'alignedspin-zprior': + prior_map["s1x"] = s_component_zprior + prior_map["s1y"] = s_component_zprior + prior_map["s2x"] = functools.partial(s_component_zprior,R=chi_small_max) + prior_map["s2y"] = functools.partial(s_component_zprior,R=chi_small_max) +elif opts.transverse_prior == 'sqrt-prior': + prior_map["s1x"] = s_component_sqrt_prior + prior_map["s1y"] = s_component_sqrt_prior + prior_map["s2x"] = functools.partial(s_component_sqrt_prior,R=chi_small_max) + prior_map["s2y"] = functools.partial(s_component_sqrt_prior,R=chi_small_max) + prior_map['chi1_perp_bar'] = s_component_sqrt_prior + prior_map['chi2_perp_bar'] = s_component_sqrt_prior +elif opts.transverse_prior == 'taper-down': + prior_map["s1x"] = triangle_prior + prior_map["s1y"] = triangle_prior + prior_map["s2x"] = functools.partial(triangle_prior,R=chi_small_max) + prior_map["s2y"] = functools.partial(triangle_prior,R=chi_small_max) + prior_map['chi1_perp_bar'] = triangle_prior + prior_map['chi2_perp_bar'] = triangle_prior +else: + print(" UNKOWN OPTION for --transverse-prior ", opts.transverse_prior) + +if opts.aligned_prior == 'volumetric': + prior_map["s1z"] = s_component_aligned_volumetricprior + prior_map["s2z"] = s_component_aligned_volumetricprior + +if opts.pseudo_uniform_magnitude_prior and opts.pseudo_uniform_magnitude_prior_alternate_sampling: + prior_map['s1x'] = s_component_gaussian_prior + prior_map['s2x'] = s_component_gaussian_prior + prior_map['s1y'] = s_component_gaussian_prior + prior_map['s2y'] = s_component_gaussian_prior + if 's1z' in low_level_coord_names: + prior_map['s1z'] = s_component_gaussian_prior + prior_map['s2z'] = s_component_gaussian_prior + elif 'chiz_plus' in low_level_coord_names: # because of rotated coordinate system. This matches in interior + print(" CODE PATH NOT YET WORKING ") + sys.exit(0) + prior_map['chiz_plus'] = s_component_gaussian_prior #lambda x: s_component_gaussian_prior(x, R=chi_max/3.) + prior_map['chiz_minus'] = s_component_gaussian_prior #lambda x: s_component_gaussian_prior(x, R=chi_max/3.) +# prior_map['s1z'] = s_component_gaussian_prior +# prior_map['s2z'] = s_component_gaussian_prior + +if opts.prior_gaussian_spin1_magnitude: + if not 'chi1' in low_level_coord_names: + print(" Incompatible options: gaussian spin1 prior requires polar coordinates") + sys.exit(0) + prior_map['chi1'] =functools.partial(gaussian_mass_prior,mu=0.7,sigma=0.1) # not fully normalized particularly if chimax <1! Dangerous, fixme eventually + +if opts.prior_tapered_spin1_magnitude: + if not 'chi1' in low_level_coord_names: + print(" Incompatible options: tapered spin1 prior requires polar coordinates") + sys.exit(0) + prior_map['s1z'] =tapered_magnitude_prior + +if opts.prior_tapered_spin1z: + if not 's1z' in low_level_coord_names: + print(" Incompatible options: tapered spin1z prior requires cartesian coordinates") + sys.exit(0) + prior_map['s1z'] =tapered_magnitude_prior + +if opts.prior_gaussian_mass_ratio: + if not 'q' in low_level_coord_names: + print(" Incompatible options: gaussian q prior requires q in coordinates (e.g., mtot,q coordinates)") + sys.exit(0) + prior_map['q'] = functools.partial(gaussian_mass_prior,mu=0.5,sigma=0.2) # not fully normalized, and very ad-hoc + +if opts.prior_tapered_mass_ratio: + if not 'q' in low_level_coord_names: + print(" Incompatible options: gaussian q prior requires q in coordinates (e.g., mtot,q coordinates)") + sys.exit(0) + prior_map['q'] = functools.partial(tapered_magnitude_prior_alt,loc=0.8,kappa=20.) # not fully normalized, and very ad-hoc + +if opts.prior_lambda_linear: + if opts.prior_lambda_power == 1: + prior_map['lambda1'] = functools.partial(mcsampler.linear_down_samp,xmin=0,xmax=lambda_max) + prior_map['lambda2'] = functools.partial(mcsampler.linear_down_samp,xmin=0,xmax=lambda_small_max) + else: + prior_map['lambda1'] = functools.partial(mcsampler.power_down_samp,xmin=0,xmax=lambda_max,alpha=opts.prior_lambda_power+1) + prior_map['lambda2'] = functools.partial(mcsampler.power_down_samp,xmin=0,xmax=lambda_small_max,alpha=opts.prior_lambda_power+1) + +# tex_dictionary = { +# "mtot": '$M$', +# "mc": '${\cal M}_c$', +# "m1": '$m_1$', +# "m2": '$m_2$', +# "q": "$q$", +# "delta" : "$\delta$", +# "DeltaOverM2_perp" : "$\Delta_\perp$", +# "DeltaOverM2_L" : "$\Delta_{||}$", +# "SOverM2_perp" : "$S_\perp$", +# "SOverM2_L" : "$S_{||}$", +# "eta": "$\eta$", +# "chi_eff": "$\chi_{eff}$", +# "xi": "$\chi_{eff}$", +# "s1z": "$\chi_{1,z}$", +# "s2z": "$\chi_{2,z}$" +# } + + +############################################################################### +### Linear fits. Resampling a quadratic. (Export me) +############################################################################### + +def fit_quadratic_stored(fname_h5,loc,L_offset=200): + import h5py + with h5py.File(fname_h5,'r') as F: + event = F(loc) # assumed to be mc_source ,eta, chi_eff for now! + mean = np.array(event["mean"]) + cov = np.matrix(event["cov"]) + cov_det = np.linalg.det(cov) + icov = np.linalg.pinv(cov) + n_params = len(mean) + def my_func(x): # Horribly slow implementation but much easier to read/understand + y_val = np.zeros(len(x)) + for indx in np.arange(len(x)): + dx = x[indx]-mean + alt = np.dot(icov,dx) + eps2 = float(np.dot(dx.T,alt)) # convert to scalar + y_val[indx] = np.log((2*np.pi)**(-n_params/2.) * np.sqrt(1./cov_det) ) -0.5* eps2 + L_offset + return y_val + return my_func + + +def fit_quadratic_alt(x,y,y_err=None,x0=None,symmetry_list=None,verbose=False,hard_regularize_negative=True): + gamma_x = None + if not (y_err is None): + gamma_x =np.diag(1./np.power(y_err,2)) + the_quadratic_results = BayesianLeastSquares.fit_quadratic( x, y,gamma_x=gamma_x,verbose=verbose,hard_regularize_negative=hard_regularize_negative)#x0=None)#x0_val_here) + peak_val_est, best_val_est, my_fisher_est, linear_term_est,fn_estimate = the_quadratic_results + + np.savetxt("lnL_peakval.dat",[peak_val_est]) # generally not very useful + np.savetxt("lnL_bestpt.dat",best_val_est) + np.savetxt("lnL_gamma.dat",my_fisher_est,header=' '.join(coord_names)) + + if y_err is None: + bic =-2*( -0.5*np.sum(np.power((y - fn_estimate(x)),2))/2 - 0.5* len(y)*np.log(len(x[0])) ) + else: + bic =-2*( -0.5*np.sum(np.power((y - fn_estimate(x)),2)/y_err**2)/2 - 0.5* len(y)*np.log(len(x[0])) ) + + print(" Fit: std :" , np.std( y-fn_estimate(x))) + print(" Fit: BIC :" , bic) + + return fn_estimate + + +# https://github.com/scikit-learn/scikit-learn/blob/14031f6/sklearn/preprocessing/data.py#L1139 +def fit_polynomial(x,y,x0=None,symmetry_list=None,y_errors=None): + """ + x = array so x[0] , x[1], x[2] are points. + """ + + clf_list = [] + bic_list = [] + for indx in np.arange(opts.fit_order+1): + poly = msf.PolynomialFeatures(degree=indx,symmetry_list=symmetry_list) + X_ = poly.fit_transform(x) + + if opts.verbose: + print(" Fit : poly: RAW :", poly.get_feature_names()) + print(" Fit : ", poly.powers_) + + # Strip things with inappropriate symmetry: IMPOSSIBLE + # powers_new = [] + # if not(symmetry_list is None): + # for line in poly.powers_: + # signature = np.prod(np.power( np.array(symmetry_list), line)) + # if signature >0: + # powers_new.append(line) + # poly.powers_ = powers_new + + # X_ = poly.fit_transform(x) # refit, with symmetry-constrained structure + + # print " Fit : poly: After symmetry constraint :", poly.get_feature_names() + # print " Fit : ", poly.powers_ + + + clf = linear_model.LinearRegression() + if y_errors is None or opts.ignore_errors_in_data: + clf.fit(X_,y) + else: + assert len(y_errors) == len(y) + clf.fit(X_,y,sample_weight=1./y_errors**2) # fit with usual weights + + clf_list.append(clf) + + print(" Fit: Testing order ", indx) + print(" Fit: std: ", np.std(y - clf.predict(X_)), "using number of features ", len(y)) # should NOT be perfect + if not (y_errors is None): + print(" Fit: weighted error ", np.std( (y - clf.predict(X_))/y_errors)) + bic = -2*( -0.5*np.sum(np.power((y - clf.predict(X_))/y_errors,2)) - 0.5*len(y)*np.log(len(x[0]))) + print(" Fit: BIC:", bic) + bic_list.append(bic) + + clf = clf_list[np.argmin(np.array(bic_list) )] + + return lambda x: clf.predict(poly.fit_transform(x)) + +#TODO: optimize to only import for fit_gp, fit_gp_pool, fit_gp_lazy (double check for other uses) +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import PairwiseKernel,RBF, WhiteKernel, ConstantKernel as C +if feedback: print(" Imported: gaussian_process stuff from sklearn") + + +def adderr(y): + val,err = y + return val+error_factor*err + +def fit_gp(x,y,x0=None,symmetry_list=None,y_errors=None,hypercube_rescale=False,fname_export="gp_fit"): + """ + x = array so x[0] , x[1], x[2] are points. + """ + + # If we are loading a fit, override everything else + if opts.fit_load_gp: + print(" WARNING: Do not re-use fits across architectures or versions : pickling is not transferrable ") + my_gp=joblib.load(opts.fit_load_gp) + if opts.protect_coordinate_conversions: + return lalsimutils.RangeProtectReduce(lambda x: my_gp.predict(x), -np.inf) + return lambda x:my_gp.predict(x) + + # Amplitude: + # - We are fitting lnL. + # - We know the scale more or less: more than 2 in the log is bad + # Scale + # - because of strong correlations with chirp mass, the length scales can be very short + # - they are rarely very long, but at high mass can be long + # - I need to allow for a RANGE + + length_scale_est = [] + length_scale_bounds_est = [] + for indx in np.arange(len(x[0])): + # These length scales have been tuned by expereience + length_scale_est.append( 2*np.nanstd(x[:,indx]) ) # auto-select range based on sampling retained + length_scale_min_here= np.max([1e-3,0.2*np.nanstd(x[:,indx]/np.sqrt(len(x)))]) + if indx == mc_index: + length_scale_min_here= 0.2*np.nanstd(x[:,indx]/np.sqrt(len(x))) + print(" Setting mc range: retained point range is ", np.nanstd(x[:,indx]), " and target min is ", length_scale_min_here) + length_scale_bounds_est.append( (length_scale_min_here , 5*np.nanstd(x[:,indx]) ) ) # auto-select range based on sampling *RETAINED* (i.e., passing cut). Note that for the coordinates I usually use, it would be nonsensical to make the range in coordinate too small, as can occasionally happens + + print(" GP: Input sample size ", len(x), len(y)) + print(" GP: Estimated length scales ") + print(length_scale_est) + print(length_scale_bounds_est) + + alpha = 1e-10 # default from sklearn docs + if not(y_errors is None): + alpha = y_errors**2 # added to diagonal of kernel, used to assign variances of measurements a priori; note also WhiteKernel also absorbs some of this + if not (hypercube_rescale): + # These parameters have been hand-tuned by experience to try to set to levels comparable to typical lnL Monte Carlo error + kernel = WhiteKernel(noise_level=0.1,noise_level_bounds=(1e-2,1))+C(0.5, (1e-3,1e1))*RBF(length_scale=length_scale_est, length_scale_bounds=length_scale_bounds_est) + gp = GaussianProcessRegressor(kernel=kernel, alpha=alpha, n_restarts_optimizer=8) + + gp.fit(x,y) + + print(" Fit: std: ", np.std(y - gp.predict(x)), "using number of features ", len(y)) + + if opts.fit_save_gp: + print(" Attempting to save fit ", opts.fit_save_gp+".pkl") + joblib.dump(gp,opts.fit_save_gp+".pkl") + + if not (opts.fit_uncertainty_added): + if opts.protect_coordinate_conversions: + return lalsimutils.RangeProtectReduce( (lambda x: gp.predict(x) ), -np.inf) + return lambda x: gp.predict(x) + else: + return lambda x: adderr(gp.predict(x,return_std=True)) + else: + x_scaled = np.zeros(x.shape) + x_center = np.zeros(len(length_scale_est)) + x_center = np.mean(x) + print(" Scaling data to central point ", x_center) + for indx in np.arange(len(x)): + x_scaled[indx] = (x[indx] - x_center)/length_scale_est # resize + + kernel = WhiteKernel(noise_level=0.1,noise_level_bounds=(1e-2,1))+C(0.5, (1e-3,1e1))*RBF( len(x_center), (1e-3,1e1)) + gp = GaussianProcessRegressor(kernel=kernel, alpha=alpha,n_restarts_optimizer=8) + + gp.fit(x_scaled,y) + print(" Fit: std: ", np.std(y - gp.predict(x_scaled)), "using number of features ", len(y)) # should NOT be perfect + + return lambda x,x0=x_center,scl=length_scale_est: gp.predict( (x-x0 )/scl) + +def map_funcs(func_list,obj): + return [func(obj) for func in func_list] +def fit_gp_pool(x,y,n_pool=10,**kwargs): + """ + Split the data into 10 parts, and return a GP that averages them + """ + x_copy = np.array(x) + y_copy = np.array(y) + indx_list =np.arange(len(x_copy)) + np.random.shuffle(indx_list) # acts in place + partition_list = np.array_split(indx_list,n_pool) + gp_fit_list =[] + for part in partition_list: + print(" Fitting partition ") + gp_fit_list.append(fit_gp(x[part],y[part],**kwargs)) + fn_out = lambda x: np.mean( map_funcs( gp_fit_list,x), axis=0) + print(" Testing ", fn_out([x[0]])) + return fn_out + +def fit_gp_lazy(x,y,y_errors=None,dy_cov=5): + """ + fit_gp_lazy : Attempts to build a quadratic form based on the highest amplitude parts of y + """ + print(" GP: Input sample size ", len(x), len(y)) + + ymax = np.max(y) + indx_1 = y>ymax-dy_cov + indx_2 = y>ymax-2*dy_cov + if (np.sum(indx_1) < 10*len(x[0])**2): # 10*# of dimensions^2, so if dimension=3, we need 90 points, etc + print(" Failure : need sufficient data within threshold to perform local covariance estimate ") + sys.exit(5) + cov1 = np.cov(x[indx_1].T)/(dy_cov**2) # shrink based on dy + cov2 = np.cov(x[indx_2].T)/(4*dy_cov**2) # ditto + Q = np.linalg.pinv(0.5*(cov1+cov2)) # average local estimate and nonlocal estimate. Take inverse for Q matrix. + Q = 0.1*Q/np.power(len(x),1./len(x[0])) # Add scale factor to smooth over smaller distance than the overall covariance. Reduce in size based on data size in some way + def my_func_diff_exp(x,y,Q=Q,**kwargs): + gamma=1 + if 'gamma' in kwargs: + gamma = kwargs['gamma'] + if x.shape == y.shape: + dx = x - y + else: + dx = x.reshape(len(x),1) -y + return np.exp(-gamma*np.dot(dx.T,np.dot(Q,dx))) + lazy_kernel= PairwiseKernel(metric=my_func_diff_exp) + lazy_kernel.gamma_bounds = [0.01,10] # control length scale range change + + # Do it all by hand +# Ktrain = my_func_diff_exp(x.T,x.T); +# myinv = np.linalg.pinv(Ktrain + y_errors**2) +# mybase = np.dot(myinv,y) # should be an array +# print(myinv.shape, mybase.shape) +# def my_ret(xtest,func=my_func_diff_exp): +# # return np.dot(func(xtest.T,x.T),mybase) # does not have quite the right shape +# ret = np.ones(len(xtest)) +# for indx in np.arange(len(ret)): # this is dumb +# val = np.dot(func(xtest[indx].T,x.T),mybase) +# print(val) +# ret[indx] = val +# return my_ret + + alpha = 1e-10 # default from sklearn docs + noise_level = 0.1 + if not(y_errors is None): + alpha = y_errors**2 # added to diagonal of kernel, used to assign variances of measurements a priori; note also WhiteKernel also absorbs some of this + noise_level = np.mean(np.abs(y_errors)) + + kernel = WhiteKernel(noise_level=noise_level,noise_level_bounds=(1e-2,1))+C(0.5, (1e-3,1e1))*lazy_kernel + gp = GaussianProcessRegressor(kernel=kernel, alpha=alpha, n_restarts_optimizer=8) + print(" Fit: std: ", np.std(y - gp.predict(x)), "using number of features ", len(y)) + + if not (opts.fit_uncertainty_added): + if opts.protect_coordinate_conversions: + return lalsimutils.RangeProtectReduce( (lambda x: gp.predict(x) ), -np.inf) + return lambda x: gp.predict(x) + else: + return lambda x: adderr(gp.predict(x,return_std=True)) + + +def fit_xg(x,y,y_errors=None,fname_export='nn_fit',verbose=False): + import xgboost as xgb + # Instantiate model. Usually not that many structures to find, don't overcomplicate + # - should scale like number of samples + rf = xgb.XGBRegressor(n_estimators=100) # no more than 5% of samples in a leaf + if y_errors is None: + rf.fit(x,y) + else: + rf.fit(x,y,sample_weight=1./y_errors**2) + + ### reject points with infinities : problems for inputs + def fn_return(x_in,rf=rf): + f_out = -lnL_default_large_negative*np.ones(len(x_in)) + # remove infinity or Nan + indx_ok = np.all(np.isfinite(x_in),axis=-1) + # rf internally uses float32, so we need to remove points > 10^37 or so ! + # ... this *should* never happen due to bounds constraints, but ... + indx_ok_size = np.all( np.logical_not(np.greater(np.abs(x_in),1e37)), axis=-1) + indx_ok = np.logical_and(indx_ok, indx_ok_size) + f_out[indx_ok] = rf.predict(x_in[indx_ok]) + return f_out +# fn_return = lambda x_in: rf.predict(x_in) + + print( " Demonstrating XGBoost") # debugging + residuals = rf.predict(x)-y + print( " std ", np.std(residuals), np.max(y), np.max(fn_return(x))) + return fn_return + +def fit_nn(x,y,y_errors=None,fname_export='nn_fit',adaptive=True): + y_packed = y[:,np.newaxis] + if not (y_errors is None): + errors_packed = y_errors[:,np.newaxis] + else: + errors_packed = None + import os + working_dir = os.getcwd() +# for indx in np.arange(len(x[0])): +# print np.min(x[:,indx]), np.max(x[:,indx]), (np.max(x[:,indx])-np.mean(x[:,indx]))/np.std(x[:,indx]) + if adaptive: + nn_interpolator = senni.AdaptiveInterpolator(x,y_packed,errors_packed,epochs=60, frac=0.2, hlayer_size=2**(1+len(x[0])), test_frac=0,working_dir=working_dir,loss_func='chi2',p_drop=0.02) # May want to adjust size of network based on data size? + else: + nn_interpolator = senni.Interpolator(x,y_packed,errors_packed,epochs=100, frac=0.2, test_frac=0,working_dir=working_dir,loss_func='chi2',p_drop=0.03,regularize=False,weight_decay=1e-2) # May want to adjust size of network based on data size? + nn_interpolator.train() + if opts.fit_save_gp: + print( " Attempting to save NN fit ", opts.fit_save_gp+".network") + nn_interpolator.save(opts.fit_save_gp+".network") + + def fn_return(x): + x_in = np.copy(x) # need to make a copy to avoid altering input/changing response + return nn_interpolator.evaluate(x_in) + + print( " Demonstrating NN") # debugging +# print x, fn_return(x),nn_interpolator.evaluate(x),y + residuals2 = fn_return(x) - y + residuals = nn_interpolator.evaluate(x)-y + print( " std ", np.std(residuals), np.std(residuals2), np.max(y), np.max(fn_return(x))) + return fn_return + + + +def fit_rf(x,y,y_errors=None,fname_export='nn_fit',verbose=False): +# from sklearn.ensemble import RandomForestRegressor + from sklearn.ensemble import ExtraTreesRegressor + # Instantiate model. Usually not that many structures to find, don't overcomplicate + # - should scale like number of samples + rf = ExtraTreesRegressor(n_estimators=100, verbose=verbose,n_jobs=-1) # no more than 5% of samples in a leaf + if y_errors is None: + rf.fit(x,y) + else: + rf.fit(x,y,sample_weight=1./y_errors**2) + + ### reject points with infinities : problems for inputs + def fn_return(x_in,rf=rf): + f_out = -lnL_default_large_negative*np.ones(len(x_in)) + # remove infinity or Nan + indx_ok = np.all(np.isfinite(x_in),axis=-1) + # rf internally uses float32, so we need to remove points > 10^37 or so ! + # ... this *should* never happen due to bounds constraints, but ... + indx_ok_size = np.all( np.logical_not(np.greater(np.abs(x_in),1e37)), axis=-1) + indx_ok = np.logical_and(indx_ok, indx_ok_size) + f_out[indx_ok] = rf.predict(x_in[indx_ok]) + return f_out +# fn_return = lambda x_in: rf.predict(x_in) + + print( " Demonstrating RF") # debugging + residuals = rf.predict(x)-y + print( " std ", np.std(residuals), np.max(y), np.max(fn_return(x))) + return fn_return + +def fit_rf_pca(x,y,y_errors=None,fname_export='nn_fit'): + # from aasim +# from sklearn.ensemble import RandomForestRegressor + from sklearn.ensemble import ExtraTreesRegressor + from sklearn.decomposition import PCA + from sklearn.preprocessing import StandardScaler + x_scaler = StandardScaler() + x_scaled = x_scaler.fit_transform(x) + pca = PCA() + x_pca = pca.fit_transform(x_scaled) + # Instantiate model. Usually not that many structures to find, don't overcomplicate + # - should scale like number of samples + rf = ExtraTreesRegressor(n_estimators=100, verbose=True,n_jobs=-1) # no more than 5% of samples in a leaf + + if y_errors is None: + rf.fit(x_pca,y) + else: + rf.fit(x_pca,y,sample_weight=1./y_errors**2) + + ### reject points with infinities : problems for inputs + def fn_return(x_in,rf=rf): + f_out = -100000*np.ones(len(x_in)) + # remove infinity or Nan + indx_ok = np.all(np.isfinite(x_in),axis=-1) + # rf internally uses float32, so we need to remove points > 10^37 or so ! + # ... this *should* never happen due to bounds constraints, but ... + indx_ok_size = np.all( np.logical_not(np.greater(np.abs(x_in),1e37)), axis=-1) + indx_ok = np.logical_and(indx_ok, indx_ok_size) + + f_out[indx_ok] = rf.predict(pca.transform(x_scaler.transform(x_in[indx_ok]))) + return f_out +# fn_return = lambda x_in: rf.predict(x_in) + + print( " Demonstrating RF") # debugging + residuals = rf.predict(pca.transform(x_scaler.transform(x)))-y + print( " std ", np.std(residuals), np.max(y), np.max(fn_return(x))) + return fn_return + +def fit_rbf(x,y,y_errors=None,fname_export='rbf_fit',verbose=False): + from scipy.interpolate import RBFInterpolator + # - should scale like number of samples + rbf = RBFInterpolator(x,y) + + print( " Demonstrating RBF") # debugging + residuals = rbf(x)-y + print( " std ", np.std(residuals), np.max(y), np.max(rbf(x))) + return rbf + +def fit_nn_rfwrapper(x,y,y_errors=None,fname_export='nn_fit'): + from sklearn.ensemble import RandomForestRegressor + # Instantiate model. Usually not that many structures to find, don't overcomplicate + # - should scale like number of samples + rf = RandomForestRegressor(n_estimators=100, random_state = 42,verbose=True,n_jobs=-1) + if y_errors is None: + rf.fit(x,y) + else: + rf.fit(x,y,sample_weight=1./y_errors**2) + + y_packed = y[:,np.newaxis] + if not (y_errors is None): + errors_packed = y_errors[:,np.newaxis] + else: + errors_packed = None + import os + working_dir = os.getcwd() +# for indx in np.arange(len(x[0])): +# print np.min(x[:,indx]), np.max(x[:,indx]), (np.max(x[:,indx])-np.mean(x[:,indx]))/np.std(x[:,indx]) + # train first with one loss, then the next? + nn_interpolator = senni.Interpolator(x,y_packed,errors_packed,epochs=10, frac=0.2, test_frac=0,working_dir=working_dir,loss_func='mape') # May want to adjust size of network based on data size? + nn_interpolator.train() + nn_interpolator.loss_func='chi2'; nn_interpolator.epochs = 50 + nn_interpolator.train() + + y_max = np.max(y) + def fn_return(x): + vals_rf = rf.predict(x) + vals_nn = nn_interpolator.evaluate(x) + return np.where(vals_rf > y_max-15, vals_nn, vals_rf) + + print( " Demonstrating NN") # debugging + residuals = fn_return(x)-y + print( " std ", np.std(residuals), np.max(y), np.max(fn_return(x))) + return fn_return + +def fit_kde(x,y,y_errors=None): + """ + Simple KDE -like fit: estimate function as lnLmax * [ w_k K(x-x_k)] where w_k are weights based on lnL values. + Problem is normalization! One way is to normalize one basically random point ... though that's an issue + + NOT INTENDED FOR ACCURACY -- this will intentionally oversmooth, with worse oversmoothing farther away. + """ + y_max = np.max(y) # [int(len(y)/2)] + y_min = np.min(y) + wts = y+y_min+1 # must be positive + x_max = x[np.argmax(y)] + d = len(x_max) +# print(x.shape,y.shape); print(x_max, y_max) + # wts = np.ones(x.shape) + # for indx in np.arange(len(x_max)): + # wts[:,indx] = y + my_kde = scipy.stats.gaussian_kde(x.T, weights=wts,bw_method=4) # +# nm = my_kde.integrate_gaussian(x_max.T, np.zeros((d,d))) +# print(nm) +# print(" KDE bandwidth {} ".format(my_kde.factor)) + val = my_kde(x.T) + my_kde_ref = np.max(val); # change reference point, so at least max agrees +# print(" Location of peak ",x[np.argmax(val)]) +# my_kde_ref = my_kde(x_max) + def fn_return(z,y_max=y_max): + return y_max *my_kde(z.T)/my_kde_ref # choose parameters so it attains maximum value : not safe given errors, but ... + + print( " Demonstrating KDE") # debugging + residuals = fn_return(x)-y +# print(fn_return(x),y,my_kde_ref,residuals) + print( " std ", np.std(residuals), np.max(y), np.max(fn_return(x))) + return fn_return + +def fit_cov(x,y,y_errors=None,soften_factor=1): + """ + Simple quadratic fit, based on *mean and covariance of points passing threshold*. + + NOT INTENDED FOR ACCURACY -- this will intentionally massively oversmooth. + Only use for *placement early on* + """ + + ymax = np.max(y) + xmean = np.mean(x, axis=0) +# dy2mean = np.mean( (y-ymax)**2) + dy2mean = 2*len(x[0]) # don't reduce covariance much based on data, so we sample the whole thing. Motivated by chisquared. + cov = soften_factor*np.cov(x.T)/dy2mean # scale to unit + Q = np.linalg.pinv(cov) # average local estimate and nonlocal estimate. Take inverse for Q matrix. + + + def fn_return(z,ymax=ymax,xmean=xmean,Q=Q): + val = np.zeros(len(z)) + for indx in np.arange(len(val)): + dx = z[indx]-xmean + val[indx] = ymax - 0.5* np.dot( dx.T,np.dot(Q,dx)) + return val + + print( " Demonstrating cov") # debugging + residuals = fn_return(x)-y +# print(fn_return(x),y,my_kde_ref,residuals) + print( " std ", np.std(residuals), np.max(y), np.max(fn_return(x))) + return fn_return + + +def fit_nearest(x,y,y_errors=None): + """ + Weighted nearest-neighbor fit. We *should* use a distance based on the covariance, but let's be simple here + """ + from sklearn.neighbors import RadiusNeighborsRegressor + + + rad_default = np.sqrt(np.trace(np.cov(x))/len(x[0]))/np.sqrt(len(x)) + print(" Default radius coordinate ", rad_default) + + nn_regressor = RadiusNeighborsRegressor(radius=rad_default, weights='distance', algorithm='ball_tree') + nn_regressor.fit(x,y) + print(" Fit: std: ", np.std(y - nn_regressor.predict(x)), "using number of features ", len(y)) + print(y, nn_regressor.predict(x)) + + if opts.protect_coordinate_conversions: + return lalsimutils.RangeProtectReduce( (lambda x: nn_regressor.predict(x) ), -np.inf) + def my_return(x_in,default_val=-100): + my_nearby = nn_regressor.radius_neighbors(x_in,rad_default)[0] # tells me if there's anything nearby +# print(len(my_nearby),my_nearby[0]) + my_nearby_len = np.array([ len(z) for z in my_nearby]) +# print(my_nearby,my_nearby_len) + val =nn_regressor.predict(x_in) + val = np.where(my_nearby_len <1, default_val, val) + # print(x,val) + return val + return my_return + #return lambda x: nn_regressor.predict(x) + + + +if not(gpytorch_ok): + def fit_gpytorch(x): + sys.exit(1) +else: + def fit_gpytorch(x,y,y_errors=None,fname_export='nn_fit',adaptive=True): + y_packed = y[:,np.newaxis] + if not (y_errors is None): + errors_packed = y_errors[:,np.newaxis] + else: + errors_packed = None + import os + working_dir = os.getcwd() + gp_interpolator = gpytorch_wrapper.Interpolator(x,y_packed,epochs=60) + gp_interpolator.train() + if opts.fit_save_gp: + print( " FAIL save gp fit - not yet implemented ") +# gp_interpolator.save(opts.fit_save_gp+".network") + + def fn_return(x): + x_in = np.copy(x) # need to make a copy to avoid altering input/changing response + return gp_interpolator.evaluate(x_in) + + print( " Demonstrating gpytorch fit ") # debugging + residuals2 = fn_return(x) - y + residuals = nn_interpolator.evaluate(x)-y + print( " std ", np.std(residuals), np.std(residuals2), np.max(y), np.max(fn_return(x))) + return fn_return + + +if internalGP_ok: + from RIFT.interpolators.internal_GP import fit_gp as fit_gp_sparse #repeat import from top +else: + def fit_gp_sparse(x): + sys.exit(1) + + +###################### DATA IMPORT ################### +# initialize +dat_mass = [] +weights = [] +n_params = -1 + +### +### Retrieve data - literally setting up hardcode values +### +# id m1 m2 lnL sigma/L neff +col_lnL = 9 +col_eccentricity = None +col_meanPerAno = None +col_lambda1 = None +col_distance = None +if opts.input_distance: + print(" Distance input") + col_lnL +=1 + col_distance = col_lnL -1 +if opts.input_tides: + print(" Tides input") + col_lnL +=2 + col_lambda1 = col_lnL -2 + if opts.input_eos_index: + print(" EOS Tides input") + col_lnL +=1 + if opts.tabular_eos_file: + coord_names += ['eos_table_index'] # temporary, will overwrite this, just use initially to simplify i/o + coord_names = list(coord_names) # force reallocation, since at times we have duplicate sets + low_level_coord_names += ['ordering'] + print(" Revised fit coord names (for lookup) : ", coord_names) # 'eos_table_index' will be overwritten here + print(" Revised sampling coord names : ", low_level_coord_names) +elif opts.use_eccentricity: + print(" Eccentricity input: [",ECC_MIN, ", ",ECC_MAX, "]") + if opts.use_meanPerAno: + print(" Also using meanPerAno ") + # perform modulus on desired row + col_lnL+=2 + col_meanPerAno = col_lnL -1 + col_eccentricity = col_lnL -2 + else: + col_lnL += 1 + col_eccentricity = col_lnL -1 +if opts.input_distance: + print(" Distance input") + col_lnL +=1 +####DATA IMPORT#### +dat_orig = dat = np.loadtxt(opts.fname) +dat_orig = dat[dat[:,col_lnL].argsort()] # sort http://stackoverflow.com/questions/2828059/sorting-arrays-in-numpy-by-column +if col_meanPerAno: + dat_orig[:,col_meanPerAno] = np.mod(dat_orig[:,col_meanPerAno], lalsimutils.periodic_params['meanPerAno'] ) # 2 *np.pi +print(" Original data size = ", len(dat), dat.shape) + +# Rescale lnL data, if requested. Note requires user have sensible understanding of zero points of likelihood, etc Appl +if opts.lnL_downscale_factor: + dat[:,col_lnL] *= opts.lnL_downscale_factor + dat_orig[:,col_lnL] *= opts.lnL_downscale_factor + + ### + ### Convert data. Use lalsimutils for flexibility + ### +P_list = [] +dat_out =[] + +extra_plot_coord_names = [ ['mtot', 'q', 'xi'], ['mc', 'eta'], ['m1', 'm2'], ['s1z','s2z'] ] # replot +# simplify/recast None -> [] so I cna use 'in' below +if opts.parameter==None: + opts.parameter = [] # force list, so avoid 'is iterable' below +if opts.parameter_implied==None: + opts.parameter_implied = [] +if 's1x' in opts.parameter: # in practice, we always use the transverse cartesian components + print(" Plotting coordinates include spin magnitude and transverse spins ") + extra_plot_coord_names += [['chi1_perp', 's1z'], ['chi2_perp','s2z'],['chi1','chi2'],['cos_theta1','cos_theta2']] +if 'lambda1' in opts.parameter or 'LambdaTilde' in opts.parameter or (not( opts.parameter_implied is None) and ( 'LambdaTilde' in opts.parameter_implied)): + print(" Plotting coordinates include tides") + extra_plot_coord_names += [['mc', 'eta', 'LambdaTilde'],['lambda1', 'lambda2'], ['LambdaTilde', 'DeltaLambdaTilde'], ['m1','lambda1'], ['m2','lambda2']] +dat_out_low_level_coord_names = [] +dat_out_extra = [] +for item in extra_plot_coord_names: + dat_out_extra.append([]) + +symmetry_list=None +if not(opts.tabular_eos_file): + if opts.fit_method == 'quadratic' or opts.fit_method == 'polynomial': + symmetry_list =lalsimutils.symmetry_sign_exchange(coord_names) # identify symmetry due to exchange +mc_min = 1e10 +mc_max = -1 + +mc_index = -1 # index of mchirp in parameter index. To help with nonstandard GP +mc_cut_range = [-np.inf, np.inf] +if opts.mc_range: + mc_cut_range = eval(opts.mc_range) # throw out samples outside this range. + mc_min = mc_cut_range[0] + mc_max = mc_cut_range[1] + if opts.source_redshift>0: + mc_cut_range =np.array(mc_cut_range)*(1+opts.source_redshift) # prevent stupidity in grid selection +print(" Stripping samples outside of ", mc_cut_range, " in mc") +P= lalsimutils.ChooseWaveformParams() +P_list_in = [] +for line in dat: + # Skip precessing binaries unless explicitly requested not to! + if not opts.use_precessing and (line[3]**2 + line[4]**2 + line[6]**2 + line[7]**2)>0.01: + print(" Skipping precessing binaries ") + continue + if line[1]+line[2] > opts.M_max_cut: + if opts.verbose: + print(" Skipping ", line, " as too massive, with mass ", line[1]+line[2]) + continue + if line[col_lnL+1] > opts.sigma_cut: +# if opts.verbose: +# print(" Skipping as large error ", line) + continue + if not (opts.lnL_cut is None): + if line[col_lnL] < opts.lnL_cut: + continue # strip worthless points. DANGEROUS + mc_here = lalsimutils.mchirp(line[1],line[2]) + if (not opts.no_downselect_grid) and (mc_here < mc_cut_range[0] or mc_here > mc_cut_range[1]): + if False and opts.verbose: + print("Stripping because sample outside of target mc range ", line) + continue + if line[col_lnL] < opts.lnL_peak_insane_cut: + P.fref = opts.fref # IMPORTANT if you are using a quantity that depends on J + P.fmin = opts.fmin + P.m1 = line[1]*lal.MSUN_SI + P.m2 = line[2]*lal.MSUN_SI + P.s1x = line[3] + P.s1y = line[4] + P.s1z = line[5] + P.s2x = line[6] + P.s2y = line[7] + P.s2z = line[8] + + if opts.input_tides: + P.lambda1 = line[col_lambda1] + P.lambda2 = line[col_lambda1+1] + if opts.input_eos_index: + P.eos_table_index = line[col_lambda1+2] + if opts.use_eccentricity: + P.eccentricity = line[col_eccentricity] # 9 + if opts.use_meanPerAno: + P.meanPerAno = line[col_meanPerAno] #10 +# P.eccentricity = line[9] +# if opts.use_meanPerAno: +# P.meanPerAno = line[10] + if opts.input_distance: + P.dist = lal.PC_SI*1e6*line[col_distance] # 9. Previously incompatible with tides when hardcoded + + if opts.contingency_unevolved_neff == "quadpuff": + P_copy = P.manual_copy() # prevent duplication + P_list_in.append(P_copy) # store it, make sure distinct + + # INPUT GRID: Evaluate binary parameters on fitting coordinates + line_out = np.zeros(len(coord_names)+2) + chi_pavg_out = 0 #initialize + for x in np.arange(len(coord_names)): + if coord_names[x] == 'chi_pavg': + chi_pavg_out = P.extract_param('chi_pavg') + line_out[x] = chi_pavg_out + elif coord_names[x] =='ordering': + continue + else: + line_out[x] = P.extract_param(coord_names[x]) + # line_out[x] = getattr(P, coord_names[x]) + line_out[-2] = line[col_lnL] + line_out[-1] = line[col_lnL+1] # adjoin error estimate + dat_out.append(line_out) + + # Alternate grids: Evaluate input binary parameters on other coordinates requested, for comparison + for indx in np.arange(len(extra_plot_coord_names)): + line_out = np.zeros(len(extra_plot_coord_names[indx])) + for x in np.arange(len(line_out)): + line_out[x] = P.extract_param( extra_plot_coord_names[indx][x]) + dat_out_extra[indx].append(line_out) + + # results using sampling coordinates (low_level_coord_names) + line_out = np.zeros(len(low_level_coord_names)) + for x in np.arange(len(line_out)): + fac = 1 + if low_level_coord_names[x] in ['ordering']: + continue + if low_level_coord_names[x] in ['mc','m1','m2','mtot']: + fac = lal.MSUN_SI + if low_level_coord_names[x] == 'chi_pavg': + line_out[x] = chi_pavg_out + else: + line_out[x] = P.extract_param(low_level_coord_names[x])/fac + if low_level_coord_names[x] in ['mc']: + mc_index = x + dat_out_low_level_coord_names.append(line_out) + + + # Update mc range + if not(opts.mc_range): + mc_here = lalsimutils.mchirp(line[1],line[2]) + if mc_here < mc_min: + mc_min = mc_here + if mc_here > mc_max: + mc_max = mc_here + + # Mirror! + if opts.mirror_points: + P.swap_components() + # INPUT GRID: Evaluate binary parameters on fitting coordinates + line_out = np.zeros(len(coord_names)+2) + for x in np.arange(len(coord_names)): + line_out[x] = P.extract_param(coord_names[x]) + line_out[-2] = line[col_lnL] + line_out[-1] = line[col_lnL+1] # adjoin error estimate + dat_out.append(line_out) + + # Alternate grids: Evaluate input binary parameters on other coordinates requested, for comparison + for indx in np.arange(len(extra_plot_coord_names)): + line_out = np.zeros(len(extra_plot_coord_names[indx])) + for x in np.arange(len(line_out)): + line_out[x] = P.extract_param( extra_plot_coord_names[indx][x]) + dat_out_extra[indx].append(line_out) + + # results using sampling coordinates (low_level_coord_names) + line_out = np.zeros(len(low_level_coord_names)) + for x in np.arange(len(line_out)): + fac = 1 + if low_level_coord_names[x] in ['mc','m1','m2','mtot']: + fac = lal.MSUN_SI + line_out[x] = P.extract_param(low_level_coord_names[x])/fac + if low_level_coord_names[x] in ['mc']: + mc_index = x + dat_out_low_level_coord_names.append(line_out) + +Pref_default = P.copy() # keep this around to fix the masses, if we don't have an inj + +# Force 32 bit dype +dat_out = np.array(dat_out,dtype=internal_dtype) +print(" Stripped size = ", dat_out.shape, " with memory usage (bytes) ", sys.getsizeof(dat_out)) +dat_out_low_level_coord_names = np.array(dat_out_low_level_coord_names) + # scale out mass units +for p in ['mc', 'm1', 'm2', 'mtot']: + if p in coord_names: + indx = coord_names.index(p) + dat_out[:,indx] /= lal.MSUN_SI + for x in np.arange(len(extra_plot_coord_names)): + dat_out_extra[x] = np.array(dat_out_extra[x],dtype=internal_dtype) # put into np form + if p in extra_plot_coord_names[x]: + indx = extra_plot_coord_names[x].index(p) + dat_out_extra[x][:,indx] /= lal.MSUN_SI + + +# EOS from tabular data: need to do after everything loaded! Need to know reference masses, etc +if opts.tabular_eos_file: + import RIFT.physics.EOSManager as EOSManager + # Find reference mass in msun: pick a TYPICAL chirp mass in grid, as all should be close enough for our purposes! + # note this is NOT DETERMINISTIC and will depend on our grid input/what survives, but for BNS should be fine + mc_ref = Pref_default.extract_param('mc') + if mc_ref > 1e10: + mc_ref = mc_ref/lal.MSUN_SI + m_ref = mc_ref*np.power(2, 1./5.) # assume equal mass + my_eos_sequence = EOSManager.EOSSequenceLandry(fname=opts.tabular_eos_file, load_ns=True, oned_order_name='Lambda', oned_order_mass=m_ref, no_sort = True) + + # Define prior, NOT NORMALIZED + prior_map['ordering'] =lambda x: np.ones(x.shape) + prior_range_map['ordering'] = [np.min(my_eos_sequence.oned_order_values),np.max(my_eos_sequence.oned_order_values)] + + # Add the ordering values for all the imported points + # - on *import*, we've imported the index quantities; instead, evaluate the ordering statistic for all of these + # - note the saved values use the FIDUCIAL ORDERING, so must be used with GREAT CARE to preserve order! + order_vals = np.zeros(len(dat_out)) + for indx in np.arange(len(order_vals)): + eos_indx_here = int(dat_out[indx,-3]) + # find the revised EOS index, after sorting, IF the EOS is using a sorted order. (this should not happen) + if my_eos_sequence.oned_order_sorted: + eos_indx_here = my_eos_sequence.oned_order_indx_sorted[eos_indx_here] + order_vals[indx] = my_eos_sequence.lambda_of_m_indx(m_ref,eos_indx_here) # last field is index value. Note we ASSUME UNSORTED here + # overwrite into the ordering statistic field + dat_out[:,-3] = order_vals # note this is last entry of DATA + # overwrite the coordinate name for the last field, so conversion is trivial/identity + coord_names[-1] = 'ordering' + +# Repack data +X =dat_out[:,0:len(coord_names)] +Y = dat_out[:,-2] +Y_err = dat_out[:,-1] +# Save copies for later (plots) +X_orig = X.copy() +Y_orig = Y.copy() + +# Plot cumulative distribution in lnL, for all points. Useful sanity check for convergence. Start with RAW +if not opts.no_plots: + Yvals_copy = Y_orig.copy() + Yvals_copy = Yvals_copy[Yvals_copy.argsort()[::-1]] + pvals = np.arange(len(Yvals_copy))*1.0/len(Yvals_copy) + plt.plot(Yvals_copy, pvals) + plt.xlabel(r"$\ln{\cal L}$") + plt.ylabel(r"evaluation fraction $(<\ln{\cal L})$") + plt.savefig("lnL_cumulative_distribution_of_input_points.png"); plt.clf() + + +# Eliminate values with Y too small +max_lnL = np.max(Y) +indx_ok = Y>np.max(Y)-opts.lnL_offset +# Provide some random points, to insure reasonable tapering behavior away from the sample +if opts.lnL_offset_n_random>0: + p_select = np.min([opts.lnL_offset_n_random*1.0/len(Y),1]) + indx_ok = np.logical_or(indx_ok, np.random.choice([True,False],size=len(indx_ok),p=[p_select,1-p_select])) +print(" Points used in fit : ", sum(indx_ok), " given max lnL ", max_lnL) +if max_lnL < 10 and np.mean(Y) > -10: # second condition to allow synthetic tests not to fail, as these often have maxlnL not large + print(" Resetting to use ALL input data -- beware ! ") + # nothing matters, we will reject it anyways + indx_ok = np.ones(len(Y),dtype=bool) +elif sum(indx_ok) < 5*len(X[0])**2: # and max_lnL > 30: + n_crit_needed = np.min([5*len(X[0])**2,len(X)]) + print(" Likely below threshold needed to fit a maximum , beware: {} {} ".format(n_crit_needed,len(X[0]))) + # mark the top elements and use them for fits + # this may be VERY VERY DANGEROUS if the peak is high and poorly sampled + idx_sorted_index = np.lexsort((np.arange(len(Y)), Y)) # Sort the array of Y, recovering index values + indx_list = np.array( [[k, Y[k]] for k in idx_sorted_index]) # pair up with the weights again + indx_list = indx_list[::-1] # reverse, so most significant are first + indx_ok = list(map(int,indx_list[:n_crit_needed,0])) + print(" Revised number of points for fit: ", sum(indx_ok), indx_ok, indx_list[:n_crit_needed]) + Y = Y[indx_ok] + Y_err = Y_err[indx_ok] + X = X[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Reset indx_ok, so it operates correctly later + indx_ok = np.ones(len(Y), dtype=bool) +X_raw = X.copy() + + +################## FIT METHODS ################### +my_fit= None +if not(opts.fit_load_quadratic is None): + print("FIT METHOD IS STORED QUADRATIC; no data used! ") + my_fit = fit_quadratic_stored(opts.fit_load_quadratic, opts.fit_load_quadratic_path) +elif opts.fit_method == "quadratic": + print(" FIT METHOD ", opts.fit_method, " IS QUADRATIC") + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + if opts.report_best_point: + my_fit = fit_quadratic_alt(X,Y,y_err=Y_err,symmetry_list=symmetry_list,verbose=opts.verbose) + pt_best_X = np.loadtxt("lnL_bestpt.dat") + for indx in np.arange(len(coord_names)): + fac = 1 + if coord_names[indx] in ['mc','m1','m2','mtot']: + fac = lal.MSUN_SI + p_to_assign = coord_names[indx] + if p_to_assign == 'xi': + p_to_assign = "chieff_aligned" + P.assign_param(p_to_assign,pt_best_X[indx]*fac) + + print(" ====BEST BINARY ====") + print(" Parameters from fit ", pt_best_X) + P.print_params() + sys.exit(0) + my_fit = fit_quadratic_alt(X,Y,y_err=Y_err,symmetry_list=symmetry_list,verbose=opts.verbose) +elif opts.fit_method == "polynomial": + print(" FIT METHOD ", opts.fit_method, " IS POLYNOMIAL") + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_polynomial(X,Y,symmetry_list=symmetry_list,y_errors=Y_err) +elif opts.fit_method == 'gp_hyper': + print(" FIT METHOD ", opts.fit_method, " IS GP with hypercube rescaling") + # some data truncation IS used for the GP, but beware + print(" Truncating data set used for GP, to reduce memory usage needed in matrix operations") + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_gp(X,Y,y_errors=Y_err,hypercube_rescale=True) +elif opts.fit_method == 'gp': + print(" FIT METHOD ", opts.fit_method, " IS GP") + # some data truncation IS used for the GP, but beware + print(" Truncating data set used for GP, to reduce memory usage needed in matrix operations") + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_gp(X,Y,y_errors=Y_err) +elif opts.fit_method == 'gp-pool': + print(" FIT METHOD ", opts.fit_method, " IS GP (pooled) with pool size ", opts.pool_size) + # some data truncation IS used for the GP, but beware + print(" Truncating data set used for GP, to reduce memory usage needed in matrix operations") + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + if opts.pool_size == None: + opts.pool_size = np.max([2,np.round(4000/len(X))]) # pick a pool size that has no more than 4000 members per pool + my_fit = fit_gp_pool(X,Y,y_errors=Y_err,n_pool=opts.pool_size) +elif opts.fit_method == 'gp-torch': + print( " FIT METHOD ", opts.fit_method, " IS gpytorch ") + # NO data truncation for NN needed? To be *consistent*, have the code function the same way as the others + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_gpytorch(X,Y,y_errors=Y_err) +elif opts.fit_method == 'gp-xgboost': + print( " FIT METHOD ", opts.fit_method, " IS xgboost ") + # NO data truncation for NN needed? To be *consistent*, have the code function the same way as the others + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_xg(X,Y,y_errors=Y_err) +elif opts.fit_method == 'gp_lazy': + print(" FIT METHOD ", opts.fit_method, " IS lazy GP") + # some data truncation IS used for the GP, but beware + print(" Truncating data set used for GP, to reduce memory usage needed in matrix operations") + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_gp_lazy(X,Y,y_errors=Y_err) +elif opts.fit_method == 'cov': + print(" FIT METHOD ", opts.fit_method, " IS cov (a placement-only approximation!). No errors used") + print(" Truncating data set used for cov: don't fit to points with likelihood less than 2") + indx_ok = np.logical_and(Y>np.max(Y)*0.1,indx_ok) # don't fit to points below 10% of peak value + indx_ok = np.logical_and(Y>2,indx_ok) # don't fit to points below 2 absolute scale + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + my_fit = fit_cov(X,Y) +elif opts.fit_method == 'nn': + print( " FIT METHOD ", opts.fit_method, " IS NN ") + # NO data truncation for NN needed? To be *consistent*, have the code function the same way as the others + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_nn(X,Y,y_errors=Y_err) +elif opts.fit_method == 'rf': + print( " FIT METHOD ", opts.fit_method, " IS RF ") + # NO data truncation for NN needed? To be *consistent*, have the code function the same way as the others + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_rf(X,Y,y_errors=Y_err) +elif opts.fit_method == 'rf_pca': + print( " FIT METHOD ", opts.fit_method, " IS RF-pca ") + # NO data truncation for NN needed? To be *consistent*, have the code function the same way as the others + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_rf_pca(X,Y,y_errors=Y_err) +elif opts.fit_method == 'rbf': + print( " FIT METHOD ", opts.fit_method, " IS RBF; **errors not used! **") + # NO data truncation for NN needed? To be *consistent*, have the code function the same way as the others + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_rbf(X,Y,y_errors=Y_err) +elif opts.fit_method == 'nn_rfwrapper': + print( " FIT METHOD ", opts.fit_method, " IS NN with RF wrapper ") + # NO data truncation for NN needed? To be *consistent*, have the code function the same way as the others + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_nn_rfwrapper(X,Y,y_errors=Y_err) +elif opts.fit_method == 'kde': + print( " FIT METHOD ", opts.fit_method, " IS KDE ") + indx_ok = np.logical_and(indx_ok, Y>0) + # modest data truncation useful... + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_kde(X,Y) +elif opts.fit_method == 'gp_sparse': + print( " FIT METHOD ", opts.fit_method, " IS gp_sparse ") + if not internalGP_ok: + print( " FAILED ") + sys.exit(1) + # NO data truncation for NN needed? To be *consistent*, have the code function the same way as the others + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_gp_sparse(X,Y,y_errors=Y_err) +elif opts.fit_method == 'weighted_nearest': + print( " FIT METHOD ", opts.fit_method, " IS weighted_nearest ") + # NO data truncation for NN needed? To be *consistent*, have the code function the same way as the others + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + dat_out_low_level_coord_names = dat_out_low_level_coord_names[indx] + my_fit = fit_nearest(X,Y,y_errors=Y_err) +else: + print(" NO KNOWN FIT METHOD ") + sys.exit(55) +############################## END FIT METHODS ################################ + + +# Sort for later convenience (scatterplots, etc) +indx = Y.argsort()#[::-1] +X=X[indx] +Y=Y[indx] +dat_out_low_level_coord_names =dat_out_low_level_coord_names[indx] + +# Make grid plots for all pairs of points, to facilitate direct validation of where posterior support lies +if not no_plots: + import itertools #duplicate import, also on line 34 (only used here) + for i, j in itertools.product( np.arange(len(coord_names)),np.arange(len(coord_names)) ): + if i < j: + plt.scatter( X[:,i],X[:,j],label='rapid_pe:'+opts.desc_ILE,c=Y); plt.legend(); plt.colorbar() + x_name = render_coord(coord_names[i]) + y_name = render_coord(coord_names[j]) + plt.xlabel( x_name) + plt.ylabel( y_name ) + plt.title("rapid_pe evaluations (=inputs); no fits") + plt.savefig("scatter_"+coord_names[i]+"_"+coord_names[j]+".png"); plt.clf() + + +############################################################################### +### Integrate posterior - eventually +### + +# Oracle for portfolio if needed +oracle_realizations = None +if opts.sampler_method == 'portfolio' and not(opts.sampler_oracle is None): + oracle_realizations = [] + from RIFT.integrators.unreliable_oracle.resampling import ResamplingOracle + from RIFT.integrators.unreliable_oracle.hill_climber import ClimbingOracle + for name in opts.sampler_oracle: + if name =='RS': + my_oracle = ResamplingOracle() + elif name =='Climb': + my_oracle = ClimbingOracle() + else: + raise Exception(" Unknown oracle type ") + print('ORACLE: adding {} '.format(name)) + oracle_realizations.append(my_oracle) + + +### +### Likelihood block +### + +likelihood_function = None +# prior p/ps rescaling, to enable prior inside integrand +# These are functions of the INTEGRATION VARIABLES, not the fit variables +def my_log_prior_scale(X): + return np.zeros(len(X)) +def my_prior_scale(X): + return np.ones(len(X)) + +if len(low_level_coord_names) ==1: + def likelihood_function(x): + if isinstance(x,float): + return np.exp(my_fit([x]))*my_prior_scale([x]) + else: +# return np.exp(my_fit(convert_coords(np.array([x],dtype=internal_dtype).T) )) + return np.exp(my_fit(convert_coords(np.c_[x])))*my_prior_scale(np.c_[x]) + def log_likelihood_function(x): + if isinstance(x,float): + return my_fit([x])+ my_log_prior_scale([x]) + else: +# return np.exp(my_fit(convert_coords(np.array([x],dtype=internal_dtype).T) )) + return my_fit(convert_coords(np.c_[x])) + my_log_prior_scale(np.c_[x]) +if len(low_level_coord_names) ==2: + def likelihood_function(x,y): + if isinstance(x,float): + return np.exp(my_fit([x,y]))*my_prior_scale([x,y]) + else: + return np.exp(my_fit(convert_coords(np.c_[x,y])))*my_prior_scale(np.c_[x,y]) + def log_likelihood_function(x,y): + if isinstance(x,float): + return my_fit([x,y])+ my_log_prior_scale(np.c_[x,y]) + else: + return my_fit(convert_coords(np.c_[x,y]))+ my_log_prior_scale(np.c_[x,y]) +if len(low_level_coord_names) ==3: + def likelihood_function(x,y,z): + if isinstance(x,float): + return np.exp(my_fit([x,y,z]))*my_prior_scale([x,y,z]) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y,z],dtype=internal_dtype).T))) + return np.exp(my_fit(convert_coords(np.c_[x,y,z])))* my_prior_scale(np.c_[x,y,z]) + def log_likelihood_function(x,y,z): + if isinstance(x,float): + return my_fit([x,y,z]) + my_log_prior_scale(np.c_[x,y,z]) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y,z],dtype=internal_dtype).T))) + return my_fit(convert_coords(np.c_[x,y,z]))+my_log_prior_scale(np.c_[x,y,z]) +if len(low_level_coord_names) ==4: + def likelihood_function(x,y,z,a): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a]))*my_prior_scale([x,y,z,a]) + else: + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a])))*my_prior_scale(np.c_[x,y,z,a]) + def log_likelihood_function(x,y,z,a): + if isinstance(x,float): + return my_fit([x,y,z,a])+ +my_log_prior_scale([x,y,z,a]) + else: + return my_fit(convert_coords(np.c_[x,y,z,a]))+ my_log_prior_scale(np.c_[x,y,z,a]) +if len(low_level_coord_names) ==5: + def likelihood_function(x,y,z,a,b): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b]))*my_prior_scale([x,y,z,a,b]) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y,z,a,b],dtype=internal_dtype).T))) + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b])))*my_prior_scale(np.c_[x,y,z,a,b]) + def log_likelihood_function(x,y,z,a,b): + if isinstance(x,float): + return my_fit([x,y,z,a,b])+ my_log_prior_scale([x,y,z,a,b]) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y,z,a,b],dtype=internal_dtype).T))) + return my_fit(convert_coords(np.c_[x,y,z,a,b]))+ my_log_prior_scale(np.c_[x,y,z,a,b]) +if len(low_level_coord_names) ==6: + def likelihood_function(x,y,z,a,b,c): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b,c]))*my_prior_scale([x,y,z,a,b,c]) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y,z,a,b,c],dtype=internal_dtype).T))) + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b,c])))*my_prior_scale(np.c_[x,y,z,a,b,c]) + def log_likelihood_function(x,y,z,a,b,c): + if isinstance(x,float): + return my_fit([x,y,z,a,b,c])+ my_log_prior_scale([x,y,z,a,b,c]) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y,z,a,b,c],dtype=internal_dtype).T))) + return my_fit(convert_coords(np.c_[x,y,z,a,b,c]))+ my_log_prior_scale(np.c_[x,y,z,a,b,c]) +if len(low_level_coord_names) ==7: + def likelihood_function(x,y,z,a,b,c,d): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b,c,d]))*my_prior_scale([x,y,z,a,b,c,d]) + else: + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b,c,d])))*my_prior_scale(np.c_[x,y,z,a,b,c,d]) + def log_likelihood_function(x,y,z,a,b,c,d): + if isinstance(x,float): + return my_fit([x,y,z,a,b,c,d])+ my_log_prior_scale([x,y,z,a,b,c,d]) + else: + return my_fit(convert_coords(np.c_[x,y,z,a,b,c,d]))+ my_log_prior_scale(np.c_[x,y,z,a,b,c,d]) +if len(low_level_coord_names) ==8: + def likelihood_function(x,y,z,a,b,c,d,e): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b,c,d,e]))*my_prior_scale([x,y,z,a,b,c,d,e]) + else: + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b,c,d,e])))*my_prior_scale(np.c_[x,y,z,a,b,c,d,e]) + def log_likelihood_function(x,y,z,a,b,c,d,e): + if isinstance(x,float): + return my_fit([x,y,z,a,b,c,d,e])+ my_log_prior_scale([x,y,z,a,b,c,d,e]) + else: + return my_fit(convert_coords(np.c_[x,y,z,a,b,c,d,e]))+ my_log_prior_scale(np.c_[x,y,z,a,b,c,d,e]) +if len(low_level_coord_names) ==9: + def likelihood_function(x,y,z,a,b,c,d,e,f): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b,c,d,e,f]))*my_prior_scale([x,y,z,a,b,c,d,e,f]) + else: + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b,c,d,e,f]))) *my_prior_scale(np.c_[x,y,z,a,b,c,d,e,f]) + def log_likelihood_function(x,y,z,a,b,c,d,e,f): + if isinstance(x,float): + return my_fit([x,y,z,a,b,c,d,e,f]) + my_log_prior_scale([x,y,z,a,b,c,d,e,f]) + else: + return my_fit(convert_coords(np.c_[x,y,z,a,b,c,d,e,f]))+ my_log_prior_scale(np.c_[x,y,z,a,b,c,d,e,f]) +if len(low_level_coord_names) ==10: + def likelihood_function(x,y,z,a,b,c,d,e,f,g): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b,c,d,e,f,g]))*my_prior_scale([x,y,z,a,b,c,d,e,f,g]) + else: + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b,c,d,e,f,g])))*my_prior_scale(np.c_[x,y,z,a,b,c,d,e,f,g]) + def log_likelihood_function(x,y,z,a,b,c,d,e,f,g): + if isinstance(x,float): + return my_fit([x,y,z,a,b,c,d,e,f,g])+ my_log_prior_scale([x,y,z,a,b,c,d,e,f,g]) + else: + return my_fit(convert_coords(np.c_[x,y,z,a,b,c,d,e,f,g]))+ my_log_prior_scale(np.c_[x,y,z,a,b,c,d,e,f,g]) + + +############################################################################### +### Prior reweight functions +### +if opts.prior_in_integrand_correction == 'uniform_over_rbar_singular': # and opts.transverse_prior == 'Rbar-singular': + print(" MODIFY INTEGRAND : Add factor to implement sample reweighting to be uniform in spin magnitude, in Rbar,zbar coordinates ") + if 'chi2_perp_bar' in low_level_coord_names: + coord_names_needed = ['chi1','chi2','chi1_perp_bar', 'chi2_perp_bar'] + def prior_fac(X): + vec = lalsimutils.convert_waveform_coordinates(X,coord_names=coord_names_needed,low_level_coord_names=low_level_coord_names) + return 4*(np.power(vec[:,2]*vec[:,3],1+(1-p_Rbar)))/(3*vec[:,0]**2 *3* vec[:,1]**2) / p_Rbar**2 + elif 'chi2_perp_u' in low_level_coord_names: + coord_names_needed = ['chi1','chi2','chi1_perp_u', 'chi2_perp_u'] + def prior_fac(X): + vec = lalsimutils.convert_waveform_coordinates(X,coord_names=coord_names_needed,low_level_coord_names=low_level_coord_names) + return 4*np.power(vec[:,2]*vec[:,3],2/p_Rbar -1)/(3*vec[:,0]**2 *3* vec[:,1]**2) / p_Rbar**2 + elif 'chi1_perp_u' in low_level_coord_names: # just one of them, for testing + coord_names_needed = ['chi1','chi1_perp_u'] + def prior_fac(X): + vec = lalsimutils.convert_waveform_coordinates(X,coord_names=coord_names_needed,low_level_coord_names=low_level_coord_names) + return 2*np.power(vec[:,1],2/p_Rbar -1)/(3*vec[:,0]**2) / p_Rbar + my_prior_scale = prior_fac + my_log_prior_scale = lambda x, f=prior_fac: np.log(f(x)) + +elif opts.prior_in_integrand_correction == 'uniform_over_volumetric': + print(" MODIFY INTEGRAND : Add factor to implement sample reweighting to be uniform in spin magnitude, ASSUMING default prior is volumetric ") + # Assume *both* spins are present + # Assume coordinate conversion is POSSIBLE given the information we've been provided + # Volmetric prior = (r^2 dr)_1 (r^2 dr)_2 + # Uniform prior + coord_names_needed = ['chi1','chi2'] + def prior_fac(X): + vec = lalsimutils.convert_waveform_coordinates(X,coord_names=coord_names_needed,low_level_coord_names=low_level_coord_names) + return 1./(3*vec[:,0]**2 *3* vec[:,1]**2) # assuming normalized to 1 + my_prior_scale = prior_fac + my_log_prior_scale = lambda x, f=prior_fac: np.log(f(x)) +elif opts.prior_in_integrand_correction == 'volumetric_over_uniform': + print(" MODIFY INTEGRAND : Add factor to implement sample reweighting to be volumetric, ASSUMING default prior is uniform in magnitude ") + coord_names_needed = ['chi1','chi2'] + def prior_fac(X): + vec = lalsimutils.convert_waveform_coordinates(X,coord_names=coord_names_needed,low_level_coord_names=low_level_coord_names) + return (3*vec[:,0]**2 *3* vec[:,1]**2) # assuming normalized to 1 + my_prior_scale = prior_fac + my_log_prior_scale = lambda x, f=prior_fac: np.log(f(x)) + + + +########################################## +### Prepare EOS for loop +### + +my_eos=None +#option to be used if gridded values not calculated assuming EOS +fake_eos = False +make_CIP_eos = False +eos_dat = None +eos_name = None +n_eos_pts = 1 +if opts.using_eos_for_prior and opts.using_eos: #doing this route - fake_eos overridden if suppl code has retrieve_eos func + # NO LOGIC HERE, PERFORM NECESSARY LOGIC LATER + fake_eos = True # don't use eos in convert_waveform_coordinates. will override later if EOS routine present in routine! + n_eos_pts = opts.n_events_to_analyze +elif opts.using_eos!=None and not(opts.using_eos_for_prior): + import RIFT.physics.EOSManager as EOSManager #test if this holds to within loop... + eos_name=opts.using_eos + if opts.verbose: + print(" Using EOS ", eos_name, opts.using_eos_index, opts.eos_param, opts.eos_param_values) + + if eos_name.startswith("file:") and not(opts.using_eos_index is None): + n_eos_pts = opts.n_events_to_analyze + + # Load in filename + fname = eos_name.replace('file:', '') + # Retrieve row with parameters + eos_dat = np.loadtxt(fname)[opts.using_eos_index:opts.using_eos_index+n_eos_pts] + + make_CIP_eos = True + #prelim check for bad eos_param opt, saves milliseconds + spec_param_array = eos_dat[0][2:] # drop first two as lnL, sigma_lnL - will always pass w/ masses present + if opts.eos_param == 'spectral': + check_ok = True + elif opts.eos_param == 'cs_spectral' and len(spec_param_array >=4): + check_ok = True + elif opts.eos_param == 'PP' and len(spec_param_array >=4): + check_ok = True + else: + raise Exception("Unknown method for parametric EOS data file {} : {} ".format(eos_name,opts.eos_param)) + elif opts.eos_param == 'spectral': + # Will not work yet -- need to modify to parse command-line arguments + spec_param_packed=eval(opts.eos_param_values) # two lists: first are 'fixed' and second are specific + fixed_param_array=spec_param_packed[0] + spec_param_array=spec_param_packed[1] + spec_params ={} + spec_params['gamma1']=spec_param_array[0] + spec_params['gamma2']=spec_param_array[1] + # not used anymore: p0, epsilon0 set by the LI interface + spec_params['p0']=fixed_param_array[0] + spec_params['epsilon0']=fixed_param_array[1] + spec_params['xmax']=fixed_param_array[2] + if len(spec_param_array) <3: + spec_params['gamma3']=spec_params['gamma4']=0 + else: + spec_params['gamma3']=spec_param_array[2] + spec_params['gamma4']=spec_param_array[3] + eos_base = EOSManager.EOSLindblomSpectral(name=eos_name,spec_params=spec_params,use_lal_spec_eos=not opts.no_use_lal_eos) + # eos_vals = eos_base.make_spec_param_eos(npts=500) + # lalsim_spec_param = eos_vals/(C_CGS**2)*7.42591549*10**(-25) # argh, Monica! + # np.savetxt("lalsim_eos/"+eos_name+"_spec_param_geom.dat", np.c_[lalsim_spec_param[:,1], lalsim_spec_param[:,0]]) + # my_eos=lalsim.SimNeutronStarEOSFromFile(path+"/lalsim_eos/"+eos_name+"_spec_param_geom.dat") + my_eos=eos_base + elif opts.eos_param == 'cs_spectral': + # Will not work yet -- need to modify to parse command-line arguments + spec_param_packed=eval(opts.eos_param_values) # two lists: first are 'fixed' and second are specific + fixed_param_array=spec_param_packed[0] + spec_param_array=spec_param_packed[1] + spec_params ={} + spec_params['gamma1']=spec_param_array[0] + spec_params['gamma2']=spec_param_array[1] + spec_params['p0']=fixed_param_array[0] + spec_params['epsilon0']=fixed_param_array[1] + spec_params['xmax']=fixed_param_array[2] + spec_params['gamma3']=spec_params['gamma4']=0 + spec_params['gamma3']=spec_param_array[2] + spec_params['gamma4']=spec_param_array[3] + eos_base = EOSManager.EOSLindblomSpectralSoundSpeedVersusPressure(name=eos_name,spec_params=spec_params,use_lal_spec_eos=not opts.no_use_lal_eos) + my_eos = eos_base + elif 'lal_' in eos_name: + eos_name = eos_name.replace('lal_','') + my_eos = EOSManager.EOSLALSimulation(name=eos_name) + elif 'pyr_' in eos_name: + eos_name = eos_name.replace('pyr_','') + my_eos = EOSManager.EOSReprimand(name=eos_name) #this will create an empty EOS object + elif 'tabular_cgs' in eos_name: + eos_name = eos_name.replace('tabular_cgs','') + # Format now defined by record structure needed : column data, header with 'baryon_density', 'pressure', and 'energy_density' in cgs + fname =EOSManager.dirEOSTablesBase+"/tabular_" + eos_name+".dat" + my_eos_dat = np.genfromtxt(fname, names=True) + my_eos = EOSManager.EOSFromTabularData(name=eos_name,eos_data=my_eos_dat) + else: + my_eos = EOSManager.EOSFromDataFile(name=eos_name,fname =EOSManager.dirEOSTablesBase+"/" + eos_name+".dat") + + +# If using an EOS, reject samples that are out of range for that EOS (remember, we interpolate!) +if my_eos: #this doesn't trigger for externally-supplied EOS, but still true! + print(" WARNING: output not enforcing masses consistent with EOS maximum mass! Returning nearly zero tides in this case, but will run. ") + + +### +### Supplemental likelihood: load (as in ILE) +### +supplemental_ln_likelihood= None +supplemental_ln_likelihood_prep =None +supplemental_ln_likelihood_parsed_ini=None +supplemental_init=None +supplemental_eos=None +eos_param_names = None +# Supplemental likelihood factor. Must have identical call sequence to 'likelihood_function'. Called with identical raw inputs (including cosines/etc) +if opts.supplementary_likelihood_factor_code and opts.supplementary_likelihood_factor_function: + print(" EXTERNAL SUPPLEMENTARY LIKELIHOOD FACTOR : {}.{} ".format(opts.supplementary_likelihood_factor_code,opts.supplementary_likelihood_factor_function)) + __import__(opts.supplementary_likelihood_factor_code) + external_likelihood_module = sys.modules[opts.supplementary_likelihood_factor_code] + supplemental_ln_likelihood = getattr(external_likelihood_module,opts.supplementary_likelihood_factor_function) + name_prep = "prepare_"+opts.supplementary_likelihood_factor_function + if opts.using_eos_for_prior: + # Load in filename & all data, save for loop + fname = opts.using_eos.replace('file:', '') + dat = np.genfromtxt(fname,names=True)[opts.using_eos_index:opts.using_eos_index+n_eos_pts] # Parse file for them, to reduce need for burden parsing, and avoid burden/confusion. + eos_param_names = dat.dtype.names + eos_dat = dat.view((float, len(eos_param_names))) + supplemental_init = getattr(external_likelihood_module, 'initialize_me') + # CHECK IF WE RETRIEVE AN EOS from these hyperparameters too, so we can do both. + if hasattr(external_likelihood_module,'retrieve_eos'): + fake_eos = False # using EOS hyperparameter conversion! + supplemental_eos = getattr(external_likelihood_module, 'retrieve_eos') + + if hasattr(external_likelihood_module,name_prep): + supplemental_ln_likelhood_prep=getattr(external_likelihood_module,name_prep) #TODO: typo + # Check for and load in ini file associated with external library + if opts.supplementary_likelihood_factor_ini: + import configparser as ConfigParser + config = ConfigParser.ConfigParser() + config.optionxform=str # force preserve case! + config.read(opts.supplementary_likelihood_factor_ini) + supplemental_ln_likelhood_parsed_ini=config + + # Call the ini file, tell it what coordinates we are using by name + supplemental_ln_likelihood_prep(config=supplemental_ln_likelihood_parsed_ini,coords=coord_names) + + +### +### Coordinate conversion tool +### +if not opts.using_eos or (fake_eos): + def convert_coords(x_in): + return lalsimutils.convert_waveform_coordinates(x_in, coord_names=coord_names,low_level_coord_names=low_level_coord_names,source_redshift=source_redshift,enforce_kerr=opts.downselect_enforce_kerr) +else: + def convert_coords(x_in): + x_out = lalsimutils.convert_waveform_coordinates_with_eos(x_in, coord_names=coord_names,low_level_coord_names=low_level_coord_names,eos_class=my_eos,no_matter1=opts.no_matter1, no_matter2=opts.no_matter2,source_redshift=source_redshift,enforce_kerr=opts.downselect_enforce_kerr) + return x_out + + + +############################################################################### +### SAMPLER LOOP +############################################################################### + +#PUT ALL OF SAMPLER STUFF INTO LOOP FOR NOW +hyper_out_list = [] +fname_output_integral_orig = opts.fname_output_integral +for i in np.arange(n_eos_pts): + print("\n---------- SAMPLING FOR EOS LINE "+str(opts.using_eos_index+i)+" ----------") + eos_dat_line = eos_dat[i] + + # Set up EOS + if opts.using_eos_for_prior and opts.using_eos and supplemental_init is not None: + # Initialize external prior code & retrieve EOS from it (if applicable) + args_init = {'input_line':eos_dat_line, 'param_names':eos_param_names, 'cip_param_names':coord_names, 'cip_faster':True} # pass the recordarray broken into parts, for convenience + failstate = supplemental_init(**args_init) + #TODO: instead of forcing a return val: make func in ext code, like retrieve_eos(), & call to fetch a fail-state global variable there; e.g., ext_lh_module.check_fail() returns True/False + #unphysical_eos_val = supplemental_failstate() - check this func exists, first; if it doesn't, CIP will have to crash + #if opts.save_unphysical_eos: eos_dat_line[0] = unphysical_eos_val + if failstate: #failstate = None means success! + print("ERROR: Could not initialize this EOS line; errorcode =",failstate) + try: + eos_dat_line[0] = np.float64(failstate) + print("Saving failstate as lnL for this eos line.") + except: + print("Cannot use provided failstate as output; saving default unphysical lnL.") + eos_dat_line[0] = -1e6 + eos_dat_line[1] = 1. + hyper_out_list.append(eos_dat_line) + continue + + if not (fake_eos): #if fake_eos is false, supplemental_eos will be True (not None) + my_eos = supplemental_eos(**args_init) #THIS CANNOT RETURN NONE, ELSE WILL CRASH + elif make_CIP_eos: + #import RIFT.physics.EOSManager as EOSManager #imported earlier, hopefully + if opts.verbose: + print(" Using EOS ", eos_name, opts.using_eos_index, opts.eos_param, opts.eos_param_values) + + spec_param_array = eos_dat_line[2:] # drop first two as lnL, sigma_lnL + try: + if opts.eos_param == 'spectral': + spec_params ={} + spec_params['gamma1']=spec_param_array[0] + spec_params['gamma2']=spec_param_array[1] + if len(spec_param_array) <3: + spec_params['gamma3']=spec_params['gamma4']=0 + else: + spec_params['gamma3']=spec_param_array[2] + spec_params['gamma4']=spec_param_array[3] + eos_base = EOSManager.EOSLindblomSpectral(name=eos_name,spec_params=spec_params,use_lal_spec_eos=not opts.no_use_lal_eos) + my_eos=eos_base + elif opts.eos_param == 'cs_spectral' and len(spec_param_array >=4): + spec_params ={} + spec_params['gamma1']=spec_param_array[0] + spec_params['gamma2']=spec_param_array[1] + spec_params['gamma3']=spec_params['gamma4']=0 + spec_params['gamma3']=spec_param_array[2] + spec_params['gamma4']=spec_param_array[3] + eos_base = EOSManager.EOSLindblomSpectralSoundSpeedVersusPressure(name=eos_name,spec_params=spec_params,use_lal_spec_eos=not opts.no_use_lal_eos) + my_eos = eos_base + elif opts.eos_param == 'PP' and len(spec_param_array >=4): + spec_params ={} + spec_params['logP1'] = spec_param_array[0] + spec_params['gamma1'] = spec_param_array[1] + spec_params['gamma2'] = spec_param_array[2] + spec_params['gamma3'] = spec_param_array[3] + eos_base = EOSManager.EOSPiecewisePolytrope(name=eos_name,params_dict=spec_params) + my_eos = eos_base + else: + raise Exception("Unknown method for parametric EOS data file {} : {} ".format(eos_name,opts.eos_param)) + except Exception as e: + print(" ERROR: EOS CREATION FAILED:",e) + #append low-lnL val for unphysical EOS (same file format as w/ suppl prior code) + eos_dat_line[0] = -1e6 + eos_dat_line[1] = 1. + hyper_out_list.append(eos_dat_line) + continue + + + # Sampler------------------------------- + sampler = mcsampler.MCSampler() + use_portfolio=False + if opts.sampler_method == "adaptive_cartesian_gpu": + sampler = mcsamplerGPU.MCSampler() + sampler.xpy = xpy_default + sampler.identity_convert=identity_convert + mcsampler = mcsamplerGPU # force use of routines in that file, for properly configured GPU-accelerated code as needed + + # if opts.sampler_xpy == "numpy": + # mcsampler.set_xpy_to_numpy() + # sampler.xpy= numpy + # sampler.identity_convert= lambda x: x + elif opts.sampler_method == "GMM": + sampler = mcsamplerEnsemble.MCSampler() + elif opts.sampler_method == "AV": + sampler = mcsamplerAdaptiveVolume.MCSampler() + opts.internal_use_lnL= True # required! + elif opts.sampler_method == "NFlow": + # expensive import, only do if requested + try: + import RIFT.integrators.mcsamplerNFlow as mcsamplerNFlow + mcsampler_NF_ok = True + except: + print(" No mcsamplerNFlow ") + sampler = mcsamplerNFlow.MCSampler() + + opts.internal_use_lnL= True # required! + elif opts.sampler_method == "portfolio": + use_portfolio=True + opts.internal_use_lnL=True # required, we only implement those scenarios right now + sampler = None + sampler_list = [] + sampler_types = opts.sampler_portfolio + for name in sampler_types: + if name =='AV': + sampler = mcsamplerAdaptiveVolume.MCSampler() + if name =='GMM': + sampler = mcsamplerEnsemble.MCSampler() + opts.sampler_method = 'GMM' # this will force the creation/parsing of GMM-specific arguments below, so they are properly passed + if name == "adaptive_cartesian_gpu" or name =='AC': + sampler = mcsamplerGPU.MCSampler() + sampler.xpy = xpy_default + sampler.identity_convert=identity_convert + if name == 'NFlow': + # expensive import, only do if requested + try: + import RIFT.integrators.mcsamplerNFlow as mcsamplerNFlow + mcsampler_NF_ok = True + except: + print(" No mcsamplerNFlow ") + continue + sampler = mcsamplerNFlow.MCSampler() + sampler.xpy = xpy_default + sampler.identity_convert=identity_convert + if sampler is None: + # Don't add unknown type + continue + print('PORTFOLIO: adding {} '.format(name)) + sampler_list.append(sampler) + sampler = mcsamplerPortfolio.MCSampler(portfolio=sampler_list) + elif mcsampler_Portfolio_ok: + if opts.sampler_method in mcsamplerPortfolio.known_pipelines: # access from plugins + sampler = mcsamplerPortfolio.known_pipelines[opts.sampler_method]() + + + ## + ## Loop over param names + ## + print(" Preparing sampling ", low_level_coord_names) + for p in low_level_coord_names: + if not(opts.parameter_implied is None): + if p in opts.parameter_implied and not(p == 'chi_pavg'): + # We do not need to sample parameters that are implied by other parameters, so we can overparameterize + continue + prior_here = prior_map[p] + range_here = prior_range_map[p] + + ## Special case: mc : provide ability to override range + if p == 'mc' and opts.mc_range: + range_here = eval(opts.mc_range) + elif p=='mc' and opts.trust_sample_parameter_box: + range_here = [np.max([range_here[0],mc_min]), np.min([range_here[1],mc_max])] + if p =='mtot' and opts.mtot_range: + range_here = eval(opts.mtot_range) + # special cases mu1,mu2: rather than try to choose intelligently, just use training box data for limits + if (p=='ln_mu1' or p == 'mu1' or p=='mu2') and opts.trust_sample_parameter_box: + # extremely special/unusual use case, looking up from *input* data : mu1 only used as coordinate if also used for fitting + indx_lnmu = coord_names.index(p) + lnmu_max = np.max(dat_out[:,indx_lnmu]) + lnmu_min = np.min(dat_out[:,indx_lnmu]) + range_here = [lnmu_min,lnmu_max] + + adapt_me = True + if p in opts.no_adapt_parameter: + adapt_me=False + # sampling prior initial normalization: normally not used, provided for sanity's sake just in case it is + fac = (range_here[1] - range_here[0]) + sampler.add_parameter(p, pdf=np.vectorize(lambda x,z=fac:1./z), prior_pdf=prior_here,left_limit=range_here[0],right_limit=range_here[1],adaptive_sampling=adapt_me) + + + ### + ### Supplemental priors: load module + ### + + if opts.supplementary_prior_code: + print(" EXTERNAL PRIOR IMPORT : {} ".format(opts.supplementary_prior_code)) + __import__(opts.supplementary_prior_code) + external_prior_module = sys.modules[opts.supplementary_prior_code] + if hasattr(external_prior_module,'prior_pdf') and hasattr(external_prior_module,'param_ranges'): + if type(external_prior_module.prior_pdf) == dict: + for name in external_prior_module.prior_pdf: + sampler.prior_pdf[name] = external_prior_module[name] + sampler.llim[name],sampler.rlim[param] = stored_param_ranges[param] + + # Import prior (does not work as implemented! Need to edit) + if not(opts.import_prior_dictionary_file is None): + dat = joblib.load(opts.import_prior_dictionary_file) + # print dat + stored_param_prior_pdf = dat[0] + stored_param_ranges =dat[1] + for param in stored_param_prior_pdf.keys(): + if param in sampler.prior_pdf.keys(): + print(" OVER-RIDE OF PRIOR FOR ", param, stored_param_prior_pdf[param], stored_param_ranges[param]) + # stored_param_pdf[param] = sampler.pdf[param] + sampler.prior_pdf[param] = stored_param_prior_pdf[param] + sampler.llim[param], sampler.rlim[param] = stored_param_ranges[param] + + + # Export prior + if not(opts.output_prior_dictionary_file is None): + stored_param_prior_pdf = {} + stored_param_ranges = {} + for param in sampler.params: + # stored_param_pdf[param] = sampler.pdf[param] + stored_param_prior_pdf[param] = sampler.prior_pdf[param] + stored_param_ranges[param] = [sampler.llim[param],sampler.rlim[param]] + joblib.dump([stored_param_prior_pdf,stored_param_ranges],opts.output_prior_dictionary_file) + # with open(opts.output_prior_dictionary_file,'w') as f: + # pickle.dump([stored_param_prior_pdf,stored_param_ranges],f) + + + ### + ### Oracles + ### + sampler_oracle = None + if opts.oracle_reference_sample_file: + # default i/o request is parameters provide. + # NOTE: saved fields don't match sampling in general, so we can barf here unless using XML i/o + ref_params = sampler.params_ordered + if (opts.oracle_reference_sample_params): + ref_params = opts.oracle_reference_sample_params.replace('[','').replace(']','').split(',') + + from RIFT.misc.reference_samples import ReferenceSamples + from RIFT.integrators.unreliable_oracle.resampling import ResamplingOracle + + rs_object = ReferenceSamples() + if opts.oracle_reference_sample_file.endswith('.dat') or opts.oracle_reference_sample_file.endswith('.txt'): + rs_object.from_ascii(opts.oracle_reference_sample_file) + elif opts.oracle_reference_sample_file.endswith('.xml.gz'): + rs_object.from_sim_xml(opts.oracle_reference_sample_file,reference_params=ref_params) + else: + raise Exception(" Unknown reference params file format") + + sampler_oracle = ResamplingOracle() + for p in sampler.params_ordered: + sampler_oracle.add_parameter(p,pdf=None, left_limit =sampler.llim[p], right_limit = sampler.rlim[p]) + sampler_oracle.setup(reference_samples=rs_object.reference_samples, reference_params=rs_object.reference_params) + + + #Set up steps, args, etc + n_step = int(opts.n_chunk) + if opts.sampler_method == 'GMM': + n_step *=3 # bigger steps for GMM + # tempering exp: most appropriate for histogram method to avoid oversampling + my_exp = np.min([1,0.8*np.log(n_step)/np.max(Y)]) # target value : scale to slightly sublinear to (n_step)^(0.8) for Ymax = 200. This means we have ~ n_step points, with peak value wt~ n_step^(0.8)/n_step ~ 1/n_step^(0.2), limiting contrast + if opts.sampler_method == 'GMM': + my_exp = np.min([1,4*np.log(n_step)/np.max(Y)]) # target value : scale to slightly sublinear to (n_step)^(0.8) for Ymax = 200. This means we have ~ n_step points, with peak value wt~ n_step^(0.8)/n_step ~ 1/n_step^(0.2), limiting contrast + if opts.sampler_method == 'NFlow': + my_exp = 1 # don't use it + #my_exp = np.max([my_exp, 1/np.log(n_step)]) # do not allow extreme contrast in adaptivity, to the point that one iteration will dominate + print(" Weight exponent ", my_exp, " and peak contrast (exp)*lnL = ", my_exp*np.max(Y), "; exp(ditto) = ", np.exp(my_exp*np.max(Y)), " which should ideally be no larger than of order the number of trials in each epoch, to insure reweighting doesn't select a single preferred bin too strongly. Note also the floor exponent also constrains the peak, de-facto") + + + extra_args={} + if opts.sampler_method == "GMM" or (opts.sampler_method == 'portfolio' and 'GMM' in opts.sampler_portfolio): + n_max_blocks = ((1.0*int(opts.n_max))/n_step) + n_comp = opts.internal_n_comp # default + def parse_corr_params(my_str): + """ + Takes a string with no spaces, and returns a tuple + """ + corr_param_names = my_str.replace(',',' ').split() + corr_param_indexes = [] + for param in corr_param_names: + try: + indx = low_level_coord_names.index(param) + corr_param_indexes.append(indx) + except: + continue + return tuple(corr_param_indexes) + if opts.internal_correlate_parameters == 'all': + gmm_dict = {tuple(range(len(low_level_coord_names))):None} # integrate *jointly* in all parameters together + elif not (opts.internal_correlate_parameters is None): + # Correlate identified parameters + my_blocks = opts.internal_correlate_parameters.split() + my_tuples = list(map( parse_corr_params, my_blocks)) + gmm_dict = {x:None for x in my_tuples} + # What about un-labelled parameters? Make a null tuple for them as well + correlated_params = set(()); correlated_params = correlated_params.union( *list(map(set,my_tuples))) + uncorrelated_params = set(np.arange(len(low_level_coord_names))); + uncorrelated_params = uncorrelated_params.difference(correlated_params) + for x in uncorrelated_params: + gmm_dict[(x,)] = None + print( " Using correlated GMM sampling on sampling variable indexes " , gmm_dict, " out of ", low_level_coord_names) + else: + param_indexes = range(len(low_level_coord_names)) + gmm_dict = {(k,):None for k in param_indexes} # no correlations + if opts.internal_gmm_memory_chisquared_factor: + lnL_offset_saving = len(low_level_coord_names)*opts.internal_gmm_memory_chisquared_factor # based on chisquared distribution, we should not be keeping more than this for output. This is PURELY FOR MEMORY MANAGEMENT + else: + lnL_offset_saving = opts.lnL_offset + extra_args = {'n_comp':n_comp,'max_iter':n_max_blocks,'L_cutoff': (np.exp(max_lnL-lnL_shift - lnL_offset_saving)),'gmm_dict':gmm_dict,'max_err':50} # made up for now, should adjust + extra_args.update({ + "n_adapt": 100, # Number of chunks to allow adaption over + "history_mult": 10, # Multiplier on 'n' - number of samples to estimate marginalized 1D histograms with, + "force_no_adapt":opts.force_no_adapt, + "tripwire_fraction":opts.tripwire_fraction, + "enforce_bounds":True # needed for any AV integrators used + }) + if opts.sampler_method == 'NFlow': + extra_args['n_adapt'] = 10 # reduce this? + tempering_adapt=True + if opts.force_no_adapt: + tempering_adapt=False + # Result shifted by lnL_shift + fn_passed = likelihood_function + if supplemental_ln_likelihood: + fn_passed = lambda *x: likelihood_function(*x)*np.exp(supplemental_ln_likelihood(*x)) + if opts.internal_use_lnL: + fn_passed = log_likelihood_function # helps regularize large values + if supplemental_ln_likelihood: + fn_passed = lambda *x: log_likelihood_function(*x) + supplemental_ln_likelihood(*x) + extra_args.update({"use_lnL":True,"return_lnI":True}) + if opts.internal_temper_log: + extra_args.update({'temper_log':True}) + if hasattr(sampler, 'setup'): + extra_args_here = {} + extra_args_here.update(extra_args) + extra_args_here['oracle_realizations'] = oracle_realizations + extra_args_here['lnL'] = fn_passed # pass it to oracle specifically + if use_portfolio: + print(" PORTFOLIO : setup") + our_breakpoints = None + if opts.sampler_portfolio_breakpoints: + print(opts.sampler_portfolio_breakpoints) + our_breakpoints = eval(opts.sampler_portfolio_breakpoints) + print(" Portfolio breakpoints ", our_breakpoints) + if opts.sampler_portfolio_args: + print(" PRE_EVAL", opts.sampler_portfolio_args) + #opts.sampler_portfolio_args = list(map(lambda x: eval(' "{}" '.format(x)), opts.sampler_portfolio_args)) + opts.sampler_portfolio_args = list(map(eval, opts.sampler_portfolio_args)) + # confirm all are dict + for indx in range(len(opts.sampler_portfolio_args)): + if not(isinstance(opts.sampler_portfolio_args[indx], dict)): + print(indx,opts.sampler_portfolio_args[indx]) + print(" ARGS ", opts.sampler_portfolio_args) + sampler.setup(portolio_args=opts.sampler_portfolio_args,portfolio_breakpoints=our_breakpoints,**extra_args_here) + + # Call oracle if provided, to initialize sampler + if sampler_oracle: # NON-PORTFOLIO SCENARIO TARGET + if hasattr(sampler, 'update_sampling_prior'): + rvs_train = {} + _, _, rv_oracle = sampler_oracle.draw_simplified(n_step) + for indx,p in enumerate(sampler.params_ordered): + rvs_train[p] = rv_oracle[:,indx] + # train with equal weight - no likelihood information + lnL_oracles = np.zeros(n_step) + sampler.update_sampling_prior(lnL_oracles, n_step, external_rvs=rvs_train,log_scale_weights=True) + else: + print(" -- ORACLE: no mechanism to update sampling prior with oracle samples ") + + + + #========================================================================== + #========================================================================== + #TODO: INTEGRAL IS HERE + ############################ + res, var, neff, dict_return = sampler.integrate(fn_passed, *low_level_coord_names, verbose=True,nmax=int(opts.n_max),n=n_step,neff=opts.n_eff, save_intg=True,tempering_adapt=tempering_adapt, floor_level=1e-3,igrand_threshold_p=1e-3,convergence_tests=test_converged,tempering_exp=my_exp,no_protect_names=True, **extra_args) # weight ecponent needs better choice. We are using arbitrary-name functions + ############################ + #TODO: INTEGRAL IS HERE + #========================================================================== + #========================================================================== + + + + # result value: be careful, if return lnL, then must not take log twice! + ln_integrand_value = None + if opts.internal_use_lnL: # eg, AV integrator, etc + ln_integrand_value = res + else: + ln_integrand_value = np.log(res) + + #if more than 1 line, update output filenames to be different + if n_eos_pts > 1: #regular RIFT runs must not be affected by this! + newout = "" + try: + last_digit = int(opts.fname_output_integral[-1]) #will work in hyperpipe + newout = opts.fname_output_integral[:-1]+str(last_digit+i) + except: + if opts.using_eos_index: + newout = opts.fname_output_integral+"-"+str(opts.using_eos_index+i) + else: #can't help you, bud + newout = opts.fname_output_integral+"-"+str(i) + if opts.fname_output_integral == opts.fname_output_samples: + opts.fname_output_samples = newout + opts.fname_output_integral = newout + + # Test n_eff threshold + if not (opts.fail_unless_n_eff is None): + if neff < opts.fail_unless_n_eff and not(opts.not_worker): # if we need the output to continue: + print(" FAILURE: n_eff too small") + continue #sys.exit(1) + if neff < opts.n_eff: + print(" ==> neff (={}) is low <==".format(neff)) + if opts.contingency_unevolved_neff == 'quadpuff' and neff < np.min([500,opts.n_eff]): # we can usually get by with about 500 points + # Add errors + # Note we only want to add errors to RETAINED points + print(" Contingency: quadpuff: take covariance of points, draw from it again, add to existing points as offsets (i.e. a puffball) ") + n_output_size = np.min([len(P_list_in),opts.n_output_samples]) + print(" Preparing to write ", n_output_size , " samples ") + + my_cov = np.cov(X.T) # covariance of data points + rv = scipy.stats.multivariate_normal(mean=np.zeros(len(X[0])), cov=my_cov,allow_singular=True) # they are just complaining about dynamic range + delta_X = rv.rvs(size=len(X)) + X_new = X+delta_X + P_out_list = [] + # Loop over points + # Jitter using the parameters we use to fit with + for indx_P in np.arange(np.min([len(P_list_in),len(X)])): # make sure no past-limits errors + include_item=True + P = P_list_in[indx_P] + for indx in np.arange(len(coord_names)): + param = coord_names[indx] + fac = 1 + if coord_names[indx] in ['mc', 'mtot', 'm1', 'm2']: + fac = lal.MSUN_SI + P.assign_param(param, (X_new[indx_P,indx]*fac)) + for p in downselect_dict.keys(): + val = P.extract_param(p) + if p in ['mc','m1','m2','mtot']: + val = val/lal.MSUN_SI + if val < downselect_dict[p][0] or val > downselect_dict[p][1]: + include_item = False + if include_item: + P_out_list.append(P) + + # Save output + lalsimutils.ChooseWaveformParams_array_to_xml(P_out_list[:n_output_size],fname=opts.fname_output_samples,fref=P.fref) + continue #sys.exit(0) + + ####### Saving results ######## + # Save result -- needed for odds ratios, etc. + #File 1/7 - just the lnL (MARG-X-X.dat) + # Warning: integral_result.dat uses *original* prior, before any reweighting + if not (opts.save_hyperfile_only) and not (opts.chunk_save): + np.savetxt(opts.fname_output_integral+".dat", [ln_integrand_value+lnL_shift]) + #elif not(opts.save_hyperfile_only) and opts.chunk_save: handled after loop + + #File 2/7 - lnL + EOS line (MARG-X-X+annotation.dat) - hyperpipe file + eos_extra = [] + annotation_header = "lnL sigmaL neff " + if opts.using_eos and not(opts.using_eos.startswith('file:')): + eos_extra = [opts.using_eos] + annotation_header += 'eos_name ' + if opts.eos_param == 'spectral': + # Should also + my_eos_params = my_eos.spec_params + eos_extra += list(map( lambda x: str(my_eos_params[x]), ["gamma1", "gamma2", "gamma3", "gamma4", "p0", "epsilon0", "xmax"])) + # eos_extra += opts.eos_param + annotation_header += "gamma1 gamma2 gamma3 gamma4 p0 epsilon0 xmax" + elif opts.using_eos and opts.using_eos.startswith('file:'): + fname = opts.using_eos.replace('file:','') + params_here = np.loadtxt(fname)[opts.using_eos_index+i][2:] #TODO: redundant == eos_dat_line[2:] + linefirst ='' + with open(fname) as f: + linefirst = f.readline() + linefirst = linefirst[2:] + annotation_header = linefirst # this will/must be lnL sigma_lnL and then parameter names, which we want to preserve + if opts.chunk_save: + if not(opts.using_eos) or not(opts.using_eos.startswith('file:')): + str_out =list( map(str,[ln_integrand_value, np.sqrt(var)/res, neff])) + hyper_out_list.append(np.float64(str_out + eos_extra)) #len = 3 + len(using_eos) + eos.spec_params + else: + eos_dat_here = eos_dat_line + eos_dat_here[0] = ln_integrand_value + eos_dat_here[1] = np.sqrt(var)/res + hyper_out_list.append(eos_dat_here) #compile for later + else: + with open(opts.fname_output_integral+"+annotation.dat", 'w') as file_out: + if not(opts.using_eos) or not(opts.using_eos.startswith('file:')): + str_out =list( map(str,[ln_integrand_value, np.sqrt(var)/res, neff])) + file_out.write("# " + annotation_header + "\n") + file_out.write(' '.join( str_out + eos_extra + ["\n"])) + else: + file_out.write("# " + annotation_header + "\n") + file_out.write(" {} {} ".format(ln_integrand_value, np.sqrt(var)/res) + ' '.join(map(str,params_here))) + #np.savetxt(opts.fname_output_integral+"+annotation.dat", np.array([[np.log(res), np.sqrt(var)/res, neff]]), header=eos_extra) + # since not EOS, can just use np.savetxt + # with open(opts.fname_output_integral+"+annotation_ESS.dat", 'w') as file_out: + # annotation_header = "lnL sigmaL neff n_ESS " + # str_out =list( map(str,[np.log(res), np.sqrt(var)/res, neff, n_ESS])) + # file_out.write("# " + annotation_header + "\n") + # file_out.write(' '.join( str_out + ["\n"])) + #np.savetxt(opts.fname_output_integral+"+annotation.dat", np.array([[np.log(res), np.sqrt(var)/res, neff]]), header=eos_extra) + + + if neff < len(low_level_coord_names): + print(" neff < # low level coords -> PLOTS WILL FAIL ") + print(" Not enough independent Monte Carlo points to generate useful contours") + + if opts.no_save_samples or opts.save_hyperfile_only: + continue #sys.exit(0) + + #TODO: chunk_save not supported for any of the remaining valid files (3/7, 4/7, 5/7) + samples = sampler._rvs + samples_type_names = list(samples.keys()) + print(samples_type_names) + n_params = len(coord_names) + dat_mass = np.zeros((len(samples[low_level_coord_names[0]]),n_params+3)) + dat_logL = np.zeros(len(samples[low_level_coord_names[0]])) + if not(opts.internal_use_lnL): + if 'log_integrand' in samples_type_names: + dat_logL = np.log(samples["log_integrand"]) + elif 'integrand' in samples_type_names: + dat_logL = np.log(samples["integrand"]) + else: + raise Exception("Failure : cannot identify lnL field") + else: + if 'log_integrand' in samples_type_names: + dat_logL = samples['log_integrand'] + else: + dat_logL = samples["integrand"] + lnLmax = np.max(dat_logL[np.isfinite(dat_logL)]) + print(" Max lnL ", np.max(dat_logL)) + + n_ESS = -1 + if True: + # Compute n_ESS. Should be done by integrator! + if 'log_joint_s_prior' in samples: + weights_scaled = np.exp(dat_logL - lnLmax + samples["log_joint_prior"] - samples["log_joint_s_prior"]) + # dictionary, write this to enable later use of it + samples["joint_s_prior"] = np.exp(samples["log_joint_s_prior"]) + samples["joint_prior"] = np.exp(samples["log_joint_prior"]) + else: + weights_scaled = np.exp(dat_logL - lnLmax)*sampler._rvs["joint_prior"]/sampler._rvs["joint_s_prior"] + weights_scaled = weights_scaled/np.max(weights_scaled) # try to reduce dynamic range + n_ESS = np.sum(weights_scaled)**2/np.sum(weights_scaled**2) + print(" n_eff n_ESS ", neff, n_ESS) + np.savetxt(opts.fname_output_integral+"+annotation_ESS.dat",[[ln_integrand_value, np.sqrt(var)/res, neff, n_ESS]],header=" lnL sigmaL neff n_ESS ") #File 3/7 + + + # Throw away stupid points that don't impact the posterior + indx_ok = np.logical_and(dat_logL > lnLmax-opts.lnL_offset ,samples["joint_s_prior"]>0) + for p in low_level_coord_names: + samples[p] = samples[p][indx_ok] + dat_logL = dat_logL[indx_ok] + samples["joint_prior"] =samples["joint_prior"][indx_ok] + samples["joint_s_prior"] =samples["joint_s_prior"][indx_ok] + + + + ### + ### 1d posteriors of the coordinates used for sampling [EQUALLY WEIGHTED, BIASED because physics cuts aren't applied] + ### + + p = samples["joint_prior"] + ps =samples["joint_s_prior"] + lnL = dat_logL + lnLmax = np.max(lnL) + weights = np.exp(lnL-lnLmax)*p/ps + + + # If we are using pseudo uniform spin magnitude, reweight + # ONLY done if we use s1x, s1y, s1z, s2x, s2y, s2z + # volumetric prior scales as a1^2 a2^2 da1 da2; we need to undo it + if opts.pseudo_uniform_magnitude_prior and 's1x' in samples.keys() and 's1z' in samples.keys(): + prior_weight = np.prod([prior_map[x](samples[x]) for x in ['s1x','s1y','s1z'] ],axis=0) + val = np.array(samples["s1z"]**2+samples["s1y"]**2 + samples["s1x"]**2,dtype=internal_dtype) + chi1 = np.sqrt(val) # weird typecasting problem + weights *= 3.*chi_max*chi_max/(chi1*chi1*prior_weight) # prior_weight accounts for the density, in cartesian coordinates + weights[ chi1>chi_max] =0 + if 's2z' in samples.keys(): + prior_weight = np.prod([prior_map[x](samples[x]) for x in ['s2x','s2y','s2z'] ],axis=0) + val = np.array(samples["s2z"]**2+samples["s2y"]**2 + samples["s2x"]**2,dtype=internal_dtype) + chi2= np.sqrt(val) + weights[ chi2>chi_small_max] =0 + weights *= 3.*chi_small_max*chi_small_max/(chi2*chi2*prior_weight) + elif opts.pseudo_uniform_magnitude_prior and 'chiz_plus' in samples.keys() and not opts.pseudo_uniform_magnitude_prior_alternate_sampling: + # Uniform sampling: simple volumetric reweight + s1z = samples['chiz_plus'] + samples['chiz_minus'] + s2z = samples['chiz_plus'] - samples['chiz_minus'] + val1 = np.array(s1z**2+samples["s1y"]**2 + samples["s1x"]**2,dtype=internal_dtype); chi1 = np.sqrt(val1) + val2 = np.array(s2z**2+samples["s2y"]**2 + samples["s2x"]**2,dtype=internal_dtype); chi2= np.sqrt(val2) + indx_ok = np.logical_and(chi1<=chi_max , chi2<=chi_small_max) + weights[ np.logical_not(indx_ok)] = 0 # Zero out failing samples. Has effect of fixing prior range! + weights[indx_ok] *= 9.*(chi_max**2 * chi_small_max**2)/(chi1*chi1*chi2*chi2)[indx_ok] + elif opts.pseudo_uniform_magnitude_prior and 'chiz_plus' in samples.keys() and not opts.pseudo_uniform_magnitude_prior_alternate_sampling: + s1z = samples['chiz_plus'] + samples['chiz_minus'] + s2z = samples['chiz_plus'] - samples['chiz_minus'] + val1 = np.array(s1z**2+samples["s1y"]**2 + samples["s1x"]**2,dtype=internal_dtype); chi1 = np.sqrt(val1) + val2 = np.array(s2z**2+samples["s2y"]**2 + samples["s2x"]**2,dtype=internal_dtype); chi2= np.sqrt(val2) + indx_ok = np.logical_and(chi1<=chi_max , chi2<=chi_small_max) + weights[ np.logical_not(indx_ok)] = 0 # Zero out failing samples. Has effect of fixing prior range! + prior_weight = np.prod([prior_map[x](samples[x]) for x in ['s1x','s1y', 's2x', 's2y','chiz_plus','chiz_minus'] ],axis=0) + weights[indx_ok] *= 9.*(chi_max**2 * chi_small_max**2)/(chi1*chi1*chi2*chi2)[indx_ok]/prior_weight[indx_ok] # undo chizplus, chizminus prior + + + # If we are using alignedspin-zprior AND chiz+, chiz-, then we need to reweight .. that prior cannot be evaluated internally + # Prevent alignedspin-zprior from being used when transverse spins are present ... no sense! + # Note we need to downslelect early in this case + if opts.aligned_prior =="alignedspin-zprior" and 'chiz_plus' in samples.keys() and (not 's1x' in samples.keys()): + prior_weight = np.prod([prior_map[x](samples[x]) for x in ['chiz_plus','chiz_minus'] ],axis=0) + s1z = samples['chiz_plus'] + samples['chiz_minus'] + s2z =samples['chiz_plus'] - samples['chiz_minus'] + indx_ok = np.logical_and(np.abs(s1z)<=chi_max , np.abs(s2z)<=chi_max) + weights[ np.logical_not(indx_ok)] = 0 # Zero out failing samples. Has effect of fixing prior range! + weights[indx_ok] *= s_component_zprior( s1z[indx_ok])*s_component_zprior(s2z[indx_ok])/(prior_weight[indx_ok]) # correct for uniform + + if opts.pseudo_gaussian_mass_prior: + # mass normalization (assuming mc, eta limits are bounds - as is invariably the case) + mass_area = 0.5*(mc_max**2 - mc_min**2)*(unscaled_eta_prior_cdf(eta_range[0]) - unscaled_eta_prior_cdf(eta_range[1])) + # Extract m1 and m2, i solar mass units + m1 = np.zeros(len(weights)) + m2 = np.zeros(len(weights)) + for indx in np.arange(len(weights)): + P=lalsimutils.ChooseWaveformParams() + for indx_name in np.arange(len(low_level_coord_names)): + p = low_level_coord_names[indx_name] + # Do not bother to scale by solar masses, only to undo it later + P.assign_param(p, samples[p][indx]) + m1[indx] = P.extract_param('m1') + m2[indx] = P.extract_param('m2') + # For speed, do this transformation to mass coordinates by hand rather than the usual loop + # m1=None + # m2=None + # if 'm1' in samples.keys(): # will never happen + # m1 = samples['m1'] + # m2 = samples['m2'] + # elif 'mc' in samples.keys(): #almost always true + # mc = samples['mc'] + # eta = None + # if 'eta' in samples.keys(): + # eta = samples['eta'] + # elif 'delta_mc' in samples.keys(): + # eta = np.array(0.25*(1-samples['delta_mc']**2)) # see definition + # else: + # print " Failed transformation " + # sys.exit(0) + # print type(mc), type(eta) + # m1 = lalsimutils.mass1(mc,eta) + # m2 = lalsimutils.mass2(mc,eta) + # else: + # print " Failed transformation" + # sys.exit(0) + # Reormalize mass region. Note normalizatoin issue introduced: no boundaries in mass region used to rescale. + weights *= mass_area*gaussian_mass_prior( m1, opts.pseudo_gaussian_mass_prior_mean,opts.pseudo_gaussian_mass_prior_std)*gaussian_mass_prior( m2, opts.pseudo_gaussian_mass_prior_mean,opts.pseudo_gaussian_mass_prior_std) + + + # Integral result v2: using modified prior. + # Note also downselects NOT applied: no range cuts, unless applied as part of aligned_prior, etc. + # - use for Bayes factors with GREAT CARE for this reason; should correct for with indx_ok + log_res_reweighted = lnLmax + np.log(np.mean(weights)) + sigma_reweighted= np.std(weights,dtype=np.float128)/np.mean(weights) + neff_reweighted = np.sum(weights)/np.max(weights) + np.savetxt(opts.fname_output_integral+"_withpriorchange.dat", [log_res_reweighted]) # should agree with the usual result, if no prior changes #File 4/7 + with open(opts.fname_output_integral+"_withpriorchange+annotation.dat", 'w') as file_out: + str_out = list(map(str,[log_res_reweighted, sigma_reweighted, neff])) + file_out.write("# " + annotation_header + "\n") + file_out.write(' '.join( str_out + eos_extra + ["\n"])) #File 5/7 + #np.savetxt(opts.fname_output_integral+"_withpriorchange+annotation.dat", np.array([[log_res_reweighted,sigma_reweighted, neff]]),header=eos_extra) + + + ########################################################################### + #PLOT STUFF FOLLOWS + ########################################################################### + ''' + Much of this does not need to be in the loop: + it will only be utilized when n = 1 + just need to keep down to sample saving (6/7, 7/7) + will need to store sampler results that get used in the analyses below + (need to check what/where they are) + + N.B.: Last loop iteration vars remain in-scope after exiting loop + + Ideal: find out what gets used below, save it to vars defined before the + loop. (If >1 eos line, all plotting banned regardless.) + ''' + + + # Load in reference parameters + Pref = lalsimutils.ChooseWaveformParams() + if opts.inj_file is not None: + Pref = lalsimutils.xml_to_ChooseWaveformParams_array(opts.inj_file)[opts.event_num] + else: + Pref.m1 = Pref_default.m1 + Pref.m2 = Pref_default.m2 + Pref.fref = opts.fref # not encoded in the XML! + Pref.print_params() + + + if not no_plots: + for indx in np.arange(len(low_level_coord_names)): + try: + dat_out = []; dat_out_LI=[] + p = low_level_coord_names[indx] + print(" -- 1d cumulative "+ str(indx)+ ":"+ low_level_coord_names[indx]+" ----") + dat_here = samples[low_level_coord_names[indx]] + range_x = [np.min(dat_here), np.max(dat_here)] + if opts.fname_lalinference: + dat_LI = extract_combination_from_LI( samples_LI, low_level_coord_names[indx]) + if not(dat_LI is None): + range_x[0] = np.min([range_x[0], np.min(dat_LI)]) + range_x[1] = np.max([range_x[1], np.max(dat_LI)]) + for x in np.linspace(range_x[0],range_x[1],200): + dat_out.append([x, np.sum( weights[ dat_here< x])/np.sum(weights)]) + # if opts.fname_lalinference: + # tmp = extract_combination_from_LI(p) + # if not (tmp == None) : + # dat_out_LI.append([x, 1.0*sum( tmp np.mean(samples[p])-2*np.std(samples[p]) ): + range_here[-1][0] = np.mean(samples[p])-2*np.std(samples[p]) + # Don't let lower limit get too extreme + if (range_here[-1][0] < np.mean(samples[p])-3*np.std(samples[p]) ): + range_here[-1][0] = np.mean(samples[p])-3*np.std(samples[p]) + # Don't let upper limit get too extreme + if (range_here[-1][1] > np.mean(samples[p])+5*np.std(samples[p]) ): + range_here[-1][1] = np.mean(samples[p])+5*np.std(samples[p]) + if range_here[-1][0] < prior_range_map[p][0]: + range_here[-1][0] = prior_range_map[p][0] + if range_here[-1][1] > prior_range_map[p][1]: + range_here[-1][1] = prior_range_map[p][1] + if opts.fname_lalinference: + # If LI samples are shown, make sure their full posterior range is plotted. + try: + print(p) + p_LI = remap_ILE_2_LI[p] + if range_here[-1][0] > np.min(samples_LI[p_LI]): + range_here[-1][0] = np.min(samples_LI[p_LI]) + if range_here[-1][1] < np.max(samples_LI[p_LI]): + range_here[-1][1] = np.max(samples_LI[p_LI]) + except: + print(" Parameter failure with LI, trying extract_combination_... ") + tmp = extract_combination_from_LI(samples_LI, p) + if range_here[-1][0] > np.min(tmp): + range_here[-1][0] = np.min(tmp) + if range_here[-1][1] < np.max(tmp): + range_here[-1][1] = np.max(tmp) + print(p, range_here[-1]) # print out range to be used in plots. + + if not no_plots: + labels_tex = list(map(lambda x: tex_dictionary[x], low_level_coord_names)) + fig_base = corner.corner(dat_mass[:,:len(low_level_coord_names)], weights=(weights/np.sum(weights)).astype(np.float64),labels=labels_tex, quantiles=quantiles_1d,plot_datapoints=False,plot_density=False,no_fill_contours=True,fill_contours=False,levels=CIs,truths=truth_here,range=range_here) + my_cmap_values = 'g' # default color + if True: + #try: + # Plot simulation points (X array): MAY NOT BE POSSIBLE if dimensionality is inconsistent + cm = plt.cm.get_cmap('RdYlBu_r') + y_span = Y.max() - Y.min() + y_min = Y.min() + # print y_span, y_min + my_cmap_values = map(tuple,cm( (Y-y_min)/y_span) ) + my_cmap_values ='g' + + fig_base = corner.corner(dat_out_low_level_coord_names,weights=np.ones(len(X))/len(X), plot_datapoints=True,plot_density=False,plot_contours=False,quantiles=None,fig=fig_base, data_kwargs={'c':my_cmap_values},hist_kwargs={'color':'g', 'linestyle':'dashed'},range=range_here) + + # TRUNCATED data set used here + indx_ok = Y > Y.max() - scipy.stats.chi2.isf(0.1,len(low_level_coord_names))/2 # approximate threshold for significant points,from inverse cdf 90% + n_ok = np.sum(indx_ok) + fig_base = corner.corner(dat_out_low_level_coord_names[indx_ok],weights=np.ones(n_ok)*1.0/n_ok, plot_datapoints=True,plot_density=False,plot_contours=False,quantiles=None,fig=fig_base, data_kwargs={'c':'b'},hist_kwargs={'color':'b', 'linestyle':'dashed'},range=range_here) + + #except: + else: + print(" Some ridiculous range error with the corner plots, again") + + if opts.fname_lalinference: + try: + corner.corner( dat_mass_LI,color='r',labels=labels_tex,weights=np.ones(len(dat_mass_LI))*1.0/len(dat_mass_LI),fig=fig_base,quantiles=quantiles_1d,no_fill_contours=True,plot_datapoints=False,plot_density=False,fill_contours=False,levels=CIs) #,range=range_here) + except: + print(" Failed !") + plt.legend(handles=line_handles, bbox_to_anchor=corner_legend_location, prop=corner_legend_prop,loc=4) + plt.savefig("posterior_corner_nocut_beware.png"); plt.clf() + + print(" ---- Subset for posterior samples (and further corner work) --- ") + + + # pick random numbers + p_threshold_size = np.min([5*opts.n_output_samples,len(weights)]) + #p_thresholds = np.random.uniform(low=0.0,high=1.0,size=p_threshold_size)#opts.n_output_samples) + if opts.verbose: + print(" output size: selected thresholds N=", p_threshold_size) + # find sample indexes associated with the random numbers + # - FIXME: first truncate the bad ones + #cum_sum = np.cumsum(weights) + #cum_sum = cum_sum/cum_sum[-1] + #indx_list = list(map(lambda x : np.sum(cum_sum < x), p_thresholds)) # this can lead to duplicates + indx_list = np.random.choice(np.arange(len(weights)),p_threshold_size,p=np.array(weights/np.sum(weights),dtype=float),replace=False) + if opts.verbose: + print(" output size: selected random indices N=", len(indx_list)) + if opts.internal_bound_factor_if_n_eff_small and neff downselect_dict[param][1]: + # print " Skipping " , line + # include_item =False + if include_item: + if Pgrid.m2 <= Pgrid.m1: # do not add grid elements with m2> m1, to avoid possible code pathologies ! + P_list.append(Pgrid) + if not(opts.internal_use_lnL): + lnL_list.append(np.log(samples["integrand"][indx_here])) + else: + lnL_list.append(samples["integrand"][indx_here]) + else: + Pgrid.swap_components() # IMPORTANT. This should NOT change the physical functionality FOR THE PURPOSES OF OVERLAP (but will for PE - beware phiref, etc!) + P_list.append(Pgrid) + if not(opts.internal_use_lnL): + lnL_list.append(np.log(samples["integrand"][indx_here])) + else: + lnL_list.append(samples["integrand"][indx_here]) + else: + True + + + # FAILURE MODE: no exportable data + if len(P_list) <1: + raise Exception(" Run failure: no export data! ") + + ### + ### Export data - files 6/7 & 7/7 + ### + n_output_size = np.min([len(P_list),opts.n_output_samples]) + lalsimutils.ChooseWaveformParams_array_to_xml(P_list[:n_output_size],fname=opts.fname_output_samples,fref=P.fref) + lnL_list = np.array(lnL_list,dtype=internal_dtype) + np.savetxt(opts.fname_output_samples+"_lnL.dat", lnL_list) + + + if not opts.no_plots: + Yvals_copy = lnL_list.copy() + Yvals_copy = Yvals_copy[Yvals_copy.argsort()[::-1]] + pvals = np.arange(len(Yvals_copy))*1.0/len(Yvals_copy) + plt.plot(Yvals_copy, pvals) + plt.xlabel(r"$\ln{\cal L}$") + plt.ylabel(r"$\hat{P}(<\ln{\cal L})$") + plt.savefig("lnL_cumulative_distribution_posterior_estimate.png"); plt.clf() + + + ### + ### STOP IF NO MORE PLOTS + ### + if no_plots: + continue #sys.exit(0) + + + ### + ### Identify, save best point + ### + + P_best = P_list[ np.argmax(lnL_list) ] + lalsimutils.ChooseWaveformParams_array_to_xml([P_best], "best_point_by_lnL") + lnL_best = lnL_list[np.argmax(lnL_list)] + np.savetxt("best_point_by_lnL_value.dat", np.array([lnL_best])); + + + + ### + ### Extract data from samples, in array form. INCLUDES any cuts (e.g., kerr limit) + ### + dat_mass_post = np.zeros( (len(P_list),len(coord_names)),dtype=np.float64) + for indx_line in np.arange(len(P_list)): + for indx in np.arange(len(coord_names)): + fac=1 + if coord_names[indx] in ['mc', 'mtot', 'm1', 'm2']: + fac = lal.MSUN_SI + dat_mass_post[indx_line,indx] = P_list[indx_line].extract_param(coord_names[indx])/fac + + + dat_extra_post = [] + for x in np.arange(len(extra_plot_coord_names)): + coord_names_here = extra_plot_coord_names[x] + feature_here = np.zeros( (len(P_list),len(coord_names_here)),dtype=np.float64) + for indx_line in np.arange(len(P_list)): + for indx in np.arange(len(coord_names_here)): + fac=1 + if coord_names_here[indx] in ['mc', 'mtot', 'm1', 'm2']: + fac = lal.MSUN_SI + feature_here[indx_line,indx] = P_list[indx_line].extract_param(coord_names_here[indx])/fac + dat_extra_post.append(feature_here) + + + + range_here=[] + for indx in np.arange(len(coord_names)): + range_here.append( [np.min(dat_mass_post[:, indx]),np.max(dat_mass_post[:, indx])]) + # Manually reset some ranges to be more useful for plotting + if coord_names[indx] in ['xi', 'chi_eff']: + range_here[-1] = [-1,1] + if coord_names[indx] in ['eta']: + range_here[-1] = [0,0.25] + if opts.eta_range: + range_here[-1] = eval(opts.eta_range) + if coord_names[indx] in ['q']: + range_here[-1] = [0,1] + if coord_names[indx] in ['s1z', 's2z']: + range_here[-1] = [-1,1] + if coord_names[indx] in ['chi1_perp', 'chi2_perp']: + range_here[-1] = [0,1] + print(coord_names[indx], range_here[-1]) + + if opts.fname_lalinference: + dat_mass_LI = np.zeros( (len(samples_LI), len(coord_names)), dtype=np.float64) + for indx in np.arange(len(coord_names)): + if coord_names[indx] in remap_ILE_2_LI.keys(): + tmp = extract_combination_from_LI(samples_LI, coord_names[indx]) + if not (tmp is None): + dat_mass_LI[:,indx] = tmp + if coord_names[indx] in ["lambda1", "lambda2"]: + print(" populating ", coord_names[indx], " via _extract ") + dat_mass_LI[:,indx] = extract_combination_from_LI(samples_LI, coord_names[indx]) # requires special extraction technique, since it needs to be converted + + if range_here[indx][0] > np.min(dat_mass_LI[:,indx]): + range_here[indx][0] = np.min(dat_mass_LI[:,indx]) + if range_here[indx][1] < np.max(dat_mass_LI[:,indx]): + range_here[indx][1] = np.max(dat_mass_LI[:,indx]) + + print(" ---- 1d cumulative on fitting coordinates (NOT biased: FROM SAMPLES, including downselect) --- ") + for indx in np.arange(len(coord_names)): + p = coord_names[indx] + dat_out = []; dat_out_LI=[] + print(" -- 1d cumulative "+ str(indx)+ ":"+ coord_names[indx]+" ----") + dat_here = dat_mass_post[:,indx] + wt_here = np.ones(len(dat_here)) + range_x = [np.min(dat_here), np.max(dat_here)] + if opts.fname_lalinference: + dat_LI = extract_combination_from_LI(samples_LI,p) + if not(dat_LI is None): + range_x[0] = np.min([range_x[0], np.min(dat_LI)]) + range_x[1] = np.max([range_x[1], np.max(dat_LI)]) + + for x in np.linspace(range_x[0],range_x[1],200): + dat_out.append([x, np.sum( wt_here[dat_here< x] )/len(dat_here)]) # NO WEIGHTS for these resampled points + # dat_out.append([x, np.sum( weights[ dat_here< x])/np.sum(weights)]) + if opts.fname_lalinference and not (dat_LI is None) : + dat_out_LI.append([x, 1.0*sum( dat_LI Y.max() - scipy.stats.chi2.isf(0.1,len(low_level_coord_names))/2 # approximate threshold for significant points,from inverse cdf 90% + n_ok = np.sum(indx_ok) + fig_base = corner.corner(X[indx_ok],weights=np.ones(n_ok)*1.0/n_ok, plot_datapoints=True,plot_density=False,plot_contours=False,quantiles=None,fig=fig_base, data_kwargs={'c':'r'},hist_kwargs={'color':'b', 'linestyle':'dashed'},range=range_here) + + + plt.legend(handles=line_handles, bbox_to_anchor=corner_legend_location, prop=corner_legend_prop,loc=4) + plt.savefig("posterior_corner_fit_coords.png"); plt.clf() + + except: + #else: + print(" No corner 2") + + + ### + ### Corner plot 3 + ### + print(" ---- Corner 3: Bonus corner plots ---- ") + for indx in np.arange(len(extra_plot_coord_names)): + if True: + # try: + fig_base =None + coord_names_here = extra_plot_coord_names[indx] + str_name = '_'.join(coord_names_here) + print(" Generating corner for ", str_name) + dat_here = dat_extra_post[indx] + dat_points_here = dat_out_extra[indx] + labels_tex = render_coordinates(coord_names_here)#map(lambda x: tex_dictionary[x], coord_names_here) + range_here=[] + dat_mass_LI = None + + can_render_LI = opts.fname_lalinference and ( set(coord_names_here) < (set(samples_LI.dtype.names) | set(remap_ILE_2_LI.keys()) | set(['lambda1','lambda2','LambdaTilde','DeltaLambdaTilde'])) ) + + if can_render_LI: + print(" - LI parameters available for ", coord_names_here) + dat_mass_LI = np.zeros( (len(samples_LI), len(coord_names_here)), dtype=np.float64) + for x in np.arange(len(coord_names_here)): + print(" ... extracting ", coord_names_here[x]) + tmp = extract_combination_from_LI(samples_LI, coord_names_here[x]) + if not (tmp is None): + dat_mass_LI[:,x] = tmp + # print " ..... ", tmp[:3] + else: + print(" ... warning, extraction failed for ", coord_names_here[x]) + for z in np.arange(len(coord_names_here)): + range_here.append( [np.min(dat_points_here[:, z]),np.max(dat_points_here[:, z])]) + if not (opts.plots_do_not_force_large_range): + # Manually reset some ranges to be more useful for plotting + if coord_names_here[z] in ['xi', 'chi_eff']: + range_here[-1] = [-1,1] # can be horrible if we are very informative + if coord_names_here[z] in ['eta']: + range_here[-1] = [0,0.25] # can be horrible if we are very informative + if coord_names_here[z] in ['q']: + range_here[-1] = [0,1] + if coord_names_here[z] in ['s1z', 's2z']: + range_here[-1] = [-1,1] + if coord_names_here[z] in ['chi1_perp', 'chi2_perp']: + range_here[-1] = [0,1] + if opts.fname_lalinference and ( set(coord_names_here) < set(remap_ILE_2_LI.keys())): + range_here[-1][0] = np.min( [np.min( dat_mass_LI[:,z]), range_here[-1][0] ]) + range_here[-1][1] = np.max( [np.max( dat_mass_LI[:,z]), range_here[-1][1] ]) + if coord_names_here[z] in prior_range_map.keys(): + range_here[-1][1] = np.min( [range_here[-1][1], prior_range_map[coord_names_here[z]][1]]) # override the limits, if I have a prior. + + print(' - Range ', coord_names_here[z], range_here[-1]) + + truth_here = [] + for z in np.arange(len(coord_names_here)): + fac=1 + if coord_names_here[z] in ['mc','m1','m2','mtot']: + fac = lal.MSUN_SI + truth_here.append(Pref.extract_param(coord_names_here[z])/fac) + + print(" Truth here for ", coord_names_here, truth_here) + + print(" Generating figure for ", extra_plot_coord_names[indx], " using ", len(dat_here), " from the posterior and ", len(dat_points_here) , len(Y_orig), " from the original data set ") + fig_base = corner.corner(dat_here, weights=np.ones(len(dat_here))*1.0/len(dat_here), labels=labels_tex, quantiles=quantiles_1d,plot_datapoints=False,plot_density=False,no_fill_contours=True,fill_contours=False,levels=CIs,range=range_here,truths=truth_here) + + if can_render_LI: + corner.corner( dat_mass_LI, weights=np.ones(len(dat_mass_LI))*1.0/len(dat_mass_LI), color='r',labels=labels_tex,fig=fig_base,quantiles=quantiles_1d,no_fill_contours=True,plot_datapoints=False,plot_density=False,fill_contours=False,levels=CIs,range=range_here) + + + print(" Rendering past samples for ", extra_plot_coord_names[indx], " based on ", len(dat_points_here)) + fig_base = corner.corner(dat_points_here,weights=np.ones(len(dat_points_here))*1.0/len(dat_points_here), plot_datapoints=True,plot_density=False,plot_contours=False,quantiles=None,fig=fig_base, data_kwargs={'color':'g'},hist_kwargs={'color':'g', 'linestyle':'dashed'},range=range_here) + # Render points available. Note we use the ORIGINAL data set, and truncate it + indx_ok = Y_orig > Y_orig.max() - scipy.stats.chi2.isf(0.1,len(low_level_coord_names))/2 # approximate threshold for significant points,from inverse cdf 90% + n_ok = np.sum(indx_ok) + print(" Adding points for figure ", n_ok, extra_plot_coord_names[indx], " drawn from original ") + fig_base = corner.corner(dat_points_here[indx_ok],weights=np.ones(n_ok)*1.0/n_ok, plot_datapoints=True,plot_density=False,plot_contours=False,quantiles=None,fig=fig_base, data_kwargs={'c':'b'},hist_kwargs={'color':'b', 'linestyle':'dashed'},range=range_here) + + + plt.legend(handles=line_handles, bbox_to_anchor=corner_legend_location, prop=corner_legend_prop,loc=4) + print(" Writing coord ", str_name) + plt.savefig("posterior_corner_extra_coords_"+str_name+".png"); plt.clf() + + # except: + else: + print(" Failed to generate corner for ", extra_plot_coord_names[indx]) + + ################ END OF LOOP ##################### + + +if not (opts.chunk_save): + sys.exit(0) #regular RIFT exit point + +### +### Post-loop saving (for hyperpipe only) - #TODO: could port to save_CIP_output.py, for consistency +### + +#if all EOS lines failed, will be nothing to save #TODO: save low-lnL val file? (NICER code does this) +if len(hyper_out_list) == 0: + print("\n WARNING: no data to save! Will not produce an output file for these lines.") + sys.exit(0) + +#reset opts.fname_output_integral +opts.fname_output_integral = fname_output_integral_orig + +#convert list to np.ndarray +hyper_out_array = np.asarray(hyper_out_list,dtype=np.float64) + +#File 1/7 +if not(opts.save_hyperfile_only) and opts.chunk_save: + lnL_shifted = hyper_out_array[:,0] + lnL_shift + np.savetxt(opts.fname_output_integral+".dat", lnL_shifted) + +#File 2/7: MARG-0-0+annotation.dat +if opts.chunk_save: + # remove invalid lines - unneeded here + #indx_ok = np.ones(len(out_grid),dtype=bool) + #indx_ok = np.logical_and(indx_ok, np.logical_not(np.isnan(out_grid[:,0]))) #check nans (shouldn't happen) + #print(' Ignoring lines with lnL = -inf : {} '.format(len(out_grid)-np.sum(indx_ok))) + #out_grid = out_grid[indx_ok] + try: + tester = annotation_header + except: + fname = opts.using_eos.replace('file:','') + linefirst ='' + with open(fname) as f: + linefirst = f.readline() + annotation_header = linefirst[2:] + #File (2/7): + np.savetxt(opts.fname_output_integral+"+annotation.dat",hyper_out_array,header=annotation_header[:-1]) #skip newline char in header + print("\nChunk file saved; length =",len(hyper_out_array)) + +sys.exit(0) + + diff --git a/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py b/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py new file mode 100644 index 000000000..66820d6d3 --- /dev/null +++ b/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py @@ -0,0 +1,977 @@ +#!/usr/bin/env python +# +# util_ConstructEOSPosterior.py modified for HyperPipe but without RoboCode +# - takes in *generic-format* hyperparameter likelihood data +# - uses *uniform* prior on hyperparameters. [non-uniform priors can be applied by the user with a supplementary function] +# - generates posterior distribution by weighted Monte Carlo +# +# EXAMPLE: +# python `which util_ConstructEOSPosterior.py` --fname fake_int_grid.dat --parameter gamma1 --parameter gamma2 --lnL-offset 50 + +#import RIFT.interpolators.BayesianLeastSquares as BayesianLeastSquares + +import argparse +import sys +import numpy as np +import numpy.lib.recfunctions +import scipy +#import scipy.stats +#import functools +#import itertools + +import joblib # http://scikit-learn.org/stable/modules/model_persistence.html + +# GPU acceleration: NOT YET, just do usual +xpy_default=numpy # just in case, to make replacement clear and to enable override +identity_convert = lambda x: x # trivial return itself +cupy_success=False + +no_plots = True +internal_dtype = np.float32 # only use 32 bit storage! Factor of 2 memory savings for GP code in high dimensions + + +# ============================================================================= +# try: +# import matplotlib.pyplot as plt +# from mpl_toolkits.mplot3d import Axes3D +# import matplotlib.lines as mlines +# import corner +# +# no_plots=False +# except ImportError: +# print(" - no matplotlib - ") +# ============================================================================= + + +from sklearn.preprocessing import PolynomialFeatures +if True: +#try: + import RIFT.misc.ModifiedScikitFit as msf # altenative polynomialFeatures +else: +#except: + print(" - Faiiled ModifiedScikitFit : No polynomial fits - ") +from sklearn import linear_model + +from igwn_ligolw import lsctables, utils, ligolw +lsctables.use_in(ligolw.LIGOLWContentHandler) + +import RIFT.integrators.mcsampler as mcsampler +try: + import RIFT.integrators.mcsamplerEnsemble as mcsamplerEnsemble + mcsampler_gmm_ok = True +except: + print(" No mcsamplerEnsemble ") + mcsampler_gmm_ok = False +try: + import RIFT.integrators.mcsamplerGPU as mcsamplerGPU + mcsampler_gpu_ok = True + mcsamplerGPU.xpy_default =xpy_default # force consistent, in case GPU present + mcsamplerGPU.identity_convert = identity_convert +except: + print( " No mcsamplerGPU ") + mcsampler_gpu_ok = False +try: + import RIFT.integrators.mcsamplerAdaptiveVolume as mcsamplerAdaptiveVolume + mcsampler_AV_ok = True +except: + print(" No mcsamplerAV ") + mcsampler_AV_ok = False +try: + import RIFT.integrators.mcsamplerPortfolio as mcsamplerPortfolio + mcsampler_Portfolio_ok = True +except: + print(" No mcsamplerPortolfio ") + + + + + +def add_field(a, descr): + """Return a new array that is like "a", but has additional fields. + + Arguments: + a -- a structured numpy array + descr -- a numpy type description of the new fields + + The contents of "a" are copied over to the appropriate fields in + the new array, whereas the new fields are uninitialized. The + arguments are not modified. + + >>> sa = numpy.array([(1, 'Foo'), (2, 'Bar')], \ + dtype=[('id', int), ('name', 'S3')]) + >>> sa.dtype.descr == numpy.dtype([('id', int), ('name', 'S3')]) + True + >>> sb = add_field(sa, [('score', float)]) + >>> sb.dtype.descr == numpy.dtype([('id', int), ('name', 'S3'), \ + ('score', float)]) + True + >>> numpy.all(sa['id'] == sb['id']) + True + >>> numpy.all(sa['name'] == sb['name']) + True + """ + if a.dtype.fields is None: + raise ValueError("`A' must be a structured numpy array") + b = numpy.empty(a.shape, dtype=a.dtype.descr + descr) + for name in a.dtype.names: + b[name] = a[name] + return b + + +parser = argparse.ArgumentParser() +parser.add_argument("--fname",help="filename of *.dat file (EOS-format: lnL sigma_lnL p1 p2 ... . ASSUME any stacking over events already performed.") +parser.add_argument("--fname-output-samples",default="output-EOS-samples",help="output grid") +parser.add_argument("--fname-output-integral",default="output-EOS-samples",help="for evidencees and pipeline compatibility") +parser.add_argument("--n-output-samples",default=2000,type=int,help="output posterior samples (default 3000)") +parser.add_argument("--eos-param", type=str, default=None, help="parameterization of equation of state [spectral only, for now]") +parser.add_argument("--parameter", action='append', help="Parameters used as fitting parameters AND varied at a low level to make a posterior. Currently can only specify gamma1,gamma2, ..., and these MUST be columns in --fname. IF NOT PROVIDED, DEFAULTS TO LIST IN FILE. ") +parser.add_argument("--parameter-implied", action='append', help="Parameter used in fit, but not independently varied for Monte Carlo. For EOS objects, only possible for physical quantities like R1.4, etc. NOT YET PROVIDED") +#parser.add_argument("--no-adapt-parameter",action='append',help="Disable adaptive sampling in a parameter. Useful in cases where a parameter is not well-constrained, and the a prior sampler is well-chosen.") +parser.add_argument("--parameter-nofit", action='append', help="Parameter used to initialize the implied parameters, and varied at a low level, but NOT the fitting parameters.") +parser.add_argument("--integration-parameter-range",action='append', help="Integration parameter ranges. Syntax is name:[a,b]") +parser.add_argument("--downselect-parameter",action='append', help='Name of parameter to be used to eliminate grid points ') +parser.add_argument("--downselect-parameter-range",action='append',type=str) +parser.add_argument("--no-downselect",action='store_true') +parser.add_argument("--aligned-prior", default="uniform",help="Options are 'uniform', 'volumetric', and 'alignedspin-zprior'") +parser.add_argument("--cap-points",default=-1,type=int,help="Maximum number of points in the sample, if positive. Useful to cap the number of points ued for GP. See also lnLoffset. Note points are selected AT RANDOM") +parser.add_argument("--lambda-max", default=4000,type=float,help="Maximum range of 'Lambda' allowed. Minimum value is ZERO, not negative.") +parser.add_argument("--lnL-shift-prevent-overflow",default=None,type=float,help="Define this quantity to be a large positive number to avoid overflows. Note that we do *not* define this dynamically based on sample values, to insure reproducibility and comparable integral results. BEWARE: If you shift the result to be below zero, because the GP relaxes to 0, you will get crazy answers.") +parser.add_argument("--lnL-offset",type=float,default=np.inf,help="lnL offset") +parser.add_argument("--lnL-cut",type=float,default=None,help="lnL cut [MANUAL]") +parser.add_argument("--sigma-cut",type=float,default=0.6,help="Eliminate points with large error from the fit.") +parser.add_argument("--ignore-errors-in-data",action='store_true',help='Ignore reported error in lnL. Helpful for testing purposes (i.e., if the error is zero)') +parser.add_argument("--lnL-peak-insane-cut",type=float,default=np.inf,help="Throw away lnL greater than this value. Should not be necessary") +parser.add_argument("--verbose", action="store_true",default=False, help="Required to build post-frame-generating sanity-test plots") +parser.add_argument("--save-plots",default=False,action='store_true', help="Write plots to file (only useful for OSX, where interactive is default") +parser.add_argument("--n-max",default=3e5,type=float) +parser.add_argument("--n-step",default=1e5,type=int) +parser.add_argument("--n-eff",default=3e3,type=int) +parser.add_argument("--pool-size",default=3,type=int,help="Integer. Number of GPs to use (result is averaged)") +parser.add_argument("--fit-method",default="rf",help="rf (default) : rf|gp|quadratic|polynomial|gp_hyper|gp_lazy|cov|kde. Note 'polynomial' with --fit-order 0 will fit a constant") +parser.add_argument("--fit-load-gp",default=None,type=str,help="Filename of GP fit to load. Overrides fitting process, but user MUST correctly specify coordinate system to interpret the fit with. Does not override loading and converting the data.") +parser.add_argument("--fit-save-gp",default=None,type=str,help="Filename of GP fit to save. ") +parser.add_argument("--fit-order",type=int,default=2,help="Fit order (polynomial case: degree)") +parser.add_argument("--no-plots",action='store_true') +parser.add_argument("--using-eos-type", type=str, default=None, help="Name of EOS parameterization (must match what is used for inputs). Will use EOS parameterization to identify appropriate field headers") +parser.add_argument("--sampler-method",default="adaptive_cartesian",help="adaptive_cartesian|GMM|adaptive_cartesian_gpu") +parser.add_argument("--sampler-portfolio",default=None,action='append',type=str,help="comma-separated strings, matching sampler methods other than portfolio") +parser.add_argument("--sampler-portfolio-args",default=None, action='append', type=str, help='eval-able dictionary to be passed to that sampler_') +parser.add_argument("--internal-use-lnL",action='store_true',help="integrator internally manipulates lnL.. ") +parser.add_argument("--internal-correlate-parameters",default=None,type=str,help="comman-separated string indicating parameters that should be sampled allowing for correlations. Must be sampling parameters. Only implemented for gmm. If string is 'all', correlate *all* parameters") +parser.add_argument("--internal-n-comp",default=1,type=int,help="number of components to use for GMM sampling. Default is 1, because we expect a unimodal posterior in well-adapted coordinates. If you have crappy coordinates, use more") +parser.add_argument("--force-no-adapt",action='store_true',help="Disable adaptation, both of the tempering exponent *and* the individual sampling prior(s)") +parser.add_argument("--tripwire-fraction",default=0.05,type=float,help="Fraction of nmax of iterations after which n_eff needs to be greater than 1+epsilon for a small number epsilon") + +# Supplemental likelihood factors: convenient way to effectively change the mass/spin prior in arbitrary ways for example +# Note this supplemental factor is passed the *fitting* arguments, directly. Use with extreme caution, since we often change the dimension in a DAG +parser.add_argument("--supplementary-likelihood-factor-code", default=None,type=str,help="Import a module (in your pythonpath!) containing a supplementary factor for the likelihood. Used to impose supplementary external priors of arbitrary complexity and external dependence (e.g., imposing alternate EOS priors)") +parser.add_argument("--supplementary-likelihood-factor-function", default=None,type=str,help="With above option, specifies the specific function used as an external likelihood. EXPERTS ONLY") +parser.add_argument("--supplementary-likelihood-factor-ini", default=None,type=str,help="With above option, specifies an ini file that is parsed (here) and passed to the preparation code, called when the module is first loaded, to configure the module. EXPERTS ONLY") +parser.add_argument("--supplementary-coordinate-code", default=None,type=str,help="Coordinate conversion/prior code. Accepts: the literal 'rift_default' (use RIFT.lalsimutils.convert_waveform_coordinates plus RIFT-standard priors); a filesystem path ending in .py (loaded as a plugin); or any importable dotted module name.") +parser.add_argument("--supplementary-coordinate-function", default=None, type=str, help="Name of the entry-point callable inside the module named by --supplementary-coordinate-code. Defaults to 'convert_coordinates'.") + +opts= parser.parse_args() + +#print(" WARNING: Always use internal_use_lnL for now ") +#opts.internal_use_lnL=True + +no_plots = no_plots | opts.no_plots +lnL_shift = 0 +lnL_default_large_negative = -500 +if opts.lnL_shift_prevent_overflow: + lnL_shift = opts.lnL_shift_prevent_overflow + + + +### Comparison data (from LI) +### + +downselect_dict = {} +dlist = [] +dlist_ranges=[] +if opts.downselect_parameter: + dlist = opts.downselect_parameter + dlist_ranges = map(eval,opts.downselect_parameter_range) +else: + dlist = [] + dlist_ranges = [] +if len(dlist) != len(dlist_ranges): + print(" downselect parameters inconsistent", dlist, dlist_ranges) +for indx in np.arange(len(dlist_ranges)): + downselect_dict[dlist[indx]] = dlist_ranges[indx] + +if opts.no_downselect: + downselect_dict={} + + +test_converged={} + +### +### Retrieve data +### +# int_sig sigma/L gamma1 gamma2 ... +col_lnL = 0 +dat_orig = dat = np.loadtxt(opts.fname) #all.net, same format as grid/consolidated files +dat_orig = dat[dat[:,col_lnL].argsort()] # sort http://stackoverflow.com/questions/2828059/sorting-arrays-in-numpy-by-column +print(" Original data size = ", len(dat), dat.shape) +dat_orig_names = None +with open(opts.fname,'r') as f: + header_str = f.readline() + header_str = header_str.rstrip() +dat_orig_names = header_str.replace('#','').split()[2:] + +### +### Parameters in use +### + +#want: + #if opts.parameter -> both fit & sampling + #if opts.param_implied -> just fit + #if opts.param_nofit -> just sampling + #if no opts.parameter -> coord_names & l_l_coord_names = dat_orig_names +coord_names = opts.parameter # coords for both fit and MC sampling +if coord_names is None: + coord_names = dat_orig_names +low_level_coord_names = coord_names # i.e., sampling coords same as data col coords + +if opts.parameter_implied: + coord_names = coord_names+opts.parameter_implied # coords for fit - wrong if opts.parameter=None + +if opts.parameter_nofit: + if opts.parameter is None: + low_level_coord_names = opts.parameter_nofit # coords for MC sampling + else: + low_level_coord_names = opts.parameter+opts.parameter_nofit # coords for MC sampling + +error_factor = len(coord_names) +name_index_dict ={} +for name in dat_orig_names: + try: + name_index_dict[name] = 2+dat_orig_names.index(name) + except: + raise Exception(" Currently fitting parameter names must match columns in data file ") +# TeX dictionary +print(" Coordinate names for fit :, ", coord_names, " from ", dat_orig_names, " indexed as ", name_index_dict) +print(" Coordinate names for Monte Carlo :, ", low_level_coord_names) + + +### +### Integration ranges +### + +param_ranges = {} +for range_code in opts.integration_parameter_range: + name, range_str = range_code.split(':') + range_expr = eval(range_str) # define. Better to split on , for example + param_ranges[name] = np.array(range_expr) + +# Add in integration range for everything else, if nothing specified +for name in dat_orig_names: + if not name in param_ranges: + vals = dat_orig[:,name_index_dict[name]] + param_ranges[name] = [np.min(vals), np.max(vals)] + +### +### Prior functions : default is UNIFORM, since it is unmodeled and generic +### + +def uniform_prior(x): + return np.ones(x.shape) + +prior_map = {} +for name in low_level_coord_names: + prior_map[name] = uniform_prior + if not(name in param_ranges): + raise Exception(" {} not provided a parameter range ".format(name)) # change later, should fall back to using prior range from above + + +prior_range_map = param_ranges + +# prior_map = { 'gamma1':eos_param_uniform_prior, 'gamma2':eos_param_uniform_prior, +# } +# # Les: somewhat more aggressive: +# # gamma1: 0.2,2 +# # gamma2: -1.67, 1.7 +# prior_range_map = { 'gamma1': [0.707899,1.31], 'gamma2':[-1.6,1.7], 'gamma3':[-0.6,0.6], 'gamma4':[-0.02,0.02] +# } + + +### +### Supplemental likelihood: load (as in ILE) +### +supplemental_ln_likelihood= None +supplemental_ln_likelihood_prep =None +supplemental_ln_likelihood_parsed_ini=None +# Supplemental likelihood factor. Must have identical call sequence to 'likelihood_function'. Called with identical raw inputs (including cosines/etc) +if opts.supplementary_likelihood_factor_code and opts.supplementary_likelihood_factor_function: + print(" EXTERNAL SUPPLEMENTARY LIKELIHOOD FACTOR : {}.{} ".format(opts.supplementary_likelihood_factor_code,opts.supplementary_likelihood_factor_function)) + __import__(opts.supplementary_likelihood_factor_code) + external_likelihood_module = sys.modules[opts.supplementary_likelihood_factor_code] + supplemental_ln_likelihood = getattr(external_likelihood_module,opts.supplementary_likelihood_factor_function) + name_prep = "prepare_"+opts.supplementary_likelihood_factor_function + if hasattr(external_likelihood_module,name_prep): + supplemental_ln_likelhood_prep=getattr(external_likelihood_module,name_prep) + # Check for and load in ini file associated with external library + if opts.supplementary_likelihood_factor_ini: + import configparser as ConfigParser + config = ConfigParser.ConfigParser() + config.optionxform=str # force preserve case! + config.read(opts.supplementary_likelihood_factor_ini) + supplemental_ln_likelhood_parsed_ini=config + + # Call the ini file, tell it what coordinates we are using by name + supplemental_ln_likelihood_prep(config=supplemental_ln_likelihood_parsed_ini,coords=coord_names) + + +supplemental_coordinate_convert = None +supplemental_coordinate_invert = None +if opts.supplementary_coordinate_code and opts.supplementary_coordinate_function: + ''' + The robot wants: The loader accepts three forms in --supplementary-coordinate-code: + -the literal 'rift_default', + -a filesystem path to a .py file, or + -an importable dotted module name. + The plugin must expose a callable named by --supplementary-coordinate-function + with args (x_in, coord_names, low_level_coord_names, **kwargs) that returns + a 2-D ndarray of shape (N, len(coord_names)). + Robot supposedly wants to allow an optional prepare() (one-shot setup, gets + the parsed ini and active coord_name lists) and a register_priors() (mutate + prior_map in place) - see RIFT.misc.coordinate_plugin. I don't care for this. + ''' + print(" EXTERNAL COORDINATE CONVERSION : {}.{} ".format(opts.supplementary_coordinate_code,opts.supplementary_coordinate_function)) + __import__(opts.supplementary_coordinate_code) + external_coordinate_module = sys.modules[opts.supplementary_coordinate_code] + if hasattr(external_coordinate_module,opts.supplementary_coordinate_function): + supplemental_coordinate_convert = getattr(external_coordinate_module,opts.supplementary_coordinate_function) + + if coord_names == low_level_coord_names: #no param_implied or param_nofit + print(" All parameters match; assuming full conversion & requiring inverted conversion.") + try: + supplemental_coordinate_invert = getattr(external_coordinate_module, "inverse_"+opts.supplementary_coordinate_function) + except: + print(" ERROR: no inverted coordinate conversion routine found!") + supplemental_coordinate_invert = None + # Send X to get converted, get converted X back -> do this later + else: + print("ERROR: could not retrieve supplied coordinate function. Will attempt default conversion (none).") + + + +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import RBF, WhiteKernel, ConstantKernel as C + +def adderr(y): + val,err = y + return val+error_factor*err + +def fit_gp(x,y,x0=None,symmetry_list=None,y_errors=None,hypercube_rescale=False,fname_export="gp_fit"): + """ + x = array so x[0] , x[1], x[2] are points. + """ + + # If we are loading a fit, override everything else + if opts.fit_load_gp: + print(" WARNING: Do not re-use fits across architectures or versions : pickling is not transferrable ") + my_gp=joblib.load(opts.fit_load_gp) + return lambda x:my_gp.predict(x) + + # Amplitude: + # - We are fitting lnL. + # - We know the scale more or less: more than 2 in the log is bad + # Scale + # - because of strong correlations with chirp mass, the length scales can be very short + # - they are rarely very long, but at high mass can be long + # - I need to allow for a RANGE + + length_scale_est = [] + length_scale_bounds_est = [] + for indx in np.arange(len(x[0])): + # These length scales have been tuned by expereience + length_scale_est.append( 2*np.std(x[:,indx]) ) # auto-select range based on sampling retained + length_scale_min_here= np.max([1e-3,0.2*np.std(x[:,indx]/np.sqrt(len(x)))]) + length_scale_bounds_est.append( (length_scale_min_here , 5*np.std(x[:,indx]) ) ) # auto-select range based on sampling *RETAINED* (i.e., passing cut). Note that for the coordinates I usually use, it would be nonsensical to make the range in coordinate too small, as can occasionally happens + + print(" GP: Input sample size ", len(x), len(y)) + print(" GP: Estimated length scales ") + print(length_scale_est) + print(length_scale_bounds_est) + + if not (hypercube_rescale): + # These parameters have been hand-tuned by experience to try to set to levels comparable to typical lnL Monte Carlo error + kernel = WhiteKernel(noise_level=0.1,noise_level_bounds=(1e-2,1))+C(0.5, (1e-3,1e1))*RBF(length_scale=length_scale_est, length_scale_bounds=length_scale_bounds_est) + gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=8) + + gp.fit(x,y) + + print(" Fit: std: ", np.std(y - gp.predict(x)), "using number of features ", len(y)) + + if opts.fit_save_gp: + print(" Attempting to save fit ", opts.fit_save_gp+".pkl") + joblib.dump(gp,opts.fit_save_gp+".pkl") + + return lambda x: gp.predict(x) + else: + x_scaled = np.zeros(x.shape) + x_center = np.zeros(len(length_scale_est)) + x_center = np.mean(x) + print(" Scaling data to central point ", x_center) + for indx in np.arange(len(x)): + x_scaled[indx] = (x[indx] - x_center)/length_scale_est # resize + + kernel = WhiteKernel(noise_level=0.1,noise_level_bounds=(1e-2,1))+C(0.5, (1e-3,1e1))*RBF( len(x_center), (1e-3,1e1)) + gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=8) + + gp.fit(x_scaled,y) + print(" Fit: std: ", np.std(y - gp.predict(x_scaled)), "using number of features ", len(y)) # should NOT be perfect + + return lambda x,x0=x_center,scl=length_scale_est: gp.predict( (x-x0 )/scl) + +def map_funcs(func_list,obj): + return [func(obj) for func in func_list] +def fit_gp_pool(x,y,n_pool=10,**kwargs): + """ + Split the data into 10 parts, and return a GP that averages them + """ + x_copy = np.array(x) + y_copy = np.array(y) + indx_list =np.arange(len(x_copy)) + np.random.shuffle(indx_list) # acts in place + partition_list = np.array_split(indx_list,n_pool) + gp_fit_list =[] + for part in partition_list: + print(" Fitting partition ") + gp_fit_list.append(fit_gp(x[part],y[part],**kwargs)) + fn_out = lambda x: np.mean( map_funcs( gp_fit_list,x), axis=0) + print(" Testing ", fn_out([x[0]])) + return fn_out + + +def fit_rf(x,y,y_errors=None,fname_export='nn_fit'): +# from sklearn.ensemble import RandomForestRegressor + from sklearn.ensemble import ExtraTreesRegressor + # Instantiate model. Usually not that many structures to find, don't overcomplicate + # - should scale like number of samples + rf = ExtraTreesRegressor(n_estimators=100, verbose=True,n_jobs=-1) # no more than 5% of samples in a leaf + if y_errors is None: + rf.fit(x,y) + else: + rf.fit(x,y,sample_weight=1./y_errors**2) + + ### reject points with infinities : problems for inputs + def fn_return(x_in,rf=rf): + f_out = -lnL_default_large_negative*np.ones(len(x_in)) + # remove infinity or Nan + indx_ok = np.all(np.isfinite(np.array(x_in,dtype=float)),axis=-1) + # rf internally uses float32, so we need to remove points > 10^37 or so ! + # ... this *should* never happen due to bounds constraints, but ... + indx_ok_size = np.all( np.logical_not(np.greater(np.abs(x_in),1e37)), axis=-1) + indx_ok = np.logical_and(indx_ok, indx_ok_size) + f_out[indx_ok] = rf.predict(x_in[indx_ok]) + return f_out +# fn_return = lambda x_in: rf.predict(x_in) + + print( " Demonstrating RF") # debugging + residuals = rf.predict(x)-y + print( " std ", np.std(residuals), np.max(y), np.max(fn_return(x))) + return fn_return + + + + + +# initialize +dat_mass = [] +weights = [] +n_params = -1 + + + ### + ### Convert data. RIGHT NOW JUST DOWNSELECTING, no intermediate fitting parameters defined + ### + +# Naive convert: no downselect. +if supplemental_coordinate_convert is None: + #old functionality + if list(low_level_coord_names) != list(coord_names): + raise ValueError(" ERROR: fit basis ({}) differs from MC sampling basis ({}), but no --supplementary-coordinate-code was supplied.".format(list(coord_names),list(low_level_coord_names))) + + indx_of_orig_names = np.array([ dat_orig_names.index(coord_names[k]) for k in range(len(coord_names))]) + + dat_out = [] + for line in dat: + dat_here= np.zeros(len(coord_names)+2) + if line[col_lnL+1] > opts.sigma_cut: + print("skipping", line) + continue + dat_here[:-2] = line[indx_of_orig_names+2]#line[2:len(coord_names)+2] # modify to use names! + dat_here[-2] = line[0] + dat_here[-1] = line[1] + dat_out.append(dat_here) + dat_out= np.array(dat_out) + + # Repack data #TODO: check this, move outside if block if both routes need it! + X =dat_out[:,0:len(coord_names)] + Y = dat_out[:,-2] + if np.max(Y)<0 and lnL_shift ==0: + lnL_shift = -100 - np.max(Y) # force it to be offset/positive -- may help some configurations. Remember our adaptivity is silly. + Y_err = dat_out[:,-1] + def convert_coords(x): #no conversion in play here + return x +else: + # Pack data using coordinate converter. Note, if not fully operating in other coord sys. + #later calculations MUST use the converter. + ''' + # The robot says: Two distinct call sites for the converter, with two different + # input bases -- this is the change that decouples the fit from + # the MC sampling basis: + # (1) The initial dat->X conversion below feeds rows whose + # columns are ordered by dat_orig_names (the data file's + # header). So we pass low_level_coord_names=dat_orig_names + # at this site. + # (2) The convert_coords def is called by the integrator on every MC + sample (like CIP). The sampler operates in low_level_coord_names, + so the def must claim its inputs are in low_level_coord_names, NOT + dat_orig_names. + Used to be hardcoded to dat_orig_names, which worked in legacy + case (low_level_coord_names == dat_orig_names). For any non-trivial + plugin where the MC samples in a different basis than the file cols, + # the old behaviour applied the rotation an extra time, which is wrong. + ''' + indx_of_orig_names = np.array([ dat_orig_names.index(coord_names[k]) for k in range(len(coord_names))]) + + dat_out = [] + for line in dat: + dat_here= np.zeros(len(coord_names)+2) + if line[col_lnL+1] > opts.sigma_cut: + print("skipping", line) + continue + dat_here[2:] = line[indx_of_orig_names+2]#line[2:len(coord_names)+2] # modify to use names! + dat_here[0] = line[0] + dat_here[1] = line[1] + dat_out.append(dat_here) + dat_out= np.array(dat_out) + X = supplemental_coordinate_convert(dat_out[:,2:], coord_names=dat_orig_names) # convert and generate X + Y = dat_out[:,0] + Y_err = dat_out[:,1] + if np.max(Y)<0 and lnL_shift ==0: + lnL_shift = -100 - np.max(Y) # force it to be offset/positive -- may help some configurations. Remember our adaptivity is silly. + if supplemental_coordinate_invert is None: + def convert_coords(x_in, _low=low_level_coord_names, _coord=coord_names): + # _low / _coord captured as defaults so the closure stays correct even if either list mutates later in the script. + return supplemental_coordinate_convert(x_in, coord_names=_low) + else: #doing everything in converted coords, so no need to convert again + def convert_coords(x): + return x + +# Save copies for later (plots) +X_orig = X.copy() +Y_orig = Y.copy() + + + +# Eliminate values with Y too small +max_lnL = np.max(Y) +if np.isinf(opts.lnL_offset): + indx_ok= np.ones(len(Y),dtype=bool) # default case, we preserve all the data +else: + indx_ok = np.array(Y>np.max(Y)-opts.lnL_offset,dtype=bool) # force cast : sometimes indx_ok is a mappable object? +n_ok = np.sum(indx_ok) +# Provide some random points, to insure reasonable tapering behavior away from the sample +print(" Points used in fit : ", n_ok, " out of ", len(indx_ok), " given max lnL ", max_lnL) +if max_lnL < 10 and np.mean(Y) > -10: # second condition to allow synthetic tests not to fail, as these often have maxlnL not large + print(" Resetting to use ALL input data -- beware ! ") + # nothing matters, we will reject it anyways + indx_ok = np.ones(len(Y),dtype=bool) +elif n_ok < 10: # and max_lnL > 30: + # mark the top 10 elements and use them for fits + # this may be VERY VERY DANGEROUS if the peak is high and poorly sampled + idx_sorted_index = np.lexsort((np.arange(len(Y)), Y)) # Sort the array of Y, recovering index values + indx_list = np.array( [[k, Y[k]] for k in idx_sorted_index]) # pair up with the weights again + indx_list = indx_list[::-1] # reverse, so most significant are first + indx_ok = list(map(int,indx_list[:10,0])) + print(" Revised number of points for fit: ", np.sum(indx_ok), len(indx_ok), indx_list[:10]) +X_raw = X.copy() + +my_fit= None +if opts.fit_method =='gp': + print(" FIT METHOD : GP") + # some data truncation IS used for the GP, but beware + print(" Truncating data set used for GP, to reduce memory usage needed in matrix operations") + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + if opts.ignore_errors_in_data: + Y_err=None + my_fit = fit_gp(X,Y,y_errors=Y_err) +elif opts.fit_method == 'rf': + print( " FIT METHOD ", opts.fit_method, " IS RF ") + # NO data truncation for NN needed? To be *consistent*, have the code function the same way as the others + X=X[indx_ok] + Y=Y[indx_ok] - lnL_shift + Y_err = Y_err[indx_ok] + # Cap the total number of points retained, AFTER the threshold cut + if opts.cap_points< len(Y) and opts.cap_points> 100: + n_keep = opts.cap_points + indx = np.random.choice(np.arange(len(Y)),size=n_keep,replace=False) + Y=Y[indx] + X=X[indx] + Y_err=Y_err[indx] + if opts.ignore_errors_in_data: + Y_err=None + my_fit = fit_rf(X,Y,y_errors=Y_err) + + +# Sort for later convenience (scatterplots, etc) +indx = Y.argsort()#[::-1] +X=X[indx] +Y=Y[indx] + + + +### +### Integrate posterior +### + + +sampler = mcsampler.MCSampler() +if opts.sampler_method == "adaptive_cartesian_gpu": + sampler = mcsamplerGPU.MCSampler() + sampler.xpy = xpy_default + sampler.identity_convert=identity_convert + mcsampler = mcsamplerGPU # force use of routines in that file, for properly configured GPU-accelerated code as needed + + # if opts.sampler_xpy == "numpy": + # mcsampler.set_xpy_to_numpy() + # sampler.xpy= numpy + # sampler.identity_convert= lambda x: x +if opts.sampler_method == "GMM": + sampler = mcsamplerEnsemble.MCSampler() +elif opts.sampler_method == "AV": + sampler = mcsamplerAdaptiveVolume.MCSampler() + opts.internal_use_lnL= True # required! +elif opts.sampler_method == "portfolio": + use_portfolio=True + sampler = None + sampler_list = [] + sampler_types = opts.sampler_portfolio + for name in sampler_types: + if name =='AV': + sampler = mcsamplerAdaptiveVolume.MCSampler() + if name =='GMM': + sampler = mcsamplerEnsemble.MCSampler() + opts.sampler_method = 'GMM' # this will force the creation/parsing of GMM-specific arguments below, so they are properly passed + if name == "adaptive_cartesian_gpu": + sampler = mcsamplerGPU.MCSampler() + sampler.xpy = xpy_default + sampler.identity_convert=identity_convert + if name == 'NFlow': + # expensive import, only do if requested + try: + import RIFT.integrators.mcsamplerNFlow as mcsamplerNFlow + mcsampler_NF_ok = True + except: + print(" No mcsamplerNFlow ") + continue + sampler = mcsamplerNFlow.MCSampler() + sampler.xpy = xpy_default + sampler.identity_convert=identity_convert + if sampler is None: + # Don't add unknown type + continue + print('PORTFOLIO: adding {} '.format(name)) + sampler_list.append(sampler) + sampler = mcsamplerPortfolio.MCSampler(portfolio=sampler_list) + + +## +## Loop over param names +## +#iterate over low_level_coord_names (sampler basis), not coord_names (fit basis) +#In legacy case the lists are equal so it doesn't matter +for p in low_level_coord_names: + prior_here = prior_map[p] + range_here = prior_range_map[p] + + sampler.add_parameter(p, pdf=np.vectorize(lambda x:1), prior_pdf=prior_here,left_limit=range_here[0],right_limit=range_here[1],adaptive_sampling=True) + +likelihood_function = None +log_likelihood_function = None +#def convert_coords(x): +# return x +def log_likelihood_function(*args): + return my_fit(convert_coords(np.array([*args]).T )) + +if len(low_level_coord_names) ==1: + def likelihood_function(x): + if isinstance(x,float): + return np.exp(my_fit([x])) + else: + return np.exp(my_fit(convert_coords(np.c_[x]))) +if len(low_level_coord_names) ==2: + def likelihood_function(x,y): + if isinstance(x,float): + return np.exp(my_fit([x,y])) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y],dtype=internal_dtype).T))) + return np.exp(my_fit(convert_coords(np.c_[x,y]))) +if len(low_level_coord_names) ==3: + def likelihood_function(x,y,z): + if isinstance(x,float): + return np.exp(my_fit([x,y,z])) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y,z],dtype=internal_dtype).T))) + return np.exp(my_fit(convert_coords(np.c_[x,y,z]))) +if len(low_level_coord_names) ==4: + def likelihood_function(x,y,z,a): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a])) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y,z,a],dtype=internal_dtype).T))) + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a]))) +if len(low_level_coord_names) ==5: + def likelihood_function(x,y,z,a,b): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b])) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y,z,a,b],dtype=internal_dtype).T))) + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b]))) +if len(low_level_coord_names) ==6: + def likelihood_function(x,y,z,a,b,c): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b,c])) + else: +# return np.exp(my_fit(convert_coords(np.array([x,y,z,a,b,c],dtype=internal_dtype).T))) + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b,c]))) +if len(low_level_coord_names) ==7: + def likelihood_function(x,y,z,a,b,c,d): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b,c,d])) + else: + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b,c,d]))) +if len(low_level_coord_names) ==8: + def likelihood_function(x,y,z,a,b,c,d,e): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b,c,d,e])) + else: + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b,c,d,e]))) +if len(low_level_coord_names) ==9: + def likelihood_function(x,y,z,a,b,c,d,e,f): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b,c,d,e,f])) + else: + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b,c,d,e,f]))) +if len(low_level_coord_names) ==10: + def likelihood_function(x,y,z,a,b,c,d,e,f,g): + if isinstance(x,float): + return np.exp(my_fit([x,y,z,a,b,c,d,e,f,g])) + else: + return np.exp(my_fit(convert_coords(np.c_[x,y,z,a,b,c,d,e,f,g]))) + + + + +n_step = opts.n_step +my_exp = np.min([1,0.8*np.log(n_step)/np.max(Y)]) # target value : scale to slightly sublinear to (n_step)^(0.8) for Ymax = 200. This means we have ~ n_step points, with peak value wt~ n_step^(0.8)/n_step ~ 1/n_step^(0.2), limiting contrast +if np.max(Y_orig) < 0: # for now, don't use a weight exponent if we are negative: can't use guess based from GW experience + my_exp = 1 +#my_exp = np.max([my_exp, 1/np.log(n_step)]) # do not allow extreme contrast in adaptivity, to the point that one iteration will dominate +print(" Weight exponent ", my_exp, " and peak contrast (exp)*lnL = ", my_exp*np.max(Y), "; exp(ditto) = ", np.exp(my_exp*np.max(Y)), " which should ideally be no larger than of order the number of trials in each epoch, to insure reweighting doesn't select a single preferred bin too strongly. Note also the floor exponent also constrains the peak, de-facto") + + +extra_args={} +if opts.sampler_method == "GMM": + n_max_blocks = ((1.0*int(opts.n_max))/n_step) + n_comp = opts.internal_n_comp # default + def parse_corr_params(my_str): + """ + Takes a string with no spaces, and returns a tuple + """ + corr_param_names = my_str.replace(',',' ').split() + corr_param_indexes = [] + for param in corr_param_names: + try: + indx = low_level_coord_names.index(param) + corr_param_indexes.append(indx) + except: + continue + return tuple(corr_param_indexes) + if opts.internal_correlate_parameters == 'all': + gmm_dict = {tuple(range(len(low_level_coord_names))):None} # integrate *jointly* in all parameters together + elif not (opts.internal_correlate_parameters is None): + # Correlate identified parameters + my_blocks = opts.internal_correlate_parameters.split() + my_tuples = list(map( parse_corr_params, my_blocks)) + gmm_dict = {x:None for x in my_tuples} + print(" GMM: Proposed correlated ", gmm_dict) + # What about un-labelled parameters? Make a null tuple for them as well + correlated_params = set(); correlated_params = correlated_params.union( *list(map(set,my_tuples))) + uncorrelated_params = set(np.arange(len(low_level_coord_names))); + uncorrelated_params = uncorrelated_params.difference(correlated_params) + for x in uncorrelated_params: + gmm_dict[(x,)] = None + print( " Using correlated GMM sampling on sampling variable indexes " , gmm_dict, " out of ", low_level_coord_names) + else: + param_indexes = range(len(low_level_coord_names)) + gmm_dict = {(k,):None for k in param_indexes} # no correlations +# lnL_offset_saving = opts.lnL_offset + lnL_offset_saving = -20 # for simplicity, hardcode for now for preserving points + print("GMM ", gmm_dict) + extra_args = {'n_comp':n_comp,'max_iter':n_max_blocks,'L_cutoff': None,'gmm_dict':gmm_dict,'max_err':50, 'lnw_failure_cut':-np.inf} # made up for now, should adjust +extra_args.update({ + "n_adapt": 100, # Number of chunks to allow adaption over + "history_mult": 10, # Multiplier on 'n' - number of samples to estimate marginalized 1D histograms with, + "force_no_adapt":opts.force_no_adapt, + "tripwire_fraction":opts.tripwire_fraction +}) + +fn_passed = likelihood_function +if supplemental_ln_likelihood: + fn_passed = lambda *x: likelihood_function(*x)*np.exp(supplemental_ln_likelihood(*x)) +if opts.internal_use_lnL: + fn_passed = log_likelihood_function # helps regularize large values + if supplemental_ln_likelihood: + fn_passed = lambda *x: log_likelihood_function(*x) + supplemental_ln_likelihood(*x) + extra_args.update({"use_lnL":True,"return_lnI":True}) + + + +res, var, neff, dict_return = sampler.integrate(fn_passed, *low_level_coord_names, verbose=True,nmax=int(opts.n_max),n=n_step,neff=opts.n_eff, save_intg=True,tempering_adapt=True, floor_level=1e-3,igrand_threshold_p=1e-3,convergence_tests=test_converged,adapt_weight_exponent=my_exp,no_protect_names=True,**extra_args) # weight ecponent needs better choice. We are using arbitrary-name functions + + +# Save result -- needed for odds ratios, etc. +np.savetxt(opts.fname_output_integral+"_result.txt", [np.log(res)]) + +if neff < len(coord_names): + print(" PLOTS WILL FAIL ") + print(" Not enough independent Monte Carlo points to generate useful contours") + + +samples = sampler._rvs +print(samples.keys()) +n_params = len(low_level_coord_names) +dat_mass = np.zeros((len(samples[low_level_coord_names[0]]),n_params+3)) +if not(opts.internal_use_lnL): + dat_logL = np.log(samples["integrand"]) +else: + if 'log_integrand' in samples: + dat_logL = samples['log_integrand'] + else: + dat_logL = samples["integrand"] +lnLmax = np.max(dat_logL[np.isfinite(dat_logL)]) +print(" Max lnL ", np.max(dat_logL)) + +n_ESS = -1 +if True: + # Compute n_ESS. Should be done by integrator! + if 'log_joint_s_prior' in samples: + weights_scaled = np.exp(dat_logL - lnLmax + samples["log_joint_prior"] - samples["log_joint_s_prior"]) + # dictionary, write this to enable later use of it + samples["joint_s_prior"] = np.exp(samples["log_joint_s_prior"]) + samples["joint_prior"] = np.exp(samples["log_joint_prior"]) + else: + weights_scaled = np.exp(dat_logL - lnLmax)*sampler._rvs["joint_prior"]/sampler._rvs["joint_s_prior"] + weights_scaled = weights_scaled/np.max(weights_scaled) # try to reduce dynamic range + n_ESS = np.sum(weights_scaled)**2/np.sum(weights_scaled**2) + print(" n_eff n_ESS ", neff, n_ESS) + + +# Throw away stupid points that don't impact the posterior +indx_ok = np.ones(len(dat_logL),dtype=bool) +if not('log_joint_s_prior' in samples): + indx_ok=samples["joint_s_prior"]>0 +indx_ok = np.logical_and(dat_logL > np.max(dat_logL)-opts.lnL_offset ,indx_ok) +for p in low_level_coord_names: + samples[p] = samples[p][indx_ok] +dat_logL = dat_logL[indx_ok] +print(samples.keys()) +samples["joint_prior"] =samples["joint_prior"][indx_ok] +samples["joint_s_prior"] =samples["joint_s_prior"][indx_ok] + + + +### +### 1d posteriors of the coordinates used for sampling [EQUALLY WEIGHTED, BIASED because physics cuts aren't applied] +### + +p = samples["joint_prior"] +ps =samples["joint_s_prior"] +lnL = dat_logL +lnLmax = np.max(lnL) +weights = np.exp(lnL-lnLmax)*p/ps + + + +print(" ---- Subset for posterior samples (and further corner work) --- ") + + +p_norm = (weights/np.sum(weights)) +indx_list = np.random.choice(np.arange(len(weights)), p=p_norm.astype(np.float64),size=opts.n_output_samples) + + +dat_out = np.zeros( (opts.n_output_samples,2+len(dat_orig_names)) ) + +if len(low_level_coord_names) < len(dat_orig_names): # not needed if all params are in fit/sampled + + if len(dat) < opts.n_output_samples: + print(" NOTE: original data shorter than requested output; adding",opts.n_output_samples-len(dat),"duplicate fill lines from original data.") + newlines = None + if opts.n_output_samples > 2*len(dat): + newlines = dat[:] + newlen = len(newlines) + while newlen < opts.n_output_samples: + newerlines = dat[:opts.n_output_samples-newlen] #will only get up to len(dat) lines + newlines = np.concatenate((newlines,newerlines), axis=0) + newlen = len(newlines) + else: + newlines = dat[:opts.n_output_samples-len(dat)] #duplicate lines to fill + dat = np.concatenate((dat,newlines), axis=0) #should be fine since dat isn't used after this + + for c in np.arange(len(dat_orig_names)): + if dat_orig_names[c] not in low_level_coord_names: + print(" Not sampled:",dat_orig_names[c],"; adding to output as constant.") + outidx = name_index_dict[dat_orig_names[c]] # write in correct place + if len(dat) > opts.n_output_samples: + dat_out[:,outidx] = dat[:opts.n_output_samples,outidx] #truncate original data to fit (not ideal) + else: #len(dat) <= n_output_samples (if dat was <, should now be =) + dat_out[:,outidx] = dat[:,outidx] + +#write out +#if not supplemental_coordinate_invert: +for name in low_level_coord_names: + if name not in name_index_dict: + print(" Sampled coord {!r} is not a data-file column; not writing to output (would only appear in corner plots).".format(name)) + continue + outindx = name_index_dict[name] + dat_out[:, outindx] = samples[name][indx_list] + +#for indx in np.arange(len(low_level_coord_names)): +# vals = samples[low_level_coord_names[indx]][indx_list] # load in data for this column +# outindx = name_index_dict[low_level_coord_names[indx]] # write in correct place +# dat_out[:,outindx] = vals + +if supplemental_coordinate_invert: + #re-convert + dat_out[:,2:] = supplemental_coordinate_invert(dat_out[:,2:], coord_names=dat_orig_names) + +#If one of the masses carried as const, re-sort to enforce m1 > m2 +#if ("m1" not in coord_names) or ("m2" not in coord_names): +# print("NOTE: re-sorting masses so m1 > m2 (precaution)") +# m1dx = name_index_dict["m1"] +# print("Minimum m1 (pre-sort):",min(dat_out[:,m1dx])) +# print("Minimum m2 (pre-sort):",min(dat_out[:,m1dx+1])) +# m1 = np.maximum(dat_out[:,m1dx], dat_out[:,m1dx+1]) #N.B.: assumes m2 col index after m1 col +# m2 = np.minimum(dat_out[:,m1dx], dat_out[:,m1dx+1]) +# dat_out[:,m1dx] = m1 +# dat_out[:,m1dx+1] = m2 +# print("Minimum m1 (post-sort):",min(dat_out[:,m1dx])) +# print("Minimum m2 (post-sort):",min(dat_out[:,m1dx+1])) + +print(" Saving to ", opts.fname_output_samples+".dat") +np.savetxt(opts.fname_output_samples+".dat",dat_out,header=" lnL sigma_lnL " + ' '.join(dat_orig_names)) \ No newline at end of file diff --git a/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py b/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py new file mode 100644 index 000000000..c93fef054 --- /dev/null +++ b/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py @@ -0,0 +1,365 @@ +#! /usr/bin/env python +''' + Updated version of util_HyperparameterPuffball.py for HyperPipe. + Handles coordinate rotation via external plugin and coordinate reflection + Supports older usage of HyperPuff via legacy code + + old usage: + --parameter m1 --downselect-parameter m1 --downselect-parameter-range [] --reflect-parameter m1 + new usage: + --parameter m1 --parameter m2 --downselect-parameter m1 --parameter-range [] --reflect-parameter m2 --parameter-range [] +''' +import argparse +import sys +import numpy as np +import numpy.lib.recfunctions +#import scipy +from scipy import stats + +parser = argparse.ArgumentParser() +parser.add_argument("--inj-file", help="Name of grid.dat file") +parser.add_argument("--inj-file-out", default="output-puffball.dat", help="Name of grid_puff.dat file") +parser.add_argument("--puff-factor", default=1,type=float) +parser.add_argument("--force-away", default=0,type=float,help="Unused legacy option. If >0, uses the icov to compute a metric, and discards points which are close to existing points") +parser.add_argument("--parameter", action='append', help="Parameters used as fitting parameters AND varied at a low level to make a posterior") +parser.add_argument("--no-correlation", type=str,action='append', help="Pairs of parameters, in format [mc,eta] The corresponding term in the covariance matrix is eliminated") +parser.add_argument("--random-parameter", action='append',help="These parameters are specified at random over the entire range, uncorrelated with the grid used for other parameters. Use for variables which correlate weakly with others; helps with random exploration") +parser.add_argument("--random-parameter-range", action='append', type=str,help="Add a range (pass as a string evaluating to a python 2-element list): --parameter-range '[0.,1000.]' MUST specify ALL parameter ranges (min and max) in order if used. ") +parser.add_argument("--downselect-parameter",action='append', help='Name of parameter to be used to eliminate grid points ') +parser.add_argument("--downselect-parameter-range",action='append',type=str,help="legacy option; prefer --parameter-range") +parser.add_argument("--reflect-parameter",action='append',type=str) +parser.add_argument("--regularize",action='store_true',help="Add some ad-hoc terms based on priors, to help with nearly-singular matricies") +parser.add_argument("--downselect-mass-range",action='store_true',help="Reject points where m2 > m1 for population models (slightly hacky; prefer diff coord sys)") +parser.add_argument("--get-range-from-external",action='store_true',help="Get/modify parameter ranges via external func in --supplementary-coordinate-code") +parser.add_argument("--external-range-args",action='append',help="Provide special args to external range func; syntax: single str 'arg_name=arg_value'.") +#to unlink downselect & reflection: +parser.add_argument("--parameter-range",action='append',type=str) +#for generalization: +parser.add_argument("--supplementary-coordinate-code", default=None,type=str,help="Coordinate conversion/prior code. Accepts: the literal 'rift_default' (use RIFT.lalsimutils.convert_waveform_coordinates plus RIFT-standard priors); a filesystem path ending in .py (loaded as a plugin); or any importable dotted module name.") +parser.add_argument("--supplementary-coordinate-function", default=None, type=str, help="Name of the entry-point callable inside the module named by --supplementary-coordinate-code. Defaults to 'convert_coordinates'.") +#to be deprecated: +parser.add_argument("--use-rotated-spectral-coords",action='store_true',help="deprecated; Apply rotated coord sys to spectral EOS params; from Wysocki 2020 https://arxiv.org/pdf/2001.01747") +parser.add_argument("--rotated-coord-buffer",default=0.0,type=float,help="deprecated; Fractional buffer (e.g., 0.1; default 0) to extend rotated hypercube space (APPLIES AS % OF BOUND VALUE)") +#parser.add_argument("--reflect-parameters",action='store_true',help="Toggle parameter reflection, even if no --reflect-parameter provided (for rotated coord reflection)") +#parser.add_argument("--use-alternate-buffer",action='store_true',help="Buffer expands hypercube by x% of its full width (2x% total expansion), instead of by x% of bound value (equivalent to (2x)% bound val buffer for [-r,+r] bounds.") + + +opts = parser.parse_args() + +opts.inj_file = "grid_test.txt" +opts.puff_factor = 0.001 +opts.parameter = ["gamma0","gamma1","gamma2","gamma3","m1","m2"] +opts.parameter_range = ["[0.2,2]","[-1.6,1.7]","[-0.6,0.6]","[-0.02,0.02]","[1,3]","[1,3]"] +opts.reflect_parameter = ["gamma0","gamma1","gamma2","gamma3","m1","m2"] +opts.supplementary_coordinate_code = "dan_rotation_conversion" +opts.supplementary_coordinate_function = "dan_rotation" +opts.get_range_from_external = True +opts.external_range_args = ["buffer=4.0"] + + +if opts.random_parameter is None: + opts.random_parameter = [] + +# Extract parameter names +coord_names = opts.parameter # Used in fit +if coord_names is None: + print(" ERROR: no coordinates provided ") + sys.exit(0) #no puff + +# match up pairs in --no-correlation +corr_list = None +if not(opts.no_correlation is None): + corr_list = [] + corr_name_list = list(map(eval,opts.no_correlation)) + #print opts.no_correlation, corr_name_list + for my_pair in corr_name_list: + + i1 = coord_names.index(my_pair[0]) + i2 = coord_names.index(my_pair[1]) + + if i1>-1 and i2 > -1: + corr_list.append([i1,i2]) +# else: +# print i1, i2 +# print opts.no_correlation, coord_names, corr_list + +downselect_dict = {} +reflect_dict = {} +param_dict = {} + +if opts.downselect_parameter_range: #to handle legacy uses + opts.parameter_range = opts.downselect_parameter_range + +#make master param dict, then filter into downselect & reflect dicts +plist = [] +if opts.downselect_parameter: + plist += opts.downselect_parameter +if opts.reflect_parameter: + plist += opts.reflect_parameter +if opts.parameter_range: + param_ranges = list(map(eval,opts.parameter_range)) + + if len(plist) < len(param_ranges): #allow unbounded params, for now (expect external bounds) + print(" parameters and parameter ranges inconsistent:",plist,param_ranges) + raise Exception(" All bounded coordinates must have a specified parameter range") + + for indx in np.arange(len(param_ranges)): + param_dict[plist[indx]] = param_ranges[indx] +#no else: allow for external bounds to be provided + +if opts.use_rotated_spectral_coords: + #legacy support - still need external rotation code for grid lines, though + plist = ["gamma0","gamma1","gamma2","gamma3"] + plist + rot_coords = {} + rot_coords["gamma0"] = [-4.37722, 4.91227] + rot_coords["gamma1"] = [-1.82240, 2.06387] + rot_coords["gamma2"] = [-0.32445, 0.36469] + rot_coords["gamma3"] = [-0.09529, 0.11046] + + for indx, param in enumerate(rot_coords.keys()): + ubound = rot_coords[param][1] + opts.rotated_coord_buffer*abs(rot_coords[param][1]) + lbound = rot_coords[param][0] - opts.rotated_coord_buffer*abs(rot_coords[param][0]) + param_dict[param] = [lbound,ubound] + +#parse args to be passed to coord convert funcs: +external_kwargs = {} +if opts.external_range_args: + for arg in opts.external_range_args: + external_kwargs[arg.split("=")[0]] = arg.split("=")[1] + +#load coordinate rotation module, if provided +supplemental_coordinate_convert = None +supplemental_coordinate_invert = None +if opts.supplementary_coordinate_code and opts.supplementary_coordinate_function: + print(" EXTERNAL COORDINATE CONVERSION : {}.{} ".format(opts.supplementary_coordinate_code,opts.supplementary_coordinate_function)) + __import__(opts.supplementary_coordinate_code) + external_coordinate_module = sys.modules[opts.supplementary_coordinate_code] + if hasattr(external_coordinate_module,opts.supplementary_coordinate_function): + supplemental_coordinate_convert = getattr(external_coordinate_module,opts.supplementary_coordinate_function) + try: + supplemental_coordinate_invert = getattr(external_coordinate_module, "inverse_"+opts.supplementary_coordinate_function) + except: + print(" ERROR: no inverted coordinate conversion found! Will not apply rotation, but supplied bounds will still be used!") + supplemental_coordinate_invert = None + supplemental_coordinate_convert = None #avoids producing output in rotated coords + else: + print("ERROR: could not retrieve supplied coordinate function. No conversion will be applied, but supplied bounds will be used!") + + if opts.get_range_from_external: #retrieve/modify param ranges externally + try: + supplemental_bound_func = getattr(external_coordinate_module, "get_bounds") + param_dict = supplemental_bound_func(plist,param_dict,**external_kwargs) + except: + print(" WARNING: external range requested but no 'get_bounds()' function found. Using supplied bounds as-is.") + + +#at this point, all given params must have ranges +if (len(plist) != len(param_dict)): + print(" Reflection/downselect params inconsistent:",plist,param_dict) + raise Exception(" Reflect/downselect coords must have valid param range") + +#set downselect +if opts.downselect_parameter: + dlist = opts.downselect_parameter + #dlist_ranges = [param_ranges[coord_names.index(p)] for p in dlist] #list(map(eval,opts.parameter_range[:len(dlist)])) + + #if len(dlist) != len(dlist_ranges): #should never happen, now + # print(" downselect parameters inconsistent", dlist, dlist_ranges) + + for indx in np.arange(len(dlist)): + if not(dlist[indx] in coord_names): + raise Exception(" Downselect only allowed for coordinates (--parameter) ") + downselect_dict[dlist[indx]] = param_dict[dlist[indx]] +else: + dlist = [] + #dlist_ranges = [] + opts.downselect_parameter = [] + +#for indx in np.arange(len(dlist_ranges)): +# downselect_dict[dlist[indx]] = dlist_ranges[indx] + +#set reflect +#indx_reflect=[] +if opts.reflect_parameter: + rlist = opts.reflect_parameter + #indx_reflect = [coord_names.index(param) for param in opts.reflect_parameter] + + if len(rlist) > len(coord_names): + print(" reflection parameters inconsistent", rlist, coord_names) + raise Exception(" Reflection only allowed for coordinates ") + + for indx in np.arange(len(rlist)): + if not(rlist[indx] in coord_names): + raise Exception(" Reflection only allowed for coordinates (--parameter) ") + #if not(rlist[indx] in downselect_dict): + # raise Exception(" Reflection requires parameter range specified as a downselection ") + if rlist[indx] in downselect_dict: + del downselect_dict[rlist[indx]] #handles legacy cases, where d-select was required to reflect + reflect_dict[rlist[indx]] = param_dict[rlist[indx]] +else: + rlist = [] + opts.reflect_parameter = [] + + +# Load data; keep parameter names +dat_raw = np.genfromtxt(opts.inj_file, names=True) +X = np.zeros((len(dat_raw), len(coord_names))) +# Copy over the parameters we use. Note we have no way to create linear combinations or alternate coordinates here +for p in coord_names: + #indx_p = list(dat_raw.dtype.names).index(p) + indx_in = coord_names.index(p) + X[:,indx_in] = dat_raw[p] + + +# Measure covariance matrix and generate random errors +if len(coord_names) >1: + cov_in = np.cov(X.T) + cov = cov_in*opts.puff_factor*opts.puff_factor + + # Check for singularities + if np.min(np.linalg.eig(cov)[0])<1e-10: + print(" ===> WARNING: SINGULAR MATRIX: are you sure you varied this parameters? <=== ") + icov_pseudo = np.linalg.pinv(cov) + # Prior range for each parameter is 1000, so icov diag terms are 10^(-6) + # This is somewhat made up, but covers most things + diag_terms = 1e-6*np.ones(len(cov)) + # + icov_proposed = icov_pseudo+np.diag(diag_terms) + cov= np.linalg.inv(icov_proposed) + + cov_orig = np.array(cov) # force copy + # Remove targeted covariances + if not(corr_list is None): + for my_pair in corr_list: + if my_pair[0] != my_pair[1]: + cov[my_pair[0],my_pair[1]]=0 + cov[my_pair[1],my_pair[0]]=0 + + + # Compute errors + rv = stats.multivariate_normal(mean=np.zeros(len(coord_names)), cov=cov,allow_singular=True) # they are just complaining about dynamic range of parameters, usually + delta_X = rv.rvs(size=len(X)) + X_out = X+delta_X + +else: + sigma = np.std(X) + cov = sigma*sigma + delta_X =np.random.normal(size=len(coord_names), scale=sigma) + X_out = X+delta_X + +# Apply coordinate rotation +if supplemental_coordinate_convert: + X_out = supplemental_coordinate_convert(X_out, coord_names, **external_kwargs) + +print(" Initial data length:",len(X_out)) + +# Reflection +for param in rlist: + indx = coord_names.index(param) + print(" Reflecting into range : {} [{}, {}]".format(param,reflect_dict[param][0],reflect_dict[param][1])) + # put in range [0,2 L] + tmp = reflect_dict[param][0] + np.mod(X_out[:,indx] - reflect_dict[param][0], 2*(reflect_dict[param][1] - reflect_dict[param][0]) ) + # final reflection + tmp = np.where( tmp > reflect_dict[param][1], 2*reflect_dict[param][1] - tmp, tmp) + print(" Increment reflect : {} {} ".format(param,np.sum(tmp != X_out[:,indx]))) + X_out[:,indx] = tmp + +# Downselect +names_downselect = list(downselect_dict.keys()) +indx_ok = np.ones(len(X_out),dtype=bool) + +# ============================================================================= +# #Apply rotation to spectral EOS params from Wysocki et. al 2020 https://arxiv.org/pdf/2001.01747 +# if opts.use_rotated_spectral_coords: +# print(" Applying rotated coordinate transformation to spectral parameters") +# dan_rot = [[0.43801, -0.53573, 0.52661, -0.49379], +# [-0.76705, 0.17169, 0.31255, -0.53336], +# [0.45143, 0.67967, -0.19454, -0.54443], +# [0.12646, 0.47070, 0.76626, 0.41868]] +# scaled_mean = [0.89421, 0.33878, -0.07894, 0.00393] +# scaled_sig = [0.35700, 0.25769, 0.05452, 0.00312] +# dan_inv = np.linalg.inv(dan_rot) +# +# #get gammas' indices in X_out(= X) from coord names +# r_tilde = np.zeros((len(X_out),4)) +# rot_cols = [] +# for i in np.arange(4): +# #do one coord at a time +# indx = coord_names.index("gamma"+str(i)) +# rot_cols.append(indx) +# +# #convert gammas to r_tilde using equation: r_tilde = (gamma - u)/sig +# r_tilde[:,i] = (X_out[:,indx] - scaled_mean[i])/scaled_sig[i] +# +# #apply transform: r_prime = S*r_tilde ( [4 x 4].([N x 4].T) ) +# r_prime = np.matmul(dan_rot,r_tilde.T).T +# +# rot_coords = {} +# rot_coords["r0"] = [-4.37722, 4.91227] +# rot_coords["r1"] = [-1.82240, 2.06387] +# rot_coords["r2"] = [-0.32445, 0.36469] +# rot_coords["r3"] = [-0.09529, 0.11046] +# +# for indx, param in enumerate(rot_coords.keys()): +# # apply hypercube buffer +# if opts.use_alternate_buffer: #new_bound = bound +/- buffer*(width of param range) -> SYMMETRIC buffer +# ubound = rot_coords[param][1] + opts.rotated_coord_buffer*abs(rot_coords[param][1]-rot_coords[param][0]) +# lbound = rot_coords[param][0] - opts.rotated_coord_buffer*abs(rot_coords[param][1]-rot_coords[param][0]) +# else: #new_bound = bound +/- buffer*|bound| -> asymmetric buffer +# ubound = rot_coords[param][1] + opts.rotated_coord_buffer*abs(rot_coords[param][1]) +# lbound = rot_coords[param][0] - opts.rotated_coord_buffer*abs(rot_coords[param][0]) +# if opts.reflect_parameters: +# # Reflection +# print(" Reflecting into range : {} [{}, {}]".format(param,lbound,ubound)) +# # put in range [0,2 L] +# #tmp = rot_coords[param][0] + np.mod(r_prime[:,indx] - rot_coords[param][0], 2*(rot_coords[param][1] - rot_coords[param][0]) ) +# tmp = lbound + np.mod(r_prime[:,indx] - lbound, 2*(ubound - lbound) ) +# # final reflection +# #tmp = np.where( tmp > rot_coords[param][1], 2*rot_coords[param][1] - tmp, tmp) +# tmp = np.where( tmp > ubound, 2*ubound - tmp, tmp) +# print(" Increment reflect : {} {} ".format(param,np.sum(tmp != r_prime[:,indx]))) +# r_prime[:,indx] = tmp +# else: +# # Downselection +# #print(" Downselecting:",name,"; indx:",indx,"[",lbound,ubound,"]; buffer =",opts.rotated_coord_buffer) +# indx_ok = np.logical_and(indx_ok, r_prime[:,indx]<= ubound ) +# indx_ok = np.logical_and(indx_ok, r_prime[:,indx]>= lbound ) +# print(' Increment downselect : {} {} '.format(param, np.sum(indx_ok) )) +# +# if opts.reflect_parameters: +# #apply inverse: S-1*r_prime = S-1*S*r_tilde = r_tilde ( [4 x 4].([N x 4].T) ) +# r_tilde_post = np.matmul(dan_inv,r_prime.T).T +# +# for i, col in enumerate(rot_cols): +# #r_tilde = (gamma - u)/sig -> gamma = r_tilde*sig + u +# X_out[:,col] = (r_tilde_post[:,i]*scaled_sig[i]) + scaled_mean[i] +# ============================================================================= + +for name in names_downselect: + indx = coord_names.index(name) + print(" Downselecting:",name,"; indx:",indx,"[",downselect_dict[name][0],downselect_dict[name][1],"]") + indx_ok = np.logical_and(indx_ok, np.logical_not(np.isnan(X_out[:,indx]))) + indx_ok = np.logical_and(indx_ok, X_out[:,indx]<= downselect_dict[name][1] ) + indx_ok = np.logical_and(indx_ok, X_out[:,indx]>= downselect_dict[name][0] ) + print(' Increment downselect : {} {} '.format(name, np.sum(indx_ok) )) + +if supplemental_coordinate_invert: + X_out = supplemental_coordinate_invert(X_out, coord_names, **external_kwargs) + +#enforce m1 >= m2 for populations - should really replace by converting m2 -> q +if opts.downselect_mass_range and ("m1" in coord_names) and ("m2" in coord_names): + m1_indx = coord_names.index("m1") + m2_indx = coord_names.index("m2") + indx_ok = np.logical_and(indx_ok, X_out[:,m1_indx]>= X_out[:,m2_indx] ) + print(' Increment downselect : {} {} '.format("m1>=m2", np.sum(indx_ok) )) + +X_out = X_out[indx_ok] +dat_raw = dat_raw[indx_ok] # must downselect here as well! + +# Write data back into correct format and save +for p in coord_names: +# indx_p = dat_raw.dtype.names.index(p) + indx_in = coord_names.index(p) + dat_raw[p] = X_out[:,indx_in] + +np.savetxt(opts.inj_file_out, dat_raw,header=" ".join(dat_raw.dtype.names)) \ No newline at end of file From b5d8a3fd1b2af4f2de53492d553c14ac32613df4 Mon Sep 17 00:00:00 2001 From: mebiri Date: Fri, 18 Sep 2026 15:00:03 -0500 Subject: [PATCH 2/8] changes to CHP to bring more in-line with CEP --- .../Code/bin/util_ConstructHyperPosterior.py | 107 +++++++++++++----- 1 file changed, 76 insertions(+), 31 deletions(-) diff --git a/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py b/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py index 66820d6d3..6c0083695 100644 --- a/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py +++ b/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py @@ -151,6 +151,7 @@ def add_field(a, descr): parser.add_argument("--fit-load-gp",default=None,type=str,help="Filename of GP fit to load. Overrides fitting process, but user MUST correctly specify coordinate system to interpret the fit with. Does not override loading and converting the data.") parser.add_argument("--fit-save-gp",default=None,type=str,help="Filename of GP fit to save. ") parser.add_argument("--fit-order",type=int,default=2,help="Fit order (polynomial case: degree)") +parser.add_argument("--fit-distance-tail",action='store_true',help="RoboCode addition: Distance-export (.dslice) runs ONLY, i.e. runs with an explicit distance fit coordinate. Beyond each intrinsic point's exported distance, make the fitted lnL decay to zero as d->infinity instead of holding its edge value (e.g., as RF/ExtraTrees fit does).") parser.add_argument("--no-plots",action='store_true') parser.add_argument("--using-eos-type", type=str, default=None, help="Name of EOS parameterization (must match what is used for inputs). Will use EOS parameterization to identify appropriate field headers") parser.add_argument("--sampler-method",default="adaptive_cartesian",help="adaptive_cartesian|GMM|adaptive_cartesian_gpu") @@ -169,7 +170,8 @@ def add_field(a, descr): parser.add_argument("--supplementary-likelihood-factor-ini", default=None,type=str,help="With above option, specifies an ini file that is parsed (here) and passed to the preparation code, called when the module is first loaded, to configure the module. EXPERTS ONLY") parser.add_argument("--supplementary-coordinate-code", default=None,type=str,help="Coordinate conversion/prior code. Accepts: the literal 'rift_default' (use RIFT.lalsimutils.convert_waveform_coordinates plus RIFT-standard priors); a filesystem path ending in .py (loaded as a plugin); or any importable dotted module name.") parser.add_argument("--supplementary-coordinate-function", default=None, type=str, help="Name of the entry-point callable inside the module named by --supplementary-coordinate-code. Defaults to 'convert_coordinates'.") - +parser.add_argument("--supplementary-coordinate-ini", default=None, type=str, help="RoboCode: Optional ini file parsed and handed to the coordinate plugin's prepare() hook so it can read its own configuration block(s).") +parser.add_argument("--supplementary-coordinate-chart", default=None, type=str, help="RoboCode: Which chart (coordinate system) defined by the plugin to use for this run. Required when the plugin's CHARTS dict has > 1 entry; ignored when the plugin doesn't define CHARTS. Different charts can share parameter names but imply different priors -- the chart name disambiguates which (name -> prior) mapping is installed.") opts= parser.parse_args() #print(" WARNING: Always use internal_use_lnL for now ") @@ -224,18 +226,24 @@ def add_field(a, descr): ### Parameters in use ### +#This supercedes the robocode attempt at this, and fixes a bug from the CIP +#version of the code where having opts.parameter = None and opts.param_implied +#would create duplicate params in coord_names #want: #if opts.parameter -> both fit & sampling #if opts.param_implied -> just fit #if opts.param_nofit -> just sampling - #if no opts.parameter -> coord_names & l_l_coord_names = dat_orig_names + #if no opts -> coord_names & l_l_coord_names = dat_orig_names coord_names = opts.parameter # coords for both fit and MC sampling if coord_names is None: coord_names = dat_orig_names low_level_coord_names = coord_names # i.e., sampling coords same as data col coords if opts.parameter_implied: - coord_names = coord_names+opts.parameter_implied # coords for fit - wrong if opts.parameter=None + if opts.parameter is None: #this code should fix the bug + coord_names = opts.parameter_implied #coords for fit + else: + coord_names = coord_names+opts.parameter_implied # coords for fit - used to be wrong if opts.parameter=None if opts.parameter_nofit: if opts.parameter is None: @@ -260,7 +268,7 @@ def add_field(a, descr): ### param_ranges = {} -for range_code in opts.integration_parameter_range: +for range_code in opts.integration_parameter_range: #robocode: (opts.integration_parameter_range or []) name, range_str = range_code.split(':') range_expr = eval(range_str) # define. Better to split on , for example param_ranges[name] = np.array(range_expr) @@ -281,6 +289,9 @@ def uniform_prior(x): prior_map = {} for name in low_level_coord_names: prior_map[name] = uniform_prior + #Robocode removes the below & delays until after coord conversion code + #Only useful if using external code to retrieve bounds via conversion code's + #get_bounds() func (which may or may not be present!) - see HyperPuffball_new for implementation if not(name in param_ranges): raise Exception(" {} not provided a parameter range ".format(name)) # change later, should fall back to using prior range from above @@ -310,14 +321,14 @@ def uniform_prior(x): supplemental_ln_likelihood = getattr(external_likelihood_module,opts.supplementary_likelihood_factor_function) name_prep = "prepare_"+opts.supplementary_likelihood_factor_function if hasattr(external_likelihood_module,name_prep): - supplemental_ln_likelhood_prep=getattr(external_likelihood_module,name_prep) + supplemental_ln_likelihood_prep=getattr(external_likelihood_module,name_prep) # Check for and load in ini file associated with external library if opts.supplementary_likelihood_factor_ini: import configparser as ConfigParser config = ConfigParser.ConfigParser() config.optionxform=str # force preserve case! config.read(opts.supplementary_likelihood_factor_ini) - supplemental_ln_likelhood_parsed_ini=config + supplemental_ln_likelihood_parsed_ini=config # Call the ini file, tell it what coordinates we are using by name supplemental_ln_likelihood_prep(config=supplemental_ln_likelihood_parsed_ini,coords=coord_names) @@ -325,21 +336,16 @@ def uniform_prior(x): supplemental_coordinate_convert = None supplemental_coordinate_invert = None -if opts.supplementary_coordinate_code and opts.supplementary_coordinate_function: - ''' - The robot wants: The loader accepts three forms in --supplementary-coordinate-code: - -the literal 'rift_default', - -a filesystem path to a .py file, or - -an importable dotted module name. - The plugin must expose a callable named by --supplementary-coordinate-function - with args (x_in, coord_names, low_level_coord_names, **kwargs) that returns - a 2-D ndarray of shape (N, len(coord_names)). - Robot supposedly wants to allow an optional prepare() (one-shot setup, gets - the parsed ini and active coord_name lists) and a register_priors() (mutate - prior_map in place) - see RIFT.misc.coordinate_plugin. I don't care for this. - ''' +if opts.supplementary_coordinate_code and opts.supplementary_coordinate_function: + #Ignoring RoboCode completely here - no absurd plugins! Reuse functionality above print(" EXTERNAL COORDINATE CONVERSION : {}.{} ".format(opts.supplementary_coordinate_code,opts.supplementary_coordinate_function)) __import__(opts.supplementary_coordinate_code) + ''' + Expect supplementary-coordinate-code to contain: + supplementary-coordinate-function(X, coord_names, **kwargs) + "inverse_"+supplementary-coordinate-function(X, coord_names, **kwargs) + Optional: get_bounds(param_list, bounds_dict, **kwargs) + ''' external_coordinate_module = sys.modules[opts.supplementary_coordinate_code] if hasattr(external_coordinate_module,opts.supplementary_coordinate_function): supplemental_coordinate_convert = getattr(external_coordinate_module,opts.supplementary_coordinate_function) @@ -455,7 +461,12 @@ def fit_rf(x,y,y_errors=None,fname_export='nn_fit'): if y_errors is None: rf.fit(x,y) else: - rf.fit(x,y,sample_weight=1./y_errors**2) + #WARNING: RoboCode! + # floor sigma so a zero error (placeholder rows can leak into the + # accumulated marg net with sigma=0) doesn't make sample_weight=1/sigma^2 + # infinite (sklearn rejects inf sample_weight). + rf.fit(x,y,sample_weight=1./np.maximum(np.asarray(y_errors,dtype=float),1e-3)**2) + #rf.fit(x,y,sample_weight=1./y_errors**2) ### reject points with infinities : problems for inputs def fn_return(x_in,rf=rf): @@ -499,14 +510,14 @@ def fn_return(x_in,rf=rf): dat_out = [] for line in dat: - dat_here= np.zeros(len(coord_names)+2) - if line[col_lnL+1] > opts.sigma_cut: - print("skipping", line) - continue - dat_here[:-2] = line[indx_of_orig_names+2]#line[2:len(coord_names)+2] # modify to use names! - dat_here[-2] = line[0] - dat_here[-1] = line[1] - dat_out.append(dat_here) + dat_here= np.zeros(len(coord_names)+2) + if line[col_lnL+1] > opts.sigma_cut: + print("skipping", line) + continue + dat_here[:-2] = line[indx_of_orig_names+2]#line[2:len(coord_names)+2] # modify to use names! + dat_here[-2] = line[0] + dat_here[-1] = line[1] + dat_out.append(dat_here) dat_out= np.array(dat_out) # Repack data #TODO: check this, move outside if block if both routes need it! @@ -627,6 +638,38 @@ def convert_coords(x): Y_err=None my_fit = fit_rf(X,Y,y_errors=Y_err) +#WARNING: RoboCode! +''' +# Distance tail: make the fit decay beyond each intrinsic point's exported distance support +# Only meaningful for a distance-export (.dslice) run, where `dist` is a FIT coordinate and the +# training set is ~50 discrete distances per intrinsic point. An RF/ExtraTrees fit is piecewise +# constant outside its training envelope, so past a point's outermost exported slice it returns +# that slice's lnL forever. The distance prior is volumetric and keeps growing like d^2, so the +# integrand grows instead of dying and the recovered distance posterior comes out ~18% too wide +# in every quantile span, median untouched. See RIFT/interpolators/distance_tail.py. +# +# This wraps whatever fit was just built and is a no-op on the support, so nothing that currently +# works changes. It is applied AFTER the cap/threshold cuts above so the tail is built from +# exactly the rows the fit itself saw. +''' +if opts.fit_distance_tail: + if my_fit is None: + raise ValueError("--fit-distance-tail: no fit was built (--fit-method %s)" % opts.fit_method) + if 'dist' not in list(coord_names): + # Fail rather than silently do nothing: a run that asked for this and did not get it would + # carry the very bias the option exists to remove, with no sign of it in the log. + raise ValueError("--fit-distance-tail requires a distance fit coordinate, but coord_names " + "is %s. This option is for distance-export (.dslice) runs." % (list(coord_names),)) + from RIFT.interpolators.distance_tail import wrap_distance_tail + tail_report = {} + # The production decay law is the parameter-free chord (ratio = u), so the wrapper's tuning + # arguments -- the per-slice edge-slope fit and its `outer_frac`, the alternate laws -- reach + # only the diagnostic branches described in RIFT/interpolators/distance_tail.py. They are + # deliberately not exposed here: a CLI knob that cannot change the posterior is worse than none. + my_fit = wrap_distance_tail(my_fit, X, Y, coord_names, y_errors=Y_err, + lnL_offset=lnL_shift, report=tail_report) + print(" DISTANCE TAIL : decay beyond exported support enabled ", tail_report) + # Sort for later convenience (scatterplots, etc) indx = Y.argsort()#[::-1] @@ -689,6 +732,8 @@ def convert_coords(x): sampler_list.append(sampler) sampler = mcsamplerPortfolio.MCSampler(portfolio=sampler_list) +#Note: O4d CEP has RoboCode complaining about enforcing --internal-use-lnL (ignored by some samplers). +#Unlike it, we aren't stupid, so that nonsense is truncated here. ## ## Loop over param names @@ -703,11 +748,10 @@ def convert_coords(x): likelihood_function = None log_likelihood_function = None -#def convert_coords(x): -# return x def log_likelihood_function(*args): return my_fit(convert_coords(np.array([*args]).T )) +#NOTE: RoboCode had a field day reorganizing this in CEP. Those changes ignored here. if len(low_level_coord_names) ==1: def likelihood_function(x): if isinstance(x,float): @@ -844,7 +888,8 @@ def parse_corr_params(my_str): res, var, neff, dict_return = sampler.integrate(fn_passed, *low_level_coord_names, verbose=True,nmax=int(opts.n_max),n=n_step,neff=opts.n_eff, save_intg=True,tempering_adapt=True, floor_level=1e-3,igrand_threshold_p=1e-3,convergence_tests=test_converged,adapt_weight_exponent=my_exp,no_protect_names=True,**extra_args) # weight ecponent needs better choice. We are using arbitrary-name functions - +# result value: be careful, if the sampler returns lnL, must not take log twice! +#NOTE: RoboCode messes with this value because of its insane plugin functionality # Save result -- needed for odds ratios, etc. np.savetxt(opts.fname_output_integral+"_result.txt", [np.log(res)]) From 97452077b5fd488e4698cd91991d8dbba80a3edd Mon Sep 17 00:00:00 2001 From: mebiri Date: Mon, 21 Sep 2026 18:23:17 -0500 Subject: [PATCH 3/8] commented out hardcoded opts in HyperPuff --- .../Code/bin/util_HyperPuffball_new.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py b/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py index c93fef054..a87b5a40e 100644 --- a/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py +++ b/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py @@ -46,15 +46,15 @@ opts = parser.parse_args() -opts.inj_file = "grid_test.txt" -opts.puff_factor = 0.001 -opts.parameter = ["gamma0","gamma1","gamma2","gamma3","m1","m2"] -opts.parameter_range = ["[0.2,2]","[-1.6,1.7]","[-0.6,0.6]","[-0.02,0.02]","[1,3]","[1,3]"] -opts.reflect_parameter = ["gamma0","gamma1","gamma2","gamma3","m1","m2"] -opts.supplementary_coordinate_code = "dan_rotation_conversion" -opts.supplementary_coordinate_function = "dan_rotation" -opts.get_range_from_external = True -opts.external_range_args = ["buffer=4.0"] +#opts.inj_file = "grid_test.txt" +#opts.puff_factor = 0.001 +#opts.parameter = ["gamma0","gamma1","gamma2","gamma3","m1","m2"] +#opts.parameter_range = ["[0.2,2]","[-1.6,1.7]","[-0.6,0.6]","[-0.02,0.02]","[1,3]","[1,3]"] +#opts.reflect_parameter = ["gamma0","gamma1","gamma2","gamma3","m1","m2"] +#opts.supplementary_coordinate_code = "dan_rotation_conversion" +#opts.supplementary_coordinate_function = "dan_rotation" +#opts.get_range_from_external = True +#opts.external_range_args = ["buffer=4.0"] if opts.random_parameter is None: @@ -362,4 +362,4 @@ indx_in = coord_names.index(p) dat_raw[p] = X_out[:,indx_in] -np.savetxt(opts.inj_file_out, dat_raw,header=" ".join(dat_raw.dtype.names)) \ No newline at end of file +np.savetxt(opts.inj_file_out, dat_raw,header=" ".join(dat_raw.dtype.names)) From ff4b7ae9a8824f852782a6eb17d6fafcc460aaad Mon Sep 17 00:00:00 2001 From: mebiri Date: Mon, 21 Sep 2026 22:45:55 -0500 Subject: [PATCH 4/8] alterations to new hyperpuff it should work now --- .../Code/bin/util_HyperPuffball_new.py | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py b/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py index a87b5a40e..263bd9854 100644 --- a/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py +++ b/MonteCarloMarginalizeCode/Code/bin/util_HyperPuffball_new.py @@ -28,35 +28,23 @@ parser.add_argument("--downselect-parameter",action='append', help='Name of parameter to be used to eliminate grid points ') parser.add_argument("--downselect-parameter-range",action='append',type=str,help="legacy option; prefer --parameter-range") parser.add_argument("--reflect-parameter",action='append',type=str) -parser.add_argument("--regularize",action='store_true',help="Add some ad-hoc terms based on priors, to help with nearly-singular matricies") -parser.add_argument("--downselect-mass-range",action='store_true',help="Reject points where m2 > m1 for population models (slightly hacky; prefer diff coord sys)") +parser.add_argument("--regularize",action='store_true',help="Add some ad-hoc terms based on priors, to help with nearly-singular matrices") parser.add_argument("--get-range-from-external",action='store_true',help="Get/modify parameter ranges via external func in --supplementary-coordinate-code") parser.add_argument("--external-range-args",action='append',help="Provide special args to external range func; syntax: single str 'arg_name=arg_value'.") #to unlink downselect & reflection: -parser.add_argument("--parameter-range",action='append',type=str) +parser.add_argument("--parameter-range",action='append',type=str,help="Recommended format: 'coord:[lower,upper]'; if no 'coord:' supplied, matches provided ranges with parameter list in cmd line order (old functionality)") #for generalization: parser.add_argument("--supplementary-coordinate-code", default=None,type=str,help="Coordinate conversion/prior code. Accepts: the literal 'rift_default' (use RIFT.lalsimutils.convert_waveform_coordinates plus RIFT-standard priors); a filesystem path ending in .py (loaded as a plugin); or any importable dotted module name.") parser.add_argument("--supplementary-coordinate-function", default=None, type=str, help="Name of the entry-point callable inside the module named by --supplementary-coordinate-code. Defaults to 'convert_coordinates'.") #to be deprecated: +parser.add_argument("--downselect-mass-range",action='store_true',help="Reject points where m2 > m1 for population models (slightly hacky; prefer diff coord sys)") parser.add_argument("--use-rotated-spectral-coords",action='store_true',help="deprecated; Apply rotated coord sys to spectral EOS params; from Wysocki 2020 https://arxiv.org/pdf/2001.01747") parser.add_argument("--rotated-coord-buffer",default=0.0,type=float,help="deprecated; Fractional buffer (e.g., 0.1; default 0) to extend rotated hypercube space (APPLIES AS % OF BOUND VALUE)") #parser.add_argument("--reflect-parameters",action='store_true',help="Toggle parameter reflection, even if no --reflect-parameter provided (for rotated coord reflection)") #parser.add_argument("--use-alternate-buffer",action='store_true',help="Buffer expands hypercube by x% of its full width (2x% total expansion), instead of by x% of bound value (equivalent to (2x)% bound val buffer for [-r,+r] bounds.") - opts = parser.parse_args() -#opts.inj_file = "grid_test.txt" -#opts.puff_factor = 0.001 -#opts.parameter = ["gamma0","gamma1","gamma2","gamma3","m1","m2"] -#opts.parameter_range = ["[0.2,2]","[-1.6,1.7]","[-0.6,0.6]","[-0.02,0.02]","[1,3]","[1,3]"] -#opts.reflect_parameter = ["gamma0","gamma1","gamma2","gamma3","m1","m2"] -#opts.supplementary_coordinate_code = "dan_rotation_conversion" -#opts.supplementary_coordinate_function = "dan_rotation" -#opts.get_range_from_external = True -#opts.external_range_args = ["buffer=4.0"] - - if opts.random_parameter is None: opts.random_parameter = [] @@ -90,21 +78,34 @@ if opts.downselect_parameter_range: #to handle legacy uses opts.parameter_range = opts.downselect_parameter_range -#make master param dict, then filter into downselect & reflect dicts +#make master param dict, then filter into downselect & reflect dicts later plist = [] if opts.downselect_parameter: plist += opts.downselect_parameter if opts.reflect_parameter: - plist += opts.reflect_parameter + plist += opts.reflect_parameter + +#parse parameter ranges if opts.parameter_range: - param_ranges = list(map(eval,opts.parameter_range)) - - if len(plist) < len(param_ranges): #allow unbounded params, for now (expect external bounds) + if len(opts.parameter_range[0].split(":")) == 1: + #legacy cmd line - no variable with range, fill in order + param_ranges = list(map(eval,opts.parameter_range)) + + for indx in np.arange(len(param_ranges)): + param_dict[plist[indx]] = param_ranges[indx] + else: + #more intelligent - adapted from ConstructEOSPosterior + for range_code in opts.parameter_range: + param_dat = range_code.split(':') + if param_dat[0] not in plist: + print(" WARNING: skipping param not in reflect/downselect lists:",param_dat[0]) + continue + str_range = param_dat[1].replace("[","").replace("]","").split(",") #safer than eval + param_range = [float(x) for x in str_range] + param_dict[param_dat[0]] = param_range #note the dict keys' order may be different from param order now + if len(plist) < len(param_dict): #allow unbounded params (expect external bounds) print(" parameters and parameter ranges inconsistent:",plist,param_ranges) - raise Exception(" All bounded coordinates must have a specified parameter range") - - for indx in np.arange(len(param_ranges)): - param_dict[plist[indx]] = param_ranges[indx] + raise Exception(" parameter ranges overspecified: missing corresponding parameters") #no else: allow for external bounds to be provided if opts.use_rotated_spectral_coords: @@ -147,6 +148,7 @@ if opts.get_range_from_external: #retrieve/modify param ranges externally try: + print(" Retrieving param bounds from external module") supplemental_bound_func = getattr(external_coordinate_module, "get_bounds") param_dict = supplemental_bound_func(plist,param_dict,**external_kwargs) except: @@ -261,7 +263,7 @@ tmp = reflect_dict[param][0] + np.mod(X_out[:,indx] - reflect_dict[param][0], 2*(reflect_dict[param][1] - reflect_dict[param][0]) ) # final reflection tmp = np.where( tmp > reflect_dict[param][1], 2*reflect_dict[param][1] - tmp, tmp) - print(" Increment reflect : {} {} ".format(param,np.sum(tmp != X_out[:,indx]))) + print(" Increment reflect : {} {} ".format(param,np.sum(np.round(tmp,decimals=8) != np.round(X_out[:,indx],decimals=8)))) X_out[:,indx] = tmp # Downselect @@ -362,4 +364,4 @@ indx_in = coord_names.index(p) dat_raw[p] = X_out[:,indx_in] -np.savetxt(opts.inj_file_out, dat_raw,header=" ".join(dat_raw.dtype.names)) +np.savetxt(opts.inj_file_out, dat_raw,header=" ".join(dat_raw.dtype.names)) \ No newline at end of file From 46231ee49eb116bda9a6122fab63815e3b64db60 Mon Sep 17 00:00:00 2001 From: mebiri Date: Tue, 22 Sep 2026 03:13:16 -0500 Subject: [PATCH 5/8] fix to error handling for external codes --- util_HyperPuffball_new.py | 368 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 util_HyperPuffball_new.py diff --git a/util_HyperPuffball_new.py b/util_HyperPuffball_new.py new file mode 100644 index 000000000..c0bdb220c --- /dev/null +++ b/util_HyperPuffball_new.py @@ -0,0 +1,368 @@ +#! /usr/bin/env python +''' + Updated version of util_HyperparameterPuffball.py for HyperPipe. + Handles coordinate rotation via external plugin and coordinate reflection + Supports older usage of HyperPuff via legacy code + + old usage: + --parameter m1 --downselect-parameter m1 --downselect-parameter-range [] --reflect-parameter m1 + new usage: + --parameter m1 --parameter m2 --downselect-parameter m1 --parameter-range [] --reflect-parameter m2 --parameter-range [] +''' +import argparse +import sys +import numpy as np +import numpy.lib.recfunctions +#import scipy +from scipy import stats + +parser = argparse.ArgumentParser() +parser.add_argument("--inj-file", help="Name of grid.dat file") +parser.add_argument("--inj-file-out", default="output-puffball.dat", help="Name of grid_puff.dat file") +parser.add_argument("--puff-factor", default=1,type=float) +parser.add_argument("--force-away", default=0,type=float,help="Unused legacy option. If >0, uses the icov to compute a metric, and discards points which are close to existing points") +parser.add_argument("--parameter", action='append', help="Parameters used as fitting parameters AND varied at a low level to make a posterior") +parser.add_argument("--no-correlation", type=str,action='append', help="Pairs of parameters, in format [mc,eta] The corresponding term in the covariance matrix is eliminated") +parser.add_argument("--random-parameter", action='append',help="These parameters are specified at random over the entire range, uncorrelated with the grid used for other parameters. Use for variables which correlate weakly with others; helps with random exploration") +parser.add_argument("--random-parameter-range", action='append', type=str,help="Add a range (pass as a string evaluating to a python 2-element list): --parameter-range '[0.,1000.]' MUST specify ALL parameter ranges (min and max) in order if used. ") +parser.add_argument("--downselect-parameter",action='append', help='Name of parameter to be used to eliminate grid points ') +parser.add_argument("--downselect-parameter-range",action='append',type=str,help="legacy option; prefer --parameter-range") +parser.add_argument("--reflect-parameter",action='append',type=str) +parser.add_argument("--regularize",action='store_true',help="Add some ad-hoc terms based on priors, to help with nearly-singular matrices") +parser.add_argument("--get-range-from-external",action='store_true',help="Get/modify parameter ranges via external func in --supplementary-coordinate-code") +parser.add_argument("--external-range-args",action='append',help="Provide special args to external range func; syntax: single str 'arg_name=arg_value'.") +#to unlink downselect & reflection: +parser.add_argument("--parameter-range",action='append',type=str,help="Recommended format: 'coord:[lower,upper]'; if no 'coord:' supplied, matches provided ranges with parameter list in cmd line order (old functionality)") +#for generalization: +parser.add_argument("--supplementary-coordinate-code", default=None,type=str,help="Coordinate conversion/prior code. Accepts: the literal 'rift_default' (use RIFT.lalsimutils.convert_waveform_coordinates plus RIFT-standard priors); a filesystem path ending in .py (loaded as a plugin); or any importable dotted module name.") +parser.add_argument("--supplementary-coordinate-function", default=None, type=str, help="Name of the entry-point callable inside the module named by --supplementary-coordinate-code. Defaults to 'convert_coordinates'.") +#to be deprecated: +parser.add_argument("--downselect-mass-range",action='store_true',help="Reject points where m2 > m1 for population models (slightly hacky; prefer diff coord sys)") +parser.add_argument("--use-rotated-spectral-coords",action='store_true',help="deprecated; Apply rotated coord sys to spectral EOS params; from Wysocki 2020 https://arxiv.org/pdf/2001.01747") +parser.add_argument("--rotated-coord-buffer",default=0.0,type=float,help="deprecated; Fractional buffer (e.g., 0.1; default 0) to extend rotated hypercube space (APPLIES AS % OF BOUND VALUE)") +#parser.add_argument("--reflect-parameters",action='store_true',help="Toggle parameter reflection, even if no --reflect-parameter provided (for rotated coord reflection)") +#parser.add_argument("--use-alternate-buffer",action='store_true',help="Buffer expands hypercube by x% of its full width (2x% total expansion), instead of by x% of bound value (equivalent to (2x)% bound val buffer for [-r,+r] bounds.") + +opts = parser.parse_args() + +if opts.random_parameter is None: + opts.random_parameter = [] + +# Extract parameter names +coord_names = opts.parameter # Used in fit +if coord_names is None: + print(" ERROR: no coordinates provided ") + sys.exit(0) #no puff + +# match up pairs in --no-correlation +corr_list = None +if not(opts.no_correlation is None): + corr_list = [] + corr_name_list = list(map(eval,opts.no_correlation)) + #print opts.no_correlation, corr_name_list + for my_pair in corr_name_list: + + i1 = coord_names.index(my_pair[0]) + i2 = coord_names.index(my_pair[1]) + + if i1>-1 and i2 > -1: + corr_list.append([i1,i2]) +# else: +# print i1, i2 +# print opts.no_correlation, coord_names, corr_list + +downselect_dict = {} +reflect_dict = {} +param_dict = {} + +if opts.downselect_parameter_range: #to handle legacy uses + opts.parameter_range = opts.downselect_parameter_range + +#make master param dict, then filter into downselect & reflect dicts later +plist = [] +if opts.downselect_parameter: + plist += opts.downselect_parameter +if opts.reflect_parameter: + plist += opts.reflect_parameter + +#parse parameter ranges +if opts.parameter_range: + if len(opts.parameter_range[0].split(":")) == 1: + #legacy cmd line - no variable with range, fill in order + param_ranges = list(map(eval,opts.parameter_range)) + + for indx in np.arange(len(param_ranges)): + param_dict[plist[indx]] = param_ranges[indx] + else: + #more intelligent - adapted from ConstructEOSPosterior + for range_code in opts.parameter_range: + param_dat = range_code.split(':') + if param_dat[0] not in plist: + print(" WARNING: skipping param not in reflect/downselect lists:",param_dat[0]) + continue + str_range = param_dat[1].replace("[","").replace("]","").split(",") #safer than eval + param_range = [float(x) for x in str_range] + param_dict[param_dat[0]] = param_range #note the dict keys' order may be different from param order now + if len(plist) < len(param_dict): #allow unbounded params (expect external bounds) + print(" parameters and parameter ranges inconsistent:",plist,param_ranges) + raise Exception(" parameter ranges overspecified: missing corresponding parameters") +#no else: allow for external bounds to be provided + +if opts.use_rotated_spectral_coords: + #legacy support - still need external rotation code for grid lines, though + plist = ["gamma0","gamma1","gamma2","gamma3"] + plist + rot_coords = {} + rot_coords["gamma0"] = [-4.37722, 4.91227] + rot_coords["gamma1"] = [-1.82240, 2.06387] + rot_coords["gamma2"] = [-0.32445, 0.36469] + rot_coords["gamma3"] = [-0.09529, 0.11046] + + for indx, param in enumerate(rot_coords.keys()): + ubound = rot_coords[param][1] + opts.rotated_coord_buffer*abs(rot_coords[param][1]) + lbound = rot_coords[param][0] - opts.rotated_coord_buffer*abs(rot_coords[param][0]) + param_dict[param] = [lbound,ubound] + +#parse args to be passed to coord convert funcs: +external_kwargs = {} +if opts.external_range_args: + for arg in opts.external_range_args: + external_kwargs[arg.split("=")[0]] = arg.split("=")[1] + +#load coordinate rotation module, if provided +supplemental_coordinate_convert = None +supplemental_coordinate_invert = None +if opts.supplementary_coordinate_code and opts.supplementary_coordinate_function: + print(" EXTERNAL COORDINATE CONVERSION : {}.{} ".format(opts.supplementary_coordinate_code,opts.supplementary_coordinate_function)) + __import__(opts.supplementary_coordinate_code) + external_coordinate_module = sys.modules[opts.supplementary_coordinate_code] + if hasattr(external_coordinate_module,opts.supplementary_coordinate_function): + supplemental_coordinate_convert = getattr(external_coordinate_module,opts.supplementary_coordinate_function) + try: + supplemental_coordinate_invert = getattr(external_coordinate_module, "inverse_"+opts.supplementary_coordinate_function) + except: + print(" ERROR: no inverted coordinate conversion found! Will not apply rotation, but supplied bounds will still be used!") + supplemental_coordinate_invert = None + supplemental_coordinate_convert = None #avoids producing output in rotated coords + else: + print("ERROR: could not retrieve supplied coordinate function. No conversion will be applied, but supplied bounds will be used!") + + if opts.get_range_from_external: #retrieve/modify param ranges externally + try: + print(" Retrieving param bounds from external module") + supplemental_bound_func = getattr(external_coordinate_module, "get_bounds") + param_dict = supplemental_bound_func(plist,param_dict,**external_kwargs) + except Exception as e: + print(" ERROR fetching external ranges:",e) + print(" WARNING: external range requested but not retrieved. Using supplied bounds as-is.") + + +#at this point, all given params must have ranges +if (len(plist) != len(param_dict)): + print(" Reflection/downselect params inconsistent:",plist,param_dict) + raise Exception(" All reflect/downselect coords must have valid param ranges") + +#set downselect +if opts.downselect_parameter: + dlist = opts.downselect_parameter + #dlist_ranges = [param_ranges[coord_names.index(p)] for p in dlist] #list(map(eval,opts.parameter_range[:len(dlist)])) + + #if len(dlist) != len(dlist_ranges): #should never happen, now + # print(" downselect parameters inconsistent", dlist, dlist_ranges) + + for indx in np.arange(len(dlist)): + if not(dlist[indx] in coord_names): + raise Exception(" Downselect only allowed for coordinates (--parameter) ") + downselect_dict[dlist[indx]] = param_dict[dlist[indx]] +else: + dlist = [] + #dlist_ranges = [] + opts.downselect_parameter = [] + +#for indx in np.arange(len(dlist_ranges)): +# downselect_dict[dlist[indx]] = dlist_ranges[indx] + +#set reflect +#indx_reflect=[] +if opts.reflect_parameter: + rlist = opts.reflect_parameter + #indx_reflect = [coord_names.index(param) for param in opts.reflect_parameter] + + if len(rlist) > len(coord_names): + print(" reflection parameters inconsistent", rlist, coord_names) + raise Exception(" Reflection only allowed for coordinates ") + + for indx in np.arange(len(rlist)): + if not(rlist[indx] in coord_names): + raise Exception(" Reflection only allowed for coordinates (--parameter) ") + #if not(rlist[indx] in downselect_dict): + # raise Exception(" Reflection requires parameter range specified as a downselection ") + if rlist[indx] in downselect_dict: + del downselect_dict[rlist[indx]] #handles legacy cases, where d-select was required to reflect + reflect_dict[rlist[indx]] = param_dict[rlist[indx]] +else: + rlist = [] + opts.reflect_parameter = [] + + +# Load data; keep parameter names +dat_raw = np.genfromtxt(opts.inj_file, names=True) +X = np.zeros((len(dat_raw), len(coord_names))) +# Copy over the parameters we use. Note we have no way to create linear combinations or alternate coordinates here +for p in coord_names: + #indx_p = list(dat_raw.dtype.names).index(p) + indx_in = coord_names.index(p) + X[:,indx_in] = dat_raw[p] + + +# Measure covariance matrix and generate random errors +if len(coord_names) >1: + cov_in = np.cov(X.T) + cov = cov_in*opts.puff_factor*opts.puff_factor + + # Check for singularities + if np.min(np.linalg.eig(cov)[0])<1e-10: + print(" ===> WARNING: SINGULAR MATRIX: are you sure you varied this parameters? <=== ") + icov_pseudo = np.linalg.pinv(cov) + # Prior range for each parameter is 1000, so icov diag terms are 10^(-6) + # This is somewhat made up, but covers most things + diag_terms = 1e-6*np.ones(len(cov)) + # + icov_proposed = icov_pseudo+np.diag(diag_terms) + cov= np.linalg.inv(icov_proposed) + + cov_orig = np.array(cov) # force copy + # Remove targeted covariances + if not(corr_list is None): + for my_pair in corr_list: + if my_pair[0] != my_pair[1]: + cov[my_pair[0],my_pair[1]]=0 + cov[my_pair[1],my_pair[0]]=0 + + + # Compute errors + rv = stats.multivariate_normal(mean=np.zeros(len(coord_names)), cov=cov,allow_singular=True) # they are just complaining about dynamic range of parameters, usually + delta_X = rv.rvs(size=len(X)) + X_out = X+delta_X + +else: + sigma = np.std(X) + cov = sigma*sigma + delta_X =np.random.normal(size=len(coord_names), scale=sigma) + X_out = X+delta_X + +# Apply coordinate rotation +if supplemental_coordinate_convert: + X_out = supplemental_coordinate_convert(X_out, coord_names, **external_kwargs) + +print(" Initial data length:",len(X_out)) + +# Reflection +for param in rlist: + indx = coord_names.index(param) + print(" Reflecting into range : {} [{}, {}]".format(param,reflect_dict[param][0],reflect_dict[param][1])) + # put in range [0,2 L] + tmp = reflect_dict[param][0] + np.mod(X_out[:,indx] - reflect_dict[param][0], 2*(reflect_dict[param][1] - reflect_dict[param][0]) ) + # final reflection + tmp = np.where( tmp > reflect_dict[param][1], 2*reflect_dict[param][1] - tmp, tmp) + print(" Increment reflect : {} {} ".format(param,np.sum(np.round(tmp,decimals=8) != np.round(X_out[:,indx],decimals=8)))) + X_out[:,indx] = tmp + +# Downselect +names_downselect = list(downselect_dict.keys()) +indx_ok = np.ones(len(X_out),dtype=bool) + +# ============================================================================= +# #Apply rotation to spectral EOS params from Wysocki et. al 2020 https://arxiv.org/pdf/2001.01747 +# if opts.use_rotated_spectral_coords: +# print(" Applying rotated coordinate transformation to spectral parameters") +# dan_rot = [[0.43801, -0.53573, 0.52661, -0.49379], +# [-0.76705, 0.17169, 0.31255, -0.53336], +# [0.45143, 0.67967, -0.19454, -0.54443], +# [0.12646, 0.47070, 0.76626, 0.41868]] +# scaled_mean = [0.89421, 0.33878, -0.07894, 0.00393] +# scaled_sig = [0.35700, 0.25769, 0.05452, 0.00312] +# dan_inv = np.linalg.inv(dan_rot) +# +# #get gammas' indices in X_out(= X) from coord names +# r_tilde = np.zeros((len(X_out),4)) +# rot_cols = [] +# for i in np.arange(4): +# #do one coord at a time +# indx = coord_names.index("gamma"+str(i)) +# rot_cols.append(indx) +# +# #convert gammas to r_tilde using equation: r_tilde = (gamma - u)/sig +# r_tilde[:,i] = (X_out[:,indx] - scaled_mean[i])/scaled_sig[i] +# +# #apply transform: r_prime = S*r_tilde ( [4 x 4].([N x 4].T) ) +# r_prime = np.matmul(dan_rot,r_tilde.T).T +# +# rot_coords = {} +# rot_coords["r0"] = [-4.37722, 4.91227] +# rot_coords["r1"] = [-1.82240, 2.06387] +# rot_coords["r2"] = [-0.32445, 0.36469] +# rot_coords["r3"] = [-0.09529, 0.11046] +# +# for indx, param in enumerate(rot_coords.keys()): +# # apply hypercube buffer +# if opts.use_alternate_buffer: #new_bound = bound +/- buffer*(width of param range) -> SYMMETRIC buffer +# ubound = rot_coords[param][1] + opts.rotated_coord_buffer*abs(rot_coords[param][1]-rot_coords[param][0]) +# lbound = rot_coords[param][0] - opts.rotated_coord_buffer*abs(rot_coords[param][1]-rot_coords[param][0]) +# else: #new_bound = bound +/- buffer*|bound| -> asymmetric buffer +# ubound = rot_coords[param][1] + opts.rotated_coord_buffer*abs(rot_coords[param][1]) +# lbound = rot_coords[param][0] - opts.rotated_coord_buffer*abs(rot_coords[param][0]) +# if opts.reflect_parameters: +# # Reflection +# print(" Reflecting into range : {} [{}, {}]".format(param,lbound,ubound)) +# # put in range [0,2 L] +# #tmp = rot_coords[param][0] + np.mod(r_prime[:,indx] - rot_coords[param][0], 2*(rot_coords[param][1] - rot_coords[param][0]) ) +# tmp = lbound + np.mod(r_prime[:,indx] - lbound, 2*(ubound - lbound) ) +# # final reflection +# #tmp = np.where( tmp > rot_coords[param][1], 2*rot_coords[param][1] - tmp, tmp) +# tmp = np.where( tmp > ubound, 2*ubound - tmp, tmp) +# print(" Increment reflect : {} {} ".format(param,np.sum(tmp != r_prime[:,indx]))) +# r_prime[:,indx] = tmp +# else: +# # Downselection +# #print(" Downselecting:",name,"; indx:",indx,"[",lbound,ubound,"]; buffer =",opts.rotated_coord_buffer) +# indx_ok = np.logical_and(indx_ok, r_prime[:,indx]<= ubound ) +# indx_ok = np.logical_and(indx_ok, r_prime[:,indx]>= lbound ) +# print(' Increment downselect : {} {} '.format(param, np.sum(indx_ok) )) +# +# if opts.reflect_parameters: +# #apply inverse: S-1*r_prime = S-1*S*r_tilde = r_tilde ( [4 x 4].([N x 4].T) ) +# r_tilde_post = np.matmul(dan_inv,r_prime.T).T +# +# for i, col in enumerate(rot_cols): +# #r_tilde = (gamma - u)/sig -> gamma = r_tilde*sig + u +# X_out[:,col] = (r_tilde_post[:,i]*scaled_sig[i]) + scaled_mean[i] +# ============================================================================= + +for name in names_downselect: + indx = coord_names.index(name) + print(" Downselecting:",name,"; indx:",indx,"[",downselect_dict[name][0],downselect_dict[name][1],"]") + indx_ok = np.logical_and(indx_ok, np.logical_not(np.isnan(X_out[:,indx]))) + indx_ok = np.logical_and(indx_ok, X_out[:,indx]<= downselect_dict[name][1] ) + indx_ok = np.logical_and(indx_ok, X_out[:,indx]>= downselect_dict[name][0] ) + print(' Increment downselect : {} {} '.format(name, np.sum(indx_ok) )) + +if supplemental_coordinate_invert: + X_out = supplemental_coordinate_invert(X_out, coord_names, **external_kwargs) + +#enforce m1 >= m2 for populations - should really replace by converting m2 -> q +if opts.downselect_mass_range and ("m1" in coord_names) and ("m2" in coord_names): + m1_indx = coord_names.index("m1") + m2_indx = coord_names.index("m2") + indx_ok = np.logical_and(indx_ok, X_out[:,m1_indx]>= X_out[:,m2_indx] ) + print(' Increment downselect : {} {} '.format("m1>=m2", np.sum(indx_ok) )) + +X_out = X_out[indx_ok] +dat_raw = dat_raw[indx_ok] # must downselect here as well! + +# Write data back into correct format and save +for p in coord_names: +# indx_p = dat_raw.dtype.names.index(p) + indx_in = coord_names.index(p) + dat_raw[p] = X_out[:,indx_in] + +np.savetxt(opts.inj_file_out, dat_raw,header=" ".join(dat_raw.dtype.names)) \ No newline at end of file From 1fb4aa2cba914692131cb4511ed48c5355abebb6 Mon Sep 17 00:00:00 2001 From: mebiri Date: Sun, 27 Sep 2026 15:17:59 -0500 Subject: [PATCH 6/8] Delete util_HyperPuffball_new.py - in wrong place --- util_HyperPuffball_new.py | 368 -------------------------------------- 1 file changed, 368 deletions(-) delete mode 100644 util_HyperPuffball_new.py diff --git a/util_HyperPuffball_new.py b/util_HyperPuffball_new.py deleted file mode 100644 index c0bdb220c..000000000 --- a/util_HyperPuffball_new.py +++ /dev/null @@ -1,368 +0,0 @@ -#! /usr/bin/env python -''' - Updated version of util_HyperparameterPuffball.py for HyperPipe. - Handles coordinate rotation via external plugin and coordinate reflection - Supports older usage of HyperPuff via legacy code - - old usage: - --parameter m1 --downselect-parameter m1 --downselect-parameter-range [] --reflect-parameter m1 - new usage: - --parameter m1 --parameter m2 --downselect-parameter m1 --parameter-range [] --reflect-parameter m2 --parameter-range [] -''' -import argparse -import sys -import numpy as np -import numpy.lib.recfunctions -#import scipy -from scipy import stats - -parser = argparse.ArgumentParser() -parser.add_argument("--inj-file", help="Name of grid.dat file") -parser.add_argument("--inj-file-out", default="output-puffball.dat", help="Name of grid_puff.dat file") -parser.add_argument("--puff-factor", default=1,type=float) -parser.add_argument("--force-away", default=0,type=float,help="Unused legacy option. If >0, uses the icov to compute a metric, and discards points which are close to existing points") -parser.add_argument("--parameter", action='append', help="Parameters used as fitting parameters AND varied at a low level to make a posterior") -parser.add_argument("--no-correlation", type=str,action='append', help="Pairs of parameters, in format [mc,eta] The corresponding term in the covariance matrix is eliminated") -parser.add_argument("--random-parameter", action='append',help="These parameters are specified at random over the entire range, uncorrelated with the grid used for other parameters. Use for variables which correlate weakly with others; helps with random exploration") -parser.add_argument("--random-parameter-range", action='append', type=str,help="Add a range (pass as a string evaluating to a python 2-element list): --parameter-range '[0.,1000.]' MUST specify ALL parameter ranges (min and max) in order if used. ") -parser.add_argument("--downselect-parameter",action='append', help='Name of parameter to be used to eliminate grid points ') -parser.add_argument("--downselect-parameter-range",action='append',type=str,help="legacy option; prefer --parameter-range") -parser.add_argument("--reflect-parameter",action='append',type=str) -parser.add_argument("--regularize",action='store_true',help="Add some ad-hoc terms based on priors, to help with nearly-singular matrices") -parser.add_argument("--get-range-from-external",action='store_true',help="Get/modify parameter ranges via external func in --supplementary-coordinate-code") -parser.add_argument("--external-range-args",action='append',help="Provide special args to external range func; syntax: single str 'arg_name=arg_value'.") -#to unlink downselect & reflection: -parser.add_argument("--parameter-range",action='append',type=str,help="Recommended format: 'coord:[lower,upper]'; if no 'coord:' supplied, matches provided ranges with parameter list in cmd line order (old functionality)") -#for generalization: -parser.add_argument("--supplementary-coordinate-code", default=None,type=str,help="Coordinate conversion/prior code. Accepts: the literal 'rift_default' (use RIFT.lalsimutils.convert_waveform_coordinates plus RIFT-standard priors); a filesystem path ending in .py (loaded as a plugin); or any importable dotted module name.") -parser.add_argument("--supplementary-coordinate-function", default=None, type=str, help="Name of the entry-point callable inside the module named by --supplementary-coordinate-code. Defaults to 'convert_coordinates'.") -#to be deprecated: -parser.add_argument("--downselect-mass-range",action='store_true',help="Reject points where m2 > m1 for population models (slightly hacky; prefer diff coord sys)") -parser.add_argument("--use-rotated-spectral-coords",action='store_true',help="deprecated; Apply rotated coord sys to spectral EOS params; from Wysocki 2020 https://arxiv.org/pdf/2001.01747") -parser.add_argument("--rotated-coord-buffer",default=0.0,type=float,help="deprecated; Fractional buffer (e.g., 0.1; default 0) to extend rotated hypercube space (APPLIES AS % OF BOUND VALUE)") -#parser.add_argument("--reflect-parameters",action='store_true',help="Toggle parameter reflection, even if no --reflect-parameter provided (for rotated coord reflection)") -#parser.add_argument("--use-alternate-buffer",action='store_true',help="Buffer expands hypercube by x% of its full width (2x% total expansion), instead of by x% of bound value (equivalent to (2x)% bound val buffer for [-r,+r] bounds.") - -opts = parser.parse_args() - -if opts.random_parameter is None: - opts.random_parameter = [] - -# Extract parameter names -coord_names = opts.parameter # Used in fit -if coord_names is None: - print(" ERROR: no coordinates provided ") - sys.exit(0) #no puff - -# match up pairs in --no-correlation -corr_list = None -if not(opts.no_correlation is None): - corr_list = [] - corr_name_list = list(map(eval,opts.no_correlation)) - #print opts.no_correlation, corr_name_list - for my_pair in corr_name_list: - - i1 = coord_names.index(my_pair[0]) - i2 = coord_names.index(my_pair[1]) - - if i1>-1 and i2 > -1: - corr_list.append([i1,i2]) -# else: -# print i1, i2 -# print opts.no_correlation, coord_names, corr_list - -downselect_dict = {} -reflect_dict = {} -param_dict = {} - -if opts.downselect_parameter_range: #to handle legacy uses - opts.parameter_range = opts.downselect_parameter_range - -#make master param dict, then filter into downselect & reflect dicts later -plist = [] -if opts.downselect_parameter: - plist += opts.downselect_parameter -if opts.reflect_parameter: - plist += opts.reflect_parameter - -#parse parameter ranges -if opts.parameter_range: - if len(opts.parameter_range[0].split(":")) == 1: - #legacy cmd line - no variable with range, fill in order - param_ranges = list(map(eval,opts.parameter_range)) - - for indx in np.arange(len(param_ranges)): - param_dict[plist[indx]] = param_ranges[indx] - else: - #more intelligent - adapted from ConstructEOSPosterior - for range_code in opts.parameter_range: - param_dat = range_code.split(':') - if param_dat[0] not in plist: - print(" WARNING: skipping param not in reflect/downselect lists:",param_dat[0]) - continue - str_range = param_dat[1].replace("[","").replace("]","").split(",") #safer than eval - param_range = [float(x) for x in str_range] - param_dict[param_dat[0]] = param_range #note the dict keys' order may be different from param order now - if len(plist) < len(param_dict): #allow unbounded params (expect external bounds) - print(" parameters and parameter ranges inconsistent:",plist,param_ranges) - raise Exception(" parameter ranges overspecified: missing corresponding parameters") -#no else: allow for external bounds to be provided - -if opts.use_rotated_spectral_coords: - #legacy support - still need external rotation code for grid lines, though - plist = ["gamma0","gamma1","gamma2","gamma3"] + plist - rot_coords = {} - rot_coords["gamma0"] = [-4.37722, 4.91227] - rot_coords["gamma1"] = [-1.82240, 2.06387] - rot_coords["gamma2"] = [-0.32445, 0.36469] - rot_coords["gamma3"] = [-0.09529, 0.11046] - - for indx, param in enumerate(rot_coords.keys()): - ubound = rot_coords[param][1] + opts.rotated_coord_buffer*abs(rot_coords[param][1]) - lbound = rot_coords[param][0] - opts.rotated_coord_buffer*abs(rot_coords[param][0]) - param_dict[param] = [lbound,ubound] - -#parse args to be passed to coord convert funcs: -external_kwargs = {} -if opts.external_range_args: - for arg in opts.external_range_args: - external_kwargs[arg.split("=")[0]] = arg.split("=")[1] - -#load coordinate rotation module, if provided -supplemental_coordinate_convert = None -supplemental_coordinate_invert = None -if opts.supplementary_coordinate_code and opts.supplementary_coordinate_function: - print(" EXTERNAL COORDINATE CONVERSION : {}.{} ".format(opts.supplementary_coordinate_code,opts.supplementary_coordinate_function)) - __import__(opts.supplementary_coordinate_code) - external_coordinate_module = sys.modules[opts.supplementary_coordinate_code] - if hasattr(external_coordinate_module,opts.supplementary_coordinate_function): - supplemental_coordinate_convert = getattr(external_coordinate_module,opts.supplementary_coordinate_function) - try: - supplemental_coordinate_invert = getattr(external_coordinate_module, "inverse_"+opts.supplementary_coordinate_function) - except: - print(" ERROR: no inverted coordinate conversion found! Will not apply rotation, but supplied bounds will still be used!") - supplemental_coordinate_invert = None - supplemental_coordinate_convert = None #avoids producing output in rotated coords - else: - print("ERROR: could not retrieve supplied coordinate function. No conversion will be applied, but supplied bounds will be used!") - - if opts.get_range_from_external: #retrieve/modify param ranges externally - try: - print(" Retrieving param bounds from external module") - supplemental_bound_func = getattr(external_coordinate_module, "get_bounds") - param_dict = supplemental_bound_func(plist,param_dict,**external_kwargs) - except Exception as e: - print(" ERROR fetching external ranges:",e) - print(" WARNING: external range requested but not retrieved. Using supplied bounds as-is.") - - -#at this point, all given params must have ranges -if (len(plist) != len(param_dict)): - print(" Reflection/downselect params inconsistent:",plist,param_dict) - raise Exception(" All reflect/downselect coords must have valid param ranges") - -#set downselect -if opts.downselect_parameter: - dlist = opts.downselect_parameter - #dlist_ranges = [param_ranges[coord_names.index(p)] for p in dlist] #list(map(eval,opts.parameter_range[:len(dlist)])) - - #if len(dlist) != len(dlist_ranges): #should never happen, now - # print(" downselect parameters inconsistent", dlist, dlist_ranges) - - for indx in np.arange(len(dlist)): - if not(dlist[indx] in coord_names): - raise Exception(" Downselect only allowed for coordinates (--parameter) ") - downselect_dict[dlist[indx]] = param_dict[dlist[indx]] -else: - dlist = [] - #dlist_ranges = [] - opts.downselect_parameter = [] - -#for indx in np.arange(len(dlist_ranges)): -# downselect_dict[dlist[indx]] = dlist_ranges[indx] - -#set reflect -#indx_reflect=[] -if opts.reflect_parameter: - rlist = opts.reflect_parameter - #indx_reflect = [coord_names.index(param) for param in opts.reflect_parameter] - - if len(rlist) > len(coord_names): - print(" reflection parameters inconsistent", rlist, coord_names) - raise Exception(" Reflection only allowed for coordinates ") - - for indx in np.arange(len(rlist)): - if not(rlist[indx] in coord_names): - raise Exception(" Reflection only allowed for coordinates (--parameter) ") - #if not(rlist[indx] in downselect_dict): - # raise Exception(" Reflection requires parameter range specified as a downselection ") - if rlist[indx] in downselect_dict: - del downselect_dict[rlist[indx]] #handles legacy cases, where d-select was required to reflect - reflect_dict[rlist[indx]] = param_dict[rlist[indx]] -else: - rlist = [] - opts.reflect_parameter = [] - - -# Load data; keep parameter names -dat_raw = np.genfromtxt(opts.inj_file, names=True) -X = np.zeros((len(dat_raw), len(coord_names))) -# Copy over the parameters we use. Note we have no way to create linear combinations or alternate coordinates here -for p in coord_names: - #indx_p = list(dat_raw.dtype.names).index(p) - indx_in = coord_names.index(p) - X[:,indx_in] = dat_raw[p] - - -# Measure covariance matrix and generate random errors -if len(coord_names) >1: - cov_in = np.cov(X.T) - cov = cov_in*opts.puff_factor*opts.puff_factor - - # Check for singularities - if np.min(np.linalg.eig(cov)[0])<1e-10: - print(" ===> WARNING: SINGULAR MATRIX: are you sure you varied this parameters? <=== ") - icov_pseudo = np.linalg.pinv(cov) - # Prior range for each parameter is 1000, so icov diag terms are 10^(-6) - # This is somewhat made up, but covers most things - diag_terms = 1e-6*np.ones(len(cov)) - # - icov_proposed = icov_pseudo+np.diag(diag_terms) - cov= np.linalg.inv(icov_proposed) - - cov_orig = np.array(cov) # force copy - # Remove targeted covariances - if not(corr_list is None): - for my_pair in corr_list: - if my_pair[0] != my_pair[1]: - cov[my_pair[0],my_pair[1]]=0 - cov[my_pair[1],my_pair[0]]=0 - - - # Compute errors - rv = stats.multivariate_normal(mean=np.zeros(len(coord_names)), cov=cov,allow_singular=True) # they are just complaining about dynamic range of parameters, usually - delta_X = rv.rvs(size=len(X)) - X_out = X+delta_X - -else: - sigma = np.std(X) - cov = sigma*sigma - delta_X =np.random.normal(size=len(coord_names), scale=sigma) - X_out = X+delta_X - -# Apply coordinate rotation -if supplemental_coordinate_convert: - X_out = supplemental_coordinate_convert(X_out, coord_names, **external_kwargs) - -print(" Initial data length:",len(X_out)) - -# Reflection -for param in rlist: - indx = coord_names.index(param) - print(" Reflecting into range : {} [{}, {}]".format(param,reflect_dict[param][0],reflect_dict[param][1])) - # put in range [0,2 L] - tmp = reflect_dict[param][0] + np.mod(X_out[:,indx] - reflect_dict[param][0], 2*(reflect_dict[param][1] - reflect_dict[param][0]) ) - # final reflection - tmp = np.where( tmp > reflect_dict[param][1], 2*reflect_dict[param][1] - tmp, tmp) - print(" Increment reflect : {} {} ".format(param,np.sum(np.round(tmp,decimals=8) != np.round(X_out[:,indx],decimals=8)))) - X_out[:,indx] = tmp - -# Downselect -names_downselect = list(downselect_dict.keys()) -indx_ok = np.ones(len(X_out),dtype=bool) - -# ============================================================================= -# #Apply rotation to spectral EOS params from Wysocki et. al 2020 https://arxiv.org/pdf/2001.01747 -# if opts.use_rotated_spectral_coords: -# print(" Applying rotated coordinate transformation to spectral parameters") -# dan_rot = [[0.43801, -0.53573, 0.52661, -0.49379], -# [-0.76705, 0.17169, 0.31255, -0.53336], -# [0.45143, 0.67967, -0.19454, -0.54443], -# [0.12646, 0.47070, 0.76626, 0.41868]] -# scaled_mean = [0.89421, 0.33878, -0.07894, 0.00393] -# scaled_sig = [0.35700, 0.25769, 0.05452, 0.00312] -# dan_inv = np.linalg.inv(dan_rot) -# -# #get gammas' indices in X_out(= X) from coord names -# r_tilde = np.zeros((len(X_out),4)) -# rot_cols = [] -# for i in np.arange(4): -# #do one coord at a time -# indx = coord_names.index("gamma"+str(i)) -# rot_cols.append(indx) -# -# #convert gammas to r_tilde using equation: r_tilde = (gamma - u)/sig -# r_tilde[:,i] = (X_out[:,indx] - scaled_mean[i])/scaled_sig[i] -# -# #apply transform: r_prime = S*r_tilde ( [4 x 4].([N x 4].T) ) -# r_prime = np.matmul(dan_rot,r_tilde.T).T -# -# rot_coords = {} -# rot_coords["r0"] = [-4.37722, 4.91227] -# rot_coords["r1"] = [-1.82240, 2.06387] -# rot_coords["r2"] = [-0.32445, 0.36469] -# rot_coords["r3"] = [-0.09529, 0.11046] -# -# for indx, param in enumerate(rot_coords.keys()): -# # apply hypercube buffer -# if opts.use_alternate_buffer: #new_bound = bound +/- buffer*(width of param range) -> SYMMETRIC buffer -# ubound = rot_coords[param][1] + opts.rotated_coord_buffer*abs(rot_coords[param][1]-rot_coords[param][0]) -# lbound = rot_coords[param][0] - opts.rotated_coord_buffer*abs(rot_coords[param][1]-rot_coords[param][0]) -# else: #new_bound = bound +/- buffer*|bound| -> asymmetric buffer -# ubound = rot_coords[param][1] + opts.rotated_coord_buffer*abs(rot_coords[param][1]) -# lbound = rot_coords[param][0] - opts.rotated_coord_buffer*abs(rot_coords[param][0]) -# if opts.reflect_parameters: -# # Reflection -# print(" Reflecting into range : {} [{}, {}]".format(param,lbound,ubound)) -# # put in range [0,2 L] -# #tmp = rot_coords[param][0] + np.mod(r_prime[:,indx] - rot_coords[param][0], 2*(rot_coords[param][1] - rot_coords[param][0]) ) -# tmp = lbound + np.mod(r_prime[:,indx] - lbound, 2*(ubound - lbound) ) -# # final reflection -# #tmp = np.where( tmp > rot_coords[param][1], 2*rot_coords[param][1] - tmp, tmp) -# tmp = np.where( tmp > ubound, 2*ubound - tmp, tmp) -# print(" Increment reflect : {} {} ".format(param,np.sum(tmp != r_prime[:,indx]))) -# r_prime[:,indx] = tmp -# else: -# # Downselection -# #print(" Downselecting:",name,"; indx:",indx,"[",lbound,ubound,"]; buffer =",opts.rotated_coord_buffer) -# indx_ok = np.logical_and(indx_ok, r_prime[:,indx]<= ubound ) -# indx_ok = np.logical_and(indx_ok, r_prime[:,indx]>= lbound ) -# print(' Increment downselect : {} {} '.format(param, np.sum(indx_ok) )) -# -# if opts.reflect_parameters: -# #apply inverse: S-1*r_prime = S-1*S*r_tilde = r_tilde ( [4 x 4].([N x 4].T) ) -# r_tilde_post = np.matmul(dan_inv,r_prime.T).T -# -# for i, col in enumerate(rot_cols): -# #r_tilde = (gamma - u)/sig -> gamma = r_tilde*sig + u -# X_out[:,col] = (r_tilde_post[:,i]*scaled_sig[i]) + scaled_mean[i] -# ============================================================================= - -for name in names_downselect: - indx = coord_names.index(name) - print(" Downselecting:",name,"; indx:",indx,"[",downselect_dict[name][0],downselect_dict[name][1],"]") - indx_ok = np.logical_and(indx_ok, np.logical_not(np.isnan(X_out[:,indx]))) - indx_ok = np.logical_and(indx_ok, X_out[:,indx]<= downselect_dict[name][1] ) - indx_ok = np.logical_and(indx_ok, X_out[:,indx]>= downselect_dict[name][0] ) - print(' Increment downselect : {} {} '.format(name, np.sum(indx_ok) )) - -if supplemental_coordinate_invert: - X_out = supplemental_coordinate_invert(X_out, coord_names, **external_kwargs) - -#enforce m1 >= m2 for populations - should really replace by converting m2 -> q -if opts.downselect_mass_range and ("m1" in coord_names) and ("m2" in coord_names): - m1_indx = coord_names.index("m1") - m2_indx = coord_names.index("m2") - indx_ok = np.logical_and(indx_ok, X_out[:,m1_indx]>= X_out[:,m2_indx] ) - print(' Increment downselect : {} {} '.format("m1>=m2", np.sum(indx_ok) )) - -X_out = X_out[indx_ok] -dat_raw = dat_raw[indx_ok] # must downselect here as well! - -# Write data back into correct format and save -for p in coord_names: -# indx_p = dat_raw.dtype.names.index(p) - indx_in = coord_names.index(p) - dat_raw[p] = X_out[:,indx_in] - -np.savetxt(opts.inj_file_out, dat_raw,header=" ".join(dat_raw.dtype.names)) \ No newline at end of file From 517b3cc1f07268cf388cd988eedf924306ec683f Mon Sep 17 00:00:00 2001 From: mebiri Date: Tue, 29 Sep 2026 14:12:44 -0500 Subject: [PATCH 7/8] upgrades to CHP now behaves like puff for rotation and param bounds --- .../Code/bin/util_ConstructHyperPosterior.py | 176 ++++++++---------- 1 file changed, 80 insertions(+), 96 deletions(-) diff --git a/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py b/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py index 6c0083695..fed5ce5c2 100644 --- a/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py +++ b/MonteCarloMarginalizeCode/Code/bin/util_ConstructHyperPosterior.py @@ -170,8 +170,10 @@ def add_field(a, descr): parser.add_argument("--supplementary-likelihood-factor-ini", default=None,type=str,help="With above option, specifies an ini file that is parsed (here) and passed to the preparation code, called when the module is first loaded, to configure the module. EXPERTS ONLY") parser.add_argument("--supplementary-coordinate-code", default=None,type=str,help="Coordinate conversion/prior code. Accepts: the literal 'rift_default' (use RIFT.lalsimutils.convert_waveform_coordinates plus RIFT-standard priors); a filesystem path ending in .py (loaded as a plugin); or any importable dotted module name.") parser.add_argument("--supplementary-coordinate-function", default=None, type=str, help="Name of the entry-point callable inside the module named by --supplementary-coordinate-code. Defaults to 'convert_coordinates'.") -parser.add_argument("--supplementary-coordinate-ini", default=None, type=str, help="RoboCode: Optional ini file parsed and handed to the coordinate plugin's prepare() hook so it can read its own configuration block(s).") -parser.add_argument("--supplementary-coordinate-chart", default=None, type=str, help="RoboCode: Which chart (coordinate system) defined by the plugin to use for this run. Required when the plugin's CHARTS dict has > 1 entry; ignored when the plugin doesn't define CHARTS. Different charts can share parameter names but imply different priors -- the chart name disambiguates which (name -> prior) mapping is installed.") +#parser.add_argument("--supplementary-coordinate-ini", default=None, type=str, help="RoboCode: Optional ini file parsed and handed to the coordinate plugin's prepare() hook so it can read its own configuration block(s).") +#parser.add_argument("--supplementary-coordinate-chart", default=None, type=str, help="RoboCode: chart (coord system) defined by the plugin to use. Required when the plugin's CHARTS dict has > 1 entry; ignored when the plugin doesn't define CHARTS. Different charts can share parameter names but imply different priors -- the chart name disambiguates the (name -> prior) mapping.") +parser.add_argument("--get-range-from-external",action='store_true',help="Get/modify parameter ranges via external func in --supplementary-coordinate-code") +parser.add_argument("--external-range-args",action='append',help="Provide special args to external range func; syntax: single str 'arg_name=arg_value'.") opts= parser.parse_args() #print(" WARNING: Always use internal_use_lnL for now ") @@ -273,6 +275,50 @@ def add_field(a, descr): range_expr = eval(range_str) # define. Better to split on , for example param_ranges[name] = np.array(range_expr) +#Do coordiante rotation stuff here, in case getting param bounds from external script +#parse args to be passed to coord convert funcs: +external_kwargs = {} +if opts.external_range_args: + for arg in opts.external_range_args: + external_kwargs[arg.split("=")[0]] = arg.split("=")[1] + +supplemental_coordinate_convert = None +supplemental_coordinate_invert = None +if opts.supplementary_coordinate_code and opts.supplementary_coordinate_function: + #Ignoring RoboCode completely here - no absurd plugins! Reuse supplemental likelihood functionality + print(" EXTERNAL COORDINATE CONVERSION : {}.{} ".format(opts.supplementary_coordinate_code,opts.supplementary_coordinate_function)) + __import__(opts.supplementary_coordinate_code) + ''' + Expect supplementary-coordinate-code to contain: + supplementary-coordinate-function(X, coord_names, **kwargs) + "inverse_"+supplementary-coordinate-function(X, coord_names, **kwargs) + Optional: get_bounds(param_list, bounds_dict, **kwargs) + ''' + external_coordinate_module = sys.modules[opts.supplementary_coordinate_code] + if hasattr(external_coordinate_module,opts.supplementary_coordinate_function): + supplemental_coordinate_convert = getattr(external_coordinate_module,opts.supplementary_coordinate_function) + + if coord_names == low_level_coord_names: #no param_implied or param_nofit + print(" All parameters match; will work fully in converted coordinates & requiring inverted conversion.") + try: + supplemental_coordinate_invert = getattr(external_coordinate_module, "inverse_"+opts.supplementary_coordinate_function) + except: + print(" ERROR: no inverted coordinate conversion routine found!") + supplemental_coordinate_invert = None + # Send X to get converted, get converted X back -> do this later + else: + print("ERROR: could not retrieve supplied coordinate function. Will attempt default conversion (none).") + + if opts.get_range_from_external: #retrieve/modify param ranges externally + try: + print(" Retrieving param bounds from external module") + supplemental_bound_func = getattr(external_coordinate_module, "get_bounds") + param_ranges = supplemental_bound_func(dat_orig_names,param_ranges,**external_kwargs) + except Exception as e: + print(" ERROR fetching external ranges:",e) + print(" WARNING: external range requested but not retrieved. Using supplied bounds as-is.") + + # Add in integration range for everything else, if nothing specified for name in dat_orig_names: if not name in param_ranges: @@ -289,9 +335,6 @@ def uniform_prior(x): prior_map = {} for name in low_level_coord_names: prior_map[name] = uniform_prior - #Robocode removes the below & delays until after coord conversion code - #Only useful if using external code to retrieve bounds via conversion code's - #get_bounds() func (which may or may not be present!) - see HyperPuffball_new for implementation if not(name in param_ranges): raise Exception(" {} not provided a parameter range ".format(name)) # change later, should fall back to using prior range from above @@ -332,35 +375,6 @@ def uniform_prior(x): # Call the ini file, tell it what coordinates we are using by name supplemental_ln_likelihood_prep(config=supplemental_ln_likelihood_parsed_ini,coords=coord_names) - - -supplemental_coordinate_convert = None -supplemental_coordinate_invert = None -if opts.supplementary_coordinate_code and opts.supplementary_coordinate_function: - #Ignoring RoboCode completely here - no absurd plugins! Reuse functionality above - print(" EXTERNAL COORDINATE CONVERSION : {}.{} ".format(opts.supplementary_coordinate_code,opts.supplementary_coordinate_function)) - __import__(opts.supplementary_coordinate_code) - ''' - Expect supplementary-coordinate-code to contain: - supplementary-coordinate-function(X, coord_names, **kwargs) - "inverse_"+supplementary-coordinate-function(X, coord_names, **kwargs) - Optional: get_bounds(param_list, bounds_dict, **kwargs) - ''' - external_coordinate_module = sys.modules[opts.supplementary_coordinate_code] - if hasattr(external_coordinate_module,opts.supplementary_coordinate_function): - supplemental_coordinate_convert = getattr(external_coordinate_module,opts.supplementary_coordinate_function) - - if coord_names == low_level_coord_names: #no param_implied or param_nofit - print(" All parameters match; assuming full conversion & requiring inverted conversion.") - try: - supplemental_coordinate_invert = getattr(external_coordinate_module, "inverse_"+opts.supplementary_coordinate_function) - except: - print(" ERROR: no inverted coordinate conversion routine found!") - supplemental_coordinate_invert = None - # Send X to get converted, get converted X back -> do this later - else: - print("ERROR: could not retrieve supplied coordinate function. Will attempt default conversion (none).") - from sklearn.gaussian_process import GaussianProcessRegressor @@ -496,76 +510,41 @@ def fn_return(x_in,rf=rf): n_params = -1 - ### - ### Convert data. RIGHT NOW JUST DOWNSELECTING, no intermediate fitting parameters defined - ### - +### +### Convert data. RIGHT NOW JUST DOWNSELECTING, no intermediate fitting parameters defined +### + # Naive convert: no downselect. +indx_of_orig_names = np.array([dat_orig_names.index(k) for k in coord_names])#coord_names[k]) for k in range(len(coord_names))]) +dat_out = [] +for line in dat: + dat_here = np.zeros(len(coord_names)+2) + if line[col_lnL+1] > opts.sigma_cut: + print("skipping", line) + continue + dat_here[:-2] = line[indx_of_orig_names+2]#line[2:len(coord_names)+2] # modify to use names! + dat_here[-2] = line[0] #lnL + dat_here[-1] = line[1] #sig_lnL + dat_out.append(dat_here) +dat_out = np.array(dat_out) + if supplemental_coordinate_convert is None: #old functionality if list(low_level_coord_names) != list(coord_names): raise ValueError(" ERROR: fit basis ({}) differs from MC sampling basis ({}), but no --supplementary-coordinate-code was supplied.".format(list(coord_names),list(low_level_coord_names))) - indx_of_orig_names = np.array([ dat_orig_names.index(coord_names[k]) for k in range(len(coord_names))]) - - dat_out = [] - for line in dat: - dat_here= np.zeros(len(coord_names)+2) - if line[col_lnL+1] > opts.sigma_cut: - print("skipping", line) - continue - dat_here[:-2] = line[indx_of_orig_names+2]#line[2:len(coord_names)+2] # modify to use names! - dat_here[-2] = line[0] - dat_here[-1] = line[1] - dat_out.append(dat_here) - dat_out= np.array(dat_out) - - # Repack data #TODO: check this, move outside if block if both routes need it! - X =dat_out[:,0:len(coord_names)] - Y = dat_out[:,-2] - if np.max(Y)<0 and lnL_shift ==0: - lnL_shift = -100 - np.max(Y) # force it to be offset/positive -- may help some configurations. Remember our adaptivity is silly. - Y_err = dat_out[:,-1] - def convert_coords(x): #no conversion in play here + # Repack data + X = dat_out[:,0:len(coord_names)] + def convert_coords(x): #no conversion needed return x else: - # Pack data using coordinate converter. Note, if not fully operating in other coord sys. - #later calculations MUST use the converter. - ''' - # The robot says: Two distinct call sites for the converter, with two different - # input bases -- this is the change that decouples the fit from - # the MC sampling basis: - # (1) The initial dat->X conversion below feeds rows whose - # columns are ordered by dat_orig_names (the data file's - # header). So we pass low_level_coord_names=dat_orig_names - # at this site. - # (2) The convert_coords def is called by the integrator on every MC - sample (like CIP). The sampler operates in low_level_coord_names, - so the def must claim its inputs are in low_level_coord_names, NOT - dat_orig_names. - Used to be hardcoded to dat_orig_names, which worked in legacy - case (low_level_coord_names == dat_orig_names). For any non-trivial - plugin where the MC samples in a different basis than the file cols, - # the old behaviour applied the rotation an extra time, which is wrong. - ''' - indx_of_orig_names = np.array([ dat_orig_names.index(coord_names[k]) for k in range(len(coord_names))]) - - dat_out = [] - for line in dat: - dat_here= np.zeros(len(coord_names)+2) - if line[col_lnL+1] > opts.sigma_cut: - print("skipping", line) - continue - dat_here[2:] = line[indx_of_orig_names+2]#line[2:len(coord_names)+2] # modify to use names! - dat_here[0] = line[0] - dat_here[1] = line[1] - dat_out.append(dat_here) - dat_out= np.array(dat_out) - X = supplemental_coordinate_convert(dat_out[:,2:], coord_names=dat_orig_names) # convert and generate X - Y = dat_out[:,0] - Y_err = dat_out[:,1] - if np.max(Y)<0 and lnL_shift ==0: - lnL_shift = -100 - np.max(Y) # force it to be offset/positive -- may help some configurations. Remember our adaptivity is silly. + # WARNING: RoboCode! + # Pack data using coordinate converter. NOTE: if not fully operating in rotated + # coords (i.e. suppl_invert == None), later calculations (e.g. integration) MUST use the converter. + #TODO: verify this works as claimed; I'm not convinced (test w/ coord_names != low_level_coord_names) + # idea is stay in Cartesian coords, convert every MC step in integration separately to make fit, + # but sample in Cartesian coords at the end (because no invert_rotation code) + X = supplemental_coordinate_convert(dat_out[:,0:len(coord_names)], coord_names=dat_orig_names) # convert and generate X if supplemental_coordinate_invert is None: def convert_coords(x_in, _low=low_level_coord_names, _coord=coord_names): # _low / _coord captured as defaults so the closure stays correct even if either list mutates later in the script. @@ -573,6 +552,11 @@ def convert_coords(x_in, _low=low_level_coord_names, _coord=coord_names): else: #doing everything in converted coords, so no need to convert again def convert_coords(x): return x + +Y = dat_out[:,-2] +if np.max(Y)<0 and lnL_shift==0: + lnL_shift = -100 - np.max(Y) # force it to be offset/positive -- may help some configurations. Remember our adaptivity is silly. +Y_err = dat_out[:,-1] # Save copies for later (plots) X_orig = X.copy() From 55943faff38b9854208753cae1eaaee4d13c4014 Mon Sep 17 00:00:00 2001 From: mebiri Date: Sat, 3 Oct 2026 12:58:49 -0500 Subject: [PATCH 8/8] Removed space in quotes in line 191 for puff_args --- .../Code/bin/create_eos_posterior_pipeline | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MonteCarloMarginalizeCode/Code/bin/create_eos_posterior_pipeline b/MonteCarloMarginalizeCode/Code/bin/create_eos_posterior_pipeline index ae1023595..35d706688 100755 --- a/MonteCarloMarginalizeCode/Code/bin/create_eos_posterior_pipeline +++ b/MonteCarloMarginalizeCode/Code/bin/create_eos_posterior_pipeline @@ -188,7 +188,7 @@ if opts.puff_args and opts.puff_exe: puff_args = ' '.join( [x.replace('\\','') for x in puff_args_list] ) puff_args = ' '.join(puff_args.split(' ')[1:]) # Some argument protection for later - puff_args = puff_args.replace('[', ' \'[') + puff_args = puff_args.replace('[', '\'[') puff_args = puff_args.replace(']', ']\'') puff_args=puff_args.rstrip() print("PUFF", puff_args)