diff --git a/PyPIC3D/__init__.py b/PyPIC3D/__init__.py index fd742b0..8001d17 100644 --- a/PyPIC3D/__init__.py +++ b/PyPIC3D/__init__.py @@ -1,4 +1,4 @@ -from .distributed import initialize_distributed +from .utilities.distributed import initialize_distributed initialize_distributed() diff --git a/PyPIC3D/__main__.py b/PyPIC3D/__main__.py index 5b6ee63..2e54849 100644 --- a/PyPIC3D/__main__.py +++ b/PyPIC3D/__main__.py @@ -1,48 +1,25 @@ # Christopher Woolford July 2024 # 3D PIC code in Python using the JAX library -import os import time -import traceback import jax from jax import block_until_ready from tqdm import tqdm -from PyPIC3D.diagnostics.plotting import write_data as _write_data -from PyPIC3D.diagnostics.async_writer import ( - create_async_tiled_openpmd_field_writer, - create_async_tiled_openpmd_particle_writer, - enqueue_openpmd_field_output, - enqueue_openpmd_particle_output, - make_tiled_field_snapshot, - make_tiled_particle_snapshot, +from PyPIC3D.diagnostics.simulation_output import ( + open_simulation_output, + write_simulation_output, + write_scalar_diagnostics, + write_work_energy_tracker, + simulation_energy_components, + print_energy, ) -from PyPIC3D.diagnostics.openPMD import ( - TiledMeshLayout, - write_tiled_field_snapshot_openpmd, - write_tiled_particle_snapshot_openpmd, -) -from PyPIC3D.diagnostics.output_adapters import build_field_output_map -from PyPIC3D.diagnostics.diagnostic_quantities import ( - compute_energy, - compute_dark_energy, - compute_total_momentum, -) -from PyPIC3D.utilities.field_helpers import add_external_fields -from PyPIC3D.utilities.simulation_helpers import setup_pmd_files from PyPIC3D.utilities.toml_helpers import ( dump_parameters_to_toml, load_config_file, ) from PyPIC3D.initialization import initialize_simulation -from PyPIC3D.relativity.interpolate_metric import particle_lorentz_factor - - -def write_data(*args, **kwargs): - """Collective diagnostics are evaluated everywhere; rank zero writes scalars.""" - if jax.process_index() == 0: - _write_data(*args, **kwargs) def _raise_if_tiled_particles_overflowed(fields): @@ -54,14 +31,6 @@ def _raise_if_tiled_particles_overflowed(fields): raise RuntimeError("tiled particle tile capacity overflowed during periodic retile") -def write_work_energy_tracker(tracker, output_dir): - """Write one synchronized sample through the scalar time-series output.""" - from PyPIC3D.diagnostics.gr_work_energy_theorem import tracker_values - - for name, value in tracker_values(tracker).items(): - write_data(f"{output_dir}/data/work_energy_{name}.txt", tracker.latest.time, value) - - def run_PyPIC3D(config_file, *, pair_injector=None, injection_key=None, injection_interval_steps=1, bssn_state=None, bssn_evolution_parameters=None, field_initializers=None): @@ -70,31 +39,21 @@ def run_PyPIC3D(config_file, *, pair_injector=None, injection_key=None, The injector follows time_loop_static_metric's callable contract. Supply an explicit RNG key and reserve enough particle slots in the initial configuration. """ - from PyPIC3D.distributed import mpi_communicator - comm = mpi_communicator() - try: - ( - loop, - particles, - fields, - static_parameters, - dynamic_parameters, - plotting_parameters, - plasma_parameters, - species_config, - ) = initialize_simulation(config_file, bssn_state=bssn_state, - bssn_evolution_parameters=bssn_evolution_parameters, - field_initializers=field_initializers) - except BaseException: - if comm is not None: - traceback.print_exc() - comm.Abort(1) - raise - - dt = dynamic_parameters.dt + ( + loop, + particles, + fields, + static_parameters, + dynamic_parameters, + plotting_parameters, + plasma_parameters, + species_config, + ) = initialize_simulation(config_file, bssn_state=bssn_state, + bssn_evolution_parameters=bssn_evolution_parameters, + field_initializers=field_initializers) + Nt = static_parameters.Nt output_dir = static_parameters.output_dir - particle_species_names = plotting_parameters.get("particle_species_names") static_metric = static_parameters.solver == "static_metric" dynamic_metric = static_parameters.solver == "dynamic_metric" gr_metric = static_metric or dynamic_metric @@ -130,175 +89,25 @@ def loop_with_static_parameters( initial_energy = None if not gr_metric: - E, B, J, rho, phi, external_fields, *rest = fields - total_E, total_B = add_external_fields(E, B, external_fields) - e_energy, b_energy, kinetic_energy = compute_energy( - particles, - total_E, - total_B, - static_parameters, - dynamic_parameters, - species_config=species_config, - ) - dark_energy = (compute_dark_energy(fields[7], static_parameters, dynamic_parameters, - None if fields[6] is None else fields[6][1]) - if static_parameters.solver == "dark_matter_yee" else 0.0) - initial_energy = e_energy + b_energy + kinetic_energy + dark_energy - - field_writer = None - particle_writer = None - if comm is not None: - layout = TiledMeshLayout( - global_shape=(int(dynamic_parameters.Nx), int(dynamic_parameters.Ny), int(dynamic_parameters.Nz)), - tile_shape=tuple(int(width) for width in static_parameters.tile_shape), - guard_cells=int(static_parameters.guard_cells), - ) - if comm is None and plotting_parameters["plot_openpmd_fields"]: - setup_pmd_files(os.path.join(output_dir, "data"), "fields", ".h5") - field_writer = create_async_tiled_openpmd_field_writer( - static_parameters, - dynamic_parameters, - os.path.join(output_dir, "data"), - filename="fields", - file_extension=".h5", - queue_size=int(plotting_parameters.get("openpmd_field_queue_size", 2)), - ) - if comm is None and plotting_parameters["plot_openpmd_particles"]: - setup_pmd_files(os.path.join(output_dir, "data"), "particles", ".h5") - particle_writer = create_async_tiled_openpmd_particle_writer( - static_parameters, - dynamic_parameters, - os.path.join(output_dir, "data"), - filename="particles", - file_extension=".h5", - queue_size=int(plotting_parameters.get("openpmd_particle_queue_size", 2)), - ) + initial_energy = sum(simulation_energy_components( + particles, species_config, fields, static_parameters, dynamic_parameters)) matched_gr = gr_metric and static_parameters.current_filter != "none" # A common-time final snapshot is needed when the work-energy budget writes its # endpoint at t = Nt, and for filtered GR qualification. final_endpoint = work_energy_theorem or matched_gr - loop_error = None - try: - if comm is not None: - if comm.rank == 0: - for name in ("fields", "particles"): - if plotting_parameters["plot_openpmd_" + name]: - setup_pmd_files(os.path.join(output_dir, "data"), name, ".h5") - comm.Barrier() + with open_simulation_output(static_parameters, dynamic_parameters, plotting_parameters) as writers: iterations = Nt + int(final_endpoint) for t in tqdm(range(iterations), disable=jax.process_index() != 0): if t % plotting_parameters["plotting_interval"] == 0 or (final_endpoint and t == Nt): - if dynamic_metric: - from PyPIC3D.diagnostics.static_metric import bssn_constraint_norms - norms = bssn_constraint_norms(particles, species_config, fields, - static_parameters, dynamic_parameters) - for name, value in norms.items(): - write_data(f"{output_dir}/data/{name}.txt", - dynamic_parameters.bssn_parameters.time, value) - if work_energy_theorem and 0 < t < Nt: - write_work_energy_tracker(dynamic_parameters.work_energy_tracker, output_dir) - - if not gr_metric: - E, B, J, rho, phi, external_fields, *rest = fields - total_E, total_B = add_external_fields(E, B, external_fields) - e_energy, b_energy, kinetic_energy = compute_energy( - particles, - total_E, - total_B, - static_parameters, - dynamic_parameters, - species_config=species_config, - ) - dark_energy = (compute_dark_energy(fields[7], static_parameters, dynamic_parameters, - None if fields[6] is None else fields[6][1]) - if static_parameters.solver == "dark_matter_yee" else 0.0) - total_energy = e_energy + b_energy + kinetic_energy + dark_energy - if static_parameters.solver == "dark_matter_yee": - write_data(f"{output_dir}/data/dark_field_energy.txt", t * dt, dark_energy) - write_data(f"{output_dir}/data/total_energy.txt", t * dt, total_energy) - write_data( - f"{output_dir}/data/energy_error.txt", - t * dt, - abs(initial_energy - total_energy) / max(initial_energy, 1e-10), - ) - write_data(f"{output_dir}/data/electric_field_energy.txt", t * dt, e_energy) - write_data(f"{output_dir}/data/magnetic_field_energy.txt", t * dt, b_energy) - write_data(f"{output_dir}/data/kinetic_energy.txt", t * dt, kinetic_energy) - - total_momentum = compute_total_momentum(particles, species_config=species_config) - write_data(f"{output_dir}/data/total_momentum.txt", t * dt, total_momentum) - - if plotting_parameters["plot_openpmd_particles"]: - gamma = None - output_particles = particles - if dynamic_metric: - from PyPIC3D.relativity.dynamical_relativity import synchronize_particles - output_particles = synchronize_particles( - particles, species_config, fields, static_parameters, dynamic_parameters) - if gr_metric: - metric = fields[6] - gamma = particle_lorentz_factor(output_particles, metric, static_parameters, dynamic_parameters) - # particle u holds covariant u_i, so Gamma needs the metric at each particle - if comm is None: - enqueue_openpmd_particle_output( - particle_writer, output_particles, dynamic_parameters, t, t, - species_config=species_config, species_names=particle_species_names, - gamma=gamma, - ) - else: - snapshot = make_tiled_particle_snapshot( - output_particles, step=t, time=float(dynamic_parameters.bssn_parameters.time), - species_names=particle_species_names, species_config=species_config, gamma=gamma, - ) - write_tiled_particle_snapshot_openpmd( - snapshot, output_dir=os.path.join(output_dir, "data"), filename="particles", - static_parameters=static_parameters, dynamic_parameters=dynamic_parameters, - file_extension=".h5", comm=comm, - ) - - if plotting_parameters["plot_openpmd_fields"]: - field_map = build_field_output_map( - fields, - particles, - species_config, - static_parameters, - dynamic_parameters, - include_fluid_velocity=bool(plotting_parameters["plotvelocities"]), - include_charge_density=bool(plotting_parameters["plotchargedensity"]), - ) - plotting_parameters["field_map"] = field_map - if comm is None: - enqueue_openpmd_field_output(field_writer, field_map, dynamic_parameters, t, t) - else: - snapshot = make_tiled_field_snapshot( - field_map, step=t, time=float(dynamic_parameters.bssn_parameters.time), - current_time_offset=0.0 if t == 0 else -float(dt)/4, - ) - write_tiled_field_snapshot_openpmd( - snapshot, output_dir=os.path.join(output_dir, "data"), filename="fields", - dynamic_parameters=dynamic_parameters, layout=layout, file_extension=".h5", comm=comm, - ) + write_scalar_diagnostics(particles, species_config, fields, static_parameters, + dynamic_parameters, t, initial_energy) + write_simulation_output(particles, species_config, fields, static_parameters, + dynamic_parameters, plotting_parameters, t, writers) if t == Nt: break - if static_parameters.benchmark and t == 0: - start_compile = time.perf_counter() - jit_loop = jit_loop.lower(particles, species_config, fields, - dynamic_parameters, injection_key, t).compile() - compile_elapsed = time.perf_counter()-start_compile - if comm is not None: - from mpi4py import MPI - compile_elapsed = comm.allreduce(compile_elapsed, op=MPI.MAX) - if jax.process_index() == 0: - print(f"Step compilation (slowest process): {compile_elapsed:.3f} s") - print(f"Compiled step memory per device: {jit_loop.memory_analysis()}") - if static_parameters.benchmark and comm is not None: - # Exclude another rank's compilation or diagnostics from the - # timed step; all ranks start after the same completed work. - comm.Barrier() - start_step = time.perf_counter() if static_parameters.benchmark else None result = jit_loop( particles, species_config, @@ -307,14 +116,6 @@ def loop_with_static_parameters( injection_key, t, ) - if static_parameters.benchmark: - block_until_ready(result) - elapsed = time.perf_counter()-start_step - if comm is not None: - from mpi4py import MPI - elapsed = comm.allreduce(elapsed, op=MPI.MAX) - if jax.process_index() == 0: - print(f"Step {t} elapsed (slowest process): {elapsed:.6f} s") if dynamic_metric: particles, fields, dynamic_parameters = result elif pair_injector is None: @@ -341,24 +142,10 @@ def loop_with_static_parameters( # appeared at any cadence. This also writes the sole Nt=0 sample. write_work_energy_tracker(dynamic_parameters.work_energy_tracker, output_dir) - except BaseException as exc: - loop_error = exc - if comm is not None: - traceback.print_exc() - comm.Abort(1) - raise - finally: - writer_error = None - for writer in (particle_writer, field_writer): - if writer is None: - continue - try: - writer.close(raise_errors=loop_error is None) - except BaseException as exc: - if writer_error is None: - writer_error = exc - if loop_error is None and writer_error is not None: - raise writer_error + if dynamic_metric: + from PyPIC3D.diagnostics.static_metric import print_bssn_constraints + print_bssn_constraints("Final", particles, species_config, fields, + static_parameters, dynamic_parameters) return ( static_parameters, @@ -399,28 +186,10 @@ def main(): ) = block_until_ready(run_PyPIC3D(toml_file)) end = time.time() - if static_parameters.solver in ("static_metric", "dynamic_metric"): - print("Skipping final flat-space energy diagnostics for GR fields and covariant particle u_i\n") - else: - E, B, J, rho, phi, external_fields, *rest = fields - total_E, total_B = add_external_fields(E, B, external_fields) - e_energy, b_energy, kinetic_energy = compute_energy( - particles, - total_E, - total_B, - static_parameters, - dynamic_parameters, - species_config=species_config, - ) - print(f"Final Electric Field Energy: {e_energy}") - print(f"Final Magnetic Field Energy: {b_energy}") - print(f"Final Kinetic Energy: {kinetic_energy}") - dark_energy = (compute_dark_energy(fields[7], static_parameters, dynamic_parameters, - None if fields[6] is None else fields[6][1]) - if static_parameters.solver == "dark_matter_yee" else 0.0) - if static_parameters.solver == "dark_matter_yee": - print(f"Final Dark Field Energy: {dark_energy}") - print(f"Total Final Energy: {e_energy + b_energy + kinetic_energy + dark_energy}\n") + if static_parameters.solver not in ("static_metric", "dynamic_metric"): + energies = simulation_energy_components(particles, species_config, fields, + static_parameters, dynamic_parameters) + print_energy("Final", energies, dark_matter=static_parameters.solver == "dark_matter_yee") duration = end - start Nt = static_parameters.Nt diff --git a/PyPIC3D/boundary_conditions/supergaussian.py b/PyPIC3D/boundary_conditions/supergaussian.py index bca38ae..ac948c2 100644 --- a/PyPIC3D/boundary_conditions/supergaussian.py +++ b/PyPIC3D/boundary_conditions/supergaussian.py @@ -82,20 +82,6 @@ def load_supergaussian_from_toml(raw_supergaussian, dynamic_parameters): return bool(layers), sg_x, sg_y, sg_z, tuple(layers) -def _shape_from_static_parameters(static_parameters): - tile_nx, tile_ny, tile_nz = [int(width) for width in static_parameters.tile_shape] - g = int(static_parameters.guard_cells) - mesh_shape = tuple(int(static_parameters.field_mesh.shape[axis]) for axis in ghost_cells.MESH_AXES) - return ( - mesh_shape[0], - mesh_shape[1], - mesh_shape[2], - tile_nx + 2 * g, - tile_ny + 2 * g, - tile_nz + 2 * g, - ) - - def _axis_envelope(shape, static_parameters, axis_index, side, width, order, sigma_max, step_dt): tile_width = int(static_parameters.tile_shape[axis_index]) g = int(static_parameters.guard_cells) @@ -135,7 +121,9 @@ def build_supergaussian_envelope(static_parameters, dynamic_parameters, step_dt, del dynamic_parameters if shape is None: - shape = _shape_from_static_parameters(static_parameters) + g = int(static_parameters.guard_cells) + mesh_shape = tuple(int(static_parameters.field_mesh.shape[axis]) for axis in ghost_cells.MESH_AXES) + shape = mesh_shape + tuple(int(width) + 2 * g for width in static_parameters.tile_shape) envelope = jnp.ones(tuple(int(value) for value in shape)) for axis_index, side, width, order, sigma_max in static_parameters.supergaussian_layers: diff --git a/PyPIC3D/diagnostics/diagnostic_quantities.py b/PyPIC3D/diagnostics/diagnostic_quantities.py index 94fdeab..d692a4d 100644 --- a/PyPIC3D/diagnostics/diagnostic_quantities.py +++ b/PyPIC3D/diagnostics/diagnostic_quantities.py @@ -1,6 +1,24 @@ import jax.numpy as jnp +def compute_energy_components(particles, E, B, external_fields, static_parameters, + dynamic_parameters, species_config, *, dark_fields=None, dark_pml=None): + """Flat-space energy including prescribed fields and optional Proca fields. + + Use the supplied particle time level: initial reporting precedes the + backward leapfrog seed, while runtime diagnostics use the evolved state. + """ + from PyPIC3D.utilities.field_helpers import add_external_fields + + total_E, total_B = add_external_fields(E, B, external_fields) + electric, magnetic, kinetic = compute_energy( + particles, total_E, total_B, static_parameters, dynamic_parameters, + species_config=species_config) + dark = (compute_dark_energy(dark_fields, static_parameters, dynamic_parameters, dark_pml) + if dark_fields is not None else 0.0) + return electric, magnetic, kinetic, dark + + def compute_dark_energy(dark_fields, static_parameters, dynamic_parameters, pml_state=None): """Physical Proca energy at integer time, using reconstructed A and B. diff --git a/PyPIC3D/diagnostics/merge_openpmd.py b/PyPIC3D/diagnostics/merge_openpmd.py new file mode 100644 index 0000000..d88f4e0 --- /dev/null +++ b/PyPIC3D/diagnostics/merge_openpmd.py @@ -0,0 +1,425 @@ +"""Assemble the per-process openPMD series a multiprocess run writes. + +Each JAX process writes its own ``_pNNN.`` series without talking to +any other process. This tool joins a group back into the single +``.`` file that a one-process run produces, which every analysis +script already reads. + +The HDF5 backend reports every dataset as one whole chunk, so a reader cannot +discover which regions a file actually wrote. ``PyPIC3D.diagnostics.openPMD`` +therefore records ownership as attributes, and this tool reads them back. +""" + +from dataclasses import dataclass +from typing import Tuple + +import argparse +import os +import re + +import numpy as np +import openpmd_api as io + +from PyPIC3D.utilities.simulation_helpers import setup_pmd_files + + +PROCESS_SUFFIX_PATTERN = re.compile(r"^(?P.+)_p(?P\d+)(?P\.h5|\.bp)$") + +PARTICLE_VECTOR_RECORDS = ("position", "positionOffset", "momentum") +PARTICLE_SCALAR_RECORDS = ("weighting", "charge", "mass", "gamma") + +# openPMD orders the seven base dimensions L, M, T, I, theta, N, J. +UNIT_DIMENSION_ORDER = ("L", "M", "T", "I", "theta", "N", "J") + +OWNERSHIP_ATTRIBUTES = frozenset({ + "pypic3dOwnedTiles", "pypic3dOwnedTileCounts", "pypic3dOwnedTileCount", + "pypic3dTileCounts", "pypic3dTileShape", "pypic3dGlobalShape", + "pypic3dProcessIndex", "pypic3dProcessCount", +}) + +STANDARD_MESH_ATTRIBUTES = frozenset({ + "geometry", "geometryParameters", "dataOrder", "axisLabels", "gridSpacing", + "gridGlobalOffset", "gridUnitSI", "unitDimension", "timeOffset", "unitSI", + "position", +}) + + +DEFAULT_MAX_PENDING_BYTES = 1 << 30 + + +@dataclass(frozen=True) +class TileGeometry: + """Global tile decomposition plus which tiles each source file owns.""" + + global_shape: Tuple[int, ...] + tile_shape: Tuple[int, ...] + tile_counts: Tuple[int, ...] + owned: Tuple[np.ndarray, ...] + + def ordered_tiles(self): + """Yield (source, tile id) in ascending global tile order.""" + pairs = [(int(tile), source) + for source, ids in enumerate(self.owned) for tile in ids] + pairs.sort() + for tile, source in pairs: + yield source, tile + + def tile_offset(self, tile): + """Global cell offset of a flattened tile id.""" + return [int(index)*int(width) for index, width + in zip(np.unravel_index(int(tile), self.tile_counts), self.tile_shape)] + + +@dataclass +class MergeReport: + output: str + sources: Tuple[str, ...] + iterations: Tuple[int, ...] + + +def _resolve_pmd(path): + """Resolve PyPIC3D's one-line .pmd pointer to the series it names.""" + with open(path) as pointer: + lines = [line.strip() for line in pointer if line.strip()] + if len(lines) != 1: + raise ValueError(f"{path} must name exactly one series, found {len(lines)} lines") + return os.path.join(os.path.dirname(path), lines[0]) + + +def discover_series_groups(inputs): + """Group ``_pNNN.`` siblings named by paths, files or directories.""" + groups = {} + + def add(path): + directory, name = os.path.split(os.path.abspath(path)) + match = PROCESS_SUFFIX_PATTERN.match(name) + if match is None: + return False + key = (directory, match.group("stem"), match.group("extension")) + groups.setdefault(key, {})[int(match.group("index"))] = os.path.join(directory, name) + return True + + for entry in inputs: + entry = str(entry) + if entry.endswith(".pmd") and os.path.isfile(entry): + entry = _resolve_pmd(entry) + if not os.path.exists(entry): + raise FileNotFoundError(entry) + # BP series are directories too; recognize members before scanning folders. + if add(entry): + # One member names its whole group; pick up its siblings. + directory, name = os.path.split(os.path.abspath(entry)) + match = PROCESS_SUFFIX_PATTERN.match(name) + prefix = match.group("stem") + "_p" + for sibling in sorted(os.listdir(directory)): + if sibling.startswith(prefix): + add(os.path.join(directory, sibling)) + elif os.path.isdir(entry): + for name in sorted(os.listdir(entry)): + add(os.path.join(entry, name)) + + ordered = [] + for (directory, stem, extension), members in sorted(groups.items()): + paths = tuple(members[index] for index in sorted(members)) + ordered.append((directory, stem, extension, paths)) + return ordered + + +def _attribute(attributable, name): + return attributable.get_attribute(name) if attributable.contains_attribute(name) else None + + +def _int_vector(attributable, name): + """openPMD returns a length-one numeric vector attribute as a scalar.""" + return np.atleast_1d(np.asarray(attributable.get_attribute(name), dtype=np.int64)) + + +def read_tile_geometry(source_iterations): + """Read ownership metadata and check the sources describe one decomposition.""" + global_shape = tile_shape = tile_counts = None + owned = [] + for index, iteration in enumerate(source_iterations): + shape = tuple(int(n) for n in iteration.get_attribute("pypic3dGlobalShape")) + widths = tuple(int(n) for n in iteration.get_attribute("pypic3dTileShape")) + counts = tuple(int(n) for n in iteration.get_attribute("pypic3dTileCounts")) + if global_shape is None: + global_shape, tile_shape, tile_counts = shape, widths, counts + elif (shape, widths, counts) != (global_shape, tile_shape, tile_counts): + raise ValueError( + f"Source {index} describes tiles {widths} of {counts} over {shape}, " + f"but source 0 describes {tile_shape} of {tile_counts} over {global_shape}.") + count = int(iteration.get_attribute("pypic3dOwnedTileCount")) + ids = (_int_vector(iteration, "pypic3dOwnedTiles") + if count else np.empty(0, dtype=np.int64)) + if ids.size != count: + raise ValueError(f"Source {index} lists {ids.size} tiles but claims {count}.") + owned.append(np.sort(ids)) + return TileGeometry(global_shape, tile_shape, tile_counts, tuple(owned)) + + +def validate_tile_coverage(geometry): + """Owned tiles must partition the global tile grid exactly once.""" + coverage = np.zeros(int(np.prod(geometry.tile_counts)), dtype=np.int64) + for ids in geometry.owned: + np.add.at(coverage, ids, 1) + missing = np.flatnonzero(coverage == 0) + duplicated = np.flatnonzero(coverage > 1) + if missing.size or duplicated.size: + raise ValueError( + f"Owned tiles do not partition the {geometry.tile_counts} tile grid: " + f"{missing.size} unwritten (first {missing[:5].tolist()}), " + f"{duplicated.size} duplicated (first {duplicated[:5].tolist()}). " + "The group is incomplete or mixes runs with different process counts.") + + +def _copy_extra_attributes(source, destination, skip=frozenset()): + for name in source.attributes: + if name in skip or name in OWNERSHIP_ATTRIBUTES: + continue + destination.set_attribute(name, source.get_attribute(name)) + + +def _copy_unit_dimension(source, destination): + values = np.asarray(source.unit_dimension, dtype=float) + exponents = {getattr(io.Unit_Dimension, symbol): float(value) + for symbol, value in zip(UNIT_DIMENSION_ORDER, values) if value} + if exponents: + destination.unit_dimension = exponents + + +def _drain(pending, destination_record, extent, source_iterations, destination_iteration): + """Complete the deferred loads, then the stores, then release the buffers.""" + if not pending: + return + for iteration in source_iterations: + iteration.series_flush() + for offset, data in pending: + destination_record.store_chunk(data, offset, extent) + destination_iteration.series_flush() + pending.clear() + + +def _merge_meshes(source_iterations, destination_iteration, geometry, + max_pending_bytes=DEFAULT_MAX_PENDING_BYTES): + """Overlay every source's owned tiles into one global-extent mesh record.""" + for name, source_mesh in source_iterations[0].meshes.items(): + destination_mesh = destination_iteration.meshes[name] + destination_mesh.geometry = source_mesh.geometry + destination_mesh.data_order = source_mesh.data_order + destination_mesh.axis_labels = source_mesh.axis_labels + destination_mesh.grid_spacing = source_mesh.grid_spacing + destination_mesh.grid_global_offset = source_mesh.grid_global_offset + destination_mesh.set_grid_unit_SI(source_mesh.grid_unit_SI) + destination_mesh.unit_SI = source_mesh.unit_SI + _copy_unit_dimension(source_mesh, destination_mesh) + if source_mesh.contains_attribute("timeOffset"): + destination_mesh.time_offset = source_mesh.time_offset + _copy_extra_attributes(source_mesh, destination_mesh, STANDARD_MESH_ATTRIBUTES) + + for component, source_record in source_mesh.items(): + destination_record = destination_mesh[component] + destination_record.reset_dataset( + io.Dataset(source_record.dtype, list(geometry.global_shape))) + destination_record.unit_SI = source_record.unit_SI + destination_record.position = source_record.position + _copy_extra_attributes(source_record, destination_record, STANDARD_MESH_ATTRIBUTES) + + # load_chunk is deferred until the source flushes, and a stored + # buffer must stay alive until the destination flushes. Tiles are + # drained in batches so one component need not fit in memory. + extent = list(geometry.tile_shape) + tile_bytes = int(np.prod(extent))*np.dtype(source_record.dtype).itemsize + per_batch = max(1, max_pending_bytes // max(1, tile_bytes)) + pending = [] + for source_index, tile in geometry.ordered_tiles(): + offset = geometry.tile_offset(tile) + record = source_iterations[source_index].meshes[name][component] + pending.append((offset, record.load_chunk(offset, extent))) + if len(pending) >= per_batch: + _drain(pending, destination_record, extent, + source_iterations, destination_iteration) + _drain(pending, destination_record, extent, + source_iterations, destination_iteration) + + +def _particle_blocks(source_iterations, species_name, geometry): + """Locate each owned tile's slice of every source's local records.""" + blocks = [] + for source_index, iteration in enumerate(source_iterations): + species = iteration.particles[species_name] + tiles = _int_vector(species, "pypic3dOwnedTiles") + counts = _int_vector(species, "pypic3dOwnedTileCounts") + if tiles.size != counts.size: + raise ValueError(f"Source {source_index} lists {tiles.size} tiles for " + f"{species_name} but {counts.size} counts.") + local = np.cumsum(counts) - counts + for tile, start, count in zip(tiles, local, counts): + if count: + blocks.append((int(tile), source_index, int(start), int(count))) + blocks.sort() + return blocks + + +def _merge_particles(source_iterations, destination_iteration, geometry): + """Concatenate locally sized records back into global tile order.""" + for species_name, source_species in source_iterations[0].particles.items(): + destination_species = destination_iteration.particles[species_name] + blocks = _particle_blocks(source_iterations, species_name, geometry) + total = sum(block[3] for block in blocks) + + present = set(dict(source_species.items())) + components = [] + for record_name in PARTICLE_VECTOR_RECORDS: + if record_name in present: + components.extend((record_name, axis) for axis in ("x", "y", "z")) + for record_name in PARTICLE_SCALAR_RECORDS: + if record_name in present: + components.append((record_name, io.Mesh_Record_Component.SCALAR)) + + for record_name, component in components: + source_record = source_species[record_name][component] + destination_record = destination_species[record_name][component] + destination_record.reset_dataset(io.Dataset(source_record.dtype, [int(total)])) + destination_record.unit_SI = source_record.unit_SI + + pending, running = [], 0 + for tile, source_index, start, count in blocks: + record = source_iterations[source_index].particles[species_name][record_name][component] + pending.append((running, count, record.load_chunk([start], [count]))) + running += count + for iteration in source_iterations: + iteration.series_flush() + for offset, count, data in pending: + destination_record.store_chunk(data, [offset], [count]) + destination_iteration.series_flush() + pending.clear() + + for record_name in PARTICLE_VECTOR_RECORDS + PARTICLE_SCALAR_RECORDS: + if record_name in present: + _copy_unit_dimension(source_species[record_name], destination_species[record_name]) + _copy_extra_attributes(source_species, destination_species) + + +def merge_series(sources, destination, *, names=(), iterations=None, log=None, + max_pending_bytes=DEFAULT_MAX_PENDING_BYTES): + """Copy owned regions from already-open sources into an open destination.""" + steps = sorted(set(sources[0].iterations)) + for index, series in enumerate(sources[1:], start=1): + if sorted(set(series.iterations)) != steps: + raise ValueError(f"Source {index} holds iterations " + f"{sorted(set(series.iterations))}, source 0 holds {steps}.") + if iterations is not None: + steps = [step for step in steps if step in set(iterations)] + + _copy_extra_attributes(sources[0], destination) + if names: + destination.set_attribute("pypic3dMergedFrom", [os.path.basename(name) for name in names]) + + for step in steps: + source_iterations = [series.iterations[step] for series in sources] + geometry = read_tile_geometry(source_iterations) + validate_tile_coverage(geometry) + + out = destination.iterations[step] + out.time = source_iterations[0].time + out.dt = source_iterations[0].dt + out.time_unit_SI = source_iterations[0].time_unit_SI + _copy_extra_attributes(source_iterations[0], out, frozenset({"time", "dt", "timeUnitSI"})) + + _merge_meshes(source_iterations, out, geometry, max_pending_bytes) + _merge_particles(source_iterations, out, geometry) + if log is not None: + log(f"iteration {step}: {int(np.prod(geometry.tile_counts))} tiles " + f"from {len(sources)} files") + return steps + + +def merge_paths(inputs, output=None, *, overwrite=False, write_pmd=True, iterations=None, + log=None, max_pending_bytes=DEFAULT_MAX_PENDING_BYTES): + """Merge every ``_pNNN`` group named by ``inputs``; return one report each.""" + groups = discover_series_groups(inputs) + if not groups: + raise FileNotFoundError( + "No _pNNN openPMD series found. A single-process run writes one unsuffixed " + "file and needs no merge.") + + if output is not None and len(groups) > 1: + raise ValueError( + f"--output names one series but {len(groups)} groups were found " + f"({', '.join(stem for _, stem, _, _ in groups)}). Name one group member, " + "or drop --output to merge each in place.") + + reports = [] + for directory, stem, extension, paths in groups: + target = output if output is not None else os.path.join(directory, stem + extension) + if os.path.exists(target) and not overwrite: + raise FileExistsError(f"{target} exists; pass --overwrite to replace it") + + sources = [io.Series(path, io.Access.read_only) for path in paths] + _validate_group(sources, paths) + destination = io.Series(target, io.Access.create) + try: + steps = merge_series(sources, destination, names=paths, iterations=iterations, + log=log, max_pending_bytes=max_pending_bytes) + finally: + destination.close() + for series in sources: + series.close() + if write_pmd: + target_directory, target_name = os.path.split(os.path.abspath(target)) + target_stem, target_extension = os.path.splitext(target_name) + setup_pmd_files(target_directory, target_stem, target_extension) + reports.append(MergeReport(target, paths, tuple(steps))) + return reports + + +def _validate_group(sources, paths): + """The group must be exactly one run's processes, each present once.""" + expected = _attribute(sources[0], "pypic3dProcessCount") + indices = [] + for series, path in zip(sources, paths): + count = _attribute(series, "pypic3dProcessCount") + if count != expected: + raise ValueError(f"{path} was written by a run of {count} processes, " + f"but {paths[0]} by a run of {expected}.") + indices.append(_attribute(series, "pypic3dProcessIndex")) + if expected is None: + raise ValueError(f"{paths[0]} carries no PyPIC3D ownership metadata; it was not " + "written by this version of PyPIC3D.") + if sorted(indices) != list(range(int(expected))): + raise ValueError(f"Expected processes 0..{int(expected)-1}, found {sorted(indices)}. " + "Stale files from a run with a different process count?") + + +def main(argv=None): + parser = argparse.ArgumentParser( + prog="PyPIC3D-merge-openpmd", + description="Join the per-process openPMD series of a multiprocess PyPIC3D run.") + parser.add_argument("inputs", nargs="*", default=["."], + help="Data directories, one group member, or a .pmd pointer") + parser.add_argument("-o", "--output", help="Merged series path (default )") + parser.add_argument("--iterations", help="Comma-separated subset of steps to merge") + parser.add_argument("--overwrite", action="store_true", help="Replace an existing merged file") + parser.add_argument("--no-pmd", action="store_true", help="Do not rewrite the .pmd pointer") + parser.add_argument("-v", "--verbose", action="store_true") + args = parser.parse_args(argv) + + steps = None + if args.iterations: + steps = [int(value) for value in args.iterations.split(",") if value.strip()] + log = print if args.verbose else None + try: + reports = merge_paths(args.inputs or ["."], args.output, overwrite=args.overwrite, + write_pmd=not args.no_pmd, iterations=steps, log=log) + except (ValueError, FileExistsError) as error: + parser.exit(1, f"{parser.prog}: {error}\n") + except FileNotFoundError as error: + parser.exit(1, f"{parser.prog}: {error}\n") + for report in reports: + print(f"merged {len(report.sources)} files into {report.output} " + f"({len(report.iterations)} iterations)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/PyPIC3D/diagnostics/openPMD.py b/PyPIC3D/diagnostics/openPMD.py index b3c959c..203d2fb 100644 --- a/PyPIC3D/diagnostics/openPMD.py +++ b/PyPIC3D/diagnostics/openPMD.py @@ -5,8 +5,8 @@ from typing import Any, Tuple, Union import importlib.metadata +import json import os -import traceback import jax import numpy as np @@ -64,50 +64,80 @@ def _ensure_openpmd_array(data, dtype=np.float64, squeeze=False): return arr -def _abort_parallel_write(comm): - """Stop peers before a rank-local failure leaves them in collective I/O.""" - if comm is not None: - traceback.print_exc() - comm.Abort(1) +def process_series_filename(filename, process_index, process_count): + """One process keeps the historical name; N>1 inserts a _pNNN suffix. + Each process writes its own series, so nothing is shared and nothing is + collective. ``PyPIC3D.diagnostics.merge_openpmd`` reassembles the group. + """ + if int(process_count) <= 1: + return filename + width = max(3, len(str(int(process_count) - 1))) + stem, extension = os.path.splitext(filename) + return f"{stem}_p{int(process_index):0{width}d}{extension}" + + +def _mesh_dataset(layout): + """Tile-shaped HDF5 chunks keep a per-process file proportional to its tiles. + + Every record keeps the global extent, so each file stays a valid whole-domain + mesh; only the chunks a process actually stores are allocated. "auto" chunks + would straddle tile boundaries and allocate cells owned by another process. + """ + dtype = np.dtype(layout.dtype) + chunks = [int(width) for width in layout.tile_shape] + options = "{}" + if all(width > 0 for width in chunks) and int(np.prod(chunks))*dtype.itemsize < (1 << 32): + options = json.dumps({"hdf5": {"dataset": {"chunks": chunks}}}) + return io.Dataset(dtype, list(layout.global_shape), options) + + +def _set_iteration_ownership(iteration, *, owned_tiles, tile_counts, tile_shape, global_shape): + """Record which global tiles this process stored. + + The HDF5 backend reports every dataset as one whole chunk, so a merge cannot + discover written regions; ownership has to be written down. + """ + owned = np.sort(np.asarray(owned_tiles, dtype=np.int64)) + iteration.set_attribute("pypic3dTileCounts", [int(n) for n in tile_counts]) + iteration.set_attribute("pypic3dTileShape", [int(n) for n in tile_shape]) + iteration.set_attribute("pypic3dGlobalShape", [int(n) for n in global_shape]) + iteration.set_attribute("pypic3dOwnedTileCount", int(owned.size)) + if owned.size: + iteration.set_attribute("pypic3dOwnedTiles", owned) + return owned + + +def _owned_tile_ids(host_shards, tile_counts): + """Flattened global tile ids covered by one component's addressable shards.""" + ids = [] + for index, data in host_shards: + origin = tuple(_slice_start(part) for part in index[:3]) + for local in np.ndindex(tuple(int(n) for n in data.shape[:3])): + ids.append(np.ravel_multi_index( + tuple(start + offset for start, offset in zip(origin, local)), tile_counts)) + return np.sort(np.asarray(ids, dtype=np.int64)) -def _open_openpmd_series(output_path, filename, file_extension=".bp", comm=None, access_mode=None): + +def _open_openpmd_series(output_path, filename, file_extension=".bp", access_mode=None): if file_extension is not None: filename = "_".join(filename.split()) + file_extension + filename = process_series_filename(filename, jax.process_index(), jax.process_count()) series_path = os.path.join(output_path, filename) - try: - if comm is None: - os.makedirs(output_path, exist_ok=True) - if access_mode is None: - access_mode = io.Access.append if os.path.exists(series_path) else io.Access.create - series = io.Series(series_path, access_mode) - else: - if not io.variants.get("mpi", False): - raise RuntimeError("Shared distributed output requires MPI-enabled openpmd-api and parallel HDF5") - if comm.rank == 0: - os.makedirs(output_path, exist_ok=True) - comm.Barrier() - if access_mode is None: - exists = comm.bcast(os.path.exists(series_path) if comm.rank == 0 else None, root=0) - access_mode = io.Access.append if exists else io.Access.create - # Ranks may own different numbers of chunks, including no particles. - # Metadata, flushes and close still execute collectively. - series = io.Series(series_path, access_mode, comm, - '{"hdf5":{"independent_stores":true}}') - series.set_attribute("software", "PyPIC3D") - series.set_attribute("softwareVersion", importlib.metadata.version("PyPIC3D")) - return series - except BaseException: - _abort_parallel_write(comm) - raise - - -def _close_openpmd_series(series, comm=None): - try: - series.close() - except BaseException: - _abort_parallel_write(comm) - raise + os.makedirs(output_path, exist_ok=True) + if access_mode is None: + access_mode = io.Access.append if os.path.exists(series_path) else io.Access.create + series = io.Series(series_path, access_mode) + series.set_attribute("software", "PyPIC3D") + series.set_attribute("softwareVersion", importlib.metadata.version("PyPIC3D")) + series.set_attribute("pypic3dProcessIndex", int(jax.process_index())) + series.set_attribute("pypic3dProcessCount", int(jax.process_count())) + return series + + +def _close_openpmd_series(series): + series.close() + def _configure_openpmd_mesh( mesh, @@ -221,28 +251,12 @@ def _field_map_to_interior(field_map): return interior_map -def _as_3tuple(value): - if isinstance(value, tuple): - return tuple(int(v) for v in value) - if isinstance(value, list): - return tuple(int(v) for v in value) - return (int(value), int(value), int(value)) - - def _slice_start(index_entry): if isinstance(index_entry, slice): return 0 if index_entry.start is None else int(index_entry.start) return int(index_entry) -def _tile_interior(tile, guard_cells): - gx, gy, gz = guard_cells - sx = slice(gx, -gx) if gx > 0 else slice(None) - sy = slice(gy, -gy) if gy > 0 else slice(None) - sz = slice(gz, -gz) if gz > 0 else slice(None) - return tile[sx, sy, sz] - - def _iter_tile_chunks_from_host_shard(shard_index, shard_data, *, layout): """ Yield ``(global_mesh_offset, tile_interior)`` chunks from one host shard. @@ -255,7 +269,12 @@ def _iter_tile_chunks_from_host_shard(shard_index, shard_data, *, layout): ) tile_shape = tuple(int(width) for width in layout.tile_shape) - guard_cells = _as_3tuple(layout.guard_cells) + if isinstance(layout.guard_cells, (tuple, list)): + guard_cells = tuple(int(g) for g in layout.guard_cells) + else: + guard_cells = (int(layout.guard_cells),) * 3 + gx, gy, gz = guard_cells + interior_slices = tuple(slice(g, -g) if g > 0 else slice(None) for g in (gx, gy, gz)) tx0 = _slice_start(shard_index[0]) ty0 = _slice_start(shard_index[1]) @@ -270,7 +289,7 @@ def _iter_tile_chunks_from_host_shard(shard_index, shard_data, *, layout): tz = tz0 + ltz raw_tile = shard_data[ltx, lty, ltz] - interior = _tile_interior(raw_tile, guard_cells) + interior = raw_tile[interior_slices] if tuple(interior.shape) != tile_shape: raise ValueError( f"Tile interior has shape {interior.shape}, expected {tile_shape}. " @@ -294,7 +313,7 @@ def _reset_scalar_mesh_record(iteration, name, *, dynamic_parameters, layout): quantity_name=name, ) record = mesh[io.Mesh_Record_Component.SCALAR] - record.reset_dataset(io.Dataset(np.dtype(layout.dtype), list(layout.global_shape))) + record.reset_dataset(_mesh_dataset(layout)) record.unit_SI = 1.0 _set_mesh_position(record, name, 0, layout.active_dims, dynamic_parameters) return record @@ -309,7 +328,7 @@ def _reset_vector_mesh_record(iteration, name, component_name, *, dynamic_parame quantity_name=name, ) record = mesh[component_name] - record.reset_dataset(io.Dataset(np.dtype(layout.dtype), list(layout.global_shape))) + record.reset_dataset(_mesh_dataset(layout)) record.unit_SI = 1.0 _set_mesh_position(record, name, "xyz".index(component_name), layout.active_dims, dynamic_parameters) return record @@ -377,6 +396,19 @@ def write_tiled_field_snapshot_to_iteration(iteration, snapshot, *, dynamic_para iteration, name, value, dynamic_parameters=dynamic_parameters, layout=layout) buffers.extend(chunks) + if snapshot.fields: + sample = snapshot.fields[sorted(snapshot.fields)[0]] + host_shards = sample[0] if isinstance(sample, tuple) and len(sample) == 3 else sample + tile_counts = tuple(int(extent)//int(width) + for extent, width in zip(layout.global_shape, layout.tile_shape)) + _set_iteration_ownership( + iteration, + owned_tiles=_owned_tile_ids(host_shards, tile_counts), + tile_counts=tile_counts, + tile_shape=tuple(int(width) for width in layout.tile_shape), + global_shape=tuple(int(extent) for extent in layout.global_shape), + ) + current_offset = getattr(snapshot, "current_time_offset", None) if current_offset is not None and "J" in snapshot.fields: iteration.meshes["J"].time_offset = current_offset @@ -391,9 +423,8 @@ def write_tiled_field_snapshot_openpmd( dynamic_parameters, layout, file_extension=".bp", - comm=None, ): - series = _open_openpmd_series(output_dir, filename, file_extension=file_extension, comm=comm) + series = _open_openpmd_series(output_dir, filename, file_extension=file_extension) try: iteration = series.iterations[int(snapshot.step)] @@ -404,11 +435,8 @@ def write_tiled_field_snapshot_openpmd( buffers = write_tiled_field_snapshot_to_iteration( iteration, snapshot, dynamic_parameters=dynamic_parameters, layout=layout) series.flush() - except BaseException: - _abort_parallel_write(comm) - raise finally: - _close_openpmd_series(series, comm) + _close_openpmd_series(series) def write_openpmd_fields_to_iteration(iteration, field_map, dynamic_parameters, active_dims=(1,1,1)): @@ -451,23 +479,6 @@ def _diagnostic_position_array(x, u, static_parameters, dynamic_parameters): ) -def _count_snapshot_particles_by_species(snapshot): - counts = [0] * len(snapshot.species_names) - - for active_index, active_chunk in snapshot.active_shards: - if active_chunk.ndim != 5: - raise ValueError( - "Expected active particle shard with shape " - "(ntx, nty, ntz, nspecies, max_particles_per_tile). " - f"Got {active_chunk.shape}." - ) - s0 = _slice_start(active_index[3]) - for local_s in range(active_chunk.shape[3]): - counts[s0 + local_s] += int(np.count_nonzero(active_chunk[:, :, :, local_s, :])) - - return counts - - def _particle_shards_by_index(shards): """Match owned tile views even when arrays group local tiles differently.""" indexed = {} @@ -627,24 +638,46 @@ def _store_particle_record_chunk( def write_tiled_particle_snapshot_to_iteration( - iteration, snapshot, static_parameters, dynamic_parameters, dtype=np.float64, comm=None, species_index=None, + iteration, snapshot, static_parameters, dynamic_parameters, dtype=np.float64, species_index=None, ): - """Compact local slots and place records in global tile order on every rank.""" + """Compact local slots into locally sized records kept in global tile order. + + Each process writes its own series, so records hold only its own particles. + ``pypic3dOwnedTiles``/``pypic3dOwnedTileCounts`` let the merge restore the + single-file global tile order without exchanging anything between processes. + """ n_species = len(snapshot.species_names) tile_counts = tuple(int(getattr(dynamic_parameters, "N" + axis)) // int(width) for axis, width in zip("xyz", static_parameters.tile_shape)) - counts = np.zeros(tile_counts + (n_species,), dtype=np.int64) + + # Owned tiles need not be contiguous in the flattened global tile order, so + # counts are keyed by tile rather than sliced out of a global array. + local_counts = {} for index, active in snapshot.active_shards: - counts[index[:4]] += np.count_nonzero(active, axis=-1) - local_counts = counts - if comm is not None: - from mpi4py import MPI - counts = np.empty_like(local_counts) - comm.Allreduce(local_counts, counts, op=MPI.SUM) - - flat_counts = counts.reshape((int(np.prod(tile_counts)), n_species)) - offsets = (np.cumsum(flat_counts, axis=0) - flat_counts).reshape(counts.shape) - totals = np.sum(flat_counts, axis=0) + origin = tuple(_slice_start(part) for part in index[:3]) + s0 = _slice_start(index[3]) + per_tile = np.count_nonzero(np.asarray(active), axis=-1) + for local in np.ndindex(per_tile.shape[:3]): + tile = tuple(start + offset for start, offset in zip(origin, local)) + entry = local_counts.setdefault(tile, np.zeros(n_species, dtype=np.int64)) + entry[s0:s0 + per_tile.shape[3]] += per_tile[local] + + owned = sorted(local_counts) + owned_ids = np.asarray([np.ravel_multi_index(tile, tile_counts) for tile in owned], dtype=np.int64) + offsets, running = {}, np.zeros(n_species, dtype=np.int64) + for tile in owned: + offsets[tile] = running.copy() + running = running + local_counts[tile] + totals = running + + _set_iteration_ownership( + iteration, + owned_tiles=owned_ids, + tile_counts=tile_counts, + tile_shape=tuple(int(width) for width in static_parameters.tile_shape), + global_shape=tuple(int(getattr(dynamic_parameters, "N" + axis)) for axis in "xyz"), + ) + selected_species = range(n_species) if species_index is None else (species_index,) for species in selected_species: species_group = iteration.particles[snapshot.species_names[species].replace(" ", "_")] @@ -653,6 +686,9 @@ def write_tiled_particle_snapshot_to_iteration( dtype=dtype, num_particles=int(totals[species]), ) + species_group.set_attribute("pypic3dOwnedTiles", owned_ids) + species_group.set_attribute("pypic3dOwnedTileCounts", np.asarray( + [local_counts[tile][species] for tile in owned], dtype=np.int64)) buffers = [] stored = [0] * n_species @@ -663,7 +699,7 @@ def write_tiled_particle_snapshot_to_iteration( species_group = iteration.particles[snapshot.species_names[species].replace(" ", "_")] _store_particle_record_chunk( species_group, - int(offsets[tile_species]), + int(offsets[tile_species[:3]][species]), x, momentum, charge, @@ -675,11 +711,10 @@ def write_tiled_particle_snapshot_to_iteration( ) stored[species] += int(x.shape[0]) - expected = local_counts.sum(axis=(0, 1, 2)) for species in selected_species: - if stored[species] != int(expected[species]): + if stored[species] != int(totals[species]): raise RuntimeError(f"Stored {stored[species]} local particles for {snapshot.species_names[species]}, " - f"expected {expected[species]}.") + f"expected {totals[species]}.") return buffers @@ -692,9 +727,8 @@ def write_tiled_particle_snapshot_openpmd( dynamic_parameters, file_extension=".bp", dtype=np.float64, - comm=None, ): - series = _open_openpmd_series(output_dir, filename, file_extension=file_extension, comm=comm) + series = _open_openpmd_series(output_dir, filename, file_extension=file_extension) try: iteration = series.iterations[int(snapshot.step)] @@ -708,15 +742,11 @@ def write_tiled_particle_snapshot_openpmd( static_parameters, dynamic_parameters, dtype=dtype, - comm=comm, ) series.flush() - except BaseException: - _abort_parallel_write(comm) - raise finally: - _close_openpmd_series(series, comm) + _close_openpmd_series(series) def write_openpmd_initial_particles( @@ -728,96 +758,80 @@ def write_openpmd_initial_particles( species_config=None, species_names=None, metric=None, - comm=None, ): """Write initial owned particle chunks to the existing per-species files.""" from PyPIC3D.diagnostics.async_writer import make_tiled_particle_snapshot - try: - gamma = None if metric is None else particle_lorentz_factor( - particles, metric, static_parameters, dynamic_parameters) - bssn = getattr(dynamic_parameters, "bssn_parameters", None) - time = float(bssn.time) if bssn is not None else 0.0 - snapshot = make_tiled_particle_snapshot( - particles, step=0, time=time, species_names=species_names, - species_config=species_config, gamma=gamma, initial=True) - # Initial positions and velocities precede the leapfrog seed. The - # snapshot's initial flag keeps x unchanged even for non-GR particles. - output_path = os.path.join(output_dir, "data", "initial_particles") - for species, name in enumerate(snapshot.species_names): - series_filename = f"{name.replace(' ', '_')}_{filename}" - series = _open_openpmd_series( - output_path, series_filename, file_extension=None, comm=comm, access_mode=io.Access.create) - try: - iteration = series.iterations[0] - iteration.time = time - iteration.dt = float(dynamic_parameters.dt) - iteration.time_unit_SI = 1.0 - buffers = write_tiled_particle_snapshot_to_iteration( - iteration, snapshot, static_parameters, dynamic_parameters, - comm=comm, species_index=species) - series.flush() - except BaseException: - _abort_parallel_write(comm) - raise - finally: - _close_openpmd_series(series, comm) - except BaseException: - _abort_parallel_write(comm) - raise + gamma = None if metric is None else particle_lorentz_factor( + particles, metric, static_parameters, dynamic_parameters) + bssn = getattr(dynamic_parameters, "bssn_parameters", None) + time = float(bssn.time) if bssn is not None else 0.0 + snapshot = make_tiled_particle_snapshot( + particles, step=0, time=time, species_names=species_names, + species_config=species_config, gamma=gamma, initial=True) + # Initial positions and velocities precede the leapfrog seed. The + # snapshot's initial flag keeps x unchanged even for non-GR particles. + output_path = os.path.join(output_dir, "data", "initial_particles") + for species, name in enumerate(snapshot.species_names): + series_filename = f"{name.replace(' ', '_')}_{filename}" + series = _open_openpmd_series( + output_path, series_filename, file_extension=None, access_mode=io.Access.create) + try: + iteration = series.iterations[0] + iteration.time = time + iteration.dt = float(dynamic_parameters.dt) + iteration.time_unit_SI = 1.0 + buffers = write_tiled_particle_snapshot_to_iteration( + iteration, snapshot, static_parameters, dynamic_parameters, species_index=species) + series.flush() + finally: + _close_openpmd_series(series) def write_openpmd_initial_fields( - field_map, static_parameters, dynamic_parameters, output_dir, filename="initial_fields.h5", comm=None, + field_map, static_parameters, dynamic_parameters, output_dir, filename="initial_fields.h5", ): """Write initial tile interiors without globally assembling tiled fields.""" from PyPIC3D.diagnostics.async_writer import make_tiled_field_snapshot + tiled_fields = {} + ordinary_fields = {} + for name, value in field_map.items(): + vector = isinstance(value, (list, tuple)) and len(value) == 3 + component = value[0] if vector else value + if component.ndim == 6: + tiled_fields[name] = value + else: + ordinary_fields[name] = value + if jax.process_count() > 1 and ordinary_fields: + raise ValueError("Multiprocess initial field output requires tile-major fields.") + + bssn = getattr(dynamic_parameters, "bssn_parameters", None) + time = float(bssn.time) if bssn is not None else 0.0 + if tiled_fields: + snapshot = make_tiled_field_snapshot( + tiled_fields, step=0, time=time, current_time_offset=0.0 if bssn is not None else None) + layout = TiledMeshLayout( + global_shape=tuple(int(getattr(dynamic_parameters, "N" + axis)) for axis in "xyz"), + tile_shape=tuple(int(width) for width in static_parameters.tile_shape), + guard_cells=int(static_parameters.guard_cells), + ) + output_path = os.path.join(output_dir, "data", "initial_fields") + series = _open_openpmd_series( + output_path, filename, file_extension=None, access_mode=io.Access.create) try: - tiled_fields = {} - ordinary_fields = {} - for name, value in field_map.items(): - vector = isinstance(value, (list, tuple)) and len(value) == 3 - component = value[0] if vector else value - if component.ndim == 6: - tiled_fields[name] = value - else: - ordinary_fields[name] = value - if comm is not None and ordinary_fields: - raise ValueError("MPI initial field output requires tile-major fields.") - - bssn = getattr(dynamic_parameters, "bssn_parameters", None) - time = float(bssn.time) if bssn is not None else 0.0 + iteration = series.iterations[0] + iteration.time = time + iteration.dt = float(dynamic_parameters.dt) + iteration.time_unit_SI = 1.0 + buffers = [] if tiled_fields: - snapshot = make_tiled_field_snapshot( - tiled_fields, step=0, time=time, current_time_offset=0.0 if bssn is not None else None) - layout = TiledMeshLayout( - global_shape=tuple(int(getattr(dynamic_parameters, "N" + axis)) for axis in "xyz"), - tile_shape=tuple(int(width) for width in static_parameters.tile_shape), - guard_cells=int(static_parameters.guard_cells), - ) - output_path = os.path.join(output_dir, "data", "initial_fields") - series = _open_openpmd_series( - output_path, filename, file_extension=None, comm=comm, access_mode=io.Access.create) - try: - iteration = series.iterations[0] - iteration.time = time - iteration.dt = float(dynamic_parameters.dt) - iteration.time_unit_SI = 1.0 - buffers = [] - if tiled_fields: - buffers.extend(write_tiled_field_snapshot_to_iteration( - iteration, snapshot, dynamic_parameters=dynamic_parameters, layout=layout)) - buffers.extend(write_openpmd_fields_to_iteration( - iteration, _field_map_to_interior(ordinary_fields), dynamic_parameters)) - if bssn is not None and "J" in field_map: - iteration.meshes["J"].time_offset = 0.0 - series.flush() - except BaseException: - _abort_parallel_write(comm) - raise - finally: - _close_openpmd_series(series, comm) - except BaseException: - _abort_parallel_write(comm) - raise + buffers.extend(write_tiled_field_snapshot_to_iteration( + iteration, snapshot, dynamic_parameters=dynamic_parameters, layout=layout)) + buffers.extend(write_openpmd_fields_to_iteration( + iteration, _field_map_to_interior(ordinary_fields), dynamic_parameters)) + if bssn is not None and "J" in field_map: + iteration.meshes["J"].time_offset = 0.0 + series.flush() + finally: + _close_openpmd_series(series) diff --git a/PyPIC3D/diagnostics/parallel_openpmd.py b/PyPIC3D/diagnostics/parallel_openpmd.py deleted file mode 100644 index da2452d..0000000 --- a/PyPIC3D/diagnostics/parallel_openpmd.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Main-thread MPI entry points using the common tiled openPMD writer.""" - -import openpmd_api as io - -from PyPIC3D.diagnostics.openPMD import ( - TiledMeshLayout, _abort_parallel_write, _open_openpmd_series, - write_tiled_field_snapshot_to_iteration, - write_tiled_particle_snapshot_to_iteration, -) - - -def open_parallel_series(output_dir, comm=None, filename="simulation.h5"): - """Open a common series; only a supplied communicator enables MPI.""" - return _open_openpmd_series(output_dir, filename, file_extension=None, comm=comm, - access_mode=io.Access.create) - - -def write_parallel_snapshot(series, comm, static, dynamic, *, fields=None, particles=None): - """Write owned chunks collectively with the serial mesh and particle schema.""" - try: - snapshot = fields if fields is not None else particles - if fields is not None and particles is not None: - if fields.step != particles.step or fields.time != particles.time: - raise ValueError("Field and particle snapshots must describe the same output time.") - iteration = series.iterations[int(snapshot.step)] - iteration.open() - iteration.time = float(snapshot.time) - iteration.dt = float(dynamic.dt) - iteration.time_unit_SI = 1.0 - buffers = [] - - if fields is not None: - layout = TiledMeshLayout( - global_shape=tuple(int(getattr(dynamic, "N" + axis)) for axis in "xyz"), - tile_shape=tuple(int(width) for width in static.tile_shape), - guard_cells=int(static.guard_cells), - ) - buffers.extend(write_tiled_field_snapshot_to_iteration( - iteration, fields, dynamic_parameters=dynamic, layout=layout)) - if particles is not None: - buffers.extend(write_tiled_particle_snapshot_to_iteration( - iteration, particles, static, dynamic, comm=comm)) - - # Empty ranks must enter the same flush and close operations. - iteration.series_flush() - iteration.close() - except BaseException: - _abort_parallel_write(comm) - raise diff --git a/PyPIC3D/diagnostics/simulation_output.py b/PyPIC3D/diagnostics/simulation_output.py new file mode 100644 index 0000000..42cbcea --- /dev/null +++ b/PyPIC3D/diagnostics/simulation_output.py @@ -0,0 +1,176 @@ +"""Simulation snapshots, scalar diagnostics, and output writer lifetime.""" + +from contextlib import contextmanager +import os + +import jax + +from PyPIC3D.diagnostics.async_writer import ( + create_async_tiled_openpmd_field_writer, + create_async_tiled_openpmd_particle_writer, + enqueue_openpmd_field_output, + enqueue_openpmd_particle_output, +) +from PyPIC3D.diagnostics.output_adapters import build_field_output_map +from PyPIC3D.diagnostics.plotting import write_data as _write_data +from PyPIC3D.diagnostics.diagnostic_quantities import compute_energy_components, compute_total_momentum +from PyPIC3D.relativity.interpolate_metric import particle_lorentz_factor +from PyPIC3D.utilities.simulation_helpers import setup_pmd_files + + +def simulation_energy_components(particles, species, fields, static, dynamic): + """Read ordinary and dark fields from the runtime tuple for flat-space diagnostics.""" + E, B, J, rho, phi, external_fields, *rest = fields + dark_fields = dark_pml = None + if static.solver == "dark_matter_yee": + dark_fields = fields[7] + dark_pml = None if fields[6] is None else fields[6][1] + return compute_energy_components(particles, E, B, external_fields, static, dynamic, + species, dark_fields=dark_fields, dark_pml=dark_pml) + + +def print_energy(label, energies, *, dark_matter=False): + """Report energies already evaluated on every process.""" + if jax.process_index() != 0: + return + electric, magnetic, kinetic, dark = energies + print(f"{label} Electric Field Energy: {electric:.6e} J") + print(f"{label} Magnetic Field Energy: {magnetic:.6e} J") + print(f"{label} Kinetic Energy: {kinetic:.6e} J") + if dark_matter: + print(f"{label} Dark Field Energy: {dark:.6e} J") + print(f"Total {label} Energy: {electric + magnetic + kinetic + dark:.6e} J\n") + + +def write_data(filename, time, value): + """Collective diagnostics run everywhere; only rank zero appends scalar files.""" + if jax.process_index() == 0: + _write_data(filename, time, value) + + +def write_work_energy_tracker(tracker, output_dir): + """Write one synchronized work-energy sample.""" + from PyPIC3D.diagnostics.gr_work_energy_theorem import tracker_values + + for name, value in tracker_values(tracker).items(): + write_data(f"{output_dir}/data/work_energy_{name}.txt", tracker.latest.time, value) + + +def write_scalar_diagnostics(particles, species, fields, static, dynamic, step, initial_energy): + """Write the active solver's scalar diagnostics at the driver's output cadence.""" + output_dir = static.output_dir + if static.solver == "dynamic_metric": + from PyPIC3D.diagnostics.static_metric import bssn_constraint_norms + + norms = bssn_constraint_norms(particles, species, fields, static, dynamic) + for name, value in norms.items(): + write_data(f"{output_dir}/data/{name}.txt", dynamic.bssn_parameters.time, value) + if static.gr_work_energy_theorem and 0 < step < static.Nt: + write_work_energy_tracker(dynamic.work_energy_tracker, output_dir) + elif static.solver != "static_metric": + electric, magnetic, kinetic, dark = simulation_energy_components( + particles, species, fields, static, dynamic) + total = electric + magnetic + kinetic + dark + time = step * dynamic.dt + if static.solver == "dark_matter_yee": + write_data(f"{output_dir}/data/dark_field_energy.txt", time, dark) + write_data(f"{output_dir}/data/total_energy.txt", time, total) + write_data(f"{output_dir}/data/energy_error.txt", time, + abs(initial_energy - total) / max(initial_energy, 1e-10)) + write_data(f"{output_dir}/data/electric_field_energy.txt", time, electric) + write_data(f"{output_dir}/data/magnetic_field_energy.txt", time, magnetic) + write_data(f"{output_dir}/data/kinetic_energy.txt", time, kinetic) + momentum = compute_total_momentum(particles, species_config=species) + write_data(f"{output_dir}/data/total_momentum.txt", time, momentum) + + +@contextmanager +def open_simulation_output(static_parameters, dynamic_parameters, plotting_parameters): + """Close both writers on failure. + + Every process queues snapshots into its own series, so nothing here is + collective. A .pmd pointer holds a single name, so a multiprocess run leaves + it to the merge tool rather than writing one that dangles until then. + """ + output_dir = static_parameters.output_dir + field_writer = particle_writer = None + loop_error = None + shared = jax.process_count() == 1 + try: + if plotting_parameters["plot_openpmd_fields"]: + if shared: + setup_pmd_files(os.path.join(output_dir, "data"), "fields", ".h5") + field_writer = create_async_tiled_openpmd_field_writer( + static_parameters, + dynamic_parameters, + os.path.join(output_dir, "data"), + filename="fields", + file_extension=".h5", + queue_size=int(plotting_parameters.get("openpmd_field_queue_size", 2)), + ) + if plotting_parameters["plot_openpmd_particles"]: + if shared: + setup_pmd_files(os.path.join(output_dir, "data"), "particles", ".h5") + particle_writer = create_async_tiled_openpmd_particle_writer( + static_parameters, + dynamic_parameters, + os.path.join(output_dir, "data"), + filename="particles", + file_extension=".h5", + queue_size=int(plotting_parameters.get("openpmd_particle_queue_size", 2)), + ) + yield field_writer, particle_writer + except BaseException as exc: + loop_error = exc + raise + finally: + writer_error = None + for writer in (particle_writer, field_writer): + if writer is None: + continue + try: + writer.close(raise_errors=loop_error is None) + except BaseException as exc: + if writer_error is None: + writer_error = exc + if loop_error is None and writer_error is not None: + raise writer_error + + +def write_simulation_output(particles, species_config, fields, static_parameters, + dynamic_parameters, plotting_parameters, t, writers): + """Submit particle and field snapshots without gathering global arrays.""" + field_writer, particle_writer = writers + particle_species_names = plotting_parameters.get("particle_species_names") + dynamic_metric = static_parameters.solver == "dynamic_metric" + gr_metric = dynamic_metric or static_parameters.solver == "static_metric" + + if plotting_parameters["plot_openpmd_particles"]: + gamma = None + output_particles = particles + if dynamic_metric: + from PyPIC3D.relativity.dynamical_relativity import synchronize_particles + output_particles = synchronize_particles( + particles, species_config, fields, static_parameters, dynamic_parameters) + if gr_metric: + metric = fields[6] + gamma = particle_lorentz_factor(output_particles, metric, static_parameters, dynamic_parameters) + # particle u holds covariant u_i, so Gamma needs the metric at each particle + enqueue_openpmd_particle_output( + particle_writer, output_particles, dynamic_parameters, t, t, + species_config=species_config, species_names=particle_species_names, + gamma=gamma, + ) + + if plotting_parameters["plot_openpmd_fields"]: + field_map = build_field_output_map( + fields, + particles, + species_config, + static_parameters, + dynamic_parameters, + include_fluid_velocity=bool(plotting_parameters["plotvelocities"]), + include_charge_density=bool(plotting_parameters["plotchargedensity"]), + ) + plotting_parameters["field_map"] = field_map + enqueue_openpmd_field_output(field_writer, field_map, dynamic_parameters, t, t) diff --git a/PyPIC3D/diagnostics/static_metric.py b/PyPIC3D/diagnostics/static_metric.py index 80cfb9f..18bec47 100644 --- a/PyPIC3D/diagnostics/static_metric.py +++ b/PyPIC3D/diagnostics/static_metric.py @@ -1,5 +1,6 @@ """Finite-difference constraints and radial particle/charge budgets.""" from typing import NamedTuple +import jax import jax.numpy as jnp from PyPIC3D.utilities.grids import grid_domain_bounds from PyPIC3D.boundary_conditions.grid_and_stencil import BC_ABSORBING, BC_CONDUCTING @@ -50,6 +51,14 @@ def bssn_constraint_norms(particles, species, fields, static, dynamic, *, source return compute_constraint_norms(violations) +def print_bssn_constraints(label, particles, species, fields, static, dynamic): + """Evaluate common-time constraints collectively and report on rank zero.""" + norms = bssn_constraint_norms(particles, species, fields, static, dynamic) + if jax.process_index() == 0: + print(f"{label} Einstein constraints: Hamiltonian RMS={float(norms['hamiltonian_l2']):.6e}, " + f"momentum RMS={float(norms['momentum_l2']):.6e}") + + def node_weights(static, template): """Nodal quadrature on reflecting endpoints; ordinary ownership elsewhere.""" weight = jnp.zeros_like(template) diff --git a/PyPIC3D/initialization.py b/PyPIC3D/initialization.py index 43149c0..b9f817f 100644 --- a/PyPIC3D/initialization.py +++ b/PyPIC3D/initialization.py @@ -9,12 +9,14 @@ from PyPIC3D.particles.particle_initialization import load_particles_from_toml from PyPIC3D.pusher.particle_push import seed_leapfrog_velocity from PyPIC3D.particles.particle_tile_communication import shard_tiled_particles -from PyPIC3D.diagnostics.diagnostic_quantities import compute_energy, compute_dark_energy +from PyPIC3D.diagnostics.diagnostic_quantities import compute_energy_components +from PyPIC3D.diagnostics.simulation_output import print_energy from PyPIC3D.utilities.field_helpers import add_external_fields, yee_push_fields from PyPIC3D.utilities.plasma_quantities import build_plasma_parameters_dict from PyPIC3D.utilities.simulation_helpers import ( convert_to_jax_compatible, courant_condition, + dark_photon_courant_condition, make_dir, particle_sanity_check, print_stats, @@ -54,7 +56,7 @@ ) from PyPIC3D.boundary_conditions.PML import initialize_tiled_pml_state, load_pml_from_toml from PyPIC3D.boundary_conditions.supergaussian import load_supergaussian_from_toml -from PyPIC3D.utilities.parameters import build_dynamic_parameters, build_static_parameters, validate_dark_parameters +from PyPIC3D.utilities.parameters import build_dynamic_parameters, build_static_parameters from PyPIC3D.relativity.core import B_FIELD_LOCATIONS, D_FIELD_LOCATIONS from PyPIC3D.relativity.metrics.flat import ( initialize_flat_cartesian_metric, @@ -107,39 +109,6 @@ def validate_field_solver(solver): ) -def _dark_timestep(static_config, dynamic_config): - """Resolve a stable explicit Maxwell-Proca timestep before JAX tracing.""" - validate_dark_parameters(static_config["sin_chi"], static_config["dark_mu"]) - c = float(dynamic_config["C"]) - eps = float(dynamic_config["eps"]) - if not math.isfinite(c) or c <= 0 or not math.isfinite(eps) or eps <= 0: - raise ValueError("dark_matter_yee requires positive finite C and eps") - inverse_spacings = [1 / float(dynamic_config["d" + axis]) for axis in "xyz" - if dynamic_config["N" + axis] > 1] - frequency = c * math.sqrt(float(static_config["dark_mu"])**2 + 4 * sum(v*v for v in inverse_spacings)) - limit = 2 / frequency if frequency else math.inf - dt = dynamic_config["dt"] - if dt is None: - cfl = float(static_config["cfl"]) - if not math.isfinite(cfl) or not 0 < cfl <= 1: - raise ValueError("dark_matter_yee requires 0 < cfl <= 1") - if not frequency: - raise ValueError("Specify dt for a massless dark field with all dimensions collapsed") - maxwell_dt = cfl / (c * sum(inverse_spacings)) if inverse_spacings else math.inf - dt = min(maxwell_dt, 0.99 * cfl * limit) - if not math.isfinite(float(dt)) or not 0 < float(dt) < limit: - raise ValueError(f"dark_matter_yee requires positive finite dt < {limit} (Proca stability limit)") - return float(dt) - - -def _tile_shape_from_static_config(static_config): - return ( - int(static_config["particle_tile_nx"]), - int(static_config["particle_tile_ny"]), - int(static_config["particle_tile_nz"]), - ) - - def _encode_current_calculation(current_calculation): if current_calculation not in ("j_from_rhov", "esirkepov", "GR_direct_deposition"): raise ValueError( @@ -189,7 +158,11 @@ def _validate_tiled_yee_configuration(static_config, dynamic_config): raise ValueError("Yee runtime currently supports only particle_pusher='boris' or 'higuera_cary'") if static_config["filter_j"] not in ("none", "digital", "bilinear"): raise ValueError("Yee runtime currently supports only filter_j='none', filter_j='digital', or 'bilinear'") - tile_shape = _tile_shape_from_static_config(static_config) + tile_shape = ( + int(static_config["particle_tile_nx"]), + int(static_config["particle_tile_ny"]), + int(static_config["particle_tile_nz"]), + ) grid_shape = ( int(dynamic_config["Nx"]), int(dynamic_config["Ny"]), @@ -234,17 +207,6 @@ def _available_cpu_threads(): return max(1, int(cpu_threads or 1)) -def _local_mesh_device_count(field_mesh): - """Count mesh devices owned by this JAX process.""" - - process_index = int(jax.process_index()) - local_devices = sum( - int(device.process_index) == process_index - for device in field_mesh.devices.flat - ) - return max(1, int(local_devices)) - - def _resolve_particle_batch_size( particles, requested_batch_size, @@ -288,7 +250,11 @@ def _resolve_particle_batch_size( if local_device_count is None: if field_mesh is None: raise ValueError("field_mesh is required to determine the local CPU device count.") - local_device_count = _local_mesh_device_count(field_mesh) + process_index = int(jax.process_index()) + local_device_count = sum( + int(device.process_index) == process_index + for device in field_mesh.devices.flat + ) available_cpu_threads = max(1, int(available_cpu_threads)) local_device_count = max(1, int(local_device_count)) threads_per_device = max(1, available_cpu_threads // local_device_count) @@ -305,9 +271,9 @@ def _resolve_particle_batch_size( resolved = max(1, min(max_active_per_tile, compute_target, tile_capacity)) if jax.process_count() > 1: - from mpi4py import MPI + from jax.experimental import multihost_utils # One static batch shape is required even on differently sized hosts. - resolved = MPI.COMM_WORLD.allreduce(resolved, op=MPI.MIN) + resolved = int(min(multihost_utils.process_allgather(resolved))) return resolved, reason @@ -394,7 +360,6 @@ def default_parameters(): "Nt": None, "relativistic": True, "particle_pusher": "boris", - "benchmark": False, "verbose": False, "GPUs": False, # Accepted for input compatibility; JAX selects devices. "metric": "flat_cartesian", @@ -508,15 +473,8 @@ def initialize_simulation(toml_file, *, bssn_state=None, bssn_evolution_paramete gr_metric = static_metric or dynamic_metric dark_matter = solver == "dark_matter_yee" distributed = jax.process_count() > 1 - from PyPIC3D.distributed import mpi_communicator - comm = mpi_communicator() if distributed and not dynamic_metric: raise ValueError("Multiprocess initialization currently supports solver='dynamic_metric'") - if distributed and any(bool(plotting_parameters[name]) for name in - ("plot_openpmd_fields", "plot_openpmd_particles", "dump_fields", "dump_particles")): - import openpmd_api as io - if not io.variants.get("mpi", False): - raise RuntimeError("Shared distributed output requires MPI-enabled openpmd-api and parallel HDF5") if "metric" in config and not dynamic_metric: raise ValueError("[metric] initialization requires solver='dynamic_metric'") if static_metric: @@ -539,12 +497,6 @@ def initialize_simulation(toml_file, *, bssn_state=None, bssn_evolution_paramete raise ValueError("dynamic_metric requires common-time D/B; previous_field history is seeded internally") if key.startswith("field") and block.get("type") in (6, 7, 8): raise ValueError("dynamic_metric seeds current from common-time particles; initial J field files are unsupported") - particle_blocks = tuple(block for key, block in config.items() if key.startswith("particle")) - field_boundaries = tuple(static_config[axis + "_bc"] for axis in "xyz") - particle_boundaries = tuple(static_config["particle_" + axis + "_bc"] for axis in "xyz") - species_boundaries = tuple(block.get(axis + "_bc", "periodic") - for block in particle_blocks for axis in "xyz") - # The GR Maxwell/pusher kernels use Gaussian units with c=1. for key, value in (("C", 1.0), ("eps", 1.0), ("mu", 1.0), ("kb", 1.0)): if key not in config.get("simulation_parameters", {}) and key not in config.get("dynamic_parameters", {}): @@ -593,7 +545,7 @@ def initialize_simulation(toml_file, *, bssn_state=None, bssn_evolution_paramete dynamic_config["dz"] = dz if dark_matter: - dt = _dark_timestep(static_config, dynamic_config) + dt = dark_photon_courant_condition(static_config, dynamic_config) dynamic_config["dt"] = dt elif dynamic_config["dt"] is not None: print(f"Using user defined dt: {dynamic_config['dt']}") @@ -762,7 +714,6 @@ def initialize_simulation(toml_file, *, bssn_state=None, bssn_evolution_paramete species_config=species_config, species_names=particle_species_names, metric=metric, - comm=comm, ) E, B, J, phi, rho = initialize_fields(static_parameters, dynamic_parameters) @@ -781,7 +732,7 @@ def initialize_simulation(toml_file, *, bssn_state=None, bssn_evolution_paramete ) E, B, J = field_components[:3], field_components[3:6], field_components[6:9] if field_initializers: - from PyPIC3D.distributed import tiled_field_from_initializer + from PyPIC3D.utilities.distributed import tiled_field_from_initializer components = list(E) + list(B) names = ("Ex", "Ey", "Ez", "Bx", "By", "Bz") for index, (name, location) in enumerate(zip(names, D_FIELD_LOCATIONS + B_FIELD_LOCATIONS)): @@ -844,25 +795,11 @@ def initialize_simulation(toml_file, *, bssn_state=None, bssn_evolution_paramete dark_fields, dark_pml = initialize_dark_pml(dark_fields, static_parameters, dynamic_parameters, pml_profiles) pml_state = pml_state, dark_pml - if gr_metric: - print("Skipping flat-space energy diagnostics for GR fields and covariant particle u_i\n") - else: - total_E, total_B = add_external_fields(E, B, external_fields) - e_energy, b_energy, kinetic_energy = compute_energy( - particles, - total_E, - total_B, - static_parameters, - dynamic_parameters, - species_config=species_config, - ) - print(f"Initial Electric Field Energy: {e_energy:.2e} J") - print(f"Initial Magnetic Field Energy: {b_energy:.2e} J") - print(f"Initial Kinetic Energy: {kinetic_energy:.2e} J") - dark_energy = compute_dark_energy(dark_fields, static_parameters, dynamic_parameters, dark_pml) if dark_matter else 0.0 - if dark_matter: - print(f"Initial Dark Field Energy: {dark_energy:.2e} J") - print(f"Total Initial Energy: {e_energy + b_energy + kinetic_energy + dark_energy:.2e} J\n") + if not gr_metric: + energies = compute_energy_components( + particles, E, B, external_fields, static_parameters, dynamic_parameters, + species_config, dark_fields=dark_fields, dark_pml=dark_pml) + print_energy("Initial", energies, dark_matter=dark_matter) if dark_matter: seed_E, seed_B = dark_photon_push_fields(E, B, dark_fields, external_fields, static_parameters, dynamic_parameters, dark_pml) @@ -917,15 +854,14 @@ def initialize_simulation(toml_file, *, bssn_state=None, bssn_evolution_paramete overflow = jnp.asarray(False) if dynamic_metric: from PyPIC3D.relativity.dynamical_relativity import initialize_dynamical_relativity - from PyPIC3D.diagnostics.static_metric import bssn_constraint_norms + from PyPIC3D.diagnostics.static_metric import print_bssn_constraints fields = (E, B, J, rho, phi, external_fields, metric, None, overflow) particles, fields, dynamic_parameters = initialize_dynamical_relativity( particles, species_config, fields, static_parameters, dynamic_parameters, ) - norms = bssn_constraint_norms(particles, species_config, fields, static_parameters, dynamic_parameters) - print(f"Initial Einstein constraints: Hamiltonian RMS={float(norms['hamiltonian_l2']):.6e}, " - f"momentum RMS={float(norms['momentum_l2']):.6e}") + print_bssn_constraints("Initial", particles, species_config, fields, + static_parameters, dynamic_parameters) elif static_metric: fields = (E, B, J, rho, phi, external_fields, metric, static_metric_state, overflow) elif electrostatic: @@ -956,7 +892,6 @@ def initialize_simulation(toml_file, *, bssn_state=None, bssn_evolution_paramete dynamic_parameters, static_parameters.output_dir, filename="initial_fields.h5", - comm=comm, ) return ( diff --git a/PyPIC3D/particles/distributed_initialization.py b/PyPIC3D/particles/distributed_initialization.py index 26be79b..eef28d3 100644 --- a/PyPIC3D/particles/distributed_initialization.py +++ b/PyPIC3D/particles/distributed_initialization.py @@ -11,14 +11,18 @@ from PyPIC3D.utilities.grids import grid_domain_bounds -def load_distributed_particles(config, static, dynamic, *, comm=None, chunk_size=262144): +def load_distributed_particles(config, static, dynamic, *, chunk_size=262144): """Load original particle IDs in bounded blocks, preserving tile ordering. All six initial_x/y/z/vx/vy/vz entries must be native .npy files. Built-in random generation is deliberately unsupported here: changing random keys per process would silently change the physical initial condition. + + Every process reads every block and keeps only the particles in tiles it + owns. Occupancy and temperature therefore come out identical everywhere + without any exchange, at the cost of each process reading the whole file. """ - rank, size = (0, 1) if comm is None else (comm.rank, comm.size) + rank = jax.process_index() mesh = static.field_mesh counts = tuple(int(n) for n in mesh.devices.shape) widths = np.asarray(static.tile_shape, dtype=np.int64) @@ -46,13 +50,10 @@ def load_distributed_particles(config, static, dynamic, *, comm=None, chunk_size if any(value.shape != (n,) for value in arrays): raise ValueError("Native particle arrays must each have shape (N_particles,)") received, velocity_square = [], 0.0 - # Every rank enters the same number of exchanges, including ranks - # without particles in the final block or in an entire species. - rounds = math.ceil(n / (size*chunk_size)) - for turn in range(rounds): - start = (turn*size + rank)*chunk_size + # Blocks are walked in file order and the kept rows keep their order + # within a block, so the concatenation below is already in file order. + for start in range(0, n, chunk_size): end = min(start + chunk_size, n) - ids = np.arange(start, max(start, end), dtype=np.int64) values = [np.asarray(a[start:end], dtype=np.float64) for a in arrays] x = np.stack(values[:3], axis=-1) u = np.stack(values[3:], axis=-1) @@ -60,22 +61,16 @@ def load_distributed_particles(config, static, dynamic, *, comm=None, chunk_size cells = np.clip(np.floor((x-minimum)/spacing).astype(np.int64), 0, shape-1) tiles = cells // widths tile_ids = np.ravel_multi_index(tiles.T, counts) - destinations = owners[tile_ids] - packets = [(ids[destinations == dest], tile_ids[destinations == dest], - x[destinations == dest], u[destinations == dest]) for dest in range(size)] - incoming = packets if comm is None else comm.alltoall(packets) - received.extend(packet for packet in incoming if len(packet[0])) + occupancy[:, species] += np.bincount(tile_ids, minlength=math.prod(counts)) + kept = owners[tile_ids] == rank + if np.any(kept): + received.append((tile_ids[kept], x[kept], u[kept])) if received: - ids, tiles, x, u = (np.concatenate([packet[k] for packet in received]) for k in range(4)) - order = np.argsort(ids, kind="stable") - tiles, x, u = tiles[order], x[order], u[order] - occupancy[:, species] = np.bincount(tiles, minlength=math.prod(counts)) + tiles, x, u = (np.concatenate([packet[k] for packet in received]) for k in range(3)) else: tiles, x, u = np.empty(0, dtype=np.int64), np.empty((0, 3)), np.empty((0, 3)) local_particles.append((tiles, x, u)) - if comm is not None: - velocity_square = comm.allreduce(velocity_square) mass, charge, kb = float(block["mass"]), float(block["charge"]), float(dynamic.kb) temperature = float(block.get("temperature", mass*velocity_square/(3*kb*n) if n else 0.0)) # Preserve the serial metadata convention: vth describes the requested @@ -93,11 +88,6 @@ def load_distributed_particles(config, static, dynamic, *, comm=None, chunk_size z_bc=block.get("z_bc", "periodic"), update_x=tuple(bool(block.get("update_" + axis, True)) for axis in "xyz"))) - if comm is not None: - from mpi4py import MPI - global_occupancy = np.empty_like(occupancy) - comm.Allreduce(occupancy, global_occupancy, op=MPI.SUM) - occupancy = global_occupancy capacity = int(math.ceil(int(occupancy.max(initial=0))*float(static.particle_tile_capacity_factor))) local = {} for tile_id in np.flatnonzero(owners == rank): diff --git a/PyPIC3D/particles/particle_initialization.py b/PyPIC3D/particles/particle_initialization.py index 1009881..49609c2 100644 --- a/PyPIC3D/particles/particle_initialization.py +++ b/PyPIC3D/particles/particle_initialization.py @@ -101,14 +101,16 @@ def _particle_tile_indices(x1, x2, x3, dynamic_parameters, tile_nx, tile_ny, til return x_cell // tile_nx, y_cell // tile_ny, z_cell // tile_nz -def _prepare_species_tile_data(species_arrays, dynamic_parameters, tile_shape): +def _pack_species_arrays_into_tiles(species_arrays, dynamic_parameters, tile_shape, capacity_factor): tile_nx, tile_ny, tile_nz = tile_shape ntx = _tile_axis_count(dynamic_parameters.Nx, tile_nx) nty = _tile_axis_count(dynamic_parameters.Ny, tile_ny) ntz = _tile_axis_count(dynamic_parameters.Nz, tile_nz) n_tiles = ntx * nty * ntz + n_species = len(species_arrays) - tile_counts = np.zeros((ntx, nty, ntz, len(species_arrays)), dtype=int) + # Count active particles in each tile before choosing the shared slot capacity. + tile_counts = np.zeros((ntx, nty, ntz, n_species), dtype=int) species_tile_data = [] for species_index, (x_array, u_array, active_mask) in enumerate(species_arrays): @@ -133,17 +135,7 @@ def _prepare_species_tile_data(species_arrays, dynamic_parameters, tile_shape): species_tile_data.append((x_np, u_np, active_np, tx, ty, tz, flat_tile, active_indices)) - return tile_counts, species_tile_data - - -def _pack_species_arrays_into_tiles(species_arrays, dynamic_parameters, tile_shape, capacity_factor): - tile_nx, tile_ny, tile_nz = tile_shape - ntx = _tile_axis_count(dynamic_parameters.Nx, tile_nx) - nty = _tile_axis_count(dynamic_parameters.Ny, tile_ny) - ntz = _tile_axis_count(dynamic_parameters.Nz, tile_nz) - n_species = len(species_arrays) - - tile_counts, species_tile_data = _prepare_species_tile_data(species_arrays, dynamic_parameters, tile_shape) + # Allocate fixed-capacity tiles, including the requested inactive headroom. max_particles_per_tile = int(np.max(tile_counts)) if tile_counts.size else 0 max_particles_per_tile = int(math.ceil(max_particles_per_tile * capacity_factor)) @@ -151,6 +143,7 @@ def _pack_species_arrays_into_tiles(species_arrays, dynamic_parameters, tile_sha u_tiles = np.zeros_like(x_tiles) active_tiles = np.zeros((ntx, nty, ntz, n_species, max_particles_per_tile), dtype=bool) + # Stable sorting preserves the input particle order within each tile. for species_index, (x_np, u_np, active_np, tx, ty, tz, flat_tile, active_indices) in enumerate(species_tile_data): if active_indices.size == 0: continue @@ -219,10 +212,8 @@ def _debye_length(N_per_cell, charge, temperature, weight, dynamic_parameters): def load_particles_from_toml(config, static_parameters, dynamic_parameters): if jax.process_count() > 1: - from PyPIC3D.distributed import mpi_communicator from .distributed_initialization import load_distributed_particles - return load_distributed_particles(config, static_parameters, dynamic_parameters, - comm=mpi_communicator()) + return load_distributed_particles(config, static_parameters, dynamic_parameters) if config is None: config = {} diff --git a/PyPIC3D/particles/particle_tile_communication.py b/PyPIC3D/particles/particle_tile_communication.py index cd78ce9..0d8a81b 100644 --- a/PyPIC3D/particles/particle_tile_communication.py +++ b/PyPIC3D/particles/particle_tile_communication.py @@ -88,37 +88,6 @@ def update_tiled_particle_positions(tiled_particles, species_config, dt): return tiled_particles._replace(x=x) -def _adjacent_tile_offset(dest_tile, source_tile, tile_count): - """ - Signed adjacent offset from the source tile to the destination tile. - - The tiled particle step assumes particles move by at most one cell, so tile - ownership can only change by one neighboring tile along any active axis. - Periodic end points are represented with the physical crossing direction: - first -> last is -1, last -> first is +1. - """ - - if tile_count == 1: - return jnp.zeros_like(dest_tile) - - offset = dest_tile - source_tile - if tile_count == 2: - return offset - - offset = jnp.where(offset == tile_count - 1, -1, offset) - offset = jnp.where(offset == -(tile_count - 1), 1, offset) - - return offset - - -def _particle_axis_tile_index(position, axis_min, cell_width, tile_width, tile_count): - """Return the owner tile along one axis for bounded particle positions.""" - - cell = jnp.floor((position - axis_min) / cell_width).astype(int) - cell = jnp.clip(cell, 0, int(tile_width) * int(tile_count) - 1) - return jnp.clip(cell // int(tile_width), 0, int(tile_count) - 1) - - def _particle_face_packet_capacity(n_slots, tile_shape, axis, n_directions=1): """ Choose a static mover capacity from the tile slot density and face area. @@ -160,12 +129,6 @@ def pack_species(species_x, species_u, species_moving): return packet, jnp.any(overflow) -def _unpack_particle_packet(packet): - """Split a homogeneous mover packet into the local particle representation.""" - - return packet[..., :3], packet[..., 3:6], packet[..., 6] != 0.0 - - def _exchange_particle_axis( x, u, @@ -188,14 +151,16 @@ def _exchange_particle_axis( axis_name = MESH_AXES[int(axis)] source_tile = jax.lax.axis_index(axis_name) - destination_tile = _particle_axis_tile_index( - x[..., int(axis)], - axis_min, - cell_width, - tile_width, - tile_count, - ) - offset = _adjacent_tile_offset(destination_tile, source_tile, tile_count) + cell = jnp.floor((x[..., int(axis)] - axis_min) / cell_width).astype(int) + cell = jnp.clip(cell, 0, int(tile_width) * tile_count - 1) + destination_tile = jnp.clip(cell // int(tile_width), 0, tile_count - 1) + + offset = destination_tile - source_tile + # Periodic first -> last crossings go left; last -> first crossings go right. + # With two tiles, both faces reach the same peer and use one combined packet. + if tile_count > 2: + offset = jnp.where(offset == tile_count - 1, -1, offset) + offset = jnp.where(offset == -(tile_count - 1), 1, offset) invalid = active & (jnp.abs(offset) > 1) moving_negative = active & (offset == -1) moving_positive = active & (offset == 1) @@ -255,7 +220,9 @@ def _exchange_particle_axis( ) packet_overflow = negative_overflow | positive_overflow - incoming_x, incoming_u, incoming_active = _unpack_particle_packet(incoming_packet) + incoming_x = incoming_packet[..., :3] + incoming_u = incoming_packet[..., 3:6] + incoming_active = incoming_packet[..., 6] != 0.0 new_x, new_u, new_active, capacity_overflow = _fill_incoming_particles( stay_x, stay_u, diff --git a/PyPIC3D/distributed.py b/PyPIC3D/utilities/distributed.py similarity index 70% rename from PyPIC3D/distributed.py rename to PyPIC3D/utilities/distributed.py index d155912..bf553b8 100644 --- a/PyPIC3D/distributed.py +++ b/PyPIC3D/utilities/distributed.py @@ -6,35 +6,27 @@ def initialize_distributed(): """Opt in with PYPIC_DISTRIBUTED=1 before importing PyPIC3D. - MPI and JAX share process numbering. Slurm may expose one GPU per task; - PYPIC_LOCAL_DEVICE_IDS overrides JAX's MPI local-rank selection in that case. + JAX detects Slurm and Open MPI launches on its own. PYPIC_COORDINATOR_ADDRESS, + PYPIC_PROCESS_ID and PYPIC_NUM_PROCESSES describe a launch it cannot detect. + Slurm may expose one GPU per task; PYPIC_LOCAL_DEVICE_IDS overrides JAX's + local-device selection in that case. """ if os.environ.get("PYPIC_DISTRIBUTED", "0") != "1": return - from mpi4py import MPI import jax - if not jax.distributed.is_initialized(): - ids = os.environ.get("PYPIC_LOCAL_DEVICE_IDS") - jax.distributed.initialize( - cluster_detection_method="mpi4py", - local_device_ids=None if ids is None else [int(v) for v in ids.split(",")], - initialization_timeout=int(os.environ.get("PYPIC_INITIALIZATION_TIMEOUT", "120")), - ) - if (jax.process_index(), jax.process_count()) != (MPI.COMM_WORLD.rank, MPI.COMM_WORLD.size): - raise ValueError("MPI and JAX must use the same process numbering") - - -def mpi_communicator(): - """Return the simulation communicator only for a multiprocess JAX run.""" - import jax - if jax.process_count() == 1: - return None - from mpi4py import MPI - comm = MPI.COMM_WORLD - if (jax.process_index(), jax.process_count()) != (comm.rank, comm.size): - raise ValueError("MPI and JAX must use the same process numbering") - return comm + if jax.distributed.is_initialized(): + return + ids = os.environ.get("PYPIC_LOCAL_DEVICE_IDS") + processes = os.environ.get("PYPIC_NUM_PROCESSES") + process = os.environ.get("PYPIC_PROCESS_ID") + jax.distributed.initialize( + coordinator_address=os.environ.get("PYPIC_COORDINATOR_ADDRESS"), + num_processes=None if processes is None else int(processes), + process_id=None if process is None else int(process), + local_device_ids=None if ids is None else [int(v) for v in ids.split(",")], + initialization_timeout=int(os.environ.get("PYPIC_INITIALIZATION_TIMEOUT", "120")), + ) def tiled_field_from_initializer(initializer, static, dynamic, *, location=("C", "C", "C")): diff --git a/PyPIC3D/utilities/grids.py b/PyPIC3D/utilities/grids.py index 0dcc001..c6ad700 100644 --- a/PyPIC3D/utilities/grids.py +++ b/PyPIC3D/utilities/grids.py @@ -122,40 +122,6 @@ def build_yee_grid(dynamic_parameters): return center_grid, vertex_grid -def _tile_grid_axis(global_axis_grid, dynamic_parameters, tile_shape, tile_counts, axis_index, num_guard_cells): - tile_width = tile_shape[axis_index] - tile_count = tile_counts[axis_index] - g = num_guard_cells - - d = jax.lax.cond( - axis_index == 0, - lambda _: dynamic_parameters.dx, - lambda _: jax.lax.cond( - axis_index == 1, - lambda _: dynamic_parameters.dy, - lambda _: dynamic_parameters.dz, - None, - ), - None, - ) - # determine the grid spacing for the given axis index - - offsets = jnp.arange(tile_width + 2 * g, dtype=global_axis_grid.dtype) - # determine the offsets for the tile including guard cells - tile_indices = jnp.arange(tile_count, dtype=global_axis_grid.dtype) - # get the tile indices for the given axis - axis_lines = global_axis_grid[0] + ( - offsets[jnp.newaxis, :] + tile_indices[:, jnp.newaxis] * tile_width - (g - 1) - ) * d - # build the local tile axes - - axis_shape = [1, 1, 1, tile_width + 2 * g] - axis_shape[axis_index] = tile_count - tiled_shape = tuple(tile_counts) + (tile_width + 2 * g,) - - return jnp.broadcast_to(axis_lines.reshape(axis_shape), tiled_shape) - - def _tile_grid_axes(grid, dynamic_parameters, tile_shape, num_guard_cells=2, mesh=None): """ Build tile-local coordinate lines for a center or vertex grid. @@ -188,10 +154,25 @@ def local_line(index, axis=axis, width=width, origin=origin, spacing=spacing): tile_counts + (width+2*g,), sharding, local_line, dtype=np.float64)) return tuple(coordinates) - return tuple( - _tile_grid_axis(grid[axis], dynamic_parameters, tile_shape, tile_counts, axis, num_guard_cells) - for axis in range(3) - ) + spacings = (dynamic_parameters.dx, dynamic_parameters.dy, dynamic_parameters.dz) + g = num_guard_cells + coordinates = [] + for axis in range(3): + global_axis_grid = grid[axis] + tile_width = tile_shape[axis] + tile_count = tile_counts[axis] + offsets = jnp.arange(tile_width + 2 * g, dtype=global_axis_grid.dtype) + tile_indices = jnp.arange(tile_count, dtype=global_axis_grid.dtype) + # Shift the global grid's first guard sample to the requested guard depth. + axis_lines = global_axis_grid[0] + ( + offsets[jnp.newaxis, :] + tile_indices[:, jnp.newaxis] * tile_width - (g - 1) + ) * spacings[axis] + + axis_shape = [1, 1, 1, tile_width + 2 * g] + axis_shape[axis] = tile_count + tiled_shape = tuple(tile_counts) + (tile_width + 2 * g,) + coordinates.append(jnp.broadcast_to(axis_lines.reshape(axis_shape), tiled_shape)) + return tuple(coordinates) def build_tiled_yee_grids(static_parameters, dynamic_parameters): diff --git a/PyPIC3D/utilities/parameters.py b/PyPIC3D/utilities/parameters.py index 3082a20..f9063b4 100644 --- a/PyPIC3D/utilities/parameters.py +++ b/PyPIC3D/utilities/parameters.py @@ -18,7 +18,6 @@ class StaticParameters(NamedTuple): output_dir: str Nt: int verbose: bool - benchmark: bool solver: str electrostatic: bool relativistic: bool @@ -84,29 +83,6 @@ def _axis_tuple(axis_values): ) -def _tile_shape(static_config): - if "tile_shape" in static_config: - return tuple(int(width) for width in static_config["tile_shape"]) - - return ( - int(static_config["particle_tile_nx"]), - int(static_config["particle_tile_ny"]), - int(static_config["particle_tile_nz"]), - ) - - -def _field_mesh(static_config, tile_shape): - if static_config.get("field_mesh") is not None: - return static_config["field_mesh"] - - tile_grid_shape = ( - int(static_config["Nx"]) // int(tile_shape[0]), - int(static_config["Ny"]) // int(tile_shape[1]), - int(static_config["Nz"]) // int(tile_shape[2]), - ) - return make_field_mesh(tile_grid_shape) - - def validate_dark_parameters(sin_chi, dark_mu): """Validate the dimensionless mixing and inverse-length Proca mass.""" if not math.isfinite(float(sin_chi)) or abs(float(sin_chi)) > 1: @@ -159,18 +135,33 @@ def build_static_parameters(static_config): raise ValueError("Tiled fields require at least one guard cell.") if hybrid and guard_cells < 3: raise ValueError("Hybrid Hermite particle metrics require guard_cells >= 3") - tile_shape = _tile_shape(static_config) + if "tile_shape" in static_config: + tile_shape = tuple(int(width) for width in static_config["tile_shape"]) + else: + tile_shape = ( + int(static_config["particle_tile_nx"]), + int(static_config["particle_tile_ny"]), + int(static_config["particle_tile_nz"]), + ) particle_batch_size = static_config.get("particle_batch_size", 1) if (isinstance(particle_batch_size, bool) or not isinstance(particle_batch_size, Integral) or particle_batch_size <= 0): raise ValueError("particle_batch_size must be a positive integer.") + field_mesh = static_config.get("field_mesh") + if field_mesh is None: + tile_grid_shape = ( + int(static_config["Nx"]) // tile_shape[0], + int(static_config["Ny"]) // tile_shape[1], + int(static_config["Nz"]) // tile_shape[2], + ) + field_mesh = make_field_mesh(tile_grid_shape) + return StaticParameters( name=static_config.get("name", "Default Simulation"), output_dir=static_config.get("output_dir", "."), Nt=int(static_config.get("Nt", 0)), verbose=bool(static_config.get("verbose", False)), - benchmark=bool(static_config.get("benchmark", False)), solver=static_config.get("solver", "electrodynamic_yee"), electrostatic=bool(static_config.get("electrostatic", False)), relativistic=bool(static_config.get("relativistic", True)), @@ -194,7 +185,7 @@ def build_static_parameters(static_config): supergaussian_layers=tuple(static_config.get("supergaussian_layers", ())), boundary_conditions=_axis_tuple(static_config["boundary_conditions"]), particle_boundary_conditions=_axis_tuple(static_config.get("particle_boundary_conditions", (0, 0, 0))), - field_mesh=_field_mesh(static_config, tile_shape), + field_mesh=field_mesh, horizon_field_cells=int(horizon_cells), sin_chi=float(static_config.get("sin_chi", 0.0)), dark_mu=float(static_config.get("dark_mu", 0.0)), diff --git a/PyPIC3D/utilities/simulation_helpers.py b/PyPIC3D/utilities/simulation_helpers.py index debf4ef..cee937e 100644 --- a/PyPIC3D/utilities/simulation_helpers.py +++ b/PyPIC3D/utilities/simulation_helpers.py @@ -1,8 +1,10 @@ import os +import math import jax.numpy as jnp from jax.tree_util import tree_map +from PyPIC3D.utilities.parameters import validate_dark_parameters from PyPIC3D.utilities.grids import grid_axis_width, grid_domain_bounds @@ -88,3 +90,28 @@ def courant_condition(courant_number, dx, dy, dz, dynamic_parameters): dt = courant_number / (C * dx_inv) return dt + + +def dark_photon_courant_condition(static_config, dynamic_config): + """Resolve a stable explicit Maxwell-Proca timestep before JAX tracing.""" + validate_dark_parameters(static_config["sin_chi"], static_config["dark_mu"]) + c = float(dynamic_config["C"]) + eps = float(dynamic_config["eps"]) + if not math.isfinite(c) or c <= 0 or not math.isfinite(eps) or eps <= 0: + raise ValueError("dark_matter_yee requires positive finite C and eps") + inverse_spacings = [1 / float(dynamic_config["d" + axis]) for axis in "xyz" + if dynamic_config["N" + axis] > 1] + frequency = c * math.sqrt(float(static_config["dark_mu"])**2 + 4 * sum(v*v for v in inverse_spacings)) + limit = 2 / frequency if frequency else math.inf + dt = dynamic_config["dt"] + if dt is None: + cfl = float(static_config["cfl"]) + if not math.isfinite(cfl) or not 0 < cfl <= 1: + raise ValueError("dark_matter_yee requires 0 < cfl <= 1") + if not frequency: + raise ValueError("Specify dt for a massless dark field with all dimensions collapsed") + maxwell_dt = cfl / (c * sum(inverse_spacings)) if inverse_spacings else math.inf + dt = min(maxwell_dt, 0.99 * cfl * limit) + if not math.isfinite(float(dt)) or not 0 < float(dt) < limit: + raise ValueError(f"dark_matter_yee requires positive finite dt < {limit} (Proca stability limit)") + return float(dt) diff --git a/PyPIC3D/utilities/toml_helpers.py b/PyPIC3D/utilities/toml_helpers.py index a6dcfb8..274fec9 100644 --- a/PyPIC3D/utilities/toml_helpers.py +++ b/PyPIC3D/utilities/toml_helpers.py @@ -130,7 +130,7 @@ def load_external_fields_from_toml(fields, external_fields, config, static_param print(f"Loading field: {field_name} from {field_path}") if static_parameters.field_mesh.size > 1: - from PyPIC3D.distributed import tiled_field_from_initializer + from PyPIC3D.utilities.distributed import tiled_field_from_initializer from PyPIC3D.relativity.core import D_FIELD_LOCATIONS, B_FIELD_LOCATIONS locations = D_FIELD_LOCATIONS + B_FIELD_LOCATIONS + D_FIELD_LOCATIONS if not 0 <= field_type < len(locations): diff --git a/demos/dynamical_relativity/charged_stellar_collapse/charged_star.toml b/demos/dynamical_relativity/charged_stellar_collapse/charged_star.toml index 1e4da0e..6ceb6f7 100644 --- a/demos/dynamical_relativity/charged_stellar_collapse/charged_star.toml +++ b/demos/dynamical_relativity/charged_stellar_collapse/charged_star.toml @@ -1,9 +1,10 @@ -# Run initial_metric.py, initial_data.py, then PyPIC3D --config charged_star.toml. +# Run initial_data.py, then PyPIC3D --config charged_star.toml. # Lapse deletion removes charge without correcting fields. # Geometrized Gaussian units: G=c=1, total Q=sum(q*weight). -# Star: prescribed ADM M=1, initial areal surface R=10. -# Keep the mesh and calibrated species values synchronized with both initializers. -# A 150M run saves 150 snapshots (roughly 14 GB); this is a demo resolution. +# Uniform coordinate sphere: rest mass=1, coordinate radius=10, total Q=0.2. +# Cartesian lattice spacing=0.25; ADM mass and areal radius are measured outputs. +# Keep the mesh and printed species values synchronized with initial_data.py. +# A run to t=150 saves 150 snapshots (roughly 14 GB); this is a demo resolution. # Constant Maxwell halos are not an isolated-radiation boundary condition. @@ -105,10 +106,10 @@ zr_bc = "sommerfeld" [particle1] name = "charged_dust" -N_particles = 262144 +N_particles = 268096 charge = 0.2 -mass = 0.99711270936837326 -weight = 4.0718187428556398e-06 +mass = 1.0 +weight = 3.7300071616137502e-06 x_bc = "absorbing" y_bc = "absorbing" z_bc = "absorbing" diff --git a/demos/dynamical_relativity/charged_stellar_collapse/initial_data.py b/demos/dynamical_relativity/charged_stellar_collapse/initial_data.py index 6ff5d29..03ae3a1 100644 --- a/demos/dynamical_relativity/charged_stellar_collapse/initial_data.py +++ b/demos/dynamical_relativity/charged_stellar_collapse/initial_data.py @@ -1,47 +1,209 @@ -"""Write quiet charged dust and common-time physical Yee D inputs. +"""Uniform lattice of charged dust with a Cartesian Hamiltonian solve. -Run after initial_metric.py. Initial_v files store covariant momenta u_i. +Edit the globals and run python initial_data.py. Keep the grid and the printed +particle count/mass/charge/weight synchronized with charged_star.toml. +Density is uniform per coordinate volume; proper density and ADM mass follow +from the solved geometry. Gaussian units have G=c=1. All inputs are at t=0; +native initialization seeds the leapfrog histories once. Initial_v* store u_i. """ + +import math from pathlib import Path -from itertools import product +import sys +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[3] +if __package__ in (None, ""): + sys.path.insert(0, str(ROOT)) + import jax import jax.numpy as jnp -from scipy.stats import qmc -from initial_metric import NX, HALF_WIDTH, OUTPUT_DIR -PARTICLE_COUNT = 262144 +if __package__ in (None, ""): + from initial_metric import BOUNDARY_TOLERANCE, metric_residual, solve_initial_metric +else: + from .initial_metric import BOUNDARY_TOLERANCE, metric_residual, solve_initial_metric + +from JAX_BSSN.evolution.spatial_derivatives import diff1_physical +from PyPIC3D.boundary_conditions.ghost_cells import update_tiled_ghost_cells +from PyPIC3D.deposition.rho import compute_rho +from PyPIC3D.diagnostics.static_metric import divergence +from PyPIC3D.particles.particle_class import SpeciesConfig, TiledParticles +from PyPIC3D.relativity.bssn_parameters import build_bssn_parameters +from PyPIC3D.relativity.dynamical_relativity import bssn_to_tiles, tiles_to_bssn, build_bssn_metric +from PyPIC3D.relativity.energy_momentum_tensor import ( + compute_particle_energy_momentum, compute_electromagnetic_energy_momentum, + compute_energy_momentum_tensor, electromagnetic_moments, +) +from PyPIC3D.utilities.grids import build_yee_grid, build_tiled_yee_grids +from PyPIC3D.utilities.parameters import build_static_parameters, build_dynamic_parameters + + +TOTAL_REST_MASS = 1.0 +SPHERE_RADIUS = 10.0 # Coordinate radius, not areal radius. +MASS = 1.0 +CHARGE_TO_MASS = 0.2 +PARTICLE_SPACING = 0.25 # Gives 268096 particles at R=10. +NX = 64 +HALF_WIDTH = 16.0 +DT = 0.05 +OUTPUT_DIR = Path(__file__).with_name("initial_data") + + +def build_parameters(nx, half_width, output_dir): + """One Cartesian tile with the demo's native boundary and gauge settings.""" + dx = 2 * half_width / nx + config = dict(Nx=nx, Ny=nx, Nz=nx, dx=dx, dy=dx, dz=dx, dt=DT, + x_min=-half_width, y_min=-half_width, z_min=-half_width, + x_wind=2*half_width, y_wind=2*half_width, z_wind=2*half_width, + C=1.0, eps=1.0, mu=1.0, kb=1.0) + static = build_static_parameters(dict( + **config, name="charged_stellar_collapse", output_dir=str(output_dir), Nt=3000, + solver="dynamic_metric", metric="numerical", relativistic=True, + particle_pusher="hybrid_boris_geodesic", current_deposition="GR_direct", + current_filter="none", shape_factor=2, guard_cells=3, tile_shape=(nx, nx, nx), + particle_batch_size=8192, boundary_conditions=(3, 3, 3), + particle_boundary_conditions=(2, 2, 2), metric_backreaction=True, + lapse_deleting=True, lapse_threshold=0.1)) + center, vertex = build_yee_grid(SimpleNamespace(**config)) + tiled_center, tiled_vertex = build_tiled_yee_grids( + static, SimpleNamespace(**config, grids=SimpleNamespace(center=center, vertex=vertex))) + config["grids"] = dict(center=center, vertex=vertex, tiled_center_grid=tiled_center, + tiled_vertex_grid=tiled_vertex) + dynamic = build_dynamic_parameters(config) + bssn = build_bssn_parameters(static, dynamic, evolution_parameters=dict( + gauge=1, zero_shift=0, eta=2.0, g=0.75, kappa=0.0, nu=0.002, + xl_bc=1, xr_bc=1, yl_bc=1, yr_bc=1, zl_bc=1, zr_bc=1)) + return static, dynamic._replace(bssn_parameters=bssn) + + +def load_dust(radius=SPHERE_RADIUS, spacing=PARTICLE_SPACING, + total_rest_mass=TOTAL_REST_MASS, mass=MASS, charge_to_mass=CHARGE_TO_MASS): + """Equal-weight, half-cell-centered Cartesian lattice clipped to a sphere.""" + half_count = math.ceil(radius / spacing) + axis = (jnp.arange(2 * half_count, dtype=jnp.float64) - half_count + 0.5) * spacing + positions = jnp.stack(jnp.meshgrid(axis, axis, axis, indexing="ij"), axis=-1).reshape(-1, 3) + positions = positions[jnp.sum(positions**2, axis=-1) < radius**2] + count = positions.shape[0] + x = positions.reshape(1, 1, 1, 1, count, 3) + particles = TiledParticles(x, jnp.zeros_like(x), jnp.ones(x.shape[:-1], dtype=bool)) + species = SpeciesConfig(jnp.asarray([mass * charge_to_mass]), jnp.asarray([mass]), + jnp.asarray([total_rest_mass / (mass * count)]), + jnp.ones((1, 3), dtype=bool)) + return particles, species + + +def sphere_displacement(coordinates, radius, charge): + """Analytic densitized D^i for uniform coordinate charge density.""" + r = jnp.sqrt(sum(x*x for x in coordinates)) + scale = charge / jnp.maximum(r, radius)**3 + return tuple(scale * x for x in coordinates) + + +def build_initial_data(*, nx=NX, half_width=HALF_WIDTH, radius=SPHERE_RADIUS, + particle_spacing=PARTICLE_SPACING, total_rest_mass=TOTAL_REST_MASS, + mass=MASS, charge_to_mass=CHARGE_TO_MASS, output_dir=".", + constraint_tolerance=None): + """Return particles, species, physical fields, static, dynamic, diagnostics.""" + jax.config.update("jax_enable_x64", True) + static, dynamic = build_parameters(nx, half_width, output_dir) + # Leave the deposited sphere clear of the outer constraint boundary. + if radius + 2 * float(dynamic.dx) >= half_width: + raise ValueError("The sphere and its deposition stencil must fit inside the metric boundary") + particles, species = load_dust(radius, particle_spacing, total_rest_mass, mass, charge_to_mass) + flat = dynamic.bssn_parameters.state + flat_metric = build_bssn_metric(flat, static, dynamic) + zero = tuple(jnp.zeros_like(component.lapse) for component in flat_metric.D) + charge = compute_rho(particles, species, zero[0], static, dynamic) + + # Evaluate each component at its own Yee face, then use the runtime halos. + axis = -half_width + jnp.arange(nx) * dynamic.dx + total_charge = total_rest_mass * charge_to_mass + displacement = [] + for component in range(3): + coordinates = jnp.meshgrid(*(axis + (0.5 * dynamic.dx if d == component else 0.0) + for d in range(3)), indexing="ij") + value = sphere_displacement(coordinates, radius, total_charge)[component] + displacement.append(bssn_to_tiles(value, static)) + displacement = update_tiled_ghost_cells(tuple(displacement), static, static.guard_cells) + moments = electromagnetic_moments(displacement, zero, static) + particle_energy = compute_particle_energy_momentum(particles, species, flat_metric, static, dynamic)[0] + electromagnetic_energy = compute_electromagnetic_energy_momentum(moments, flat)[0] + + state, history, boundary_history, tolerance = solve_initial_metric( + dynamic.bssn_parameters, particle_energy, electromagnetic_energy, + constraint_tolerance=constraint_tolerance) + dynamic = dynamic._replace(bssn_parameters=dynamic.bssn_parameters._replace(state=state)) + metric = build_bssn_metric(state, static, dynamic) + rho, momentum, _ = compute_energy_momentum_tensor(particles, species, moments, state, static, dynamic) + predicted = state.conformal_factor**3 * particle_energy + state.conformal_factor**4 * electromagnetic_energy + source_error = float(jnp.max(jnp.abs(rho - predicted))) + if not bool(jnp.all(jnp.isfinite(rho))) or source_error > 1.0e-12 * max(float(jnp.max(jnp.abs(rho))), 1.0e-30): + raise RuntimeError("Production energy density disagrees with the Newton source") + H, robin, boundary = metric_residual(jnp.log(state.conformal_factor), dynamic.bssn_parameters, + particle_energy, electromagnetic_energy) + h_norm = float(jnp.max(jnp.where(boundary, 0.0, jnp.abs(H)))) + boundary_error = float(jnp.max(jnp.where(boundary, jnp.abs(robin), 0.0))) + if h_norm > tolerance or boundary_error > BOUNDARY_TOLERANCE: + raise RuntimeError("Initial data failed the final Hamiltonian/Robin check") + gauss = tiles_to_bssn(divergence(displacement, dynamic) - 4*jnp.pi*charge, static) + + # ADM surface integral on the finite outer cube. Trapezoidal face weights + # count shared edges/corners correctly. Exterior EM energy is not included. + psi = state.conformal_factor**-0.5 + mass_flux = 0.0 + weights = jnp.ones(nx).at[0].set(0.5).at[-1].set(0.5) + for d in range(3): + derivative = diff1_physical(psi, d, dynamic.bssn_parameters.evolution_parameters) + flux = jnp.take(derivative, -1, axis=d) - jnp.take(derivative, 0, axis=d) + mass_flux = mass_flux + jnp.sum(flux * weights[:, None] * weights[None, :]) * dynamic.dx**2 + adm_mass = float(-mass_flux / (2*jnp.pi)) + # Six coordinate-axis surface samples give a simple spherical radius estimate. + middle = nx // 2 + surface_psi = [] + for d in range(3): + ray = jnp.moveaxis(psi, d, 0)[:, middle, middle] + surface_psi.extend((jnp.interp(radius, axis, ray), jnp.interp(-radius, axis, ray))) + diagnostics = dict(total_mass=adm_mass, total_charge=total_charge, total_rest_mass=total_rest_mass, + coordinate_radius=radius, surface_areal_radius=float(radius*jnp.mean(jnp.asarray(surface_psi)**2)), + hamiltonian_linf=h_norm, hamiltonian_boundary_linf=float(jnp.max(jnp.where(boundary, jnp.abs(H), 0.0))), + robin_linf=boundary_error, source_linf=source_error, + momentum_source_linf=float(jnp.max(jnp.abs(momentum))), + gauss_linf=float(jnp.max(jnp.abs(gauss))), + gauss_interior_linf=float(jnp.max(jnp.abs(gauss[1:-1, 1:-1, 1:-1]))), + newton_history=jnp.asarray(history), robin_history=jnp.asarray(boundary_history), + constraint_tolerance=tolerance) + physical_D = tuple(value / component.sqrt_gamma for value, component in zip(displacement, metric.D)) + fields = (physical_D, zero, zero, charge, zero[0], (zero, zero), metric, None, jnp.asarray(False)) + return particles, species, fields, static, dynamic, diagnostics + + +def save_initial_data(output_dir, particles, fields, static, dynamic, diagnostics): + """Write native inputs and the plotter's mass/charge metadata.""" + directory = Path(output_dir) + directory.mkdir(parents=True, exist_ok=True) + for component, axis in enumerate("xyz"): + jnp.save(directory / f"charged_dust_initial_{axis}.npy", particles.x[..., component].reshape(-1)) + jnp.save(directory / f"charged_dust_initial_v{axis}.npy", particles.u[..., component].reshape(-1)) + jnp.save(directory / f"physical_D_{axis}.npy", tiles_to_bssn(fields[0][component], static)) + for name in ("conformal_factor", "conformal_metric", "lapse"): + jnp.save(directory / f"{name}.npy", getattr(dynamic.bssn_parameters.state, name)) + # Keep the filename consumed by the existing RN plotter; no radial solve + # or radial profiles are needed. total_mass is a finite-cube ADM estimate. + jnp.savez(directory / "radial_profile.npz", **diagnostics) def main(): - jax.config.update('jax_enable_x64',True) - data = jnp.load(OUTPUT_DIR/'radial_profile.npz') - r,psi = data['r'],data['psi'] - count = PARTICLE_COUNT//8 - sample = jnp.asarray(qmc.Sobol(3,scramble=False).random_base2(count.bit_length()-1))+.5/count - R = jnp.interp(sample[:,0]*data['seed_mass'][-1],data['seed_mass'],data['seed_R']) - radius = jnp.interp(R,jnp.concatenate((jnp.zeros(1),r*psi**2)),jnp.concatenate((jnp.zeros(1),r))) - cosine = sample[:,1] - sine = jnp.sqrt(1-cosine**2) - phi = .5*jnp.pi*sample[:,2] - octant = radius[:,None]*jnp.stack((sine*jnp.cos(phi),sine*jnp.sin(phi),cosine),axis=-1) - signs = jnp.asarray(tuple(product((-1.,1.),repeat=3))) - positions = (signs[:,None,:]*octant[None,:,:]).reshape((-1,3)) - for i,axis in enumerate('xyz'): - jnp.save(OUTPUT_DIR/f'charged_dust_initial_{axis}.npy',positions[:,i]) - jnp.save(OUTPUT_DIR/f'charged_dust_initial_v{axis}.npy',jnp.zeros(PARTICLE_COUNT)) - dx = 2*HALF_WIDTH/NX - nodes = -HALF_WIDTH+jnp.arange(NX)*dx - W = jnp.load(OUTPUT_DIR/'conformal_factor.npy') - for i,axis in enumerate('xyz'): - coordinates = jnp.meshgrid(*(nodes+.5*dx if j==i else nodes for j in range(3)),indexing='ij') - radius = jnp.sqrt(sum(x*x for x in coordinates)) - density = jnp.interp(radius,r,psi**2*data['E_r'])*coordinates[i]/radius - # Native C-to-face interpolation, with constant outer metric halos. - upper = jnp.take(W,jnp.minimum(jnp.arange(NX)+1,NX-1),axis=i) - w_face = .5*(W+upper) - jnp.save(OUTPUT_DIR/f'physical_D_{axis}.npy',density*w_face**3) - print(f'Saved {PARTICLE_COUNT} particles; mass={float(data["mass_factor"]):.17g}, weight={float(data["seed_mass"][-1])/PARTICLE_COUNT:.17g}') - - -if __name__ == '__main__': + particles, species, fields, static, dynamic, diagnostics = build_initial_data() + save_initial_data(OUTPUT_DIR, particles, fields, static, dynamic, diagnostics) + print(f"Saved initial arrays in {OUTPUT_DIR}") + print(f"Particles={particles.active.size}, mass={float(species.mass[0]):.17g}, " + f"charge={float(species.charge[0]):.17g}, weight={float(species.weight[0]):.17g}") + for name, value in diagnostics.items(): + if "history" not in name: + print(f" {name}: {value:.8g}") + print("ADM mass is estimated on the finite outer cube; Gauss error includes sphere sampling and field boundaries.") + + +if __name__ == "__main__": main() diff --git a/demos/dynamical_relativity/charged_stellar_collapse/initial_metric.py b/demos/dynamical_relativity/charged_stellar_collapse/initial_metric.py index df64bed..06d9ab3 100644 --- a/demos/dynamical_relativity/charged_stellar_collapse/initial_metric.py +++ b/demos/dynamical_relativity/charged_stellar_collapse/initial_metric.py @@ -1,396 +1,122 @@ -"""Time-symmetric radial Hamiltonian initial data, ported from RadiShPICR. +"""Cartesian, time-symmetric Hamiltonian solve for fixed charged dust. -Gaussian G=c=1 units. Cartesian sampling is not a Cartesian constraint solve. -Run this before initial_data.py; copy the printed species mass into the TOML. +Called by initial_data.py after loading particles and the spherical electric +field. JAX_BSSN supplies the discrete Hamiltonian and its spatial derivatives. """ -from pathlib import Path -from types import SimpleNamespace + import math + import jax import jax.numpy as jnp -import numpy as np -from scipy import sparse -from scipy.sparse.linalg import spsolve - -TOTAL_MASS = 1.0 -SURFACE_AREAL_RADIUS = 10.0 -CHARGE_TO_MASS = 0.2 -NX = 64 -HALF_WIDTH = 16.0 -OUTPUT_DIR = Path(__file__).with_name("initial_data") - -def _radial_laplacian(r, dr): - """Match the Z4C derivative stencils, omitting the final boundary row.""" - - num_cells = r.size - rows = np.repeat(np.arange(num_cells - 2), 5) - columns = rows + np.tile(np.arange(-2, 3), num_cells - 2) - # Even parity on a half-cell-centered grid: -1 -> 0 and -2 -> 1. - columns = np.where(columns < 0, -columns - 1, columns) - d1 = np.asarray((1.0, -8.0, 0.0, 8.0, -1.0)) / (12.0 * dr) - d2 = np.asarray((-1.0, 16.0, -30.0, 16.0, -1.0)) / (12.0 * dr**2) - values = (d2 + 2.0 * d1 / r[:-2, None]).ravel() - - # The penultimate row uses the same one-sided derivatives as evolution. - d1_outer = np.asarray((0.0, -1/12, 1/2, -3/2, 5/6, 1/4)) / dr - d2_outer = np.asarray((1/12, -1/2, 7/6, -1/3, -5/4, 5/6)) / dr**2 - rows = np.concatenate((rows, np.full(6, num_cells - 2))) - columns = np.concatenate((columns, np.arange(num_cells - 6, num_cells))) - values = np.concatenate((values, d2_outer + 2.0 * d1_outer / r[-2])) - - return sparse.csr_matrix( - (values, (rows, columns)), shape=(num_cells - 1, num_cells), - ) +from jax.scipy.sparse.linalg import gmres -def _linear_shape_weight(delta): - """Evaluate the one-dimensional linear CIC shape function.""" +from JAX_BSSN.bssn.constraints import compute_hamiltonian_constraint_with_matter +from JAX_BSSN.evolution.spatial_derivatives import diff1_physical - return jnp.maximum(1.0 - jnp.abs(delta), 0.0) -def _quadratic_shape_weight(delta): - """Evaluate the one-dimensional quadratic TSC shape function.""" +MAX_NEWTON_ITERATIONS = 20 +BOUNDARY_TOLERANCE = 1.0e-10 - absolute_delta = jnp.abs(delta) - center_weight = 0.75 - delta**2 - outer_weight = 0.5 * (1.5 - absolute_delta) ** 2 - return jnp.where( - absolute_delta <= 0.5, - center_weight, - jnp.where(absolute_delta <= 1.5, outer_weight, 0.0), - ) +def metric_residual(log_W, bssn, particle_energy, electromagnetic_energy): + """Return Hamiltonian, asymptotic Robin residual, and outer-node mask. -def _unbounded_raw_radial_shape_stencil( - radial_positions, - radial_grid, - dr, - shape_mode="nearest", -): - """Return raw compact radial indices and weights before boundary handling.""" - - floating_index = (radial_positions - radial_grid[0]) / dr - - if shape_mode == "nearest": - anchor = jnp.floor(floating_index + 0.5).astype(jnp.int32) - raw_indices = anchor[jnp.newaxis, :] - # The anchor already makes a deterministic right-sided choice at an - # exact half-cell tie, so every particle carries one full NGP weight. - raw_weights = jnp.ones_like(raw_indices, dtype=radial_positions.dtype) - elif shape_mode == "linear": - anchor = jnp.floor(floating_index).astype(jnp.int32) - offsets = jnp.asarray([0, 1], dtype=anchor.dtype) - raw_indices = anchor[jnp.newaxis, :] + offsets[:, jnp.newaxis] - delta = ( - floating_index[jnp.newaxis, :] - - raw_indices.astype(radial_positions.dtype) - ) - raw_weights = _linear_shape_weight(delta) - else: - anchor = jnp.rint(floating_index).astype(jnp.int32) - offsets = jnp.asarray([-1, 0, 1], dtype=anchor.dtype) - raw_indices = anchor[jnp.newaxis, :] + offsets[:, jnp.newaxis] - delta = ( - floating_index[jnp.newaxis, :] - - raw_indices.astype(radial_positions.dtype) - ) - raw_weights = _quadratic_shape_weight(delta) - - return raw_indices, raw_weights - -def apply_stencil_boundaries( - raw_indices, raw_weights, num_cells, inner_open=False, inner_boundary_index=0, -): - """Fold at the half-cell origin or discard open-boundary ghost shares. - - No retained weights are renormalized. Odd radial quantities change sign - under origin reflection; open-inner weights carry their original sign. + Fixed particles at rest have rho_dust = W^3 rho_dust_flat. Fixed + densitized electric fields have rho_EM = W^4 rho_EM_flat. The asymptotic + condition x.grad(psi) + psi - 1 = 0 permits an unknown ADM mass; at a + finite outer boundary it neglects higher powers of 1/r, including charge. """ - - reflected_indices = jnp.where(raw_indices < 0, -raw_indices - 1, raw_indices) - grid_indices = jnp.where(inner_open, raw_indices, reflected_indices) - first_index = jnp.where(inner_open, inner_boundary_index, 0) - physical = (grid_indices >= first_index) & (grid_indices < num_cells) - - indices = jnp.clip(grid_indices, 0, num_cells - 1) - even_weights = jnp.where(physical, raw_weights, 0.0) - reflection_sign = jnp.where(raw_indices < 0, -1.0, 1.0) - odd_weights = even_weights * jnp.where(inner_open, 1.0, reflection_sign) - - return indices, even_weights, odd_weights - -def proper_radial_shell_volume(metric): - inner_radius = jnp.maximum(metric.r - 0.5 * metric.dr, 0.0) - outer_radius = metric.r + 0.5 * metric.dr - coordinate_volume = (4.0 * jnp.pi / 3.0) * ( - outer_radius**3 - inner_radius**3 - ) - - proper_volume_factor = ( - jnp.sqrt(metric.conformal_grr) - * metric.conformal_gt - / metric.chi**1.5 - ) - - return coordinate_volume * proper_volume_factor - -def proper_radial_shell_quadrature(metric): - """Return proper-volume quadrature on both halves of every shell.""" - - gauss_abscissae = jnp.asarray( - (-jnp.sqrt(3.0 / 5.0), 0.0, jnp.sqrt(3.0 / 5.0)), - dtype=metric.r.dtype, - ) - gauss_weights = jnp.asarray( - (5.0 / 9.0, 8.0 / 9.0, 5.0 / 9.0), - dtype=metric.r.dtype, - ) - - inner_radius = jnp.maximum(metric.r - 0.5 * metric.dr, 0.0) - outer_radius = metric.r + 0.5 * metric.dr - - # The Ruyten transfer changes intervals at a grid center. Integrate the - # two polynomial pieces separately with the same cell-centered metric - # factor used by the production finite-volume shell. - segment_inner = jnp.stack((inner_radius, metric.r), axis=1) - segment_outer = jnp.stack((metric.r, outer_radius), axis=1) - segment_center = 0.5 * (segment_inner + segment_outer) - segment_half_width = 0.5 * (segment_outer - segment_inner) - - quadrature_radius = ( - segment_center[:, :, jnp.newaxis] - + segment_half_width[:, :, jnp.newaxis] - * gauss_abscissae[jnp.newaxis, jnp.newaxis, :] - ) - proper_volume_factor = ( - jnp.sqrt(metric.conformal_grr) - * metric.conformal_gt - / metric.chi**1.5 - ) - quadrature_volume = ( - 4.0 - * jnp.pi - * proper_volume_factor[:, jnp.newaxis, jnp.newaxis] - * quadrature_radius**2 - * segment_half_width[:, :, jnp.newaxis] - * gauss_weights[jnp.newaxis, jnp.newaxis, :] - ) - - return quadrature_radius.reshape(-1), quadrature_volume.reshape(-1) - -def metric_correction_coefficients(metric, inner_open, shape_mode="quadratic"): - """Match deposited proper volume with a limited Ruyten transfer. - - CIC uses |beta| < 1; quadratic TSC uses |beta| < 3/2. Limiting keeps - weights nonnegative but can leave a shell-volume residual. The final - shell is excluded from the recurrence, preserving outer truncation. - """ - - quadrature_radius, quadrature_volume = proper_radial_shell_quadrature( - metric - ) - raw_indices, raw_weights = _unbounded_raw_radial_shape_stencil( - quadrature_radius, - metric.r, - metric.dr, - shape_mode=shape_mode, - ) - - # Calibrate on the full metric grid. The moving absorbing boundary is - # applied only to the final particle stencil, not to this recurrence. - indices, retained_weights, _ = apply_stencil_boundaries( - raw_indices, raw_weights, metric.r.size, inner_open, - ) - uncorrected_shell_volume = jnp.zeros_like(metric.r).at[indices].add( - retained_weights * quadrature_volume[jnp.newaxis, :] - ) - - floating_index = (quadrature_radius - metric.r[0]) / metric.dr - lower_index = jnp.floor(floating_index).astype(jnp.int32) - upper_fraction = floating_index - lower_index.astype(metric.r.dtype) - valid_interval = jnp.logical_and( - lower_index >= 0, - lower_index < metric.r.shape[0] - 1, - ) - interval_index = jnp.clip(lower_index, 0, metric.r.shape[0] - 2) - interval_moment = jnp.zeros_like(metric.r[:-1]).at[interval_index].add( - jnp.where( - valid_interval, - quadrature_volume * upper_fraction * (1.0 - upper_fraction), - 0.0, - ) - ) - - shell_volume_error = ( - proper_radial_shell_volume(metric)[:-1] - - uncorrected_shell_volume[:-1] - ) - beta_limit = (1.0 if shape_mode == "linear" else 1.5) * ( - 1.0 - 8.0 * jnp.finfo(metric.r.dtype).eps - ) - - def solve_one_interval(previous_transfer, interval_data): - volume_error, volume_moment = interval_data - unconstrained_beta = ( - volume_error + previous_transfer - ) / volume_moment - beta = jnp.clip( - unconstrained_beta, - -beta_limit, - beta_limit, - ) - transfer = beta * volume_moment - return transfer, beta - - _, beta = jax.lax.scan( - solve_one_interval, - jnp.asarray(0.0, dtype=metric.r.dtype), - (shell_volume_error, interval_moment), - ) - - return beta - -def apply_metric_transfer(radial_positions, metric, beta, raw_indices, raw_weights): - """Transfer beta*f*(1-f) from the upper center to the lower center. - - The equal and opposite changes preserve each raw stencil's total weight. - Mirror beta with odd parity on negative-radius intervals; leave the - interval straddling the origin and exterior intervals uncorrected. - """ - - floating_index = (radial_positions - metric.r[0]) / metric.dr - lower_index = jnp.floor(floating_index).astype(jnp.int32) - upper_fraction = floating_index - lower_index.astype(metric.r.dtype) - - # Reflection maps interval i to -i-2 on a half-cell-centered grid. - # The origin interval maps to -1 and has no correction coefficient. - beta_index = jnp.where(lower_index >= 0, lower_index, -lower_index - 2) - valid_interval = (beta_index >= 0) & (beta_index < beta.size) - reflection_sign = jnp.where(lower_index >= 0, 1.0, -1.0) - interval_beta = jnp.where( - valid_interval, - reflection_sign * beta[jnp.clip(beta_index, 0, beta.size - 1)], - 0.0, - ) - transfer = interval_beta * upper_fraction * (1.0 - upper_fraction) - - corrected_weights = raw_weights + jnp.where( - raw_indices == lower_index[jnp.newaxis, :], - transfer[jnp.newaxis, :], - 0.0, - ) - corrected_weights = corrected_weights - jnp.where( - raw_indices == lower_index[jnp.newaxis, :] + 1, - transfer[jnp.newaxis, :], - 0.0, - ) - - return corrected_weights - -def seed_profile(count=120000): - R = jnp.linspace(0., SURFACE_AREAL_RADIUS, count) - density = 3*TOTAL_MASS/(4*jnp.pi*SURFACE_AREAL_RADIUS**3) - integrand = 4*jnp.pi*R**2*density/jnp.sqrt( - 1-2*TOTAL_MASS*R**2/SURFACE_AREAL_RADIUS**3) - cumulative = jnp.concatenate((jnp.zeros(1), jnp.cumsum( - .5*(integrand[1:]+integrand[:-1])*jnp.diff(R)))) - return R, cumulative - - -def solve_initial_metric(num_cells=4000, radial_limit=100., tolerance=1.e-12, - max_iterations=100): - """Return radial coordinates, psi, covariant electric field, mass factor. - - Shell-stratified quadrature retains fixed areal radii and charges while - the completed map and a uniform dust mass multiplier are solved together. + W = jnp.exp(log_W) + state = bssn.state._replace(conformal_factor=W) + params = bssn.evolution_parameters + rho = W**3 * particle_energy + W**4 * electromagnetic_energy + hamiltonian = compute_hamiltonian_constraint_with_matter(state, params, rho) + + psi = W**-0.5 + robin = psi - 1.0 + boundary = jnp.zeros_like(W, dtype=bool) + for axis, origin in enumerate((params.x_min, params.y_min, params.z_min)): + shape = [1, 1, 1] + shape[axis] = W.shape[axis] + index = jnp.arange(W.shape[axis]).reshape(shape) + coordinate = origin + index * params.dx + robin = robin + coordinate * diff1_physical(psi, axis, params) + boundary = boundary | (index == 0) | (index == W.shape[axis] - 1) + return hamiltonian, robin, boundary + + +def solve_initial_metric(bssn, particle_energy, electromagnetic_energy, + *, constraint_tolerance=None, + max_newton_iterations=MAX_NEWTON_ITERATIONS): + """Return solved BSSN state, H/Robin histories, and Hamiltonian tolerance. + + Newton corrections use autodiff Jacobian-vector products, keeping memory + proportional to the grid rather than forming a dense 3D Jacobian. """ - dr = radial_limit/num_cells - r = (jnp.arange(num_cells)+.5)*dr - R_seed, cumulative = seed_profile() - rest_mass = float(cumulative[-1]) - surface = .5*(SURFACE_AREAL_RADIUS-TOTAL_MASS+math.sqrt( - SURFACE_AREAL_RADIUS*(SURFACE_AREAL_RADIUS-2*TOTAL_MASS))) - root = jnp.sqrt(1-2*TOTAL_MASS*R_seed**2/SURFACE_AREAL_RADIUS**3) - seed_r = surface*(R_seed/(1+root))/(SURFACE_AREAL_RADIUS/( - 1+math.sqrt(1-2*TOTAL_MASS/SURFACE_AREAL_RADIUS))) - edges = jnp.minimum(jnp.arange(math.ceil(surface/dr)+1)*dr,surface) - edge_mass = jnp.interp(jnp.interp(edges,seed_r,R_seed),R_seed,cumulative) - counts = np.full(len(edges)-1,90000//(len(edges)-1),dtype=int) - counts[:90000 % len(counts)] += 1 - mass_samples = jnp.concatenate([a+(jnp.arange(n)+.5)/n*(b-a) - for a,b,n in zip(edge_mass[:-1],edge_mass[1:],counts)]) - weights = jnp.concatenate([jnp.full(n,(b-a)/n) - for a,b,n in zip(edge_mass[:-1],edge_mass[1:],counts)]) - particle_R = jnp.interp(mass_samples,cumulative,R_seed) - charge = CHARGE_TO_MASS*rest_mass - rn = jnp.sqrt((1+TOTAL_MASS/(2*r[-2:]))**2-charge**2/(4*r[-2:]**2)) - B = TOTAL_MASS/(2*surface**3) - psi = jnp.where(r<=surface,(1+TOTAL_MASS/(2*surface))**1.5/ - jnp.sqrt(1+B*r*r),1+TOTAL_MASS/(2*r)) + dx = float(bssn.evolution_parameters.dx) + length = dx * max(bssn.state.conformal_factor.shape) + source_scale = max(float(16 * jnp.pi * jnp.max( + jnp.abs(particle_energy) + jnp.abs(electromagnetic_energy))), 1.0 / length**2) + tolerance = (max(1.0e-12 / length**2, 1.0e-8 * source_scale) + if constraint_tolerance is None else float(constraint_tolerance)) + + def residual(unknown): + hamiltonian, robin, boundary = metric_residual( + unknown, bssn, particle_energy, electromagnetic_energy) + return jnp.where(boundary, robin, hamiltonian / source_scale) + + evaluate = jax.jit(residual) + unknown = jnp.log(bssn.state.conformal_factor) + _, _, boundary = metric_residual(unknown, bssn, particle_energy, electromagnetic_energy) @jax.jit - def sources(psi): - metric = SimpleNamespace(r=r,dr=dr,chi=psi**-4, - conformal_grr=jnp.ones_like(r),conformal_gt=jnp.ones_like(r)) - particle_r = jnp.interp(particle_R,jnp.concatenate((jnp.zeros(1),r*psi**2)), - jnp.concatenate((jnp.zeros(1),r))) - indices, shape = _unbounded_raw_radial_shape_stencil(particle_r,r,dr,'quadratic') - beta = metric_correction_coefficients(metric,True) - shape = apply_metric_transfer(particle_r,metric,beta,indices,shape) - indices, shape, _ = apply_stencil_boundaries(indices,shape,num_cells,True) - volume = proper_radial_shell_volume(metric) - shell_mass = jnp.zeros_like(r).at[indices].add(shape*weights[None,:]) - rho = shell_mass/volume - flux = jnp.concatenate((jnp.zeros(1),jnp.cumsum(CHARGE_TO_MASS*shell_mass))) - chi_faces = jnp.concatenate((metric.chi[:1],.5*(metric.chi[:-1]+metric.chi[1:]),metric.chi[-1:])) - faces = jnp.arange(num_cells+1)*dr - E_faces = jnp.where(faces>0,flux*jnp.sqrt(chi_faces)/jnp.maximum(faces,dr)**2,0.) - E_r = .5*(E_faces[:-1]+E_faces[1:]) - return rho, psi**-4*E_r**2/(8*jnp.pi), E_r - - laplacian = _radial_laplacian(np.asarray(r),dr) - exterior = sparse.csr_matrix((np.ones(2),(np.arange(2),np.arange(num_cells-2,num_cells))),shape=(2,num_cells)) - factor = 1. - for iteration in range(max_iterations): - if not bool(jnp.all(jnp.diff(r*psi**2)>0)) or not bool(jnp.all(psi>0)): - raise RuntimeError('Hamiltonian iteration lost the monotone positive radial map') - rho, rho_em, _ = sources(psi) - dust = np.asarray(2*jnp.pi*psi**5*rho) - electric = np.asarray(2*jnp.pi*psi**5*rho_em) - operator = sparse.bmat([[laplacian,sparse.csr_matrix(dust[:-1,None])], - [exterior,sparse.csr_matrix((2,1))]],format='csr') - solution = spsolve(operator,np.concatenate((-electric[:-1],np.asarray(rn)))) - change = max(float(jnp.max(jnp.abs(solution[:-1]-psi))),abs(solution[-1]-factor)) - psi = .5*(psi+solution[:-1]) - factor = .5*(factor+solution[-1]) - if change 1.0e-6 * float(jnp.linalg.norm(value)): + raise RuntimeError(f"Initial metric GMRES failed: residual={float(linear_error):.3e}") + merit = max(h_norm / tolerance, boundary_error / BOUNDARY_TOLERANCE) + for backtrack in range(20): + trial = unknown + 0.5**backtrack * correction + trial_value = evaluate(trial) + trial_merit = max( + float(jnp.max(jnp.where(boundary, 0.0, jnp.abs(trial_value)))) * source_scale / tolerance, + float(jnp.max(jnp.where(boundary, jnp.abs(trial_value), 0.0))) / BOUNDARY_TOLERANCE) + if bool(jnp.all(jnp.isfinite(trial_value))) and math.isfinite(trial_merit) and trial_merit < merit: + unknown = trial + break + else: + raise RuntimeError(f"Initial metric Newton step failed: H_linf={h_norm:.3e}, " + f"Robin_linf={boundary_error:.3e}") + raise RuntimeError(f"Initial metric did not converge: H_linf={h_norm:.3e}, " + f"Robin_linf={boundary_error:.3e}") diff --git a/demos/dynamical_relativity/charged_stellar_collapse/plot_analytical_comparison.py b/demos/dynamical_relativity/charged_stellar_collapse/plot_analytical_comparison.py index 20176f6..ddbb4f8 100644 --- a/demos/dynamical_relativity/charged_stellar_collapse/plot_analytical_comparison.py +++ b/demos/dynamical_relativity/charged_stellar_collapse/plot_analytical_comparison.py @@ -1,4 +1,4 @@ -"""Compare the final charged OS radial and time metrics with analytical RN. +"""Compare the final charged-dust radial and time metrics with analytical RN. Use the stationary reconstruction g_rr = gamma_rr + beta_r**2 / (alpha**2 - beta_r * beta^r), @@ -9,8 +9,10 @@ coordinate transformation of a still-evolving spacetime. Run with no arguments. Paths and plot limits are set below. Only the final -iteration is read, and M and Q are the initial stellar values, without fits -or boundary rescaling. PyPIC3D uses Gaussian G = c = 1 units, with RN charge term Q**2/R**2. +iteration is read, and M and Q come from the initial-data metadata, without +fits or boundary rescaling. For the lattice initializer M is an ADM surface +estimate on the finite outer cube, not an exactly prescribed asymptotic mass. +PyPIC3D uses Gaussian G = c = 1 units, with RN charge term Q**2/R**2. """ from pathlib import Path diff --git a/demos/dynamical_relativity/neutral_os_collapse/initial_metric.py b/demos/dynamical_relativity/neutral_os_collapse/initial_metric.py index 8e88508..62a611a 100644 --- a/demos/dynamical_relativity/neutral_os_collapse/initial_metric.py +++ b/demos/dynamical_relativity/neutral_os_collapse/initial_metric.py @@ -1,18 +1,12 @@ -"""Analytic time-symmetric Oppenheimer-Snyder geometry on global C nodes. - -Writes the three native ``[metric]`` arrays that os_collapse.toml loads. The -prescribed ADM mass fixes the continuum geometry; no discrete constraint solve -adjusts it to the deposited particle density. The stored conformal factor is -**W = psi**-2**, not chi, and the conformal metric is exactly the identity, so -the slice is momentarily static (A_ij = K = Gamma^i = 0, alpha = 1, beta^i = 0). +""" +Christopher Woolford November 2026 -Those four zero variables stay literal ``false`` in the TOML, which creates -zeros. ``conformal_metric`` and ``lapse`` cannot be omitted the same way: -omission would create zeros and fail the positive-definite and W > 0 checks in -PyPIC3D.relativity.bssn_parameters. +Lapse and conformal metric are minkowski, the conformal factor is the +analytically defined OS solution given by: -Edit the globals below and run ``python initial_metric.py``. Keep them -consistent with os_collapse.toml; nothing synchronizes the two automatically. +Wu, J., & Most, E. R. (2026). Numerical-Relativity Particle-in-Cell Simulations: +Self-Gravitating Collisionless Matter in Dynamical Spacetimes. arXiv preprint +arXiv:2611.12831. """ from pathlib import Path @@ -76,30 +70,25 @@ def build_initial_metric(*, nx=NX, half_width=HALF_WIDTH, total_mass=TOTAL_MASS, x, y, z = jnp.meshgrid(axis, axis, axis, indexing="ij") radius = jnp.sqrt(x**2 + y**2 + z**2) - one = jnp.ones((nx, nx, nx), dtype=jnp.float64) - conformal_factor = conformal_factor_psi(radius, total_mass, surface_areal_radius)**-2 - conformal_metric = jnp.eye(3, dtype=jnp.float64)[:, :, None, None, None] * one - return dict(conformal_metric=conformal_metric, - conformal_factor=jnp.broadcast_to(conformal_factor, one.shape), - lapse=one) - - -def save_initial_metric(output_dir, arrays): - """Write the native .npy inputs, replacing files from an earlier run.""" - directory = Path(output_dir) - directory.mkdir(parents=True, exist_ok=True) - for name, value in arrays.items(): - jnp.save(directory / f"{name}.npy", value) + lapse = jnp.ones((nx, nx, nx), dtype=jnp.float64) + conformal_factor = 1.0 / ( conformal_factor_psi(radius, total_mass, surface_areal_radius) )**2 + conformal_metric = jnp.eye(3, dtype=jnp.float64)[:, :, None, None, None] * lapse + return lapse, conformal_metric, conformal_factor def main(): jax.config.update("jax_enable_x64", True) - arrays = build_initial_metric() - save_initial_metric(OUTPUT_DIR, arrays) - factor = arrays["conformal_factor"] + lapse, conformal_metric, conformal_factor = build_initial_metric() + # define the initial metric arrays + + jnp.save(OUTPUT_DIR / "lapse.npy", lapse) + jnp.save(OUTPUT_DIR / "conformal_metric.npy", conformal_metric) + jnp.save(OUTPUT_DIR / "conformal_factor.npy", conformal_factor) + # save the arrays to .npy files in the output directory + print(f"Saved initial metric in {Path(OUTPUT_DIR).resolve()}") print(f" surface isotropic radius {surface_isotropic_radius():.16g}") - print(f" W in [{float(jnp.min(factor)):.16g}, {float(jnp.max(factor)):.16g}]") + print(f" W in [{float(jnp.min(conformal_factor)):.16g}, {float(jnp.max(conformal_factor)):.16g}]") if __name__ == "__main__": diff --git a/demos/dynamical_relativity/neutral_os_collapse/os_collapse.toml b/demos/dynamical_relativity/neutral_os_collapse/os_collapse.toml index 7e06c10..6430051 100644 --- a/demos/dynamical_relativity/neutral_os_collapse/os_collapse.toml +++ b/demos/dynamical_relativity/neutral_os_collapse/os_collapse.toml @@ -32,8 +32,8 @@ z_wind = 16.0 dt = 0.025 t_wind = 60.0 # Nt = 2400 macro steps to t = 60 M. -# One tile: a single-process run. Reduce these to the per-tile width and launch -# under MPI to engage the sharded openPMD writer; see README.md. +# One tile: a single-process run. Reduce these to the per-tile width to engage +# the sharded openPMD writer; see docs/distributed_cartesian.md. particle_tile_nx = 64 particle_tile_ny = 64 particle_tile_nz = 64 diff --git a/demos/static_metric_relativity/meisser_effect/wald_solution.py b/demos/static_metric_relativity/meisser_effect/wald_solution.py index f9a26d5..bd100ec 100644 --- a/demos/static_metric_relativity/meisser_effect/wald_solution.py +++ b/demos/static_metric_relativity/meisser_effect/wald_solution.py @@ -112,7 +112,6 @@ def build_pypic_parameters(config): output_dir=config["output"]["directory"], Nt=total_steps, verbose=False, - benchmark=False, solver="static_metric", electrostatic=False, relativistic=True, diff --git a/docs/architecture.rst b/docs/architecture.rst index 7688629..fc4f9ea 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -73,13 +73,20 @@ Execution Flow 3. ``run_PyPIC3D`` closes over the static parameters, JIT-compiles the selected timestep, advances the simulation, checks particle-retile overflow, and schedules diagnostics. -4. Asynchronous openPMD writers consume tile-local snapshots without changing - the live solver state. +4. ``diagnostics.simulation_output`` manages writer lifetime, scalar diagnostics, + and tile-local snapshots. Every process writes its own series through + asynchronous writers, without changing solver state. Core Module Map --------------- -- ``PyPIC3D/__main__.py``: CLI, run loop, writer scheduling, and diagnostics. +- ``PyPIC3D/__main__.py``: CLI, run loop, and diagnostic scheduling. +- ``PyPIC3D/diagnostics/simulation_output.py``: scalar output, snapshots, and + writer setup and cleanup. +- ``PyPIC3D/utilities/distributed.py``: JAX distributed startup and + addressable-only field initialization. +- ``PyPIC3D/diagnostics/merge_openpmd.py``: joins the per-process openPMD series + a multiprocess run writes into one file. - ``PyPIC3D/initialization.py``: configuration defaults, grid/device setup, particle and field initialization, and solver selection. - ``PyPIC3D/parameters.py``: ``StaticParameters``, ``DynamicParameters``, and diff --git a/docs/distributed_cartesian.md b/docs/distributed_cartesian.md index f5d3cb0..2aa3973 100644 --- a/docs/distributed_cartesian.md +++ b/docs/distributed_cartesian.md @@ -9,30 +9,33 @@ Multiprocess simulation initialization currently supports `dynamic_metric` only. The same output routines also handle single-process runs on one or several devices; their use does not add multiprocess support to other solvers. +PyPIC3D processes never exchange anything during output: +each one writes its own openPMD series, and +[`PyPIC3D.diagnostics.merge_openpmd`](#assembling-a-multiprocess-run) joins them +afterwards. + ## Runtime and launch -Use the same Python/JAX versions on every node, double precision, and an -MPI-enabled openPMD installation with parallel HDF5. `mpi4py`, openPMD and HDF5 -must link to the same MPI implementation. The `distributed` optional dependency -installs mpi4py; the ordinary openPMD PyPI wheel can be serial and is not sufficient. -Check `openpmd_api.variants['mpi']` before allocating a production job. Disable a -user site that would shadow the intended cluster packages. +Use the same Python/JAX versions on every node and double precision. The +ordinary openPMD PyPI wheel is all that is needed. Install both updated checkouts into that same execution environment (the tested JAX/JAXlib version is 0.9.2): ```bash python -m pip install -e /path/to/updated/JAX_BSSN -python -m pip install -e '/path/to/updated/PyPIC3D[distributed]' +python -m pip install -e /path/to/updated/PyPIC3D ``` -Replace these example paths with the actual checkouts containing the parallel -implementation. PyPIC3D's existing `relativity` extra points to the upstream -GitHub package, which can predate these changes; installing that extra alone -does not ensure that `JAX_BSSN.parallel` is available. Select the MPI-enabled -openPMD build after dependency installation so a serial wheel does not shadow it. +Replace these example paths with the actual checkouts. PyPIC3D's existing +`relativity` extra points to the upstream GitHub package, which can predate +these changes; installing that extra alone does not ensure that +`JAX_BSSN.parallel` is available. -An allocation on Perlmutter with two GPU nodes can launch four ranks per node: +Set `PYPIC_DISTRIBUTED=1` before importing PyPIC3D: package startup initializes +JAX's distributed runtime before arrays are constructed. JAX detects Slurm and +Open MPI launches on its own, so an allocation on Perlmutter with two GPU nodes +needs no coordinator settings: ```bash export PYPIC_DISTRIBUTED=1 @@ -45,56 +48,24 @@ srun --nodes=2 --ntasks-per-node=4 --gpus-per-task=1 --gpu-bind=single:1 \ The scheduler must expose one selected GPU to each task for this command's `PYPIC_LOCAL_DEVICE_IDS=0`. If all local GPUs are visible, omit that variable and -JAX selects the MPI local-rank device. Set `PYPIC_DISTRIBUTED` before importing -PyPIC3D: package startup initializes MPI/JAX before arrays are constructed. -The default initialization timeout is 120 seconds; the environment variable -`PYPIC_INITIALIZATION_TIMEOUT` overrides it. Run all ranks through the same -simulation and output schedule. MPI output is synchronous on the main thread; -single-process output uses the same chunk writers through bounded asynchronous -queues. No background writer thread enters MPI. - -### Building an MPI-enabled openPMD runtime - -Use the cluster's MPI wrappers and parallel HDF5 installation, with mpi4py -compiled against that same MPI. For openPMD 0.17.1, an isolated prefix can be -built as follows (set `HDF5_ROOT` to the parallel installation first): - -```bash -curl -L https://github.com/openPMD/openPMD-api/archive/refs/tags/0.17.1.tar.gz \ - -o /tmp/openpmd-api-0.17.1.tar.gz -tar -xf /tmp/openpmd-api-0.17.1.tar.gz -C /tmp -cmake -S /tmp/openPMD-api-0.17.1 -B /tmp/openpmd-mpi-build \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/tmp/openpmd-mpi-install \ - -DopenPMD_USE_MPI=ON -DopenPMD_USE_HDF5=ON -DopenPMD_USE_ADIOS2=OFF \ - -DopenPMD_USE_PYTHON=ON -DopenPMD_BUILD_TESTING=OFF \ - -DopenPMD_BUILD_EXAMPLES=OFF -DopenPMD_BUILD_CLI_TOOLS=OFF \ - -DPython_EXECUTABLE="$(command -v python)" -DHDF5_ROOT="$HDF5_ROOT" \ - -DMPI_C_COMPILER="$(command -v mpicc)" -DMPI_CXX_COMPILER="$(command -v mpic++)" -cmake --build /tmp/openpmd-mpi-build --parallel 2 -cmake --install /tmp/openpmd-mpi-build -``` +JAX selects the local-rank device. -Put that prefix's `lib/python/site-packages` first on `PYTHONPATH` and -its `lib` first on `LD_LIBRARY_PATH`; include the MPI/HDF5 library directory as -required by the selected runtime. On a cluster, use an equivalent shared prefix -accessible from every node. Verify the selected module rather than relying on -the package name: +Where JAX cannot detect the launcher, describe it explicitly. Every process needs +the same `PYPIC_COORDINATOR_ADDRESS` and `PYPIC_NUM_PROCESSES`, and its own +`PYPIC_PROCESS_ID`: ```bash -python -c 'import openpmd_api as io; print(io.__file__, io.variants); assert io.variants["mpi"]' +PYPIC_DISTRIBUTED=1 PYPIC_COORDINATOR_ADDRESS=10.0.0.1:12345 \ + PYPIC_NUM_PROCESSES=2 PYPIC_PROCESS_ID=0 JAX_ENABLE_X64=true \ + python -m PyPIC3D --config coupled.toml ``` -The historical local two-process validation described below used Python 3.14, JAX/JAXlib 0.9.2, -OpenMPI 5.0.10, parallel HDF5 1.14.6, and the above openPMD 0.17.1 build. -The build/install directories were `/tmp/openpmd-mpi-build` and -`/tmp/openpmd-mpi-install`; `/tmp/pypic-mpi-overlay` supplied the existing -MPI-matched mpi4py package. Those temporary paths are validation artifacts. -The ordinary installed openPMD package remained serial (`mpi=False`); a -production environment must select its own MPI-enabled build explicitly. +The default initialization timeout is 120 seconds; `PYPIC_INITIALIZATION_TIMEOUT` +overrides it. Run all processes through the same simulation and output schedule. ## Native inputs and local callbacks -Existing field `.npy` files keep shape `(Nx, Ny, Nz)` without guards. Each rank +Existing field `.npy` files keep shape `(Nx, Ny, Nz)` without guards. Each process memory-maps these files and reads only its addressable tile interiors. The ordinary boundary preparation supplies physical wall values and tile halos. Metric files keep the existing global C-node shape, including owned upper @@ -110,37 +81,43 @@ same physical vectors as native field inputs, before GR densitization/seeding. Distributed particles currently require all six native one-dimensional files: `initial_x`, `initial_y`, `initial_z`, `initial_vx`, `initial_vy`, `initial_vz`. -Their lengths must equal `N_particles`. Bounded blocks are read once by assigned -input ranks, routed to the owning spatial tiles, and ordered by original file -index. Capacity and weights retain the serial convention. Built-in random -particle generation is explicitly unsupported for multiprocess initialization; +Their lengths must equal `N_particles`. Every process reads every bounded block +in file order and keeps only the particles whose spatial tile it owns, so tile +occupancy, capacity and the derived temperature come out identical everywhere +without an exchange. The cost is that each process reads the whole file. +Capacity and weights retain the serial convention. Built-in random particle +generation is explicitly unsupported for multiprocess initialization; pre-generate the same native files when a random realization is required. Empty particle storage is supported. ## Sharded field and particle output `plot_openpmd_fields` writes `data/fields.h5`, and `plot_openpmd_particles` writes -`data/particles.h5`. These paths and record names are the same for one process -and for MPI runs. Each rank copies only its addressable device shards to host -buffers, then stores their chunks at the corresponding global offsets. It does -not assemble global field or particle arrays on a host or device. Single-process -queues retain at most the configured number of pending local snapshots; -`openpmd_field_queue_size` and `openpmd_particle_queue_size` do not limit file -counts and are not used by synchronous MPI output. - -Every mesh component has shape `(Nx, Ny, Nz)`, including singleton axes and -mixed periodic/conducting boundaries. Writers strip guards and retain the -existing physical-interior output selection and component-position metadata. -The writer does not change component extents or add upper endpoint planes. -Read the resulting files with the ordinary +`data/particles.h5`. A single-process run writes exactly those names, as before. +With N processes, each one writes its own `data/fields_p000.h5` … +`data/fields_pNNN.h5`; the same `_pNNN` suffix applies to the initial dumps. +Each process copies only its addressable device shards to host buffers, then +stores their chunks through bounded asynchronous queues. It does not assemble +global field or particle arrays on a host or device, and it does not wait on any +other process. `openpmd_field_queue_size` and `openpmd_particle_queue_size` +bound the pending local snapshots per process. + +Every mesh component keeps the full `(Nx, Ny, Nz)` shape in every per-process +file, including singleton axes and mixed periodic/conducting boundaries, so each +file is a valid whole-domain mesh that ordinary readers open directly. Only the +tiles a process owns are stored; field records use tile-shaped HDF5 chunks, so a +file costs space proportional to the tiles it actually holds. Writers strip +guards and retain the existing physical-interior output selection and +component-position metadata. Read any file with the ordinary `io.Series(path, io.Access.read_only)` constructor. -Active particle records follow global tile order, independently of rank order -or local shard enumeration. Only integer per-tile/species counts are exchanged -to determine global record sizes and offsets; particle coordinates and momenta -remain local. Ranks with no active particles still participate in collective -metadata, flush, and close operations. Scalar diagnostics use solver reductions -and are written by rank zero. +Particle records are sized to the particles a process actually holds and are +kept in global tile order within each file. Each file records which global tiles +it owns, and per species how many particles each of those tiles holds, as +`pypic3dOwnedTiles` / `pypic3dOwnedTileCounts` attributes. The HDF5 backend +reports every dataset as one whole chunk, so this metadata is the only way a +reader can tell which regions a file wrote. Scalar diagnostics use solver +reductions and are written by process zero. `dump_fields` writes `data/initial_fields/initial_fields.h5`, and `dump_particles` writes one `data/initial_particles/_initial_particles.h5` file per @@ -154,116 +131,57 @@ the existing solver time-centering and record time offsets. Output cadence and the solver's final-endpoint rules are unchanged. These files are scientific diagnostics, not complete restart checkpoints. -MPI output requires an MPI-enabled openPMD build with parallel HDF5 and fails -early when that support is absent. A rank-local output failure aborts the MPI -communicator so other ranks cannot remain waiting in collective I/O. All ranks -must use the same output flags and cadence. +## Assembling a multiprocess run -## Validation - -Run local input/chunk and initial histogram tests on two logical CPU devices: +A single-process run needs no assembly. After a multiprocess run, join each +group into the unsuffixed file every analysis script already reads: ```bash -JAX_PLATFORMS=cpu JAX_NUM_CPU_DEVICES=2 JAX_ENABLE_X64=true \ - python -m unittest tests.distributed_tests.distributed_io_test \ - tests.code_tests.sharded_histogram_test +python -m PyPIC3D.diagnostics.merge_openpmd outputs/data ``` -The MPI output acceptance worker requires an MPI-enabled openPMD build and a -directory visible to all ranks. Run it with one and then two local devices per -rank, using separate fresh directories: - -```bash -PYPIC_DISTRIBUTED=1 JAX_PLATFORMS=cpu JAX_NUM_CPU_DEVICES=1 JAX_ENABLE_X64=true \ - timeout 120s mpiexec -n 2 python -m tests.distributed_tests.sharded_openpmd_worker \ - --directory /tmp/pypic-mpi-output-one-device - -PYPIC_DISTRIBUTED=1 JAX_PLATFORMS=cpu JAX_NUM_CPU_DEVICES=2 JAX_ENABLE_X64=true \ - timeout 120s mpiexec -n 2 python -m tests.distributed_tests.sharded_openpmd_worker \ - --directory /tmp/pypic-mpi-output-two-devices -``` +The installed console script `PyPIC3D-merge-openpmd` does the same. Point it at +a data directory, one `.h5` or `.bp` group member, or a `.pmd` pointer to a +member. BP series directories are recognized directly, including through a +pointer. `--output` names the merged series, `--iterations` restricts it to +listed steps, and `--overwrite` replaces an existing merged file. + +The `.pmd` pointer is written beside the merged series with the same basename: +`--output /results/custom.h5` creates `/results/custom.pmd` containing +`custom.h5`. Existing source-directory pointers are left untouched when the +destination has a different pointer path. Without `--output`, the usual +`data/fields.pmd` still points to `fields.h5`. Use `--no-pmd` to leave all +pointers untouched. + +The merge rejects a group whose process indices are not exactly +`0 .. pypic3dProcessCount - 1`, which catches stale files left by an earlier run +with a different process count, and it checks that the owned tiles partition the +global tile grid exactly once. It then overlays each file's owned tiles into one +global-extent mesh, concatenates the particle records back into global tile order +independent of which process wrote what, and rewrites the `.pmd` pointer. A +multiprocess run leaves the pointer to the merge rather than writing one that +dangles until then. -The worker checks fields-only, particles-only and combined output, repeated -iterations, original initial-dump paths, unequal particle counts, an empty -species and a species absent on one rank. It also permutes local snapshot shard -lists, rejects transfers of global distributed arrays, and reads the files with -ordinary openPMD readers. The field fixture includes a singleton axis and mixed -boundary conditions. +## Validation -Check failure propagation separately. This command must print -`WRITER_ABORT rank=1` and terminate promptly with a nonzero MPI exit status. -The marker verifies the writer's failure handler ran. An exit status of `124` -means the timeout fired and the failure test did not pass: +Run local input/chunk and initial histogram tests on two logical CPU devices: ```bash -PYPIC_DISTRIBUTED=1 JAX_PLATFORMS=cpu JAX_NUM_CPU_DEVICES=2 JAX_ENABLE_X64=true \ - timeout 120s mpiexec -n 2 python -m tests.distributed_tests.sharded_openpmd_worker \ - --directory /tmp/pypic-mpi-output-failure --fail-rank 1 +JAX_PLATFORMS=cpu JAX_NUM_CPU_DEVICES=2 JAX_ENABLE_X64=true \ + python -m unittest tests.distributed_tests.distributed_io_test \ + tests.code_tests.sharded_histogram_test ``` -On a two-node Perlmutter allocation, exercise the output worker with one GPU on -each node and a directory visible to both nodes: +Per-process naming, ownership metadata and the merge round trip run on one +process and need no launcher: ```bash -export PYPIC_DISTRIBUTED=1 PYPIC_LOCAL_DEVICE_IDS=0 JAX_ENABLE_X64=true -export PYTHONNOUSERSITE=1 JAX_PLATFORMS=cuda -srun --nodes=2 --ntasks-per-node=1 --gpus-per-task=1 --gpu-bind=single:1 \ - python -m tests.distributed_tests.sharded_openpmd_worker \ - --directory "$SCRATCH/pypic-two-node-output-test" +JAX_PLATFORMS=cpu JAX_NUM_CPU_DEVICES=8 JAX_ENABLE_X64=true \ + python -m unittest tests.distributed_tests.merge_openpmd_test \ + tests.code_tests.sharded_openpmd_test ``` -An output worker does not establish coupled-evolution accuracy or scaling, and -a local MPI run does not establish two-node GPU operation. Record hostnames, -global/local device ownership and readback results when running on two real nodes. - -### Unified writer validation - -The unified writers passed 43 focused serial/distributed regression tests on -logical CPU devices, including initial and runtime GR time metadata. Actual -MPI output passed on two ranks with both one and two local devices per rank: -initial and runtime readback, unequal counts, empty species/ranks, reordered -shards, and fields-only/particles-only output. The injected rank-local failure -printed `WRITER_ABORT rank=1` and exited with status 1 before the timeout. - -The updated `multihost_io_worker --require-mpi-output --evolve` also completed -two coupled steps with initial and runtime output. An otherwise identical run -with output disabled produced bitwise-identical final particle, field and BSSN -arrays on each rank. The BSSN state retained one addressable tile and 373,248 -local bytes per rank. This used the MPI-enabled openPMD 0.17.1 runtime described -above and the current local JAX_BSSN checkout. A five-minute cold-compilation -limit was insufficient; the successful comparison used a 15-minute limit and -four separate physical CPU cores per rank. No multi-node or GPU validation was -performed for this writer change. - -### Historical distributed-evolution validation - -The earlier MPI draft passed a local run on 2026-11-10 with two CPU processes -on one host. That draft used `data/simulation.h5` and component-specific upper -endpoint extents; its output-format results do not validate the current unified -writers. The following measurements describe that earlier execution only. The -actual `initialize_simulation` / `run_PyPIC3D` path advanced two steps on an -`8 x 4 x 4` periodic mesh split into two `4 x 4 x 4` tiles, starting from flat -geometry, `metric_backreaction=True`, seven neutral unit-mass native-file -particles, and an analytic electric field. One rank initially owned all -particles. Matter sourcing produced nonzero `trace_K`; all final BSSN leaves -remained finite and distributed, with one addressable tile per process. This -fixture does not solve the initial Einstein constraints and tests execution and -communication rather than physical accuracy. The shared -series contained all 20 EM/BSSN records (30 components) at both saved iterations, -with finite values and the expected particle count. Separate analytic MPI -snapshots checked the physical upper endpoint planes and native E-component -extents. The final scalar particle-count metadata also passed a two-process -round trip. A separate two-process initialization/output check enabled both -`plotvelocities` and `plotchargedensity`; all four optional output components -were finite, retained local shards, and passed the shared-series readback. - -For that tiny CPU case, explicit step compilation took 180.027 seconds using -the slowest process time. An MPI barrier preceded each timed step, and all -returned arrays were synchronized before stopping its timer. The first step -took 0.059005 seconds and the second 0.057823 seconds, again using the slowest -process time. The compiled per-device memory estimate was 1,358,745 bytes of -arguments, 2,055,766 bytes of outputs, and 20,896,936 bytes of temporary storage -(zero alias bytes). The local BSSN state occupied 373,248 bytes per process. -These figures exclude initialization and diagnostics, and do not establish -production scaling, accuracy, or GPU memory requirements. The ordinary -installed serial openPMD runtime was not changed. +A real two-process check needs two shells sharing a coordinator address and a +directory visible to both, with `PYPIC_PROCESS_ID` 0 and 1. Confirm the run +wrote `fields_p000.h5` and `fields_p001.h5` and no bare `fields.h5`, then merge +and read the result with an unmodified analysis script. diff --git a/docs/solvers.rst b/docs/solvers.rst index d5c6883..a6ddad9 100644 --- a/docs/solvers.rst +++ b/docs/solvers.rst @@ -232,8 +232,7 @@ performs densitization and the leapfrog offset once. Nonzero prescribed external fields are rejected. The default pusher is ``hybrid_boris_geodesic``, current deposition is ``esirkepov``, and current filtering is disabled. Initial geometry is exactly flat, with harmonic lapse -and initially zero frozen shift. This is not a matter constraint solve: -the initial Hamiltonian and momentum residuals are reported explicitly. +and initially zero frozen shift. Python callers may provide ``bssn_state`` and ``bssn_evolution_parameters`` to ``initialize_simulation`` or ``run_PyPIC3D``. The latter may be the native diff --git a/docs/usage.rst b/docs/usage.rst index 15fe2c0..f69e5f3 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -230,21 +230,19 @@ At ``plotting_interval`` cadence, the driver appends: Set ``plot_openpmd_fields = true`` to write ``data/fields.h5`` and ``plot_openpmd_particles = true`` to write ``data/particles.h5``. Writers copy only addressable device shards and store their chunks at global offsets, -without assembling full field or particle arrays. A single-process run uses -bounded asynchronous queues, including when it spans several devices. +without assembling full field or particle arrays. Every run uses bounded +asynchronous queues, including when it spans several devices or processes. ``openpmd_field_queue_size`` and ``openpmd_particle_queue_size`` limit pending host snapshots, not the number of output files. A full queue waits for disk writes; larger queues retain more host memory. -Multiprocess output uses those same chunk writers synchronously on the main -thread with an MPI-enabled openPMD/parallel-HDF5 installation. Queue-size -settings do not apply to that path. Only particle-count metadata is exchanged -to assign global particle offsets, and ranks with empty particle chunks still -participate in collective writes. Missing MPI output support fails early; -rank-local output failures abort the communicator rather than leaving peers -waiting. Multiprocess simulation initialization currently supports -``dynamic_metric`` only; see the distributed Cartesian guide for launch and -validation commands. +Multiprocess output needs no MPI and exchanges nothing: each process writes its +own ``data/fields_pNNN.h5`` and ``data/particles_pNNN.h5`` through the same +queues, and ``python -m PyPIC3D.diagnostics.merge_openpmd outputs/data`` joins a +group afterwards into the unsuffixed file these analysis paths already read. A +single-process run keeps the unsuffixed names and needs no merge. Multiprocess +simulation initialization currently supports ``dynamic_metric`` only; see the +distributed Cartesian guide for launch and validation commands. Each snapshot is identified by its simulation step number, so openPMD iteration IDs are spaced by ``plotting_interval`` rather than numbered consecutively. diff --git a/pyproject.toml b/pyproject.toml index 2f20008..51699ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,9 +21,6 @@ dependencies = [ ] [project.optional-dependencies] -distributed = [ - "mpi4py", -] relativity = [ "JAX_BSSN @ git+https://github.com/cwoolfo1/JAX_BSSN.git", ] @@ -36,3 +33,4 @@ include = ["PyPIC3D*"] [project.scripts] PyPIC3D = "PyPIC3D.__main__:main" +PyPIC3D-merge-openpmd = "PyPIC3D.diagnostics.merge_openpmd:main" diff --git a/tests/code_tests/gr_work_energy_output_test.py b/tests/code_tests/gr_work_energy_output_test.py index 16361ab..52a9f66 100644 --- a/tests/code_tests/gr_work_energy_output_test.py +++ b/tests/code_tests/gr_work_energy_output_test.py @@ -47,8 +47,9 @@ def run_driver(self, directory, *, steps, cadence, gravity_boundary=None): data_dir.mkdir() with patch("PyPIC3D.__main__.initialize_simulation", return_value=runtime), \ - patch("PyPIC3D.__main__.tqdm", side_effect=lambda values: values), \ - patch("PyPIC3D.diagnostics.static_metric.bssn_constraint_norms", return_value={}): + patch("PyPIC3D.__main__.tqdm", side_effect=lambda values, **kwargs: values), \ + patch("PyPIC3D.diagnostics.static_metric.bssn_constraint_norms", + return_value={"hamiltonian_l2": 0., "momentum_l2": 0.}): result = run_PyPIC3D({}) final = result[1].work_energy_tracker diff --git a/tests/code_tests/initialization_test.py b/tests/code_tests/initialization_test.py index 6577987..6d664ed 100644 --- a/tests/code_tests/initialization_test.py +++ b/tests/code_tests/initialization_test.py @@ -1,5 +1,7 @@ """Single-device numerical tests.""" +from types import SimpleNamespace + from tests.support.initialization_fixtures import ( BC_ABSORBING, BC_CONDUCTING, @@ -133,6 +135,18 @@ def test_automatic_particle_batch_size_uses_cpu_threads_per_local_device(self): self.assertEqual(narrower, 256) + def test_automatic_particle_batch_size_counts_only_local_mesh_devices(self): + particles = self._particles_with_active(jnp.ones((3, 1, 1, 1, 900), dtype=bool)) + devices = tuple(SimpleNamespace(process_index=process) for process in (0, 1, 0)) + mesh = SimpleNamespace(devices=SimpleNamespace(flat=devices)) + with patch("PyPIC3D.initialization.jax.process_index", return_value=0): + batch_size, reason = _resolve_particle_batch_size( + particles, None, field_mesh=mesh, platform="cpu", available_cpu_threads=8, + ) + self.assertEqual(batch_size, 512) + self.assertIn("2 local devices", reason) + + def test_available_cpu_threads_falls_back_to_os_cpu_count(self): with patch("PyPIC3D.initialization.os.sched_getaffinity", side_effect=OSError): with patch("PyPIC3D.initialization.os.cpu_count", return_value=12): diff --git a/tests/code_tests/parameters_test.py b/tests/code_tests/parameters_test.py index 7f92a33..bc9ec27 100644 --- a/tests/code_tests/parameters_test.py +++ b/tests/code_tests/parameters_test.py @@ -11,6 +11,7 @@ from PyPIC3D.deposition.J_from_rhov import J_from_rhov from PyPIC3D.deposition.rho import compute_rho from PyPIC3D.utilities.parameters import ( + build_static_parameters, DynamicParameters, GridParameters, StaticParameters, @@ -19,6 +20,21 @@ class TestKernelParameters(unittest.TestCase): + def test_tile_configuration_preserves_legacy_shape_and_explicit_mesh(self): + config = dict(Nx=4, Ny=2, Nz=1, particle_tile_nx=4, particle_tile_ny=2, + particle_tile_nz=1, shape_factor=1, boundary_conditions=(0, 0, 0)) + legacy = build_static_parameters(config) + self.assertEqual(legacy.tile_shape, (4, 2, 1)) + self.assertEqual(legacy.field_mesh.devices.shape, (1, 1, 1)) + + # An explicit shape overrides legacy values; a supplied mesh needs no grid dimensions. + config.update(tile_shape=[4, 2, 1], particle_tile_nx=1, field_mesh=legacy.field_mesh) + for name in ("Nx", "Ny", "Nz"): + del config[name] + explicit = build_static_parameters(config) + self.assertEqual(explicit.tile_shape, (4, 2, 1)) + self.assertIs(explicit.field_mesh, legacy.field_mesh) + def test_static_and_dynamic_parameters_split_kernel_contract(self): static_parameters, dynamic_parameters = kernel_parameters( dt=0.1, diff --git a/tests/code_tests/particle_refresh_test.py b/tests/code_tests/particle_refresh_test.py index ef23ecc..ca7ea1d 100644 --- a/tests/code_tests/particle_refresh_test.py +++ b/tests/code_tests/particle_refresh_test.py @@ -1,9 +1,11 @@ """Single-device numerical tests.""" +from PyPIC3D.boundary_conditions.ghost_cells import make_field_mesh + from tests.support.particle_refresh_fixtures import ( TiledParticleRefreshFixtures, - _adjacent_tile_offset, build_tiled_particles, + refresh_tiled_particle_tiles, jnp, unittest, update_tiled_particle_positions, @@ -34,13 +36,21 @@ def test_update_tiled_particle_positions_respects_active_and_update_flags(self): self.assertTrue(jnp.allclose(fixed_moved.x, fixed.x)) - def test_adjacent_tile_offset_handles_periodic_edges(self): - source = jnp.array([0, 0, 1, 1]) - dest = jnp.array([0, 1, 0, 1]) + def test_refresh_wraps_particles_on_one_tile_without_changing_momenta(self): + parameter_set = self._build_parameter_values() + species = self._species(parameter_set, x1=[-1.75, 1.75], v1=[0.25, -0.5]) + static_parameters, dynamic_parameters = self._particle_parameters(parameter_set) + static_parameters = static_parameters._replace(field_mesh=make_field_mesh((1, 1, 1))) + particles, _ = build_tiled_particles([species], static_parameters, dynamic_parameters) + particles = particles._replace(x=particles.x.at[0, 0, 0, 0, :, 0].set(jnp.array([2.25, -2.25]))) - offset = _adjacent_tile_offset(dest, source, tile_count=2) + refreshed, overflow = refresh_tiled_particle_tiles(particles, static_parameters, dynamic_parameters) - self.assertTrue(jnp.array_equal(offset, jnp.array([0, 1, -1, 0]))) + x, u = self._active_rows(refreshed) + self.assertFalse(bool(overflow)) + self.assertEqual(int(jnp.sum(refreshed.active)), 2) + self.assertTrue(jnp.allclose(x[:, 0], jnp.array([-1.75, 1.75]))) + self.assertTrue(jnp.array_equal(u[:, 0], jnp.array([0.25, -0.5]))) diff --git a/tests/code_tests/sharded_openpmd_test.py b/tests/code_tests/sharded_openpmd_test.py index 54c3ac5..c7a5bf9 100644 --- a/tests/code_tests/sharded_openpmd_test.py +++ b/tests/code_tests/sharded_openpmd_test.py @@ -110,6 +110,24 @@ def require_two_devices(self): def assert_close(self, actual, expected): self.assertTrue(bool(jnp.allclose(jnp.asarray(actual), jnp.asarray(expected), rtol=1.e-12, atol=1.e-12))) + def test_tile_chunks_strip_scalar_and_per_axis_guards(self): + tile_shape = (2, 3, 1) + reference = jnp.arange(12, dtype=jnp.float64).reshape((2, 1, 1) + tile_shape) + shard_index = (slice(2, 4), slice(0, 1), slice(0, 1)) + (slice(None),) * 3 + for guards, widths in ((0, (0, 0, 0)), (2, (2, 2, 2)), + ((0, 1, 2), (0, 1, 2)), ([2, 0, 1], (2, 0, 1))): + with self.subTest(guards=guards): + data = jnp.pad(reference, ((0, 0),) * 3 + tuple((g, g) for g in widths), + constant_values=-9999.) + layout = SimpleNamespace(tile_shape=tile_shape, guard_cells=guards) + chunks = list(openPMD._iter_tile_chunks_from_host_shard(shard_index, data, layout=layout)) + + self.assertEqual(len(chunks), 2) + for local_tile, (offset, interior) in enumerate(chunks): + self.assertEqual(offset, [(2 + local_tile) * tile_shape[0], 0, 0]) + self.assertEqual(interior.shape, tile_shape) + self.assertTrue(jnp.array_equal(interior, reference[local_tile, 0, 0])) + def test_mesh_chunks_cover_periodic_mixed_and_singleton_grids(self): for num_tiles in (1, 2): if num_tiles > jax.local_device_count(): diff --git a/tests/code_tests/simulation_output_test.py b/tests/code_tests/simulation_output_test.py new file mode 100644 index 0000000..bd57ede --- /dev/null +++ b/tests/code_tests/simulation_output_test.py @@ -0,0 +1,100 @@ +"""Output cleanup and endpoint reports through the simulation driver.""" + +from contextlib import redirect_stdout, redirect_stderr +from io import StringIO +from pathlib import Path +from types import SimpleNamespace +import tempfile +import unittest +from unittest.mock import Mock, patch + +import jax.numpy as jnp + +from PyPIC3D.__main__ import run_PyPIC3D +from PyPIC3D.diagnostics import simulation_output +from PyPIC3D.diagnostics.static_metric import bssn_constraint_norms, print_bssn_constraints +from PyPIC3D.initialization import initialize_simulation + + +class SimulationOutputTests(unittest.TestCase): + def test_writers_close_after_step_or_writer_failure(self): + for failure in ("step", "close", "setup"): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: + static = SimpleNamespace(output_dir=directory) + plotting = dict(plot_openpmd_fields=True, plot_openpmd_particles=True) + field_writer, particle_writer = Mock(), Mock() + if failure in ("step", "close"): + particle_writer.close.side_effect = OSError("writer failed") + expected_error = RuntimeError if failure == "step" else OSError + with patch.object(simulation_output, "setup_pmd_files"), \ + patch.object(simulation_output, "create_async_tiled_openpmd_field_writer", + return_value=field_writer), \ + patch.object(simulation_output, "create_async_tiled_openpmd_particle_writer", + side_effect=OSError("setup failed") if failure == "setup" else None, + return_value=particle_writer): + with self.assertRaises(expected_error): + with simulation_output.open_simulation_output(static, None, plotting): + if failure == "step": + raise RuntimeError("step failed") + field_writer.close.assert_called_once_with(raise_errors=failure == "close") + if failure != "setup": + particle_writer.close.assert_called_once_with(raise_errors=failure == "close") + + def test_constraint_reduction_runs_on_nonprinting_ranks(self): + with patch("PyPIC3D.diagnostics.static_metric.bssn_constraint_norms", + return_value={"hamiltonian_l2": 2., "momentum_l2": 3.}) as compute, \ + patch("jax.process_index", return_value=1), redirect_stdout(StringIO()) as output: + print_bssn_constraints("Final", None, None, None, None, None) + compute.assert_called_once() + self.assertEqual(output.getvalue(), "") + + def test_initial_and_final_constraints_match_common_time_states(self): + try: + import JAX_BSSN + except ModuleNotFoundError: + self.skipTest("Optional JAX_BSSN dependency is not installed") + + for steps in (0, 1): + with self.subTest(steps=steps), tempfile.TemporaryDirectory() as directory: + config = { + "simulation_parameters": dict( + solver="dynamic_metric", output_dir=directory, Nt=steps, dt=.01, + Nx=4, Ny=4, Nz=4, x_wind=4., y_wind=4., z_wind=4.), + "plotting": dict(plotting_interval=3), + } + initial_states = [] + + def initialize(*args, **kwargs): + initial = initialize_simulation(*args, **kwargs) + initial_states.append(initial) + return initial + + with patch("PyPIC3D.__main__.initialize_simulation", side_effect=initialize), \ + patch("PyPIC3D.diagnostics.static_metric.print_bssn_constraints", + wraps=print_bssn_constraints) as report, \ + redirect_stdout(StringIO()) as output, redirect_stderr(StringIO()): + result = run_PyPIC3D(config, field_initializers={ + "Ex": lambda x, y, z: .02 * jnp.sin(jnp.pi * x / 2)}) + + _, particles, fields, static, dynamic, _, _, species = initial_states[0] + initial_norms = bssn_constraint_norms(particles, species, fields, static, dynamic) + static, dynamic, _, _, particles, fields, species = result + final_norms = bssn_constraint_norms(particles, species, fields, static, dynamic) + self.assertEqual(float(dynamic.bssn_parameters.time), steps * .01) + self.assertEqual([call.args[0] for call in report.call_args_list], ["Initial", "Final"]) + self.assertEqual(float(report.call_args.args[-1].bssn_parameters.time), steps * .01) + for label, norms in (("Initial", initial_norms), ("Final", final_norms)): + expected = (f"{label} Einstein constraints: Hamiltonian RMS={float(norms['hamiltonian_l2']):.6e}, " + f"momentum RMS={float(norms['momentum_l2']):.6e}") + self.assertIn(expected, output.getvalue()) + self.assertNotIn("Field Energy", output.getvalue()) + self.assertNotIn("Step compilation", output.getvalue()) + self.assertNotIn("slowest process", output.getvalue()) + # The final console report must not add a new scalar-file endpoint. + path = Path(directory) / "data" / "hamiltonian_l2.txt" + self.assertEqual(len(path.read_text().splitlines()) if path.exists() else 0, + int(steps > 0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/code_tests/test_dark_photon.py b/tests/code_tests/test_dark_photon.py index 2de2ac5..3a54238 100644 --- a/tests/code_tests/test_dark_photon.py +++ b/tests/code_tests/test_dark_photon.py @@ -3,7 +3,7 @@ from tests.support.dark_photon_fixtures import ( D_FIELD_LOCATIONS, DarkPhotonFixtures, - _dark_timestep, + dark_photon_courant_condition, build_field_output_map, build_static_parameters, build_tiled_particles, @@ -260,17 +260,17 @@ def test_configuration_errors_and_timestep_bound(self): dynamic.update(Nx=8, Ny=1, Nz=1, dx=.2, dy=1., dz=1., C=1.3, dt=None) static.update(dark_mu=20., sin_chi=.1) limit = 2/(1.3*math.sqrt(20**2+4/.2**2)) - self.assertAlmostEqual(_dark_timestep(static, dynamic), .99*limit) + self.assertAlmostEqual(dark_photon_courant_condition(static, dynamic), .99*limit) dynamic["dt"] = limit with self.assertRaisesRegex(ValueError, "stability limit"): - _dark_timestep(static, dynamic) + dark_photon_courant_condition(static, dynamic) dynamic.update(Nx=1, dt=None) - self.assertAlmostEqual(_dark_timestep(static, dynamic), .99*2/(1.3*20)) + self.assertAlmostEqual(dark_photon_courant_condition(static, dynamic), .99*2/(1.3*20)) static["dark_mu"] = 0. with self.assertRaisesRegex(ValueError, "Specify dt"): - _dark_timestep(static, dynamic) + dark_photon_courant_condition(static, dynamic) dynamic["dt"] = .1 - self.assertEqual(_dark_timestep(static, dynamic), .1) + self.assertEqual(dark_photon_courant_condition(static, dynamic), .1) def test_invalid_initial_field_shapes_and_types(self): diff --git a/tests/distributed_tests/distributed_io_test.py b/tests/distributed_tests/distributed_io_test.py index f87f78a..17e78e4 100644 --- a/tests/distributed_tests/distributed_io_test.py +++ b/tests/distributed_tests/distributed_io_test.py @@ -10,14 +10,14 @@ import openpmd_api as io from tests.kernel_fixtures import kernel_parameters -from PyPIC3D.distributed import tiled_field_from_initializer +from PyPIC3D.utilities.distributed import tiled_field_from_initializer from PyPIC3D.initialization import initialize_fields from PyPIC3D.utilities.toml_helpers import load_external_fields_from_toml from PyPIC3D.particles.distributed_initialization import load_distributed_particles from PyPIC3D.particles.particle_initialization import load_particles_from_toml from PyPIC3D.diagnostics.async_writer import make_tiled_field_snapshot -from PyPIC3D.diagnostics.parallel_openpmd import write_parallel_snapshot -from PyPIC3D.diagnostics.openPMD import TiledMeshLayout, _iter_tile_chunks_from_host_shard +from PyPIC3D.diagnostics.openPMD import ( + TiledMeshLayout, _iter_tile_chunks_from_host_shard, write_tiled_field_snapshot_openpmd) class TestDistributedInputOutput(unittest.TestCase): @@ -106,15 +106,20 @@ def test_owned_chunks_cover_the_output_mesh_without_guards_or_endpoints(self): coverage = coverage.at[region].add(1) self.assertTrue(bool(jnp.all(coverage == 1))) - def test_parallel_writer_preserves_uniform_record_shapes_for_default_readers(self): + def test_tiled_writer_preserves_uniform_record_shapes_for_default_readers(self): static = self.static._replace(boundary_conditions=(1, 1, 1)) field = tiled_field_from_initializer(lambda x, y, z: x+2*y+3*z, static, self.dynamic) snapshot = make_tiled_field_snapshot({"lapse": field, "E": (field, field, field)}, step=4, time=0.2) + layout = TiledMeshLayout( + global_shape=tuple(int(getattr(self.dynamic, "N" + axis)) for axis in "xyz"), + tile_shape=tuple(int(width) for width in static.tile_shape), + guard_cells=int(static.guard_cells), + ) with tempfile.TemporaryDirectory() as folder: path = os.path.join(folder, "shared.h5") - series = io.Series(path, io.Access.create) - write_parallel_snapshot(series, None, static, self.dynamic, fields=snapshot) - series.close() + write_tiled_field_snapshot_openpmd( + snapshot, output_dir=folder, filename="shared", + dynamic_parameters=self.dynamic, layout=layout, file_extension=".h5") series = io.Series(path, io.Access.read_only) scalar = series.iterations[4].meshes["lapse"][io.Mesh_Record_Component.SCALAR] electric = series.iterations[4].meshes["E"]["x"] diff --git a/tests/distributed_tests/distributed_particle_refresh_test.py b/tests/distributed_tests/distributed_particle_refresh_test.py index bf33566..52b4085 100644 --- a/tests/distributed_tests/distributed_particle_refresh_test.py +++ b/tests/distributed_tests/distributed_particle_refresh_test.py @@ -119,6 +119,35 @@ def test_refresh_moves_diagonal_particle_through_two_axis_permute(self): self.assert_allclose(refreshed.x[1, 1, 0, 0, 0, :2], jnp.asarray([0.25, 0.25])) + def test_refresh_routes_both_periodic_crossings_on_two_and_four_tiles(self): + for tile_count in (2, 4): + with self.subTest(tile_count=tile_count): + mesh_shape = (tile_count, 1, 1) + tile_shape = (2, 1, 1) + static_parameters = _static_parameters(mesh_shape, tile_shape) + dynamic_parameters = _dynamic_parameters(mesh_shape, tile_shape) + half_width = float(tile_count) + particles = _empty_particles(mesh_shape) + particles = _put_particle(particles, (0, 0, 0), 0, + (-half_width - 0.25, 0.0, 0.0), u=(-1.0, 2.0, 3.0)) + particles = _put_particle(particles, (tile_count - 1, 0, 0), 0, + (half_width + 0.25, 0.0, 0.0), u=(4.0, 5.0, 6.0)) + particles = _shard_particles(particles, static_parameters) + + refreshed, overflow = jax.jit( + lambda p: particle_comm.refresh_tiled_particle_tiles(p, static_parameters, dynamic_parameters) + )(particles) + + self.assertFalse(bool(overflow)) + self.assertEqual(int(jnp.sum(refreshed.active)), 2) + self.assertEqual(int(jnp.sum(refreshed.active[0, 0, 0])), 1) + self.assertEqual(int(jnp.sum(refreshed.active[-1, 0, 0])), 1) + self.assert_allclose(refreshed.x[0, 0, 0, 0, 0], jnp.array([-half_width + 0.25, 0.0, 0.0])) + self.assert_allclose(refreshed.x[-1, 0, 0, 0, 0], jnp.array([half_width - 0.25, 0.0, 0.0])) + self.assert_allclose(refreshed.u[0, 0, 0, 0, 0], jnp.array([4.0, 5.0, 6.0])) + self.assert_allclose(refreshed.u[-1, 0, 0, 0, 0], jnp.array([-1.0, 2.0, 3.0])) + + def test_refresh_reports_destination_capacity_overflow(self): mesh_shape = (2, 1, 1) tile_shape = (2, 1, 1) diff --git a/tests/distributed_tests/merge_openpmd_test.py b/tests/distributed_tests/merge_openpmd_test.py new file mode 100644 index 0000000..7ab6221 --- /dev/null +++ b/tests/distributed_tests/merge_openpmd_test.py @@ -0,0 +1,312 @@ +"""Per-process openPMD naming, ownership metadata, and the merge round trip.""" + +from dataclasses import replace +from pathlib import Path +import os +import tempfile +import unittest +from unittest.mock import patch + +import jax +import jax.numpy as jnp +import openpmd_api as io + +from PyPIC3D.diagnostics import async_writer, merge_openpmd, openPMD +from tests.code_tests.sharded_openpmd_test import ( + field_with_guard_sentinels, + parameters, + particle_fixture, + read_particle_records, +) + + +def _tile_of(index): + return openPMD._slice_start(index[0]) + + +def _split(host_shards, process): + """Keep the shards whose global tile belongs to one process.""" + return [(index, data) for index, data in host_shards if _tile_of(index) == process] + + +def _split_field_snapshot(snapshot, process): + fields = {} + for name, value in snapshot.fields.items(): + if isinstance(value, tuple) and len(value) == 3: + fields[name] = tuple(_split(component, process) for component in value) + else: + fields[name] = _split(value, process) + return replace(snapshot, fields=fields) + + +def _split_particle_snapshot(snapshot, process): + changed = {name: _split(getattr(snapshot, name), process) + for name in ("x_shards", "u_shards", "active_shards")} + if snapshot.gamma_shards is not None: + changed["gamma_shards"] = _split(snapshot.gamma_shards, process) + return replace(snapshot, **changed) + + +class SeriesDiscoveryTests(unittest.TestCase): + def test_directory_member_and_pointer_discover_the_same_group(self): + for extension in (".h5", ".bp"): + with self.subTest(extension=extension), tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + data = folder / "data" + data.mkdir() + members = [data / f"fields_p{process:03d}{extension}" for process in range(2)] + for member in members: + if extension == ".bp": + member.mkdir() + (member / "md.idx").touch() + else: + member.touch() + pointer = folder / "fields.pmd" + pointer.write_text(f"data/{members[0].name}\n") + expected = [(str(data), "fields", extension, tuple(str(path) for path in members))] + for entry in (data, members[0], pointer): + self.assertEqual(merge_openpmd.discover_series_groups([entry]), expected) + + def test_missing_member_and_pointer_target_raise(self): + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + for extension in (".h5", ".bp"): + missing = folder / f"fields_p000{extension}" + pointer = folder / "fields.pmd" + pointer.write_text(missing.name + "\n") + for entry in (missing, pointer): + with self.subTest(entry=entry, extension=extension): + with self.assertRaises(FileNotFoundError): + merge_openpmd.discover_series_groups([entry]) + + +class MergeOpenPMDTests(unittest.TestCase): + def setUp(self): + if jax.device_count() < 2: + self.skipTest("Run with JAX_NUM_CPU_DEVICES>=2") + self.static, self.dynamic = parameters(num_tiles=2, shape=(8, 2, 1)) + self.layout = openPMD.TiledMeshLayout( + global_shape=(8, 2, 1), + tile_shape=tuple(int(width) for width in self.static.tile_shape), + guard_cells=int(self.static.guard_cells), + ) + + def test_process_suffix_only_appears_for_multiprocess_runs(self): + self.assertEqual(openPMD.process_series_filename("fields.h5", 0, 1), "fields.h5") + self.assertEqual(openPMD.process_series_filename("fields.h5", 3, 4), "fields_p003.h5") + self.assertEqual( + openPMD.process_series_filename("ion_initial_particles.h5", 2, 16), + "ion_initial_particles_p002.h5") + + def _write_per_process(self, folder, snapshot, kind, extension=".h5"): + """Write one series per process the way a real multiprocess run does.""" + for process in range(2): + with patch("jax.process_index", return_value=process), \ + patch("jax.process_count", return_value=2): + if kind == "fields": + openPMD.write_tiled_field_snapshot_openpmd( + _split_field_snapshot(snapshot, process), + output_dir=str(folder), filename="fields", + dynamic_parameters=self.dynamic, layout=self.layout, + file_extension=extension) + else: + openPMD.write_tiled_particle_snapshot_openpmd( + _split_particle_snapshot(snapshot, process), + output_dir=str(folder), filename="particles", + static_parameters=self.static, dynamic_parameters=self.dynamic, + file_extension=extension) + + def test_custom_output_pointer_lives_beside_the_merged_series(self): + field, reference = field_with_guard_sentinels(self.static, (8, 2, 1)) + snapshot = async_writer.make_tiled_field_snapshot({"lapse": field}, step=4, time=0.2) + for location in ("same", "other", "relative"): + with self.subTest(location=location), tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + source = folder / "data" + self._write_per_process(source, snapshot, "fields") + source_pointer = source / "fields.pmd" + source_pointer.write_text("previous.h5\n") + destination = {"same": source, "other": folder / "results", "relative": folder}[location] + destination.mkdir(exist_ok=True) + target = destination / "custom.h5" + previous_directory = os.getcwd() + try: + os.chdir(folder) + merge_openpmd.merge_paths([source], output="custom.h5" if location == "relative" else str(target)) + finally: + os.chdir(previous_directory) + + pointer = target.with_suffix(".pmd") + self.assertEqual(pointer.read_text(), "custom.h5\n") + self.assertEqual(source_pointer.read_text(), "previous.h5\n") + series = io.Series(merge_openpmd._resolve_pmd(pointer), io.Access.read_only) + try: + iteration = series.iterations[4] + self.assertAlmostEqual(iteration.time, 0.2) + record = iteration.meshes["lapse"][io.Mesh_Record_Component.SCALAR] + values = record.load_chunk() + series.flush() + self.assertTrue(bool(jnp.array_equal(jnp.asarray(values), reference))) + finally: + series.close() + + def test_no_pmd_preserves_existing_pointers(self): + field, _ = field_with_guard_sentinels(self.static, (8, 2, 1)) + snapshot = async_writer.make_tiled_field_snapshot({"lapse": field}, step=0, time=0.) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + self._write_per_process(folder, snapshot, "fields") + pointer = folder / "fields.pmd" + pointer.write_text("previous.h5\n") + for output in (None, str(folder / "custom.h5")): + merge_openpmd.merge_paths([folder], output=output, write_pmd=False) + self.assertTrue(Path(output or folder / "fields.h5").exists()) + self.assertEqual(pointer.read_text(), "previous.h5\n") + self.assertFalse((folder / "custom.pmd").exists()) + + @unittest.skipUnless(io.variants.get("adios2", False), "ADIOS2 backend is unavailable") + def test_bp_member_and_pointer_merge_preserve_values_and_metadata(self): + field, reference = field_with_guard_sentinels(self.static, (8, 2, 1)) + snapshot = async_writer.make_tiled_field_snapshot( + {"J": (field, field, field)}, step=4, time=0.2, current_time_offset=-0.05) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + self._write_per_process(folder, snapshot, "fields", extension=".bp") + member = folder / "fields_p000.bp" + pointer = folder / "input.pmd" + pointer.write_text(member.name + "\n") + source = io.Series(str(member), io.Access.read_only) + try: + mesh = source.iterations[4].meshes["J"] + position = tuple(mesh["x"].position) + dimensions = tuple(mesh.unit_dimension) + finally: + source.close() + for entry, name in ((member, "direct"), (pointer, "via_pointer")): + with self.subTest(entry=entry): + target = folder / f"{name}.bp" + merge_openpmd.merge_paths([entry], output=str(target)) + merged_pointer = target.with_suffix(".pmd") + self.assertEqual(merged_pointer.read_text(), target.name + "\n") + series = io.Series(merge_openpmd._resolve_pmd(merged_pointer), io.Access.read_only) + try: + iteration = series.iterations[4] + self.assertAlmostEqual(iteration.time, 0.2) + self.assertAlmostEqual(iteration.dt, float(self.dynamic.dt)) + mesh = iteration.meshes["J"] + self.assertAlmostEqual(mesh.time_offset, -0.05) + self.assertEqual(tuple(mesh.unit_dimension), dimensions) + record = mesh["x"] + self.assertEqual(tuple(record.shape), (8, 2, 1)) + self.assertEqual(tuple(record.position), position) + values = record.load_chunk() + series.flush() + self.assertTrue(bool(jnp.array_equal(jnp.asarray(values), reference))) + finally: + series.close() + + def test_merged_fields_reproduce_the_whole_domain_mesh(self): + field, reference = field_with_guard_sentinels(self.static, (8, 2, 1)) + snapshot = async_writer.make_tiled_field_snapshot( + {"lapse": field, "E": (field, field, field)}, step=4, time=0.2) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + self._write_per_process(folder, snapshot, "fields") + self.assertTrue((folder/"fields_p000.h5").exists()) + self.assertTrue((folder/"fields_p001.h5").exists()) + self.assertFalse((folder/"fields.h5").exists()) + + merge_openpmd.merge_paths([str(folder)]) + self.assertTrue((folder/"fields.h5").exists()) + # The merge restores the pointer a single-process run writes itself. + self.assertEqual((folder/"fields.pmd").read_text().strip(), "fields.h5") + + series = io.Series(str(folder/"fields.h5"), io.Access.read_only) + try: + iteration = series.iterations[4] + self.assertAlmostEqual(float(iteration.time), 0.2) + scalar = iteration.meshes["lapse"][io.Mesh_Record_Component.SCALAR] + electric = iteration.meshes["E"]["x"] + self.assertEqual(tuple(scalar.shape), (8, 2, 1)) + values = scalar.load_chunk() + vector = electric.load_chunk() + series.flush() + self.assertTrue(bool(jnp.allclose(jnp.asarray(values), reference))) + self.assertTrue(bool(jnp.allclose(jnp.asarray(vector), reference))) + finally: + series.close() + + def test_merged_particles_match_a_single_process_write(self): + particles, species, names = particle_fixture(self.static) + snapshot = async_writer.make_tiled_particle_snapshot( + particles, step=0, time=0.5, species_names=names, species_config=species) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + openPMD.write_tiled_particle_snapshot_openpmd( + snapshot, output_dir=str(folder), filename="reference", + static_parameters=self.static, dynamic_parameters=self.dynamic, + file_extension=".h5") + expected, _ = read_particle_records(folder/"reference.h5") + + self._write_per_process(folder, snapshot, "particles") + merge_openpmd.merge_paths([str(folder/"particles_p000.h5")]) + merged, time = read_particle_records(folder/"particles.h5") + + self.assertAlmostEqual(float(time), 0.5) + self.assertEqual(set(merged), set(expected)) + for name, records in expected.items(): + for record_name, values in records.items(): + # Global tile order must survive the split, independent of which + # process held which tile. + self.assertTrue(bool(jnp.array_equal(merged[name][record_name], values)), + f"{name}/{record_name} differs after merge") + + def test_merged_fields_survive_batched_draining(self): + field, reference = field_with_guard_sentinels(self.static, (8, 2, 1)) + snapshot = async_writer.make_tiled_field_snapshot({"lapse": field}, step=0, time=0.) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + self._write_per_process(folder, snapshot, "fields") + # One tile per batch forces more than one load/store/flush cycle. + merge_openpmd.merge_paths([str(folder)], max_pending_bytes=1) + series = io.Series(str(folder/"fields.h5"), io.Access.read_only) + try: + record = series.iterations[0].meshes["lapse"][io.Mesh_Record_Component.SCALAR] + values = record.load_chunk() + series.flush() + self.assertTrue(bool(jnp.allclose(jnp.asarray(values), reference))) + finally: + series.close() + + def test_merge_rejects_an_incomplete_group(self): + field, _ = field_with_guard_sentinels(self.static, (8, 2, 1)) + snapshot = async_writer.make_tiled_field_snapshot({"lapse": field}, step=0, time=0.) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + self._write_per_process(folder, snapshot, "fields") + (folder/"fields_p001.h5").unlink() + with self.assertRaisesRegex(ValueError, "Expected processes 0..1"): + merge_openpmd.merge_paths([str(folder)]) + + def test_ownership_metadata_partitions_the_tile_grid(self): + field, _ = field_with_guard_sentinels(self.static, (8, 2, 1)) + snapshot = async_writer.make_tiled_field_snapshot({"lapse": field}, step=0, time=0.) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + self._write_per_process(folder, snapshot, "fields") + sources = [io.Series(str(folder/f"fields_p00{index}.h5"), io.Access.read_only) + for index in range(2)] + try: + geometry = merge_openpmd.read_tile_geometry( + [series.iterations[0] for series in sources]) + self.assertEqual(geometry.global_shape, (8, 2, 1)) + self.assertEqual(geometry.tile_counts, (2, 1, 1)) + merge_openpmd.validate_tile_coverage(geometry) + self.assertEqual([ids.tolist() for ids in geometry.owned], [[0], [1]]) + finally: + for series in sources: + series.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/distributed_tests/multihost_io_worker.py b/tests/distributed_tests/multihost_io_worker.py deleted file mode 100644 index a7062da..0000000 --- a/tests/distributed_tests/multihost_io_worker.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Two-rank native-input/shared-output integration test, launched with MPI. - -PYPIC_DISTRIBUTED=1 JAX_PLATFORMS=cpu JAX_NUM_CPU_DEVICES=1 mpiexec -n 2 \ - python -m tests.distributed_tests.multihost_io_worker --directory /tmp/test-io \ - --require-mpi-output -""" - -import argparse -import os -import socket -from mpi4py import MPI -import jax -import jax.numpy as jnp - -jax.config.update("jax_enable_x64", True) - -from tests.kernel_fixtures import kernel_parameters -from PyPIC3D.distributed import tiled_field_from_initializer -from PyPIC3D.relativity.core import D_FIELD_LOCATIONS -from PyPIC3D.particles.distributed_initialization import load_distributed_particles -from PyPIC3D.diagnostics.async_writer import make_tiled_field_snapshot, make_tiled_particle_snapshot -from PyPIC3D.diagnostics.parallel_openpmd import open_parallel_series, write_parallel_snapshot -import openpmd_api as io - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--directory", required=True) - parser.add_argument("--require-mpi-output", action="store_true") - parser.add_argument("--evolve", action="store_true") - args = parser.parse_args() - comm = MPI.COMM_WORLD - assert comm.size == 2 and jax.process_count() == 2 - print(f"rank={comm.rank} host={socket.gethostname()} local_devices={jax.local_devices()} " - f"global_devices={jax.devices()}", flush=True) - static, dynamic = kernel_parameters(Nx=8, Ny=4, Nz=4, tile_shape=(4, 4, 4), guard_cells=2, - boundary_conditions=(1, 1, 1)) - block = dict(name="ions", N_particles=7, charge=1.0, mass=1.0, temperature=0.0) - for name in ("x", "y", "z", "vx", "vy", "vz"): - block["initial_"+name] = os.path.join(args.directory, name+".npy") - if comm.rank == 0: - os.makedirs(args.directory, exist_ok=True) - for name in ("x", "y", "z", "vx", "vy", "vz"): - values = jnp.asarray([-0.5, -1.5, -1.0, -0.2, -0.7, -1.8, -0.3]) if name == "x" else jnp.zeros(7) - jnp.save(block["initial_"+name], values) - comm.Barrier() - particles, species, names, metadata = load_distributed_particles( - {"particle1": block}, static, dynamic, comm=comm, chunk_size=2) - assert len(particles.x.addressable_shards) == 1 - shard = particles.active.addressable_shards[0] - tile = shard.index[0].start - assert int(jnp.sum(shard.data)) == (7 if tile == 0 else 0) - expected = jnp.asarray([-0.5, -1.5, -1.0, -0.2, -0.7, -1.8, -0.3]) - if tile == 0: - assert bool(jnp.array_equal(particles.x.addressable_shards[0].data[0, 0, 0, 0, :7, 0], expected)) - field = tiled_field_from_initializer(lambda x, y, z: x+2*y+3*z, static, dynamic) - assert len(field.addressable_shards) == 1 - assert field.addressable_shards[0].device.process_index == comm.rank - from PyPIC3D.utilities.toml_helpers import dump_parameters_to_toml - if comm.rank == 0: - os.makedirs(os.path.join(args.directory, "data"), exist_ok=True) - comm.Barrier() - dump_parameters_to_toml({}, static._replace(output_dir=args.directory), dynamic, {}, - {"particle_species_names": names, "particle_species_metadata": metadata}, particles) - comm.Barrier() - if comm.rank == 0: - import toml - output = toml.load(os.path.join(args.directory, "data", "output.toml")) - assert output["particles"][0]["active_particles"] == 7 - if args.require_mpi_output or io.variants.get("mpi", False): - electric = tuple(tiled_field_from_initializer(lambda x, y, z: x+2*y+3*z, static, dynamic, - location=location) for location in D_FIELD_LOCATIONS) - field_series = open_parallel_series(args.directory, comm, filename="fields.h5") - particle_series = open_parallel_series(args.directory, comm, filename="particles.h5") - for step in range(2): - fields = make_tiled_field_snapshot({"lapse": field+step, "E": electric}, - step=step, time=step*float(dynamic.dt)) - snapshot = make_tiled_particle_snapshot(particles, step=step, time=fields.time, - species_names=names, species_config=species) - write_parallel_snapshot(field_series, comm, static, dynamic, fields=fields) - write_parallel_snapshot(particle_series, comm, static, dynamic, particles=snapshot) - field_series.close() - particle_series.close() - comm.Barrier() - if comm.rank == 0: - series = io.Series(os.path.join(args.directory, "fields.h5"), io.Access.read_only) - lapse = series.iterations[1].meshes["lapse"][io.Mesh_Record_Component.SCALAR] - assert tuple(lapse.shape) == (8, 4, 4) - values = lapse.load_chunk() - electric_values = [] - for component in "xyz": - record = series.iterations[1].meshes["E"][component] - assert tuple(record.shape) == (8, 4, 4) - electric_values.append(record.load_chunk()) - particle_series = io.Series(os.path.join(args.directory, "particles.h5"), io.Access.read_only) - record = particle_series.iterations[1].particles["ions"]["position"]["x"] - assert tuple(record.shape) == (7,) - positions = record.load_chunk() - series.flush() - particle_series.flush() - coordinates = [axis[1] + jnp.arange(n)*spacing for axis, n, spacing in - zip(dynamic.grids.center, (8, 4, 4), (dynamic.dx, dynamic.dy, dynamic.dz))] - analytical = (coordinates[0][:, None, None] + 2*coordinates[1][None, :, None] - + 3*coordinates[2][None, None, :] + 1) - assert bool(jnp.array_equal(jnp.asarray(values), analytical)) - for component, data in enumerate(electric_values): - coordinates = [axis[1] + (jnp.arange(n)+0.5*(a == component))*spacing - for a, (axis, n, spacing) in enumerate(zip( - dynamic.grids.center, data.shape, (dynamic.dx, dynamic.dy, dynamic.dz)))] - analytical = (coordinates[0][:, None, None] + 2*coordinates[1][None, :, None] - + 3*coordinates[2][None, None, :]) - assert bool(jnp.array_equal(jnp.asarray(data), analytical)) - assert bool(jnp.array_equal(jnp.asarray(positions), expected)) - series.close() - particle_series.close() - print("MPI output roundtrip passed: uniform mesh chunks and empty-rank particles", flush=True) - comm.Barrier() - elif comm.rank == 0: - print("Native MPI input passed; shared MPI output not run because openPMD has mpi=False", flush=True) - if args.evolve: - from PyPIC3D.__main__ import run_PyPIC3D - write = args.require_mpi_output or io.variants.get("mpi", False) - config = { - "simulation_parameters": dict(solver="dynamic_metric", output_dir=os.path.join(args.directory, "evolution"), - Nx=8, Ny=4, Nz=4, x_wind=4.0, y_wind=2.0, z_wind=2.0, - particle_tile_nx=4, particle_tile_ny=4, particle_tile_nz=4, - Nt=2, dt=1.e-4, metric_backreaction=True, benchmark=True), - "plotting": dict(plot_openpmd_fields=write, plot_openpmd_particles=write, plotting_interval=1, - dump_fields=write, dump_particles=write), - "particle1": dict(block, charge=0.0), - } - result = run_PyPIC3D(config, field_initializers={"Ex": lambda x, y, z: 1.e-3*jnp.sin(jnp.pi*x/2)}) - state = result[1].bssn_parameters.state - assert len(state.conformal_factor.addressable_shards) == 1 - assert not state.conformal_factor.is_fully_addressable - assert float(result[1].bssn_parameters.time) == 2.e-4 - assert all(bool(jnp.all(jnp.isfinite(leaf.addressable_shards[0].data))) for leaf in state) - assert float(jnp.max(jnp.abs(state.trace_K))) > 0 - # Writing diagnostics must not change any evolved field or particle state. - quiet_config = {**config, "simulation_parameters": {**config["simulation_parameters"], - "output_dir": os.path.join(args.directory, "evolution_without_output")}, - "plotting": dict(plotting_interval=1)} - quiet = run_PyPIC3D(quiet_config, field_initializers={"Ex": lambda x, y, z: 1.e-3*jnp.sin(jnp.pi*x/2)}) - for left, right in zip(jax.tree.leaves((result[4], result[5], state)), - jax.tree.leaves((quiet[4], quiet[5], quiet[1].bssn_parameters.state))): - for a, b in zip(left.addressable_shards, right.addressable_shards): - assert bool(jnp.array_equal(a.data, b.data)) - local_bytes = sum(shard.data.nbytes for leaf in state for shard in leaf.addressable_shards) - print(f"rank={comm.rank} BSSN local state bytes={local_bytes}", flush=True) - from PyPIC3D.utilities.toml_helpers import dump_parameters_to_toml - dump_parameters_to_toml({"total_iterations": 2}, result[0], result[1], result[3], result[2], result[4]) - comm.Barrier() - if comm.rank == 0: - import toml - output = toml.load(os.path.join(args.directory, "evolution", "data", "output.toml")) - assert output["particles"][0]["active_particles"] == 7 - if write: - output_dir = os.path.join(args.directory, "evolution", "data") - assert os.path.isfile(os.path.join(output_dir, "initial_fields", "initial_fields.h5")) - assert os.path.isfile(os.path.join(output_dir, "initial_particles", "ions_initial_particles.h5")) - assert os.path.isfile(os.path.join(output_dir, "particles.h5")) - assert not os.path.exists(os.path.join(output_dir, "simulation.h5")) - series = io.Series(os.path.join(output_dir, "fields.h5"), io.Access.read_only) - expected_records = {"E", "B", "J", "conformal_factor", "trace_K", "lapse", - "conformal_connection", "shift"} - expected_records.update(prefix+"_"+suffix for prefix in ("conformal_metric", "traceless_K") - for suffix in ("xx", "xy", "xz", "yy", "yz", "zz")) - assert list(series.iterations) == [0, 1] - for step in series.iterations: - iteration = series.iterations[step] - assert set(iteration.meshes) == expected_records - assert iteration.time == step*1.e-4 - values = [] - for name in expected_records: - components = ("x", "y", "z") if name in ("E", "B", "J", "conformal_connection", "shift") else (io.Mesh_Record_Component.SCALAR,) - assert set(iteration.meshes[name]) == set(components) - for component in components: - record = iteration.meshes[name][component] - assert tuple(record.shape) == (8, 4, 4) - values.append(record.load_chunk()) - series.flush() - assert all(bool(jnp.all(jnp.isfinite(jnp.asarray(value)))) for value in values) - series.close() - print("Actual two-process initialize_simulation/run_PyPIC3D steps passed", flush=True) - comm.Barrier() - if comm.rank == 0: - print("Two-process native input/local-shard checks passed", flush=True) - jax.distributed.shutdown() - - -if __name__ == "__main__": - try: - main() - except BaseException: - import traceback - traceback.print_exc() - MPI.COMM_WORLD.Abort(1) diff --git a/tests/distributed_tests/sharded_openpmd_worker.py b/tests/distributed_tests/sharded_openpmd_worker.py deleted file mode 100644 index a6edf81..0000000 --- a/tests/distributed_tests/sharded_openpmd_worker.py +++ /dev/null @@ -1,236 +0,0 @@ -"""MPI output acceptance: two ranks, one or several addressable tiles per rank.""" - -import argparse -from dataclasses import replace -import os -from unittest.mock import patch - -from mpi4py import MPI -import jax -import jax.numpy as jnp -from jax.sharding import NamedSharding, PartitionSpec as P -import openpmd_api as io - -from tests.kernel_fixtures import kernel_parameters -from PyPIC3D.distributed import tiled_field_from_initializer -from PyPIC3D.particles.particle_class import TiledParticles, SpeciesConfig -from PyPIC3D.diagnostics.async_writer import make_tiled_field_snapshot, make_tiled_particle_snapshot -from PyPIC3D.diagnostics import openPMD -from PyPIC3D.diagnostics.openPMD import ( - TiledMeshLayout, write_tiled_field_snapshot_openpmd, write_tiled_particle_snapshot_openpmd, - write_openpmd_initial_fields, write_openpmd_initial_particles, -) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--directory", required=True) - parser.add_argument("--fail-rank", type=int, default=-1) - parser.add_argument("--local-devices", type=int, default=int(os.environ.get("JAX_NUM_CPU_DEVICES", "1"))) - parser.add_argument("--require-gpu", action="store_true", - help="Fail if JAX uses a CPU backend instead of the assigned GPUs.") - args = parser.parse_args() - comm = MPI.COMM_WORLD - assert comm.size == jax.process_count() == 2 - assert jax.local_device_count() == args.local_devices - local_devices = jax.local_devices() - if args.require_gpu: - assert all(device.platform == "gpu" for device in local_devices), \ - f"--require-gpu requires GPU devices; JAX selected {local_devices}" - device_info = [(device.id, device.local_hardware_id, device.platform, device.device_kind) - for device in local_devices] - print(f"MPI_SHARD_START rank={comm.rank} host={MPI.Get_processor_name()} " - f"jax_process={jax.process_index()}/{jax.process_count()} " - f"devices(id,local_hardware_id,platform,kind)={device_info} " - f"CUDA_VISIBLE_DEVICES={os.environ.get('CUDA_VISIBLE_DEVICES', 'unset')} " - f"jax={jax.__version__} openpmd={io.__version__} openpmd_mpi={io.variants.get('mpi', False)}", - flush=True) - ntiles = jax.device_count() - static, dynamic = kernel_parameters( - Nx=4*ntiles, Ny=1, Nz=4, x_wind=2*ntiles, y_wind=1., z_wind=2., - tile_shape=(4, 1, 4), guard_cells=2, boundary_conditions=(1, 0, 1), C=10., - ) - layout = TiledMeshLayout((4*ntiles, 1, 4), (4, 1, 4), guard_cells=2) - field = tiled_field_from_initializer(lambda x, y, z: x+2*y+3*z, static, dynamic) - assert len(field.addressable_shards) == args.local_devices - names = ("one tile", "uneven", "empty") - species = SpeciesConfig(jnp.array([-1., 1., 0.]), jnp.array([2., 3., 1.]), - jnp.array([4., 5., 1.]), jnp.ones((3, 3), dtype=bool)) - - def positions(index): - tile = index[0].start - value = jnp.zeros((1, 1, 1, 3, 3, 3), dtype=jnp.float64) - value = value.at[..., 0].set(-ntiles+2*tile+.25+.25*jnp.arange(3)) - return jax.device_get(value) - - def velocities(index): - value = jnp.zeros((1, 1, 1, 3, 3, 3), dtype=jnp.float64) - return jax.device_get(value.at[..., 0].set(.2)) - - def active(index): - tile = index[0].start - value = jnp.zeros((1, 1, 1, 3, 3), dtype=bool) - value = value.at[0, 0, 0, 0, 0].set(tile == 0) - value = value.at[0, 0, 0, 1, :].set(jnp.arange(3) < 1+tile % 3) - return jax.device_get(value) - - sharding = NamedSharding(static.field_mesh, P("tile_x", "tile_y", "tile_z")) - shape = (ntiles, 1, 1, 3, 3, 3) - particles = TiledParticles( - jax.make_array_from_callback(shape, sharding, positions), - jax.make_array_from_callback(shape, sharding, velocities), - jax.make_array_from_callback(shape[:-1], sharding, active), - ) - assert not particles.x.is_fully_addressable - assert len(particles.x.addressable_shards) == args.local_devices - print(f"MPI_SHARD_ARRAYS rank={comm.rank} global_tiles={ntiles} " - f"field_devices={[str(shard.device) for shard in field.addressable_shards]} " - f"particle_devices={[str(shard.device) for shard in particles.x.addressable_shards]}", - flush=True) - # Reject full simulation-array transfers; tiny replicated species metadata is allowed. - device_get = jax.device_get - def local_device_get(value): - if isinstance(value, jax.Array) and not value.is_fully_addressable: - raise AssertionError("output transferred a global distributed array") - return device_get(value) - - field_reads = [(os.path.join(args.directory, "data", "initial_fields", "initial_fields.h5"), 0, field)] - with patch("jax.device_get", side_effect=local_device_get): - initial = make_tiled_particle_snapshot(particles, step=0, time=0., species_names=names, - species_config=species, initial=True) - local_bytes = sum(data.nbytes for _, data in initial.x_shards) - assert local_bytes == sum(shard.data.nbytes for shard in particles.x.addressable_shards) - assert local_bytes < particles.x.nbytes - # Each component has an independently permuted host-shard list. - initial = replace(initial, x_shards=list(reversed(initial.x_shards)), - active_shards=list(reversed(initial.active_shards))) - if args.fail_rank >= 0: - abort = openPMD._abort_parallel_write - def record_writer_abort(communicator): - print(f"WRITER_ABORT rank={comm.rank}", flush=True) - abort(communicator) - with patch.object(openPMD, "_abort_parallel_write", side_effect=record_writer_abort): - if comm.rank == args.fail_rank: - with patch.object(openPMD, "_iter_snapshot_particle_chunks", - side_effect=RuntimeError("injected rank-local output failure")): - write_tiled_particle_snapshot_openpmd( - initial, output_dir=args.directory, filename="failure", static_parameters=static, - dynamic_parameters=dynamic, file_extension=".h5", comm=comm) - else: - write_tiled_particle_snapshot_openpmd( - initial, output_dir=args.directory, filename="failure", static_parameters=static, - dynamic_parameters=dynamic, file_extension=".h5", comm=comm) - raise AssertionError("injected failure did not abort MPI") - - write_openpmd_initial_fields({"rho": field}, static, dynamic, args.directory, comm=comm) - write_openpmd_initial_particles(particles, static, dynamic, args.directory, - species_config=species, species_names=names, comm=comm) - for mode in ("fields", "particles", "both"): - folder = os.path.join(args.directory, mode) - for step in range(2): - if mode != "particles": - output_field = field+step - snapshot = make_tiled_field_snapshot({"rho": output_field}, step=step, time=step*dynamic.dt) - write_tiled_field_snapshot_openpmd(snapshot, output_dir=folder, filename="fields", - dynamic_parameters=dynamic, layout=layout, file_extension=".h5", comm=comm) - field_reads.append((os.path.join(folder, "fields.h5"), step, output_field)) - if mode != "fields": - snapshot = replace(initial, step=step, time=step*float(dynamic.dt), initial=False) - write_tiled_particle_snapshot_openpmd(snapshot, output_dir=folder, filename="particles", - static_parameters=static, dynamic_parameters=dynamic, file_extension=".h5", comm=comm) - - comm.Barrier() - # Exact I/O fidelity is checked against the arrays actually submitted to - # the writer. Each rank reads only its own tile interiors, without a gather. - for path, step, source in field_reads: - series = io.Series(path, io.Access.read_only) - record = series.iterations[step].meshes["rho"][io.Mesh_Record_Component.SCALAR] - pending = [] - for shard in source.addressable_shards: - offset = [(part.start or 0)*width for part, width in zip(shard.index[:3], static.tile_shape)] - data = record.load_chunk(offset, list(static.tile_shape)) - g = static.guard_cells - expected = shard.data[(0, 0, 0) + tuple(slice(g, g+width) for width in static.tile_shape)] - pending.append((data, expected)) - series.flush() - for data, expected in pending: - assert bool(jnp.array_equal(jnp.asarray(data), expected)), \ - f"Field readback changed source values: rank={comm.rank}, step={step}, path={path}" - series.close() - print(f"MPI_SHARD_EXACT rank={comm.rank} field_snapshots={len(field_reads)} " - f"local_tiles={args.local_devices} bitwise_readback=passed", flush=True) - comm.Barrier() - if comm.rank == 0: - x = -ntiles + .5*jnp.arange(4*ntiles) - y = -0.5+jnp.arange(1) - z = -1.+.5*jnp.arange(4) - field_reference = x[:, None, None]+2*y[None, :, None]+3*z[None, None, :] - particle_reference = ( - jnp.array([-ntiles+.25]), - jnp.array([-ntiles+2*tile+.25+.25*slot for tile in range(ntiles) - for slot in range(1+tile % 3)]), - jnp.empty(0), - ) - series = io.Series(os.path.join(args.directory, "data", "initial_fields", "initial_fields.h5"), - io.Access.read_only) - record = series.iterations[0].meshes["rho"][io.Mesh_Record_Component.SCALAR] - assert tuple(record.shape) == layout.global_shape - data = record.load_chunk() - series.flush() - # GPU coordinate construction can differ from the independent analytic - # grid by float64 roundoff; source-to-file equality is checked above. - assert bool(jnp.allclose(jnp.asarray(data), field_reference, rtol=0, atol=1.e-14)) - series.close() - for mode in ("fields", "particles", "both"): - folder = os.path.join(args.directory, mode) - for kind in ("fields", "particles"): - path = os.path.join(folder, kind+".h5") - if kind != mode and mode != "both": - assert not os.path.exists(path) - continue - series = io.Series(path, io.Access.read_only) - assert series.get_attribute("software") == "PyPIC3D" - assert list(series.iterations) == [0, 1] - for step in series.iterations: - iteration = series.iterations[step] - assert iteration.time == step*float(dynamic.dt) - if kind == "fields": - record = iteration.meshes["rho"][io.Mesh_Record_Component.SCALAR] - assert tuple(record.shape) == layout.global_shape - data = record.load_chunk() - series.flush() - assert bool(jnp.allclose(jnp.asarray(data), field_reference+step, rtol=0, atol=1.e-14)) - else: - for name, expected in zip(names, particle_reference): - group = iteration.particles[name.replace(" ", "_")] - record = group["position"]["x"] - assert tuple(record.shape) == expected.shape - if expected.size: - data = record.load_chunk() - series.flush() - assert bool(jnp.allclose(jnp.asarray(data), expected-.2*dynamic.dt/2, - rtol=0, atol=1.e-14)) - series.close() - for name, expected in zip(names, particle_reference): - path = os.path.join(args.directory, "data", "initial_particles", - name.replace(" ", "_")+"_initial_particles.h5") - series = io.Series(path, io.Access.read_only) - record = series.iterations[0].particles[name.replace(" ", "_")]["position"]["x"] - assert tuple(record.shape) == expected.shape - if expected.size: - data = record.load_chunk() - series.flush() - assert bool(jnp.array_equal(jnp.asarray(data), expected)) - series.close() - print(f"MPI shard output passed: {comm.size} ranks, {ntiles} tiles, fields/particles/initial readback", flush=True) - comm.Barrier() - jax.distributed.shutdown() - - -if __name__ == "__main__": - try: - main() - except BaseException: - import traceback - traceback.print_exc() - MPI.COMM_WORLD.Abort(1) diff --git a/tests/distributed_tests/yee_test.py b/tests/distributed_tests/yee_test.py index da1352b..a56662e 100644 --- a/tests/distributed_tests/yee_test.py +++ b/tests/distributed_tests/yee_test.py @@ -4,6 +4,7 @@ SimpleNamespace, YeeTiledFixtures, assemble_tiled_vector_field, + ghost_cells, build_tiled_yee_grids, build_yee_grid, jnp, @@ -29,35 +30,32 @@ def test_tile_vector_field_assembles_to_original_ghost_celled_field(self): def test_tile_grid_axes_include_configured_guard_cells(self): parameter_set = self._build_parameter_values() + parameter_set.update(x_min=1.25, y_min=-3.5, z_min=2.0, dx=0.25, dy=0.5, dz=0.75) tile_shape = (4, 3, 2) - g = 2 center_grid, vertex_grid = build_yee_grid(SimpleNamespace(**parameter_set)) - parameter_set["grids"] = {"center": center_grid, "vertex": vertex_grid} - static_parameters = SimpleNamespace(tile_shape=tile_shape, guard_cells=g) dynamic_parameters = SimpleNamespace( dx=parameter_set["dx"], dy=parameter_set["dy"], dz=parameter_set["dz"], grids=SimpleNamespace(center=center_grid, vertex=vertex_grid), ) - tiled_center_grid, tiled_vertex_grid = build_tiled_yee_grids(static_parameters, dynamic_parameters) - - self.assertEqual(tiled_center_grid[0].shape, (2, 2, 2, tile_shape[0] + 2 * g)) - self.assertEqual(tiled_center_grid[1].shape, (2, 2, 2, tile_shape[1] + 2 * g)) - self.assertEqual(tiled_center_grid[2].shape, (2, 2, 2, tile_shape[2] + 2 * g)) - - for tx in range(2): - for ty in range(2): - for tz in range(2): - center_x = parameter_set["grids"]["center"][0][0] + ( - jnp.arange(tile_shape[0] + 2 * g) + tx * tile_shape[0] - (g - 1) - ) * parameter_set["dx"] - vertex_y = parameter_set["grids"]["vertex"][1][0] + ( - jnp.arange(tile_shape[1] + 2 * g) + ty * tile_shape[1] - (g - 1) - ) * parameter_set["dy"] - - self.assertTrue(jnp.allclose(tiled_center_grid[0][tx, ty, tz], center_x)) - self.assertTrue(jnp.allclose(tiled_vertex_grid[1][tx, ty, tz], vertex_y)) + mesh = ghost_cells.make_field_mesh((2, 2, 2)) + for field_mesh in (None, mesh): + for g in (1, 2, 3): + with self.subTest(sharded=field_mesh is not None, guard_cells=g): + static_parameters = SimpleNamespace(tile_shape=tile_shape, guard_cells=g, field_mesh=field_mesh) + tiled_grids = build_tiled_yee_grids(static_parameters, dynamic_parameters) + for tiled_grid, stagger in zip(tiled_grids, (0.0, 0.5)): + for axis, name in enumerate("xyz"): + width = tile_shape[axis] + coordinates = tiled_grid[axis] + self.assertEqual(coordinates.shape, (2, 2, 2, width + 2 * g)) + for tile in range(2): + expected = parameter_set[name + "_min"] + ( + tile * width + jnp.arange(-g, width + g) + stagger + ) * parameter_set["d" + name] + actual = jnp.take(coordinates, tile, axis=axis) + self.assertTrue(jnp.allclose(actual, expected, rtol=1.e-12, atol=1.e-12)) def test_update_E_matches_single_tile_yee_update(self): diff --git a/tests/kernel_fixtures.py b/tests/kernel_fixtures.py index 0235135..7a4eabc 100644 --- a/tests/kernel_fixtures.py +++ b/tests/kernel_fixtures.py @@ -71,7 +71,6 @@ def kernel_parameters( output_dir=".", Nt=1, verbose=False, - benchmark=False, pml_active=False, supergaussian_active=False, supergaussian_layers=(), @@ -107,7 +106,6 @@ def kernel_parameters( output_dir=output_dir, Nt=int(Nt), verbose=bool(verbose), - benchmark=bool(benchmark), solver=solver, electrostatic=bool(electrostatic), relativistic=bool(relativistic), @@ -263,7 +261,6 @@ def kernel_parameters_from_values(parameter_set, dynamic_values=None): output_dir=parameter_set.get("output_dir", "."), Nt=parameter_set.get("Nt", 1), verbose=parameter_set.get("verbose", False), - benchmark=parameter_set.get("benchmark", False), pml_active=parameter_set.get("pml_active", False), supergaussian_active=parameter_set.get("supergaussian", (False, False, False, False, ()))[0], supergaussian_layers=parameter_set.get("supergaussian", (False, False, False, False, ()))[-1], @@ -301,7 +298,6 @@ def particle_parameters_from_values(parameter_set, tile_shape=None, dynamic_valu output_dir=parameter_set.get("output_dir", "."), Nt=int(parameter_set.get("Nt", 1)), verbose=bool(parameter_set.get("verbose", False)), - benchmark=bool(parameter_set.get("benchmark", False)), solver=parameter_set.get("solver", "electrodynamic_yee"), electrostatic=bool(parameter_set.get("electrostatic", False)), relativistic=bool(parameter_set.get("relativistic", True)), diff --git a/tests/support/dark_photon_fixtures.py b/tests/support/dark_photon_fixtures.py index 4678e32..77f9b5f 100644 --- a/tests/support/dark_photon_fixtures.py +++ b/tests/support/dark_photon_fixtures.py @@ -18,7 +18,8 @@ from PyPIC3D.__main__ import run_PyPIC3D from PyPIC3D.boundary_conditions.ghost_cells import update_tiled_ghost_cells, update_tiled_ghost_cells -from PyPIC3D.initialization import initialize_simulation, initialize_fields, _dark_timestep, default_parameters +from PyPIC3D.utilities.simulation_helpers import dark_photon_courant_condition +from PyPIC3D.initialization import initialize_simulation, initialize_fields, default_parameters from PyPIC3D.diagnostics.diagnostic_quantities import compute_dark_energy from PyPIC3D.deposition.rho import compute_rho from PyPIC3D.diagnostics.output_adapters import build_field_output_map, field_map_for_output, scalar_field_for_output diff --git a/tests/support/openpmd_fixtures.py b/tests/support/openpmd_fixtures.py index 40f4669..a8cb94e 100644 --- a/tests/support/openpmd_fixtures.py +++ b/tests/support/openpmd_fixtures.py @@ -26,7 +26,30 @@ ) -class FakeRecord: +class FakeAttributable: + """openPMD attribute surface the writers and the merge tool both use.""" + + def _attributes(self): + store = self.__dict__.get("_attribute_store") + if store is None: + store = self.__dict__["_attribute_store"] = {} + return store + + def set_attribute(self, name, value): + self._attributes()[name] = value + + def get_attribute(self, name): + return self._attributes()[name] + + def contains_attribute(self, name): + return name in self._attributes() + + @property + def attributes(self): + return list(self._attributes()) + + +class FakeRecord(FakeAttributable): def __init__(self): self.shape = None self.dataset_shape = None @@ -43,6 +66,10 @@ def reset_dataset(self, dataset): def store_chunk(self, array, offset, extent): self.chunks.append((tuple(offset), tuple(extent), array.copy())) + def load_chunk(self, offset, extent): + region = tuple(slice(start, start + length) for start, length in zip(offset, extent)) + return np.asarray(self.data)[region] + @property def data(self): values = jnp.zeros(self.dataset_shape) @@ -52,7 +79,7 @@ def data(self): return values -class FakeMesh: +class FakeMesh(FakeAttributable): def __init__(self): self.records = {} self.axis_labels = None @@ -80,7 +107,7 @@ def __getitem__(self, component): return dict.__getitem__(self, component) -class FakeParticleSpecies(dict): +class FakeParticleSpecies(FakeAttributable, dict): def __getitem__(self, record_name): if record_name not in self: if record_name in ("position", "positionOffset", "momentum"): @@ -97,7 +124,7 @@ def __getitem__(self, species_name): return dict.__getitem__(self, species_name) -class FakeIteration: +class FakeIteration(FakeAttributable): def __init__(self): self.meshes = FakeMeshes() self.particles = FakeParticles() @@ -113,13 +140,10 @@ def __getitem__(self, iteration): return dict.__getitem__(self, iteration) -class FakeSeries: +class FakeSeries(FakeAttributable): def __init__(self): self.iterations = FakeIterations() - def set_attribute(self, name, value): - pass - def flush(self): pass diff --git a/tests/support/particle_refresh_fixtures.py b/tests/support/particle_refresh_fixtures.py index 55ec902..c31c471 100644 --- a/tests/support/particle_refresh_fixtures.py +++ b/tests/support/particle_refresh_fixtures.py @@ -9,7 +9,6 @@ from tests.kernel_fixtures import build_tiled_particles, particle_parameters_from_tile_values, particle_species from tests.kernel_fixtures import kernel_parameters_from_values from PyPIC3D.particles.particle_tile_communication import ( - _adjacent_tile_offset, refresh_tiled_particle_tiles, update_tiled_particle_positions, )