Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
337 changes: 337 additions & 0 deletions PyPIC3D/collisions.py

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions PyPIC3D/diagnostics/gr_work_energy_theorem.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,10 +263,10 @@ def sample_energy(particles, species, fields, static, dynamic):
from PyPIC3D.relativity.dynamical_relativity import bssn_tile_layout
lapse_gradient = map_tiled(
lambda lapse, params: jnp.stack(tuple(diff1_physical(lapse, axis, params) for axis in range(3))),
state.lapse, bssn.evolution_parameters, **bssn_tile_layout(static),
state.lapse, static.bssn_evolution_parameters, bssn_tile_layout(static),
output_template=jnp.zeros((3,) + state.lapse.shape, dtype=state.lapse.dtype))
else:
lapse_gradient = jnp.stack(tuple(diff1_physical(state.lapse, axis, bssn.evolution_parameters)
lapse_gradient = jnp.stack(tuple(diff1_physical(state.lapse, axis, static.bssn_evolution_parameters)
for axis in range(3)))
curvature_power = volume_integral(
volume * state.lapse * jnp.einsum("ij...,ij...->...", K, stress_up), static, dynamic)
Expand Down
6 changes: 3 additions & 3 deletions PyPIC3D/diagnostics/static_metric.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ def bssn_constraint_norms(particles, species, fields, static, dynamic, *, source
if tiled:
from JAX_BSSN.parallel import tiled_constraint_norms
from PyPIC3D.relativity.dynamical_relativity import bssn_tile_layout
return tiled_constraint_norms(bssn.state, bssn.evolution_parameters, rho=rho,
momentum=momentum, **bssn_tile_layout(static))
return tiled_constraint_norms(bssn.state, static.bssn_evolution_parameters,
bssn_tile_layout(static), rho=rho, momentum=momentum)
violations = compute_all_constraints_with_matter(
bssn.state, bssn.evolution_parameters, rho, momentum)
bssn.state, static.bssn_evolution_parameters, rho, momentum)
return compute_constraint_norms(violations)


Expand Down
11 changes: 9 additions & 2 deletions PyPIC3D/initialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import jax.numpy as jnp

from PyPIC3D.relativity.field_state import densitize_vector
from PyPIC3D.collisions import load_collisions_from_toml
from PyPIC3D.particles.particle_initialization import load_particles_from_toml
from PyPIC3D.pusher.particle_push import seed_leapfrog_velocity
from PyPIC3D.particles.particle_tile_communication import shard_tiled_particles
Expand Down Expand Up @@ -649,7 +650,9 @@ def initialize_simulation(toml_file, *, bssn_state=None, bssn_evolution_paramete
from PyPIC3D.relativity.bssn_parameters import load_bssn_parameters_from_toml
from PyPIC3D.relativity.dynamical_relativity import build_bssn_metric

bssn = load_bssn_parameters_from_toml(
# The evolution controls are run constants that JAX_BSSN reads concretely,
# so the loader hands them back on static_parameters.
static_parameters, bssn = load_bssn_parameters_from_toml(
config, static_parameters, dynamic_parameters, state=bssn_state,
evolution_parameters=bssn_evolution_parameters,
)
Expand All @@ -676,7 +679,11 @@ def initialize_simulation(toml_file, *, bssn_state=None, bssn_evolution_paramete
"Reducing particle_batch_size from "
f"{static_parameters.particle_batch_size} to tile capacity {effective_batch_size}."
)
static_parameters = static_parameters._replace(particle_batch_size=effective_batch_size)
static_parameters = static_parameters._replace(
particle_batch_size=effective_batch_size,
collisions=load_collisions_from_toml(config, particle_species_names, species_config.weight),
)
particles = particles._replace(key=jax.random.PRNGKey(int(config.get("simulation_parameters", {}).get("seed", 0))))
particles = shard_tiled_particles(particles, static_parameters)
if static_parameters.lapse_deleting:
# Initial output and leapfrog histories must use the surviving ensemble.
Expand Down
2 changes: 2 additions & 0 deletions PyPIC3D/particles/particle_class.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,5 @@ class TiledParticles(NamedTuple):
# velocities at half time steps, u^{n-1/2}; covariant u_i for static_metric
active: jax.Array # (ntx, nty, ntz, species, max_particles_per_tile)
# boolean array indicating whether the particle is active or not
key: jax.Array = None
# PRNG key advanced by stochastic operators such as collisions
4 changes: 2 additions & 2 deletions PyPIC3D/particles/particle_tile_communication.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def shard_tiled_particles(tiled_particles, static_parameters):
state_sharding = NamedSharding(mesh, PARTICLE_STATE_TILE_SPEC)
active_sharding = NamedSharding(mesh, PARTICLE_ACTIVE_TILE_SPEC)

return TiledParticles(
return tiled_particles._replace(
x=jax.device_put(tiled_particles.x, state_sharding),
u=jax.device_put(tiled_particles.u, state_sharding),
active=jax.device_put(tiled_particles.active, active_sharding),
Expand Down Expand Up @@ -451,7 +451,7 @@ def refresh(tiled_particles, dynamic_parameters):
tiled_particles.active,
dynamic_parameters,
)
return TiledParticles(x=x, u=u, active=active), overflow
return tiled_particles._replace(x=x, u=u, active=active), overflow

return refresh

Expand Down
5 changes: 2 additions & 3 deletions PyPIC3D/pusher/hybrid_boris_geodesic.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import jax.numpy as jnp
from jax.sharding import PartitionSpec as P

from PyPIC3D.particles.particle_class import TiledParticles
from PyPIC3D.particles.particle_batching import (
prepare_particle_batches, particle_batch_indices, number_of_particle_batches,
)
Expand Down Expand Up @@ -318,6 +317,6 @@ def batches_remaining(batch_state):
particles.x, particles.u, particles.active, prepared_fields,
(species_q_over_m, species_config.update_x, dt))

full_step = TiledParticles(x=x_new, u=u_new, active=particles.active)
half_step = TiledParticles(x=x_half, u=u_new, active=particles.active)
full_step = particles._replace(x=x_new, u=u_new)
half_step = particles._replace(x=x_half, u=u_new)
return full_step, half_step
7 changes: 1 addition & 6 deletions PyPIC3D/pusher/particle_push.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import jax
import jax.numpy as jnp

from PyPIC3D.particles.particle_class import TiledParticles
from PyPIC3D.particles.particle_batching import (
number_of_particle_batches,
particle_batch_indices,
Expand Down Expand Up @@ -236,11 +235,7 @@ def batches_remaining(batch_state):
Bz_tiles,
)

return TiledParticles(
x=particles.x,
u=new_u,
active=particles.active,
)
return particles._replace(u=new_u)


def seed_leapfrog_velocity(
Expand Down
53 changes: 35 additions & 18 deletions PyPIC3D/relativity/bssn_parameters.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,21 @@ class BSSNParameters(NamedTuple):
"""

state: object
evolution_parameters: object
time: object


BSSN_FACE_NAMES = ("xl_bc", "xr_bc", "yl_bc", "yr_bc", "zl_bc", "zr_bc")


def _face_code(name, value):
"""Normalize one [metric] face entry to the JAX_BSSN code 0 or 1."""
if isinstance(value, str) and value in ("periodic", "sommerfeld"):
return {"periodic": 0, "sommerfeld": 1}[value]
if isinstance(value, Integral) and not isinstance(value, bool) and value in (0, 1):
return int(value)
raise ValueError(f"[metric] {name} must be 'periodic', 'sommerfeld', 0, or 1")


def load_bssn_parameters_from_toml(config, static_parameters, dynamic_parameters,
*, state=None, evolution_parameters=None):
"""Read native ``[metric]`` initial arrays and gauge/evolution controls.
Expand All @@ -36,6 +47,10 @@ def load_bssn_parameters_from_toml(config, static_parameters, dynamic_parameters

Without ``[metric]``, retain the legacy flat default, ``[bssn_parameters]``
table, and Python override behavior.

Returns ``(static_parameters, bssn)``: the evolution controls are run
constants that JAX_BSSN reads concretely, so they ride on the static
parameters rather than through jit as traced leaves.
"""
if "metric" not in config:
legacy = dict(config.get("bssn_parameters", {}))
Expand All @@ -54,7 +69,7 @@ def load_bssn_parameters_from_toml(config, static_parameters, dynamic_parameters
raise ValueError("[metric] must be a TOML table")

state_names = BSSNVariables._fields
boundary_names = ("xl_bc", "xr_bc", "yl_bc", "yr_bc", "zl_bc", "zr_bc")
boundary_names = BSSN_FACE_NAMES
numeric_names = ("time", "eta", "kappa", "nu", "g", "mad_q")
derived_names = {"dx", "dt", "x_min", "y_min", "z_min"}
forbidden = set(metric_config) & derived_names
Expand Down Expand Up @@ -85,13 +100,7 @@ def load_bssn_parameters_from_toml(config, static_parameters, dynamic_parameters
for name in boundary_names:
if name not in metric_config:
continue
value = metric_config[name]
if isinstance(value, str) and value in ("periodic", "sommerfeld"):
controls[name] = {"periodic": 0, "sommerfeld": 1}[value]
elif isinstance(value, Integral) and not isinstance(value, bool) and value in (0, 1):
controls[name] = int(value)
else:
raise ValueError(f"[metric] {name} must be 'periodic', 'sommerfeld', 0, or 1")
controls[name] = _face_code(name, metric_config[name])

time = controls.pop("time", 0.0)
if static_parameters.field_mesh.size > 1:
Expand All @@ -100,15 +109,15 @@ def load_bssn_parameters_from_toml(config, static_parameters, dynamic_parameters
for name, value in sources.items():
if value is not False and not isinstance(value, str):
raise ValueError(f"[metric] {name} must be an initialization filename or literal false")
parameters = build_bssn_parameters(
static_parameters, parameters = build_bssn_parameters(
static_parameters, dynamic_parameters, state=sources,
evolution_parameters=controls, time=time)
state = parameters.state
from JAX_BSSN.parallel import map_tiled, owned_mask
from PyPIC3D.relativity.dynamical_relativity import bssn_tile_layout
mask = map_tiled(lambda value, params: owned_mask(value.shape, params),
state.conformal_factor, parameters.evolution_parameters,
**bssn_tile_layout(static_parameters))
state.conformal_factor, static_parameters.bssn_evolution_parameters,
bssn_tile_layout(static_parameters))
if not all(bool(jnp.all(jnp.where(mask, jnp.isfinite(v), True))) for v in state):
raise ValueError("[metric] initial geometry must contain finite values")
if not bool(jnp.all(jnp.where(mask, state.conformal_factor > 0, True))):
Expand All @@ -119,11 +128,11 @@ def load_bssn_parameters_from_toml(config, static_parameters, dynamic_parameters
raise ValueError("[metric] conformal_metric must be symmetric")
if not bool(jnp.all(jnp.where(mask[..., None], jnp.linalg.eigvalsh(gamma) > 0, True))):
raise ValueError("[metric] conformal_metric must be positive definite")
return parameters
return static_parameters, parameters

# Keep the established serial global-array input contract.

parameters = build_bssn_parameters(
static_parameters, parameters = build_bssn_parameters(
static_parameters, dynamic_parameters,
evolution_parameters=controls, time=time,
)
Expand Down Expand Up @@ -157,7 +166,7 @@ def load_bssn_parameters_from_toml(config, static_parameters, dynamic_parameters
raise ValueError("[metric] conformal_metric must be symmetric")
if not bool(jnp.all(jnp.linalg.eigvalsh(gamma) > 0)):
raise ValueError("[metric] conformal_metric must be positive definite; omitted/false creates zeros")
return parameters._replace(state=state)
return static_parameters, parameters._replace(state=state)


def build_bssn_parameters(static_parameters, dynamic_parameters, *, state=None,
Expand Down Expand Up @@ -201,14 +210,20 @@ def build_bssn_parameters(static_parameters, dynamic_parameters, *, state=None,
for name in ("xl_bc", "xr_bc", "yl_bc", "yr_bc", "zl_bc", "zr_bc")):
raise ValueError("Matched GR filtering requires all six BSSN boundaries to be periodic")

evolution_parameters = jax.tree.map(jnp.asarray, evolution_parameters)
# Native Python scalars, the inverse of an asarray sweep: these are run
# constants carried on static_parameters, JAX_BSSN reads several of them with
# int()/float(), and an array leaf would make StaticParameters unhashable and
# so unusable as a static jit argument.
evolution_parameters = jax.tree.map(
lambda value: value.item() if hasattr(value, "item") else value,
evolution_parameters)
if tiled:
from JAX_BSSN.parallel import initialize_tiled_bssn
from PyPIC3D.relativity.dynamical_relativity import bssn_tile_layout

layout = bssn_tile_layout(static_parameters)
if not hasattr(state, "conformal_factor") or state.conformal_factor.ndim != 6:
state = initialize_tiled_bssn(state, evolution_parameters, **layout)
state = initialize_tiled_bssn(state, evolution_parameters, layout)
else:
expected = tuple(tile_counts) + tuple(width + 2 * layout["guard_cells"]
for width in static_parameters.tile_shape)
Expand All @@ -224,4 +239,6 @@ def build_bssn_parameters(static_parameters, dynamic_parameters, *, state=None,
elif state.conformal_factor.shape != shape:
raise ValueError("BSSN state must match the physical PIC center-node allocation")

return BSSNParameters(state, evolution_parameters, jnp.asarray(time))
static_parameters = static_parameters._replace(
bssn_evolution_parameters=evolution_parameters)
return static_parameters, BSSNParameters(state, jnp.asarray(time))
25 changes: 14 additions & 11 deletions PyPIC3D/relativity/dynamical_relativity.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,15 @@

def bssn_tile_layout(static_parameters):
"""Share PIC ownership and devices, with enough halo for C-to-Yee transfer."""
from JAX_BSSN.parallel import TileLayout

endpoints = tuple(bc == 1 for bc in static_parameters.boundary_conditions)
# An owned upper endpoint occupies one of the nominal upper ghost slots.
guard = max(4 + int(any(endpoints)), static_parameters.guard_cells + 1)
return dict(mesh=static_parameters.field_mesh,
tile_shape=static_parameters.tile_shape,
guard_cells=guard, upper_endpoints=endpoints)
# A layout is the cache key of the halo sweep, so every field must be hashable.
return TileLayout(mesh=static_parameters.field_mesh,
tile_shape=tuple(static_parameters.tile_shape),
guard_cells=guard, upper_endpoints=endpoints)


def resize_bssn_halos(values, static_parameters, dynamic_parameters, *, exterior="linear",
Expand All @@ -36,19 +39,19 @@ def resize_bssn_halos(values, static_parameters, dynamic_parameters, *, exterior

layout = bssn_tile_layout(static_parameters)
old_guard = (values.shape[-3] - static_parameters.tile_shape[0]) // 2
difference = layout["guard_cells"] - old_guard
difference = layout.guard_cells - old_guard
if difference > 0:
values = jnp.pad(values, ((0, 0),) * (values.ndim - 3) + ((difference, difference),) * 3)
elif difference < 0:
trim = -difference
values = values[..., trim:-trim, trim:-trim, trim:-trim]
parameters = dynamic_parameters.bssn_parameters.evolution_parameters
parameters = static_parameters.bssn_evolution_parameters
if boundary_conditions is not None:
boundary = tuple(int(bc != 0) for bc in boundary_conditions)
parameters = parameters._replace(xl_bc=boundary[0], xr_bc=boundary[0],
yl_bc=boundary[1], yr_bc=boundary[1],
zl_bc=boundary[2], zr_bc=boundary[2])
return refresh_tiled_field(values, parameters, **layout, exterior=exterior)
return refresh_tiled_field(values, parameters, layout, exterior=exterior)


def crop_bssn_halos(values, static_parameters, *, extra_guard=0):
Expand Down Expand Up @@ -120,11 +123,11 @@ def geometry_tiles(value):
# The established C-to-Yee exterior follows the EM domain, which
# can differ from the BSSN evolution boundary codes.
boundaries = tuple(int(bc != 0) for bc in static_parameters.boundary_conditions)
parameters = dynamic_parameters.bssn_parameters.evolution_parameters._replace(
parameters = static_parameters.bssn_evolution_parameters._replace(
xl_bc=boundaries[0], xr_bc=boundaries[0],
yl_bc=boundaries[1], yr_bc=boundaries[1],
zl_bc=boundaries[2], zr_bc=boundaries[2])
value = refresh_tiled_field(value, parameters, **bssn_tile_layout(static_parameters))
value = refresh_tiled_field(value, parameters, bssn_tile_layout(static_parameters))
return crop_bssn_halos(value, static_parameters, extra_guard=1)
return bssn_to_tiles(value, static_parameters, extra_guard=1)

Expand Down Expand Up @@ -204,13 +207,13 @@ def initialize_dynamical_relativity(particles, species, fields, static_parameter
from PyPIC3D.relativity.filtering import validate_gr_filter

validate_gr_filter(static_parameters)
if any(int(getattr(bssn.evolution_parameters, name)) != 0
if any(int(getattr(static_parameters.bssn_evolution_parameters, name)) != 0
for name in ("xl_bc", "xr_bc", "yl_bc", "yr_bc", "zl_bc", "zr_bc")):
raise ValueError("Matched GR filtering requires all six BSSN boundaries to be periodic")
if bssn.state.conformal_factor.ndim == 6:
from JAX_BSSN.parallel import prepare_tiled_state
state = prepare_tiled_state(bssn.state, bssn.evolution_parameters,
**bssn_tile_layout(static_parameters))
state = prepare_tiled_state(bssn.state, static_parameters.bssn_evolution_parameters,
bssn_tile_layout(static_parameters))
else:
state = enforce_algebraic_constraints(bssn.state)
dynamic_parameters = dynamic_parameters._replace(bssn_parameters=bssn._replace(state=state))
Expand Down
2 changes: 1 addition & 1 deletion PyPIC3D/relativity/energy_momentum_tensor.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ def compute_energy_momentum_tensor(
if tiled:
from PyPIC3D.relativity.dynamical_relativity import bssn_tile_layout
filter_parameters = static_parameters._replace(
guard_cells=bssn_tile_layout(static_parameters)["guard_cells"])
guard_cells=bssn_tile_layout(static_parameters).guard_cells)
components = filter_gr_grid(volume * components, filter_parameters, dynamic_parameters,
tiled=tiled) / volume
stress = jnp.stack((
Expand Down
3 changes: 1 addition & 2 deletions PyPIC3D/relativity/pair_injection.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import jax
import jax.numpy as jnp

from PyPIC3D.particles.particle_class import TiledParticles
from PyPIC3D.pusher.particle_push import seed_leapfrog_velocity
from PyPIC3D.relativity.interpolate_metric import (
interpolate_metric, particle_active_axes, safe_inactive_positions,
Expand Down Expand Up @@ -89,7 +88,7 @@ def one_tile(x, u, active, positions, momenta, requested, primitive_metric, grid
metric.center, dynamic.grids.tiled_center_grid)
# Insertion only activates slots; identify births before any push or migration.
newborn = active & ~particles.active
result = TiledParticles(x, u, active)
result = particles._replace(x=x, u=u, active=active)
seeded = seed_leapfrog_velocity(
result._replace(active=newborn), species, D, B, static, dynamic, metric)
result = result._replace(u=jnp.where(newborn[..., None], seeded.u, result.u))
Expand Down
Loading
Loading