diff --git a/CHANGELOG.md b/CHANGELOG.md index b87d9c0d..23398b8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,11 @@ SPDX-License-Identifier: CC-BY-4.0 * `--numba-threads` is now checked against the number of threads Numba accepts, i.e. the CPUs the rank may run on (CPU affinity from MPI binding or the Slurm allocation, e.g. `--cpus-per-task`) or the `NUMBA_NUM_THREADS` environment variable. Exceeding it stops EBFM with an error explaining the cause, instead of a Numba `ValueError` traceback. Exceeding the suggested thread count for the ranks sharing a node is now only a warning; the suggestion no longer divides the CPUs of the whole node by the number of EBFM ranks. https://github.com/EBFMorg/EBFM/pull/190 * A fatal error now ends EBFM with exit status 1 whenever mpi4py is installed, both standalone and under `mpirun`. EBFM previously aborted MPI with error code 0, so the run, and `mpirun` with it, reported success after logging "Fatal Error in EBFM". The same applies to an MPI handshake version mismatch. A fatal error before the MPI communicator splitting no longer leaves the other components of an MPMD run waiting forever. https://github.com/EBFMorg/EBFM/pull/191 * Add an example for `--netcdf-dem-mesh-unstructured`: surface elevation and ice thickness from Elmer/Ice XIOS output for the Greenland mesh (`examples/greenland_mesh_v0/xios_dem.nc`). `examples/README.md` describes the contents EBFM expects in such a file. https://github.com/EBFMorg/EBFM/pull/198 +* Add safeguards for illegal MPI runs. https://github.com/EBFMorg/EBFM/pull/201 + * EBFM on several MPI ranks now stops at startup with an error unless it runs on a partitioned Elmer mesh (`--is-partitioned-elmer-mesh`), one partition per rank. With `--matlab-mesh`, `--netcdf-mesh`, or `--elmer-mesh` without `--is-partitioned-elmer-mesh`, every rank previously computed the whole grid, and the run failed later, e.g. while the ranks wrote the same output file. + * EBFM on several MPI ranks now stops at startup with an error if `--use-part` is given. Every rank previously computed the same partition. + * EBFM on several MPI ranks now stops at startup with an error if forcing files (`--forcing-dir`) or restart files (`--restart-dir`, `--restart-init`) are used, also on a partitioned Elmer mesh. On a partitioned mesh, `--forcing-dir` previously failed when reading the first forcing file, `--restart-init` failed at the shape check of the restart file, and `--restart-dir` finished without writing a restart file, with a warning at the end. EBFM on a single rank, e.g. as one component of a coupled MPMD run, is unaffected. + * EBFM started by an MPI launcher on several processes (e.g. `mpirun -np 4 ebfm ...`) without mpi4py now stops with an error asking to install `ebfm[mpi]`. Every process previously ran the whole model on its own, and the processes then failed while writing the same output file. Detected via the launcher's `OMPI_COMM_WORLD_SIZE` (Open MPI) or `PMI_SIZE` (MPICH, Intel MPI, `srun --mpi=pmi2`). # v0.8.0 diff --git a/README.md b/README.md index d34a6a5a..fdf881e5 100644 --- a/README.md +++ b/README.md @@ -216,9 +216,17 @@ path/to/your/elmer/mesh Usage example for partitioned mesh: ```sh -ebfm --elmer-mesh examples/greenland_mesh_v0/MESH/partitioning.128/ --netcdf-dem-mesh examples/BedMachineGreenland-v5_lo.nc --is-partitioned-elmer-mesh --use-part 42 +ebfm --elmer-mesh examples/greenland_mesh_v0/MESH/partitioning.128/ --netcdf-dem-mesh examples/BedMachineGreenland-v5_lo.nc --is-partitioned-elmer-mesh --use-part 42 --elmer-mesh-crs-epsg 3413 ``` +On several MPI ranks, `--use-part` is not allowed: rank i computes the partition i + 1, e.g. the partitions 1 to 4 with: + +```sh +mpirun -n 4 ebfm --elmer-mesh examples/greenland_mesh_v0/MESH/partitioning.128/ --netcdf-dem-mesh examples/BedMachineGreenland-v5_lo.nc --is-partitioned-elmer-mesh --elmer-mesh-crs-epsg 3413 +``` + +A partitioned mesh is the only grid EBFM runs on several MPI ranks. + Note: The partitioned mesh is not included in this repository. ### Getting the example data diff --git a/src/ebfm/core/cli.py b/src/ebfm/core/cli.py index 55822e55..7a2c4f97 100644 --- a/src/ebfm/core/cli.py +++ b/src/ebfm/core/cli.py @@ -12,6 +12,7 @@ """ import argparse +import os from argparse import ArgumentParser, Namespace from datetime import datetime from enum import Enum @@ -20,6 +21,7 @@ import isodate import ebfm.core +import ebfm.core.comm from ebfm.core import INIT from ebfm.core.comm import mpi_available from ebfm.core.config import Calendar, FieldValidationLevel, GridConfig, DEFAULT_TZ, iso8601 @@ -38,6 +40,19 @@ def _cli_error(msg: str) -> None: _parser.error(msg) +def mpi_launcher_size() -> int: + """Number of processes an MPI launcher started, read from the environment it sets; 1 if none is found. + + Covers Open MPI's mpirun (OMPI_COMM_WORLD_SIZE) and PMI-based launchers such as MPICH's mpiexec, Intel MPI and + srun --mpi=pmi2 (PMI_SIZE). Works without mpi4py. + """ + for variable in ("OMPI_COMM_WORLD_SIZE", "PMI_SIZE"): + value = os.environ.get(variable, "") + if value.isdigit(): + return int(value) + return 1 + + class CliDefaults(Enum): START_TIME = datetime(1979, 1, 1, 0, 0, tzinfo=DEFAULT_TZ) END_TIME = datetime(1979, 1, 2, 0, 0, tzinfo=DEFAULT_TZ) @@ -331,7 +346,7 @@ def parse_time_step(value: str) -> float | str: type=int, default=argparse.SUPPRESS, # To not print (default: ...) in --help help="If using a partitioned Elmer mesh, allows to specify which partition ID to use for this run. " - "If not provided, the MPI rank + 1 will be used as partition ID.", + "If not provided, the MPI rank + 1 will be used as partition ID. Only allowed in runs on a single MPI rank.", ) logger_group = parser.add_argument_group("logging configuration") @@ -471,6 +486,14 @@ def parse_time_step(value: str) -> float | str: if active_mpi_opts and not mpi_available: parser.error(f"{', '.join(active_mpi_opts)} require MPI. Install via: pip install 'ebfm[mpi]'") + n_launched = mpi_launcher_size() + if n_launched > 1 and not mpi_available: + parser.error( + f"EBFM was started on {n_launched} MPI processes, but mpi4py cannot be imported " + f"({getattr(ebfm.core.comm, 'mpi_import_error', 'not installed')}), so every process would run the whole " + "model on its own. Install via: pip install 'ebfm[mpi]'" + ) + return args @@ -504,3 +527,75 @@ def validate_shading_coupling_compat(grid_config, coupling_config) -> None: "Shading routine not implemented for coupled runs. " "Please deactivate shading via --no-shading or deactivate coupling." ) + + +def validate_grid_parallel_compat(grid_config, comm) -> None: + """Check that a run on several MPI ranks uses a partitioned Elmer mesh, of which each rank computes one partition. + + Must be called on every rank of comm. + + @param[in] grid_config resolved GridConfig + @param[in] comm EBFM's MPI communicator + """ + if comm.size == 1: + return + + if grid_config.grid_type is GridInputType.MATLAB: + rejected_grid = "--matlab-mesh" + elif grid_config.grid_type is GridInputType.NETCDF: + rejected_grid = "--netcdf-mesh" + elif not grid_config.is_partitioned: + rejected_grid = "--elmer-mesh without --is-partitioned-elmer-mesh" + else: + return + + _cli_error( + f"{rejected_grid} is not supported for runs on several MPI ranks ({comm.size} ranks). " + "Only a partitioned Elmer mesh (--is-partitioned-elmer-mesh) runs on several ranks, one partition per rank. " + "Please run on a single rank." + ) + + +def validate_use_part_parallel_compat(args: Namespace, comm) -> None: + """Check that --use-part is not given in a run on several MPI ranks, where every rank would compute it. + + Must be called on every rank of comm, before args.use_part defaults to the rank's partition. + + @param[in] args parsed CLI arguments + @param[in] comm EBFM's MPI communicator + """ + if hasattr(args, "use_part") and comm.size > 1: + _cli_error( + f"--use-part is not supported for runs on several MPI ranks ({comm.size} ranks), since every rank would " + f"compute partition {args.use_part}. Please leave out --use-part: rank i then computes partition i + 1." + ) + + +def validate_forcing_parallel_compat(args: Namespace, comm) -> None: + """Check that no forcing files are read in a run on several MPI ranks. + + Must be called on every rank of comm. + + @param[in] args parsed CLI arguments + @param[in] comm EBFM's MPI communicator + """ + if args.forcing_dir is not None and comm.size > 1: + _cli_error( + f"Forcing files (--forcing-dir) are not supported for runs on several MPI ranks ({comm.size} ranks). " + "Please run on a single rank." + ) + + +def validate_restart_parallel_compat(args: Namespace, comm) -> None: + """Check that no restart files are read or written in a run on several MPI ranks. + + Must be called on every rank of comm. + + @param[in] args parsed CLI arguments + @param[in] comm EBFM's MPI communicator + """ + if (args.restart_dir is not None or args.restart_init) and comm.size > 1: + _cli_error( + f"Restart files (--restart-dir, --restart-init) are not supported for runs on several MPI ranks " + f"({comm.size} ranks). Please run on a single rank." + ) diff --git a/src/ebfm/main.py b/src/ebfm/main.py index 7ffbadec..e959f188 100644 --- a/src/ebfm/main.py +++ b/src/ebfm/main.py @@ -25,6 +25,10 @@ parse_cli_args, resolve_numba_threads, validate_shading_coupling_compat, + validate_grid_parallel_compat, + validate_use_part_parallel_compat, + validate_forcing_parallel_compat, + validate_restart_parallel_compat, ) import ebfm.coupling @@ -138,6 +142,9 @@ def _main_impl(): forcing_config = ForcingConfig(coupling_config, args) + # Ensure --use-part is only used in runs on a single rank. + validate_use_part_parallel_compat(args, ebfm_comm) + if not hasattr(args, "use_part"): # If not provided via command line option --use-part, set to rank + 1 (assuming partition IDs start at 1). args.use_part = ebfm_comm.rank + 1 @@ -172,6 +179,15 @@ def _main_impl(): # See https://github.com/EBFMorg/EBFM/issues/11 for details. validate_shading_coupling_compat(grid_config, coupling_config) + # Ensure a run on several ranks uses a partitioned mesh. + validate_grid_parallel_compat(grid_config, ebfm_comm) + + # Ensure forcing files are only read in runs on a single rank. + validate_forcing_parallel_compat(args, ebfm_comm) + + # Ensure restart files are only read and written in runs on a single rank. + validate_restart_parallel_compat(args, ebfm_comm) + logger.debug("Successfully completed consistency checks.") # Model setup & initialization diff --git a/tests/core/test_cli.py b/tests/core/test_cli.py index 7a6d4f4d..896d0528 100644 --- a/tests/core/test_cli.py +++ b/tests/core/test_cli.py @@ -6,24 +6,32 @@ import unittest import io -from contextlib import redirect_stdout +import os +from contextlib import redirect_stderr, redirect_stdout from pathlib import Path -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from ebfm.core.cli import ( CliDefaults, extract_active_coupling_features, + mpi_launcher_size, parse_cli_args, + validate_forcing_parallel_compat, + validate_grid_parallel_compat, + validate_restart_parallel_compat, validate_shading_coupling_compat, + validate_use_part_parallel_compat, ) from ebfm.core.comm import mpi_available +from ebfm.core.grid import GridInputType from ebfm.core.config import FieldValidationLevel, Calendar, TimeConfig, iso8601 # Minimal valid args for each primary grid type. _MATLAB = ["--matlab-mesh", "mesh.mat"] _ELMER = ["--elmer-mesh", "mesh/", "--elmer-mesh-crs-epsg", "3413"] _CARRA2 = ["--netcdf-mesh", "mesh.nc"] +_PARTITIONED_ELMER = _ELMER + ["--netcdf-dem-mesh", "mesh.nc", "--is-partitioned-elmer-mesh"] def _parse(*extra: str, base: list[str] = _MATLAB): @@ -213,6 +221,32 @@ def test_gpu_vendor_without_with_gpu_rejected(self): self.assertEqual(ctx.exception.code, 2) +class TestMpiLauncherWithoutMpi4py(unittest.TestCase): + """Started by an MPI launcher on several processes, EBFM needs mpi4py; otherwise every process runs serially.""" + + def _parse(self, environment: dict, mpi_available: bool): + with patch.dict(os.environ), patch("ebfm.core.cli.mpi_available", mpi_available): + for variable in ("OMPI_COMM_WORLD_SIZE", "PMI_SIZE"): + os.environ.pop(variable, None) + os.environ.update(environment) + self.assertEqual(mpi_launcher_size(), int(next(iter(environment.values()), 1))) + return parse_cli_args(_MATLAB) + + def test_several_processes_without_mpi4py_rejected(self): + for environment in ({"OMPI_COMM_WORLD_SIZE": "4"}, {"PMI_SIZE": "2"}): + with self.subTest(environment=environment), self.assertRaises(SystemExit) as ctx: + self._parse(environment, mpi_available=False) + self.assertEqual(ctx.exception.code, 2) + + def test_single_process_without_mpi4py_ok(self): + for environment in ({}, {"OMPI_COMM_WORLD_SIZE": "1"}): + with self.subTest(environment=environment): + self._parse(environment, mpi_available=False) # must not raise + + def test_several_processes_with_mpi4py_ok(self): + self._parse({"OMPI_COMM_WORLD_SIZE": "4"}, mpi_available=True) # must not raise + + class TestLocalGroupLabel(unittest.TestCase): """--local-group-label overrides the default derived from --component-name.""" @@ -348,6 +382,133 @@ def test_shading_with_coupling_rejected(self): self.assertEqual(ctx.exception.code, 2) +class TestValidateGridParallelCompat(unittest.TestCase): + """validate_grid_parallel_compat raises a CLI error on several ranks for every grid but a partitioned Elmer mesh.""" + + # name: (grid_type, is_partitioned, grid option named by the error on several ranks, or None if accepted) + _CASES = { + "--matlab-mesh": (GridInputType.MATLAB, False, "--matlab-mesh"), + "--netcdf-mesh": (GridInputType.NETCDF, False, "--netcdf-mesh"), + "--elmer-mesh": (GridInputType.ELMER, False, "--elmer-mesh"), + "--elmer-mesh --netcdf-dem-mesh": (GridInputType.CUSTOM, False, "--elmer-mesh"), + "--elmer-mesh --netcdf-dem-mesh-unstructured": (GridInputType.ELMERXIOS, False, "--elmer-mesh"), + "--elmer-mesh --netcdf-dem-mesh --is-partitioned-elmer-mesh": (GridInputType.CUSTOM, True, None), + } + + def setUp(self): + # Ensure _parser is set by running a valid parse first. + parse_cli_args(_MATLAB) + + def _validate(self, grid_type, is_partitioned: bool, comm_size: int): + grid_config = MagicMock(grid_type=grid_type, is_partitioned=is_partitioned) + validate_grid_parallel_compat(grid_config, MagicMock(size=comm_size)) + + def test_single_rank_ok(self): + for name, (grid_type, is_partitioned, _) in self._CASES.items(): + with self.subTest(name): + self._validate(grid_type, is_partitioned, comm_size=1) # must not raise + + def test_several_ranks(self): + for name, (grid_type, is_partitioned, rejected_grid) in self._CASES.items(): + with self.subTest(name): + if rejected_grid is None: + self._validate(grid_type, is_partitioned, comm_size=4) # must not raise + else: + stderr = io.StringIO() + with redirect_stderr(stderr), self.assertRaises(SystemExit) as ctx: + self._validate(grid_type, is_partitioned, comm_size=4) + self.assertEqual(ctx.exception.code, 2) + error = stderr.getvalue().splitlines()[-1] + self.assertIn(f"error: {rejected_grid} ", error) + self.assertIn("Only a partitioned Elmer mesh (--is-partitioned-elmer-mesh)", error) + + +@unittest.skipUnless(mpi_available, "requires MPI (pip install 'ebfm[mpi]')") +class TestValidateUsePartParallelCompat(unittest.TestCase): + """validate_use_part_parallel_compat raises a CLI error for --use-part in a run on several ranks.""" + + def setUp(self): + # Ensure _parser is set by running a valid parse first. + parse_cli_args(_MATLAB) + + def _validate(self, use_part, comm_size: int): + args = _parse(*(["--use-part", use_part] if use_part else []), base=_PARTITIONED_ELMER) + validate_use_part_parallel_compat(args, MagicMock(size=comm_size)) + + def test_use_part_on_single_rank_ok(self): + self._validate(use_part="42", comm_size=1) # must not raise + + def test_no_use_part_on_several_ranks_ok(self): + self._validate(use_part=None, comm_size=4) # must not raise + + def test_use_part_on_several_ranks_rejected(self): + with self.assertRaises(SystemExit) as ctx: + self._validate(use_part="42", comm_size=4) + self.assertEqual(ctx.exception.code, 2) + + +class TestValidateForcingParallelCompat(unittest.TestCase): + """validate_forcing_parallel_compat raises a CLI error when forcing files are read in a run on several ranks.""" + + def setUp(self): + # Ensure _parser is set by running a valid parse first. + parse_cli_args(_MATLAB) + + def _validate(self, forcing_dir, comm_size: int): + args = _parse(*(["--forcing-dir", forcing_dir] if forcing_dir else [])) + validate_forcing_parallel_compat(args, MagicMock(size=comm_size)) + + def test_forcing_on_single_rank_ok(self): + self._validate(forcing_dir="forcing/", comm_size=1) # must not raise + + def test_no_forcing_on_several_ranks_ok(self): + self._validate(forcing_dir=None, comm_size=4) # must not raise + + def test_forcing_on_several_ranks_rejected(self): + with self.assertRaises(SystemExit) as ctx: + self._validate(forcing_dir="forcing/", comm_size=4) + self.assertEqual(ctx.exception.code, 2) + + +class TestValidateRestartParallelCompat(unittest.TestCase): + """validate_restart_parallel_compat raises a CLI error when restart files are used in a run on several ranks.""" + + _RESTART_ARGS = { + "--restart-dir": ["--restart-dir", "restart/"], + "--restart-init": ["--restart-init"], + "--restart-dir --restart-init": ["--restart-dir", "restart/", "--restart-init"], + } + + def setUp(self): + # Ensure _parser is set by running a valid parse first. + parse_cli_args(_MATLAB) + + def _validate(self, restart_args, comm_size: int, base: list[str] = _MATLAB): + validate_restart_parallel_compat(_parse(*restart_args, base=base), MagicMock(size=comm_size)) + + def test_restart_on_single_rank_ok(self): + for name, restart_args in self._RESTART_ARGS.items(): + with self.subTest(name): + self._validate(restart_args, comm_size=1) # must not raise + + def test_no_restart_on_several_ranks_ok(self): + self._validate([], comm_size=4) # must not raise + + def test_restart_on_several_ranks_rejected(self): + for name, restart_args in self._RESTART_ARGS.items(): + with self.subTest(name), self.assertRaises(SystemExit) as ctx: + self._validate(restart_args, comm_size=4) + self.assertEqual(ctx.exception.code, 2) + + @unittest.skipUnless(mpi_available, "requires MPI (pip install 'ebfm[mpi]')") + def test_restart_on_several_ranks_rejected_for_partitioned_mesh(self): + """A partitioned mesh runs on several ranks, but without restart files.""" + for name, restart_args in self._RESTART_ARGS.items(): + with self.subTest(name), self.assertRaises(SystemExit) as ctx: + self._validate(restart_args, comm_size=4, base=_PARTITIONED_ELMER) + self.assertEqual(ctx.exception.code, 2) + + class TestCliCalendarArgument(unittest.TestCase): def test_calendar_default(self): args = parse_cli_args(_MATLAB)