From d438e57bfb83327a434a1850b719da9ce7240e89 Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Wed, 29 Oct 2025 12:00:44 -0400 Subject: [PATCH 1/9] ci: test new devel branch against EXP devel Gala devel will test against EXP devel. Gala main will test against a tagged EXP version. --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a4097db35..95d71690e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,7 +39,8 @@ jobs: !contains(github.event.pull_request.labels.*.name, 'docs only') && !contains(needs.check_skip_flags.outputs.head-commit-message, '[skip tests]') }} env: - EXP_REF: ${{ github.event_name == 'schedule' && 'devel' || 'v7.8.5' }} + # Run tests against a tagged EXP version, unless this is the devel branch or a PR into devel, in which case test against EXP devel + EXP_REF: ${{ (github.ref == 'refs/heads/devel' || (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'devel')) && 'devel' || 'v7.8.5' }} MACOSX_DEPLOYMENT_TARGET: "15.0" strategy: fail-fast: true From 9d431ef6a08a35f57d9fe134ced5491a6dfa470a Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Wed, 29 Oct 2025 12:03:10 -0400 Subject: [PATCH 2/9] build: use EXP install dir Previously, we had a frankenstein build that pulled from the EXP repo, build dir, and install dir. Now with EXP-code/EXP#170, everything we need is in the EXP install dir, including EXP's vendored dependencies. --- .github/workflows/tests.yml | 18 ++------ setup.py | 43 +++++-------------- .../potential/potential/builtin/exp_fields.cc | 6 +-- .../potential/potential/builtin/exp_fields.h | 4 +- 4 files changed, 18 insertions(+), 53 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 95d71690e..9d30cae62 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -189,28 +189,16 @@ jobs: PYTHON_LIBRARY=${PYTHON_LIBRARY_PATH}/libpython${PYTHON_VERSION}.${so_ext} cmake -G Ninja -B build \ - -DCMAKE_INSTALL_RPATH=$PWD/install/lib \ --install-prefix $PWD/install \ - -DENABLE_PYEXP=on \ + -DENABLE_PYEXP_ONLY=on \ -DPYTHON_EXECUTABLE=${PYTHON_EXEC} \ -DPYTHON_LIBRARY=${PYTHON_LIBRARY} \ -DCMAKE_CXX_FLAGS="${CXXFLAGS} -flto=auto" \ -DCMAKE_EXE_LINKER_FLAGS="${LDFLAGS}" \ -DCMAKE_SHARED_LINKER_FLAGS="${LDFLAGS}" - # Build EXP - # We are avoiding doing a CMake install as a hack to speed up the build - cmake --build build -v -t libexpui.${so_ext} libexputil.${so_ext} libyaml-cpp.${so_ext} pyEXP - - # The EXP CMake install doesn't offer a way to install just the shared libraries, so - # we are manually copying them to the install dir here. - # Users should do a proper "cmake --install" instead! - mkdir -p install/lib - if [[ "$RUNNER_OS" == "macOS" ]]; then - cp -P $(find build -name "*.so*" -o -name "*.dylib*") install/lib/ - else - cp -P $(find build -name "*.so*") install/lib/ - fi + cmake --build build -v + cmake --install build # Add the pyEXP dir as a pyEXP.pth file in Python site-packages: SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") diff --git a/setup.py b/setup.py index eb7e6a4a4..1c1328db0 100755 --- a/setup.py +++ b/setup.py @@ -97,10 +97,8 @@ gsl_version = os.environ.get("GALA_GSL_VERSION", None) gsl_prefix = os.environ.get("GALA_GSL_PREFIX", None) -# The root directory of EXP (i.e., the repository root) +# The EXP installation prefix. This directory should contain 'include' and 'lib' subdirs. exp_prefix = os.environ.get("GALA_EXP_PREFIX", None) -# The path to the built/installed EXP libraries -exp_lib_path = os.environ.get("GALA_EXP_LIB_PATH", None) try: import pybind11 @@ -521,31 +519,20 @@ def base_cfg(): if "cyexp" in ext.name: if exp_prefix is not None: - if exp_lib_path is None: - # NOTE: this assumes user installed EXP to $GALA_EXP_PREFIX/install - lib_path_tmp = os.path.join(exp_prefix, "install", "lib") - if os.path.exists(lib_path_tmp): - exp_lib_path = lib_path_tmp - else: - msg = ( - "GALA_EXP_LIB_PATH not set, and no EXP libraries found in " - f"{lib_path_tmp}. Please set GALA_EXP_LIB_PATH to the path " - "where EXP libraries are installed." - ) - raise RuntimeError(msg) - - print(f"Gala: installing with EXP libraries at {exp_lib_path}") + exp_lib_path = os.path.join(exp_prefix, "lib") + if not os.path.exists(exp_lib_path): + msg = ( + f"No EXP libraries found in {exp_lib_path}. " + "Please set GALA_EXP_PREFIX to the directory that contains the 'lib' and 'include' " + "subdirectories of your EXP installation." + ) + raise RuntimeError(msg) ext.include_dirs.append(pybind11.get_include()) if extra_incl_flags is not None: ext.extra_compile_args.extend(extra_incl_flags) if "exp" not in ext.libraries: - # TODO: we're compiling against installed EXP libraries, - # but headers from the source, because EXP doesn't install - # its headers. It would also need to install its vendored - # headers. - ext.libraries.extend( ( "exputil", @@ -555,17 +542,7 @@ def base_cfg(): ) ext.library_dirs.append(exp_lib_path) ext.runtime_library_dirs.append(exp_lib_path) - ext.include_dirs.extend( - ( - os.path.join(exp_prefix, "include"), - # TODO: requires build in $GALA_EXP_PREFIX/build - os.path.join(exp_prefix, "build"), - os.path.join(exp_prefix, "expui"), - os.path.join(exp_prefix, "extern", "HighFive", "include"), - os.path.join(exp_prefix, "extern", "yaml-cpp", "include"), - ) - ) - + ext.include_dirs.extend(os.path.join(exp_prefix, "include")) else: # Skip cyexp extension if EXP is not found continue diff --git a/src/gala/potential/potential/builtin/exp_fields.cc b/src/gala/potential/potential/builtin/exp_fields.cc index 96f4f3f6b..b674f97e0 100644 --- a/src/gala/potential/potential/builtin/exp_fields.cc +++ b/src/gala/potential/potential/builtin/exp_fields.cc @@ -10,9 +10,9 @@ namespace fs = std::filesystem; // EXP headers -#include -#include -#include +#include +#include +#include #include "exp_fields.h" #include "src/vectorization.h" diff --git a/src/gala/potential/potential/builtin/exp_fields.h b/src/gala/potential/potential/builtin/exp_fields.h index da3780323..98e605b4b 100644 --- a/src/gala/potential/potential/builtin/exp_fields.h +++ b/src/gala/potential/potential/builtin/exp_fields.h @@ -4,8 +4,8 @@ #include #include -#include -#include +#include +#include namespace gala_exp { From 095d8bbff6a78604f37d8982221af6a2682459d2 Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Wed, 29 Oct 2025 12:19:42 -0400 Subject: [PATCH 3/9] docs: update EXP install instructions for cmake updates --- docs/tutorials/exp.rst | 45 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/docs/tutorials/exp.rst b/docs/tutorials/exp.rst index b9ee9b9fa..ad3f5b4df 100644 --- a/docs/tutorials/exp.rst +++ b/docs/tutorials/exp.rst @@ -13,7 +13,7 @@ simulation snapshots. This requires: #. and setting up a `~gala.potential.potential.EXPPotential` object using the user's EXP config and coefficient files. -Note that EXP support currently requires building Gala (and EXP) from source. +Note that EXP support currently requires building Gala from source. Additionally, this workflow has only been tested on Linux and MacOS with the setups seen in the `GitHub actions test config file `_. @@ -23,10 +23,9 @@ Building EXP ------------ The `EXP documentation `_ -is the authoritative source on how to build EXP. Currently, the only Gala-specific -addition to the instructions is that Gala expects the ``build`` directory to be present -in the EXP root directory. The ``install`` directory will be looked for in the EXP root -directory too, or one can set ``GALA_EXP_LIB_PATH`` (see below). +is the best place to read about how to build EXP. Gala doesn't have any special +requirements for the EXP build, except that the user must actually "install" EXP, +rather than just build it. This is demonstrated below. To install EXP's dependencies, here is one recipe that we have found to work on Ubuntu 24.04:: @@ -36,7 +35,7 @@ To install EXP's dependencies, here is one recipe that we have found to work on Here is another recipe using modules that has been found to work on Flatiron Institute's rusty cluster:: - module load modules/2.3 cmake gcc openmpi hdf5 libtirpc eigen fftw git python + module load modules/2.4 cmake gcc openmpi hdf5 libtirpc eigen fftw git python uv EXP also builds on Mac by installing the dependencies with Homebrew:: @@ -46,10 +45,13 @@ After installing the dependencies, one can download and build EXP on Linux with: git clone --recursive https://github.com/EXP-code/EXP.git cd EXP - cmake -G Ninja -B build -DCMAKE_INSTALL_RPATH=$PWD/install/lib --install-prefix $PWD/install + cmake -G Ninja -B build --install-prefix $PWD/install cmake --build build cmake --install build +In this case, we installed EXP to the ``EXP/install/`` directory, but this can be any +directory. This will become the ``GALA_EXP_PREFIX`` directory in the next step. + For a full example of how to build EXP on Mac, see `this build recipe `_. @@ -60,24 +62,17 @@ present. Building Gala with EXP support ------------------------------ -Building Gala with the ``GALA_EXP_PREFIX`` environment variable set to the EXP root dir +Building Gala with the ``GALA_EXP_PREFIX`` environment variable set to the EXP install dir will trigger compilation of the Gala's EXP Cython extensions. For example:: git clone https://github.com/adrn/gala.git cd gala - export GALA_EXP_PREFIX=/path/to/EXP - -If you build and install EXP following the instructions above, the EXP libraries will be -located in ``EXP/install/lib`` and the Gala build process knows to look there by default. If -you installed EXP to a different location, you can set the ``GALA_EXP_LIB_PATH`` -environment variable to point to the lib directory of the EXP install:: - - # Only do this if the install location is not $GALA_EXP_PREFIX/install - # export GALA_EXP_LIB_PATH=/path/to/EXP-install/lib + export GALA_EXP_PREFIX=/path/to/EXP/install/ -That is, ``GALA_EXP_LIB_PATH`` can be set if the CMake ``--install-prefix`` was set to a -location other than ``GALA_EXP_PREFIX/install``. ``GALA_EXP_LIB_PATH`` should be the -directory that contains the ``.so`` or ``.dylib`` files. +If you build and install EXP following the instructions above, the EXP installation will be +located in ``EXP/install/``. If you installed EXP to a different location, you can set the +``GALA_EXP_PREFIX`` to that location. In either case, ``GALA_EXP_PREFIX`` must be the directory +that contains the subdirectories ``lib`` and ``include``. Now you can run the Gala build. For example, using uv:: @@ -90,8 +85,14 @@ Or using venv:: . .venv/bin/activate python -m pip install -ve . -In either case, the pip output should show a message like ``Gala: installing with EXP -support``. +In either case, the output should show a message like ``Gala: installing with EXP support``. + +Note that in previous versions of Gala, the ``GALA_EXP_PREFIX`` was supposed to point to the +EXP repo root, rather than the EXP installation directory. This is no longer the case. The +EXP repo and build directories are not needed to build Gala with EXP support. + +Likewise, ``GALA_EXP_LIB_PATH`` was used in past Gala versions but not anymore. + ---------------------------------- Running Gala with an EXP potential From 41d096c00a8419f3b69d2eebf62cf53fdadfb1f3 Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Wed, 29 Oct 2025 12:49:17 -0400 Subject: [PATCH 4/9] changelog: new EXP CMake --- CHANGES.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 0e62ed1d0..05c734ef9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,11 @@ +devel +===== + +Build changes +------------- +- EXP: the instructions to build Gala against EXP have changed. Only the EXP install + dir is now used. + 1.11.0 (unreleased) =================== From 26d1fb0f4eed142dfd4befaa248c343d5740f108 Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Wed, 29 Oct 2025 14:21:21 -0400 Subject: [PATCH 5/9] ci --- .github/workflows/tests.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9d30cae62..5165c6718 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -187,9 +187,11 @@ jobs: so_ext="so" fi PYTHON_LIBRARY=${PYTHON_LIBRARY_PATH}/libpython${PYTHON_VERSION}.${so_ext} + GALA_EXP_PREFIX=/opt/EXP/ + echo "GALA_EXP_PREFIX=${GALA_EXP_PREFIX}" >> $GITHUB_ENV cmake -G Ninja -B build \ - --install-prefix $PWD/install \ + --install-prefix $GALA_EXP_PREFIX \ -DENABLE_PYEXP_ONLY=on \ -DPYTHON_EXECUTABLE=${PYTHON_EXEC} \ -DPYTHON_LIBRARY=${PYTHON_LIBRARY} \ @@ -206,9 +208,6 @@ jobs: - name: Install package and dependencies run: | - if [[ "${{ matrix.gala-exp }}" == "1" ]]; then - export GALA_EXP_PREFIX=$PWD/EXP - fi uv pip install --system -ve .[${{ matrix.python-extras }}] env: GALA_NOGSL: ${{ matrix.gala-nogsl }} From 51841c2b111c4e61d69c506d87f1ab48701ac8a2 Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Wed, 29 Oct 2025 14:32:25 -0400 Subject: [PATCH 6/9] ci --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5165c6718..71f177ca8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -187,7 +187,7 @@ jobs: so_ext="so" fi PYTHON_LIBRARY=${PYTHON_LIBRARY_PATH}/libpython${PYTHON_VERSION}.${so_ext} - GALA_EXP_PREFIX=/opt/EXP/ + GALA_EXP_PREFIX=$HOME/EXP-install/ echo "GALA_EXP_PREFIX=${GALA_EXP_PREFIX}" >> $GITHUB_ENV cmake -G Ninja -B build \ @@ -204,7 +204,7 @@ jobs: # Add the pyEXP dir as a pyEXP.pth file in Python site-packages: SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") - echo "$PWD/install/lib/" > "${SITE_PACKAGES}/pyEXP.pth" + echo "$GALA_EXP_PREFIX/lib/" > "${SITE_PACKAGES}/pyEXP.pth" - name: Install package and dependencies run: | From 7d7480d77719d92e170051f74dae0d07d03ad38d Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Wed, 29 Oct 2025 14:43:57 -0400 Subject: [PATCH 7/9] ci --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1c1328db0..0b1def268 100755 --- a/setup.py +++ b/setup.py @@ -542,7 +542,7 @@ def base_cfg(): ) ext.library_dirs.append(exp_lib_path) ext.runtime_library_dirs.append(exp_lib_path) - ext.include_dirs.extend(os.path.join(exp_prefix, "include")) + ext.include_dirs.append(os.path.join(exp_prefix, "include")) else: # Skip cyexp extension if EXP is not found continue From 7f4f671689091b6e812c6d954e4ec13fe30df159 Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Wed, 29 Oct 2025 15:03:12 -0400 Subject: [PATCH 8/9] ci --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 71f177ca8..0ae069a34 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -204,7 +204,7 @@ jobs: # Add the pyEXP dir as a pyEXP.pth file in Python site-packages: SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") - echo "$GALA_EXP_PREFIX/lib/" > "${SITE_PACKAGES}/pyEXP.pth" + echo "$GALA_EXP_PREFIX/lib/python${PYTHON_VERSION}/site-packages/" > "${SITE_PACKAGES}/pyEXP.pth" - name: Install package and dependencies run: | From 788d4eabd17aa0448258bb72d8843982b5befef7 Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Wed, 29 Oct 2025 15:24:11 -0400 Subject: [PATCH 9/9] ci --- .github/workflows/tests.yml | 1 + docs/tutorials/exp.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0ae069a34..417cb2f30 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -193,6 +193,7 @@ jobs: cmake -G Ninja -B build \ --install-prefix $GALA_EXP_PREFIX \ -DENABLE_PYEXP_ONLY=on \ + -DCMAKE_INSTALL_RPATH=${GALA_EXP_PREFIX}/lib \ -DPYTHON_EXECUTABLE=${PYTHON_EXEC} \ -DPYTHON_LIBRARY=${PYTHON_LIBRARY} \ -DCMAKE_CXX_FLAGS="${CXXFLAGS} -flto=auto" \ diff --git a/docs/tutorials/exp.rst b/docs/tutorials/exp.rst index ad3f5b4df..7efe8386e 100644 --- a/docs/tutorials/exp.rst +++ b/docs/tutorials/exp.rst @@ -45,7 +45,7 @@ After installing the dependencies, one can download and build EXP on Linux with: git clone --recursive https://github.com/EXP-code/EXP.git cd EXP - cmake -G Ninja -B build --install-prefix $PWD/install + cmake -G Ninja -B build -DCMAKE_INSTALL_RPATH="$PWD/install/lib" --install-prefix $PWD/install cmake --build build cmake --install build