diff --git a/bin/build_supp.sh b/bin/build_supp.sh index 7b0ce97af..f3d71c3f5 100755 --- a/bin/build_supp.sh +++ b/bin/build_supp.sh @@ -52,7 +52,7 @@ $ $AOMP_SUPP/build/cmdlog File with log of all components built EOF } -SUPPLEMENTAL_COMPONENTS=${SUPPLEMENTAL_COMPONENTS:-openmpi silo hdf5 fftw ninja xpmem ucx ucc rocmopenmpi hdf5-parallel} +SUPPLEMENTAL_COMPONENTS=${SUPPLEMENTAL_COMPONENTS:-openmpi silo hdf5 netcdf-c netcdf-fortran ecbuild fiat fftw ninja xpmem ucx ucc rocmopenmpi hdf5-parallel} # rocsmilib and hwloc build invocation moved to build_rocm_sysdeps.sh (libdrm support). rocmsmi is dependent on libdrm and hwloc depends on rocmsmi. PREREQUISITE_COMPONENTS=${PREREQUISITE_COMPONENTS:-cmake aqlprofile rocm-core} @@ -680,6 +680,251 @@ function buildhdf5(){ echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE" } +function buildnetcdfc(){ + local _cname="netcdf-c" + local _version=4.10.0 + local _installdir=$AOMP_SUPP_INSTALL/$_cname-$_version + local _linkfrom=$AOMP_SUPP/$_cname + local _builddir=$AOMP_SUPP_BUILD/$_cname + BUILD_TYPE=${BUILD_TYPE:-Release} + + if checkversion "$_cname" "$_version" "$_linkfrom"; then + return + fi + + if [ ! -d "$AOMP_SUPP/hdf5" ] ; then + echo "Info: HDF5 not found at $AOMP_SUPP/hdf5, building it first..." + buildhdf5 + fi + local _hdf5_path=$AOMP_SUPP/hdf5 + + declare -a MYCMAKEOPTS + MYCMAKEOPTS=(-DCMAKE_BUILD_TYPE="$BUILD_TYPE" + -DCMAKE_INSTALL_PREFIX="$_installdir" + -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON + -DCMAKE_C_COMPILER="$AOMP/bin/clang" + -DCMAKE_PREFIX_PATH="$_hdf5_path" + -DBUILD_SHARED_LIBS=ON + -DNETCDF_ENABLE_HDF5=ON + -DNETCDF_ENABLE_CDF5=ON + -DNETCDF_ENABLE_DAP=OFF + -DNETCDF_ENABLE_BYTERANGE=OFF + -DNETCDF_ENABLE_S3=OFF + -DNETCDF_ENABLE_NCZARR=OFF + -DNETCDF_ENABLE_FILTER_SZIP=OFF + -DNETCDF_BUILD_UTILITIES=ON + -DNETCDF_ENABLE_EXAMPLES=OFF + -DNETCDF_ENABLE_TESTS=OFF + -DNETCDF_ENABLE_PARALLEL_TESTS=OFF) + + if [ -d "$_builddir" ] ; then + runcmd "rm -rf $_builddir" + fi + runcmd "mkdir -p $_builddir" + runcmd "cd $_builddir" + runcmd "wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v$_version.tar.gz" + runcmd "tar -xzf v$_version.tar.gz" + runcmd "cd netcdf-c-$_version" + if [ -d "$_installdir" ] ; then + runcmd "rm -rf $_installdir" + fi + runcmd "mkdir -p $_installdir" + runcmd "mkdir -p build" + runcmd "cd build" + echo + echo " -----Running cmake ---- " + echo "${AOMP_CMAKE}" "$(shquot "${MYCMAKEOPTS[@]}")" \ + "../" + + if ! ${AOMP_CMAKE} "${MYCMAKEOPTS[@]}" ../; then + echo "ERROR cmake failed. Cmake flags" + echo " $(shquot "${MYCMAKEOPTS[@]}")" + exit 1 + fi + runcmd "make -j${AOMP_JOB_THREADS}" + runcmd "make install" + if [ -L "$_linkfrom" ] ; then + runcmd "rm $_linkfrom" + fi + runcmd "ln -sfr $_installdir $_linkfrom" + echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE" +} + +function buildnetcdffortran(){ + local _cname="netcdf-fortran" + local _version=4.6.2 + local _installdir=$AOMP_SUPP_INSTALL/$_cname-$_version + local _linkfrom=$AOMP_SUPP/$_cname + local _builddir=$AOMP_SUPP_BUILD/$_cname + BUILD_TYPE=${BUILD_TYPE:-Release} + + if checkversion "$_cname" "$_version" "$_linkfrom"; then + return + fi + + if [ ! -d "$AOMP_SUPP/netcdf-c" ] ; then + echo "Info: NetCDF-C not found at $AOMP_SUPP/netcdf-c, building it first..." + buildnetcdfc + fi + local _netcdfc_path=$AOMP_SUPP/netcdf-c + local _hdf5_path=$AOMP_SUPP/hdf5 + + declare -a MYCMAKEOPTS + MYCMAKEOPTS=(-DCMAKE_BUILD_TYPE="$BUILD_TYPE" + -DCMAKE_INSTALL_PREFIX="$_installdir" + -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON + -DCMAKE_C_COMPILER="$AOMP/bin/clang" + -DCMAKE_Fortran_COMPILER="$AOMP/bin/flang" + -DCMAKE_PREFIX_PATH="$_netcdfc_path;$_hdf5_path" + -DBUILD_SHARED_LIBS=ON + -DENABLE_TESTS=OFF + -DBUILD_EXAMPLES=OFF) + + if [ -d "$_builddir" ] ; then + runcmd "rm -rf $_builddir" + fi + runcmd "mkdir -p $_builddir" + runcmd "cd $_builddir" + runcmd "wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v$_version.tar.gz" + runcmd "tar -xzf v$_version.tar.gz" + runcmd "cd netcdf-fortran-$_version" + if [ -d "$_installdir" ] ; then + runcmd "rm -rf $_installdir" + fi + runcmd "mkdir -p $_installdir" + runcmd "mkdir -p build" + runcmd "cd build" + echo + echo " -----Running cmake ---- " + echo "${AOMP_CMAKE}" "$(shquot "${MYCMAKEOPTS[@]}")" \ + "../" + + if ! ${AOMP_CMAKE} "${MYCMAKEOPTS[@]}" ../; then + echo "ERROR cmake failed. Cmake flags" + echo " $(shquot "${MYCMAKEOPTS[@]}")" + exit 1 + fi + runcmd "make -j${AOMP_JOB_THREADS}" + runcmd "make install" + if [ -L "$_linkfrom" ] ; then + runcmd "rm $_linkfrom" + fi + runcmd "ln -sfr $_installdir $_linkfrom" + echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE" +} + +function buildecbuild(){ + local _cname="ecbuild" + local _version=3.16.0 + local _installdir=$AOMP_SUPP_INSTALL/$_cname-$_version + local _linkfrom=$AOMP_SUPP/$_cname + local _builddir=$AOMP_SUPP_BUILD/$_cname + + if checkversion "$_cname" "$_version" "$_linkfrom"; then + return + fi + + declare -a MYCMAKEOPTS + MYCMAKEOPTS=(-DCMAKE_INSTALL_PREFIX="$_installdir" + -DENABLE_TESTS=OFF) + + if [ -d "$_builddir" ] ; then + runcmd "rm -rf $_builddir" + fi + runcmd "mkdir -p $_builddir" + runcmd "cd $_builddir" + runcmd "wget https://github.com/ecmwf/ecbuild/archive/refs/tags/$_version.tar.gz" + runcmd "tar -xzf $_version.tar.gz" + runcmd "cd ecbuild-$_version" + if [ -d "$_installdir" ] ; then + runcmd "rm -rf $_installdir" + fi + runcmd "mkdir -p $_installdir" + runcmd "mkdir -p build" + runcmd "cd build" + echo + echo " -----Running cmake ---- " + echo "${AOMP_CMAKE}" "$(shquot "${MYCMAKEOPTS[@]}")" \ + "../" + + if ! ${AOMP_CMAKE} "${MYCMAKEOPTS[@]}" ../; then + echo "ERROR cmake failed. Cmake flags" + echo " $(shquot "${MYCMAKEOPTS[@]}")" + exit 1 + fi + runcmd "make install" + if [ -L "$_linkfrom" ] ; then + runcmd "rm $_linkfrom" + fi + runcmd "ln -sfr $_installdir $_linkfrom" + echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE" +} + +function buildfiat(){ + local _cname="fiat" + local _version=2.0.1 + local _installdir=$AOMP_SUPP_INSTALL/$_cname-$_version + local _linkfrom=$AOMP_SUPP/$_cname + local _builddir=$AOMP_SUPP_BUILD/$_cname + BUILD_TYPE=${BUILD_TYPE:-Release} + + if checkversion "$_cname" "$_version" "$_linkfrom"; then + return + fi + + if [ ! -d "$AOMP_SUPP/ecbuild" ] ; then + echo "Info: ecbuild not found at $AOMP_SUPP/ecbuild, building it first..." + buildecbuild + fi + local _ecbuild_path=$AOMP_SUPP/ecbuild + + declare -a MYCMAKEOPTS + MYCMAKEOPTS=(-DCMAKE_BUILD_TYPE="$BUILD_TYPE" + -DCMAKE_INSTALL_PREFIX="$_installdir" + -Decbuild_ROOT="$_ecbuild_path" + -DCMAKE_C_COMPILER="$AOMP/bin/clang" + -DCMAKE_Fortran_COMPILER="$AOMP/bin/flang" + "-DCMAKE_C_FLAGS=-fopenmp -fPIC" + "-DCMAKE_Fortran_FLAGS=-fopenmp -fPIC" + -DENABLE_MPI=OFF + -DENABLE_DR_HOOK_ROCTX=OFF + -DENABLE_TESTS=OFF) + + if [ -d "$_builddir" ] ; then + runcmd "rm -rf $_builddir" + fi + runcmd "mkdir -p $_builddir" + runcmd "cd $_builddir" + runcmd "wget https://github.com/ecmwf-ifs/fiat/archive/refs/tags/$_version.tar.gz" + runcmd "tar -xzf $_version.tar.gz" + runcmd "cd fiat-$_version" + if [ -d "$_installdir" ] ; then + runcmd "rm -rf $_installdir" + fi + runcmd "mkdir -p $_installdir" + runcmd "mkdir -p build" + runcmd "cd build" + echo + echo " -----Running cmake ---- " + echo "${AOMP_CMAKE}" "$(shquot "${MYCMAKEOPTS[@]}")" \ + "../" + + if ! ${AOMP_CMAKE} "${MYCMAKEOPTS[@]}" ../; then + echo "ERROR cmake failed. Cmake flags" + echo " $(shquot "${MYCMAKEOPTS[@]}")" + exit 1 + fi + runcmd "make -j${AOMP_JOB_THREADS}" + runcmd "make install" + if [ -L "$_linkfrom" ] ; then + runcmd "rm $_linkfrom" + fi + runcmd "ln -sfr $_installdir $_linkfrom" + echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE" +} + function buildsilo(){ local _cname="silo" local _version=4.11.1 @@ -916,6 +1161,14 @@ for _component in $_components ; do buildhdf5 elif [ "$_component" == "hdf5-parallel" ] ; then buildhdf5 "-parallel" + elif [ "$_component" == "netcdf-c" ] ; then + buildnetcdfc + elif [ "$_component" == "netcdf-fortran" ] ; then + buildnetcdffortran + elif [ "$_component" == "ecbuild" ] ; then + buildecbuild + elif [ "$_component" == "fiat" ] ; then + buildfiat elif [ "$_component" == "fftw" ] ; then buildfftw elif [ "$_component" == "hwloc" ] ; then diff --git a/bin/run_ecrad.sh b/bin/run_ecrad.sh new file mode 100755 index 000000000..4c4f7c397 --- /dev/null +++ b/bin/run_ecrad.sh @@ -0,0 +1,278 @@ +#!/bin/bash +# +# run_ecrad.sh: +# Build ecRad (ECMWF atmospheric radiation scheme) with the AOMP +# OpenMP-offload toolchain (amdclang, amdflang), run the upstream CTest +# suite on the GPU, then run the benchmark case and compare its output +# against a reference. +# +# The ecRad sources are cloned into $AOMP_REPOS_TEST by clone_test.sh +# (see manifests/test_.xml). Source repo: +# https://github.com/ecmwf-ifs/ecrad (branch master-omp) +# +# ecRad requires serial NetCDF-Fortran, ecbuild and fiat. Build the AOMP +# supplemental dependencies first with: +# ./build_supp.sh netcdf-fortran fiat +# +# By default the CTest suite includes the bit-identity checks, which compare +# the GPU drivers against the standalone driver. CMake creates a Python venv. +# Set ECRAD_BITIDENTITY=OFF to only check that the drivers run. +# +# The benchmark needs a data directory (not in the repo) at $ECRAD_DATA_DIR, +# default $AOMP_REPOS_TEST/ecrad-data, with: +# rad4NN_inputs_2020010100_0c0.nc input +# ecrad_ifs_blocked_dp_B200-master-acc.nc reference output +# config.nam namelist +# The comparison uses the nccmp tool of the ecRad build (needs +# ECRAD_BITIDENTITY=ON). Set ECRAD_BENCHMARK=OFF to skip the benchmark. +# +# Usage: +# ./run_ecrad.sh configure + build ecRad + run tests + benchmark +# ./run_ecrad.sh nocmake skip CMake configure, rebuild + tests + benchmark +# ./run_ecrad.sh rerun skip build, just re-run tests + benchmark +# + +# --- Start standard header to set AOMP environment variables ---- +realpath=$(realpath "$0") +thisdir=$(dirname "$realpath") +export AOMP_USE_CCACHE=0 + +# shellcheck disable=SC1091 +. "$thisdir"/aomp_common_vars +# --- end standard header ---- + +# Offload kernel tracing is very noisy across a full test-suite; default off. +export LIBOMPTARGET_KERNEL_TRACE=${LIBOMPTARGET_KERNEL_TRACE:-0} + +# Fail instead of silently falling back to the host if offloading does not work. +export OMP_TARGET_OFFLOAD=${OMP_TARGET_OFFLOAD:-MANDATORY} + +# Setup AOMP variables +AOMP=${AOMP:-$HOME/rocm/srock/llvm} + +# Use function to set and test AOMP_GPU +setaompgpu + +# --- ecRad specific variables (all overridable from the environment) ------- + +# Directory name the ecRad repo is cloned into (matches manifest path=). +AOMP_ECRAD_REPO_NAME=${AOMP_ECRAD_REPO_NAME:-ecrad} +ECRAD_REPO=${ECRAD_REPO:-$AOMP_REPOS_TEST/$AOMP_ECRAD_REPO_NAME} +ECRAD_BUILD=${ECRAD_BUILD:-$ECRAD_REPO/build-aomp-$AOMP_GPU} + +# GPU arch passed to the ecRad offload flags. Defaults to detected AOMP_GPU. +GPU_ARCH=${GPU_ARCH:-$AOMP_GPU} + +# Supplemental components built by build_supp.sh. +HDF5_INSTALL=${HDF5_INSTALL:-$AOMP_SUPP/hdf5} +NETCDF_C_INSTALL=${NETCDF_C_INSTALL:-$AOMP_SUPP/netcdf-c} +NETCDF_F_INSTALL=${NETCDF_F_INSTALL:-$AOMP_SUPP/netcdf-fortran} +ECBUILD_INSTALL=${ECBUILD_INSTALL:-$AOMP_SUPP/ecbuild} +FIAT_INSTALL=${FIAT_INSTALL:-$AOMP_SUPP/fiat} + +# ROCm root used to find HIP/OpenMP runtime support. For an AOMP install under +# a ROCm tree, $AOMP is usually /lib/llvm. +ROCM_PATH=${ROCM_PATH:-"$(realpath -m "$(realpath -m "$AOMP")/../..")"} + +# Compiler flags. Same as ecRad AMD GPU build, with offload arch taken from GPU_ARCH. +ECRAD_Fortran_FLAGS=${ECRAD_Fortran_FLAGS:-"-march=native -fdefault-real-8 -O3 --offload-arch=$GPU_ARCH -ffast-real-mod -fopenmp"} +ECRAD_LINKER_FLAGS=${ECRAD_LINKER_FLAGS:-"--offload-arch=$GPU_ARCH -fopenmp -march=native -O3"} + +# Bit-identity checks of the test outputs (needs Python venv). +ECRAD_BITIDENTITY=${ECRAD_BITIDENTITY:-ON} + +# Benchmark: data directory +ECRAD_BENCHMARK=${ECRAD_BENCHMARK:-ON} +ECRAD_DATA_DIR=${ECRAD_DATA_DIR:-$AOMP_REPOS_TEST/ecrad-data} +ECRAD_BENCH_DIR=${ECRAD_BENCH_DIR:-$ECRAD_BUILD/benchmark} +ECRAD_BENCH_INPUT=${ECRAD_BENCH_INPUT:-rad4NN_inputs_2020010100_0c0.nc} +ECRAD_BENCH_REFERENCE=${ECRAD_BENCH_REFERENCE:-ecrad_ifs_blocked_dp_B200-master-acc.nc} +ECRAD_NBLOCKSIZE=${ECRAD_NBLOCKSIZE:-8480} +ECRAD_THRESHOLD=${ECRAD_THRESHOLD:-0.001} + +# Offload runtime settings used by the ecRad app team for the benchmark. +export LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES=${LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES:-0} +export LIBOMPTARGET_AMDGPU_TEAMS_PER_CU=${LIBOMPTARGET_AMDGPU_TEAMS_PER_CU:-4} + +# Make the AOMP compilers and supplemental libraries visible to CMake and tests. +export PATH=$AOMP/bin:$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:$PATH +export LD_LIBRARY_PATH=$NETCDF_F_INSTALL/lib:$NETCDF_C_INSTALL/lib:$HDF5_INSTALL/lib:$AOMP/lib:$ROCM_PATH/lib:$ROCM_PATH/lib64:$LD_LIBRARY_PATH +export LIBRARY_PATH=$AOMP/lib:$ROCM_PATH/lib:$ROCM_PATH/lib64:$LIBRARY_PATH + +AOMP_CMAKE=${AOMP_CMAKE:-cmake} +AOMP_CTEST=${AOMP_CTEST:-"$(dirname "$AOMP_CMAKE")/ctest"} +ECRAD_CTEST_ARGS=${ECRAD_CTEST_ARGS:-"--output-on-failure"} + +if [ ! -d "$ECRAD_REPO" ]; then + echo "ERROR: ecRad sources not found at $ECRAD_REPO" + echo " Run clone_test.sh first (needs a manifest entry for ecrad)." + exit 1 +fi + +if [ ! -d "$NETCDF_F_INSTALL" ]; then + echo "ERROR: NetCDF-Fortran not found at $NETCDF_F_INSTALL" + echo " Build AOMP supplemental NetCDF first with build_supp.sh netcdf-c netcdf-fortran," + echo " or set NETCDF_F_INSTALL and NETCDF_C_INSTALL." + exit 1 +fi + +if [ ! -d "$ECBUILD_INSTALL" ] || [ ! -d "$FIAT_INSTALL" ]; then + echo "ERROR: ecbuild or fiat not found at $ECBUILD_INSTALL, $FIAT_INSTALL" + echo " Build AOMP supplemental fiat first with build_supp.sh fiat," + echo " or set ECBUILD_INSTALL and FIAT_INSTALL." + exit 1 +fi + +ulimit -s unlimited + +run_ecrad_ctest() { + cd "$ECRAD_BUILD" || exit 1 + + echo "=== ecRad test configuration ===" + echo "Repo: $ECRAD_REPO" + echo "Build: $ECRAD_BUILD" + echo "fiat: $FIAT_INSTALL" + echo "GPU arch: $GPU_ARCH" + echo "NetCDF: $NETCDF_F_INSTALL" + echo "Bit-identity: $ECRAD_BITIDENTITY" + echo "CTest args: $ECRAD_CTEST_ARGS" + echo "=== End ecRad test configuration ===" + + # shellcheck disable=SC2086 + "$AOMP_CTEST" --test-dir "$ECRAD_BUILD" $ECRAD_CTEST_ARGS + ret=$? + + if [ "$ret" -ne 0 ]; then + echo "ecrad" >> "$ECRAD_REPO"/failing-tests.txt + else + echo "ecrad" >> "$ECRAD_REPO"/passing-tests.txt + fi + return "$ret" +} + +# Run ecrad_ifs_blocked_dp driver on benchmark input and compare output +# against reference. Timings are printed (but not checked). +run_ecrad_benchmark() { + local nccmp=$ECRAD_BUILD/test/common/nccmp + local exe=$ECRAD_BUILD/bin/ecrad_ifs_blocked_dp + local out=$ECRAD_BENCH_DIR/out-ecrad_ifs_blocked_dp-np$ECRAD_NBLOCKSIZE.nc + local file + + echo "=== ecRad benchmark configuration ===" + echo "Data: $ECRAD_DATA_DIR" + echo "Work dir: $ECRAD_BENCH_DIR" + echo "Blocksize: $ECRAD_NBLOCKSIZE" + echo "Reference: $ECRAD_BENCH_REFERENCE" + echo "Threshold: $ECRAD_THRESHOLD" + echo "=== End ecRad benchmark configuration ===" + + for file in "$ECRAD_BENCH_INPUT" "$ECRAD_BENCH_REFERENCE" config.nam; do + if [ ! -f "$ECRAD_DATA_DIR/$file" ]; then + echo "ERROR: ecRad benchmark data $file not found in $ECRAD_DATA_DIR" + echo " Set ECRAD_DATA_DIR, or ECRAD_BENCHMARK=OFF to skip the benchmark." + echo "ecrad-benchmark" >> "$ECRAD_REPO"/failing-tests.txt + return 1 + fi + done + if [ ! -x "$nccmp" ]; then + echo "ERROR: $nccmp not found, the benchmark comparison needs ECRAD_BITIDENTITY=ON." + echo "ecrad-benchmark" >> "$ECRAD_REPO"/failing-tests.txt + return 1 + fi + + rm -rf "$ECRAD_BENCH_DIR" + mkdir -p "$ECRAD_BENCH_DIR" + cd "$ECRAD_BENCH_DIR" || exit 1 + + # The namelist reads the lookup tables from ./data. + ln -s "$ECRAD_REPO/data" data + sed -E "s/^[[:space:]]*nblocksize[[:space:]]*=[[:space:]]*[0-9]+,/nblocksize=$ECRAD_NBLOCKSIZE,/" \ + "$ECRAD_DATA_DIR/config.nam" > config.nam + + "$exe" config.nam "$ECRAD_DATA_DIR/$ECRAD_BENCH_INPUT" "$out" 2>&1 | tee benchmark.log + ret=${PIPESTATUS[0]} + if [ "$ret" -eq 0 ]; then + echo "=== ecRad benchmark timings ===" + grep "time elapsed" benchmark.log + "$nccmp" --longwave-threshold="$ECRAD_THRESHOLD" --shortwave-threshold="$ECRAD_THRESHOLD" \ + "$out" "$ECRAD_DATA_DIR/$ECRAD_BENCH_REFERENCE" + ret=$? + fi + + if [ "$ret" -ne 0 ]; then + echo "ecRad benchmark FAILED (run or comparison against $ECRAD_BENCH_REFERENCE)" + echo "ecrad-benchmark" >> "$ECRAD_REPO"/failing-tests.txt + else + echo "ecRad benchmark passed, output matches $ECRAD_BENCH_REFERENCE within $ECRAD_THRESHOLD W/m^2" + echo "ecrad-benchmark" >> "$ECRAD_REPO"/passing-tests.txt + fi + return "$ret" +} + +run_ecrad_tests() { + local status=0 + run_ecrad_ctest || status=1 + if [ "$ECRAD_BENCHMARK" == "ON" ]; then + run_ecrad_benchmark || status=1 + fi + return "$status" +} + +if [ "$1" == "rerun" ]; then + cd "$ECRAD_REPO" || exit 1 + rm -f failing-tests.txt passing-tests.txt + run_ecrad_tests + exit $? +fi + +cd "$ECRAD_REPO" || exit 1 +rm -f make-fail.txt failing-tests.txt passing-tests.txt + +if [ "$1" != "nocmake" ]; then + echo "Configuring ecRad with AOMP OpenMP offload for $GPU_ARCH" + rm -rf "$ECRAD_BUILD" + mkdir -p "$ECRAD_BUILD" + + cmake_prefix_path="$NETCDF_F_INSTALL;$NETCDF_C_INSTALL;$HDF5_INSTALL;$ROCM_PATH" + cmake_args=( + -S "$ECRAD_REPO" + -B "$ECRAD_BUILD" + -Decbuild_ROOT="$ECBUILD_INSTALL" + -Dfiat_ROOT="$FIAT_INSTALL" + -DCMAKE_PREFIX_PATH="$cmake_prefix_path" + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=amdclang + -DCMAKE_Fortran_COMPILER=amdflang + -DECBUILD_Fortran_FLAGS="$ECRAD_Fortran_FLAGS" + -DCMAKE_EXE_LINKER_FLAGS_INIT="$ECRAD_LINKER_FLAGS" + -DBUILD_SHARED_LIBS=OFF + -DENABLE_OMP=ON + -DENABLE_GPU=ON + -DENABLE_CPU=OFF + -DENABLE_ACC=OFF + -DENABLE_NVTX=OFF + -DENABLE_ROCTX=OFF + -DENABLE_SINGLE_PRECISION=ON + -DENABLE_DOUBLE_PRECISION=ON + -DENABLE_BITIDENTITY_TESTING="$ECRAD_BITIDENTITY" + ) + + echo "$AOMP_CMAKE ${cmake_args[*]}" + "$AOMP_CMAKE" "${cmake_args[@]}" + ret=$? + if [ "$ret" -ne 0 ]; then + echo "ecrad-configure" >> make-fail.txt + exit 1 + fi +fi + +echo "Building ecRad" +"$AOMP_CMAKE" --build "$ECRAD_BUILD" -j"$AOMP_JOB_THREADS" +ret=$? +if [ "$ret" -ne 0 ]; then + echo "ecrad" >> make-fail.txt + exit 1 +fi + +run_ecrad_tests +exit $? diff --git a/manifests/test_24.0.xml b/manifests/test_24.0.xml index ff821484b..cfc430050 100644 --- a/manifests/test_24.0.xml +++ b/manifests/test_24.0.xml @@ -25,6 +25,7 @@ + @@ -53,5 +54,6 @@ +