From f2b3e81d1fcfa7ac1123d2dcc0de087d4410d1e3 Mon Sep 17 00:00:00 2001 From: Saiyedul Islam Date: Mon, 12 Oct 2026 19:31:35 +0530 Subject: [PATCH 1/3] Fix the warp_id check for GPUs with few CUs The old test advanced the expected warp every MAX_TEAMS iterations, so a 4-CU GPU with 16 default teams failed at iteration 16. The warp id is the thread index within the team divided by the wavefront size. --- test/smoke-limbo/aomp_mappings_newrt/aomp_mappings_newrt.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/smoke-limbo/aomp_mappings_newrt/aomp_mappings_newrt.c b/test/smoke-limbo/aomp_mappings_newrt/aomp_mappings_newrt.c index 174ebafb51..3c77a7d7c1 100644 --- a/test/smoke-limbo/aomp_mappings_newrt/aomp_mappings_newrt.c +++ b/test/smoke-limbo/aomp_mappings_newrt/aomp_mappings_newrt.c @@ -242,10 +242,8 @@ int main() recordError(&errors, "DEVICE NUMBER", i, default_dev, NULL); //check warp # - if (i % MAX_TEAMS * num_threads[i] == 0){ - correctWarpId++; - correctWarpId = correctWarpId % (MAX_THREADS_PER_TEAM/WARP_SIZE); - } + // Warp id is local to the team: thread index divided by wavefront size. + correctWarpId = thread_num[i] / WARP_SIZE; if (warp_id[i] != correctWarpId) recordError(&errors, "WARP NUMBER", i, warp_id, NULL); From 0c7036d42c01ece79f611a87c841ddd88dc13610 Mon Sep 17 00:00:00 2001 From: dpalermo Date: Mon, 12 Oct 2026 15:56:19 -0500 Subject: [PATCH 2/3] [build_supp.sh] buildhdf5 add "-parallel" variant (#2614) * [build_supp.sh] buildhdf5 add "-parallel" variant - Refactors buildhdf5 & buildhdf5parallel into a single routine - Avoids version drift between the two builds * move getaompcompiler to utils section --- bin/build_supp.sh | 145 +++++++++++++++------------------------------- 1 file changed, 47 insertions(+), 98 deletions(-) diff --git a/bin/build_supp.sh b/bin/build_supp.sh index 654cddd56f..7b0ce97afd 100755 --- a/bin/build_supp.sh +++ b/bin/build_supp.sh @@ -168,6 +168,23 @@ function derive_rocm_path(){ return 0 } +function getaompcompiler(){ + if [ "$#" -ne "2" ]; then + echo "Usage: ${FUNCNAME[0]} " + return 1 + fi + local primary=$1 + local fallback=$2 + if [ -x "$AOMP/bin/$primary" ]; then + echo "$AOMP/bin/$primary" + elif [ -x "$AOMP/bin/$fallback" ]; then + echo "$AOMP/bin/$fallback" + else + echo "ERROR: Cannot find $primary or $fallback under $AOMP/bin" >&2 + return 1 + fi +} + ################################################################################ # XPMEM - Cross-Process Memory Access for high-performance shared memory ################################################################################ @@ -573,119 +590,49 @@ function getrocmpackage(){ } function buildhdf5(){ - local _cname="hdf5" + local variant=$1 + local _cname="hdf5$variant" local _version=2.2.0 local _release=hdf5-2.2.0 - local _installdir=$AOMP_SUPP_INSTALL/hdf5-$_version - local _linkfrom=$AOMP_SUPP/hdf5 - local _builddir=$AOMP_SUPP_BUILD/hdf5 + local _installdir=$AOMP_SUPP_INSTALL/$_cname-$_version + local _linkfrom=$AOMP_SUPP/$_cname + local _builddir=$AOMP_SUPP_BUILD/$_cname BUILD_TYPE=${BUILD_TYPE:-Release} - declare -a MYCMAKEOPTS - MYCMAKEOPTS=(-DCMAKE_BUILD_TYPE="$BUILD_TYPE" - -DCMAKE_INSTALL_PREFIX="$_installdir" - -DCMAKE_CXX_COMPILER="$AOMP/bin/clang++" - -DCMAKE_C_COMPILER="$AOMP/bin/clang" - -DCMAKE_Fortran_COMPILER="$AOMP/bin/flang" - -DHDF5_BUILD_FORTRAN=ON - -DHDF5_BUILD_HL_LIB=ON - -DHDF5_BUILD_TOOLS=ON - -DBUILD_SHARED_LIBS=ON - -DBUILD_STATIC_LIBS=ON - -DHDF5_ENABLE_ZLIB_SUPPORT=ON - -DHDF5_ENABLE_SZIP_SUPPORT=OFF - -DHDF5_ENABLE_THREADSAFE=OFF - -DBUILD_TESTING=OFF - -DHDF5_INSTALL_MOD_FORTRAN=STATIC) - - if checkversion "$_cname" "$_version" "$_linkfrom"; then - return - fi - if [ -d "$_builddir" ] ; then - runcmd "rm -rf $_builddir" - fi - runcmd "mkdir -p $_builddir" - runcmd "cd $_builddir" - #runcmd " wget https://support.hdfgroup.org/ftp/HDF5/releases/$_release/hdf5-$_version/src/hdf5-$_version.tar.bz2" - runcmd " wget https://github.com/HDFGroup/hdf5/releases/download/$_version/hdf5-$_version.tar.gz" - runcmd "tar -xzf hdf5-$_version.tar.gz" - runcmd "cd hdf5-$_version" - if [ -d "$_installdir" ] ; then - runcmd "rm -rf $_installdir" - fi - runcmd "mkdir -p $_installdir" - runcmd "mkdir -p build" - runcmd "cd build" - #runcmd "./configure --enable-fortran --prefix=$_installdir" - echo - echo " -----Running cmake ---- " - echo "${AOMP_CMAKE}" "$(shquot "${MYCMAKEOPTS[@]}")" \ - "../" + declare -a MYVAROPTS + MYVAROPTS=(-DCMAKE_CXX_COMPILER="$AOMP/bin/clang++" + -DCMAKE_C_COMPILER="$AOMP/bin/clang" + -DCMAKE_Fortran_COMPILER="$AOMP/bin/flang") - if ! ${AOMP_CMAKE} "${MYCMAKEOPTS[@]}" ../; then - echo "ERROR cmake failed. Cmake flags" - echo " $(shquot "${MYCMAKEOPTS[@]}")" - exit 1 - fi - runcmd "make -j${AOMP_JOB_THREADS}" - runcmd "make install" - if [ -L "$_linkfrom" ] ; then - runcmd "rm $_linkfrom" - fi - runcmd "ln -sfr $_installdir $_linkfrom" - echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE" -} + if [ "$variant" == "-parallel" ]; then + OPENMPI_INSTALL=${OPENMPI_INSTALL:-$AOMP_SUPP/rocmopenmpi} -function getaompcompiler(){ - if [ "$#" -ne "2" ]; then - echo "Usage: ${FUNCNAME[0]} " - return 1 - fi - local primary=$1 - local fallback=$2 - if [ -x "$AOMP/bin/$primary" ]; then - echo "$AOMP/bin/$primary" - elif [ -x "$AOMP/bin/$fallback" ]; then - echo "$AOMP/bin/$fallback" - else - echo "ERROR: Cannot find $primary or $fallback under $AOMP/bin" >&2 - return 1 - fi -} + if [ ! -x "$OPENMPI_INSTALL/bin/mpicc" ] || [ ! -x "$OPENMPI_INSTALL/bin/mpifort" ]; then + echo "ERROR: hdf5-parallel requires ROCm-aware OpenMPI at $OPENMPI_INSTALL" + echo " Build it first with: $0 rocmopenmpi" + exit 1 + fi -function buildhdf5parallel(){ - local _cname="hdf5-parallel" - local _version=2.2.0 - local _release=hdf5-2.2.0 - local _installdir=$AOMP_SUPP_INSTALL/$_cname-$_version - local _linkfrom=$AOMP_SUPP/$_cname - local _builddir=$AOMP_SUPP_BUILD/$_cname - BUILD_TYPE=${BUILD_TYPE:-Release} - OPENMPI_INSTALL=${OPENMPI_INSTALL:-$AOMP_SUPP/rocmopenmpi} + AOMP_CC=${AOMP_CC:-$(getaompcompiler amdclang clang)} + AOMP_CXX=${AOMP_CXX:-$(getaompcompiler amdclang++ clang++)} + AOMP_FC=${AOMP_FC:-$(getaompcompiler amdflang "$FLANG")} + export OMPI_CC=${OMPI_CC:-$AOMP_CC} + export OMPI_CXX=${OMPI_CXX:-$AOMP_CXX} + export OMPI_FC=${OMPI_FC:-$AOMP_FC} - if [ ! -x "$OPENMPI_INSTALL/bin/mpicc" ] || [ ! -x "$OPENMPI_INSTALL/bin/mpifort" ]; then - echo "ERROR: hdf5-parallel requires ROCm-aware OpenMPI at $OPENMPI_INSTALL" - echo " Build it first with: $0 rocmopenmpi" - exit 1 + MYVAROPTS=(-DCMAKE_CXX_COMPILER="$OPENMPI_INSTALL/bin/mpicxx" + -DCMAKE_C_COMPILER="$OPENMPI_INSTALL/bin/mpicc" + -DCMAKE_Fortran_COMPILER="$OPENMPI_INSTALL/bin/mpifort" + -DHDF5_ENABLE_PARALLEL=ON) fi - AOMP_CC=${AOMP_CC:-$(getaompcompiler amdclang clang)} - AOMP_CXX=${AOMP_CXX:-$(getaompcompiler amdclang++ clang++)} - AOMP_FC=${AOMP_FC:-$(getaompcompiler amdflang "$FLANG")} - export OMPI_CC=${OMPI_CC:-$AOMP_CC} - export OMPI_CXX=${OMPI_CXX:-$AOMP_CXX} - export OMPI_FC=${OMPI_FC:-$AOMP_FC} - declare -a MYCMAKEOPTS MYCMAKEOPTS=(-DCMAKE_BUILD_TYPE="$BUILD_TYPE" -DCMAKE_INSTALL_PREFIX="$_installdir" - -DCMAKE_CXX_COMPILER="$OPENMPI_INSTALL/bin/mpicxx" - -DCMAKE_C_COMPILER="$OPENMPI_INSTALL/bin/mpicc" - -DCMAKE_Fortran_COMPILER="$OPENMPI_INSTALL/bin/mpifort" + "${MYVAROPTS[@]}" -DHDF5_BUILD_FORTRAN=ON -DHDF5_BUILD_HL_LIB=ON -DHDF5_BUILD_TOOLS=ON - -DHDF5_ENABLE_PARALLEL=ON -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DHDF5_ENABLE_ZLIB_SUPPORT=ON @@ -703,6 +650,7 @@ function buildhdf5parallel(){ fi runcmd "mkdir -p $_builddir" runcmd "cd $_builddir" + #runcmd " wget https://support.hdfgroup.org/ftp/HDF5/releases/$_release/hdf5-$_version/src/hdf5-$_version.tar.bz2" runcmd " wget https://github.com/HDFGroup/hdf5/releases/download/$_version/hdf5-$_version.tar.gz" runcmd "tar -xzf hdf5-$_version.tar.gz" runcmd "cd hdf5-$_version" @@ -712,6 +660,7 @@ function buildhdf5parallel(){ runcmd "mkdir -p $_installdir" runcmd "mkdir -p build" runcmd "cd build" + #runcmd "./configure --enable-fortran --prefix=$_installdir" echo echo " -----Running cmake ---- " echo "${AOMP_CMAKE}" "$(shquot "${MYCMAKEOPTS[@]}")" \ @@ -966,7 +915,7 @@ for _component in $_components ; do elif [ "$_component" == "hdf5" ] ; then buildhdf5 elif [ "$_component" == "hdf5-parallel" ] ; then - buildhdf5parallel + buildhdf5 "-parallel" elif [ "$_component" == "fftw" ] ; then buildfftw elif [ "$_component" == "hwloc" ] ; then From fc2dfc18ae398ec11ebd77c0e5e77d7a7c7ae650 Mon Sep 17 00:00:00 2001 From: Raghu Maddhipatla Date: Fri, 16 Oct 2026 18:21:07 -0500 Subject: [PATCH 3/3] Update GESTS script to use flang-new (#2630) * Update GESTS script to use flang-new instead of hipfc and its equivalent flags * Fix shellcheck warnings on this script. --- bin/run_gests.sh | 167 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 116 insertions(+), 51 deletions(-) diff --git a/bin/run_gests.sh b/bin/run_gests.sh index 62cd6bb2ab..29a6888a3f 100755 --- a/bin/run_gests.sh +++ b/bin/run_gests.sh @@ -1,88 +1,153 @@ #!/bin/bash -# -# run_nekbone.sh: +# +# run_gests.sh +# +# ./run_gests.sh [buildonly|runonly] +# +# GESTS_NP, GESTS_JOBS, OMP_NUM_THREADS can be set. +# The run input is $REPO_DIR/files/input. # # --- Start standard header to set AOMP environment variables ---- -realpath=`realpath $0` -thisdir=`dirname $realpath` +realpath=$(realpath "$0") +thisdir=$(dirname "$realpath") export AOMP_USE_CCACHE=0 - -. $thisdir/aomp_common_vars -# --- end standard header ---- - -# Setup AOMP variables -AOMP=${AOMP:-/usr/lib/aomp} +AOMP=${AOMP:-$HOME/rocm/aomp/llvm} AOMP_SUPP=${AOMP_SUPP:-$HOME/local} -HIPFORT=${HIPFORT:-$HOME/rocm/aomp/hipfort/} +ROCM=${ROCM:-/opt/rocm} +HIPFORT_INCLUDE=${HIPFORT_INCLUDE:-$HOME/rocm/aomp} +HIPFORT_LIB=${HIPFORT_LIB:-$HOME/rocm/aomp/lib} +AOMPHIP=${AOMPHIP:-$(realpath -m "$(realpath -m "$AOMP")/../..")} -# Use function to set and test AOMP_GPU -setaompgpu +# shellcheck disable=SC1091 +. "$thisdir/aomp_common_vars" +# --- end standard header ---- REPO_DIR=$AOMP_REPOS_TEST/GESTS +RUN_DIR=$AOMP_REPOS_TEST/run/GESTS +EXECUTABLE=DNS_PEN_GPU_p4.x -if [ -f $HOME/GESTS.tar ] && [ ! -d $REPO_DIR ] ; then - cp $HOME/GESTS.tar $AOMP_REPOS_TEST - cd $AOMP_REPOS_TEST - tar xvf $HOME/GESTS.tar +if [ -d "$REPO_DIR" ] ; then + echo "$REPO_DIR exists." +else + echo "Please contact Raghu Maddhipatla to get a copy of GESTS.tar and place the tar file at $HOME" + exit fi -if [ -d $REPO_DIR ] ; then - echo $REPO_DIR " exists." +# rocFFT is not in $ROCM. Use it from the system ROCm if needed. +if [ -f "$ROCM/lib/librocfft.so" ] ; then + ROCFFT_LIB=$ROCM/lib else - echo "Please contact Raghu Maddhipatla to get a copy of GESTS.tar and place the tar file at $HOME" - exit + ROCFFT_LIB=${ROCFFT_LIB:-/opt/rocm/lib} fi -cd $REPO_DIR -#patchrepo $REPO_DIR +export FFTW_DIR=$AOMP_SUPP/fftw +export OPENMPI_DIR=$AOMP_SUPP/openmpi -if [ -d $HIPFORT ] ; then - echo $HIPFORT " exists." -else - echo "Install latest hipfort and try again!" - exit +echo "AOMP = $AOMP" +echo "ROCM = $ROCM" +echo "AOMP_SUPP = $AOMP_SUPP" +echo "HIPFORT_INCLUDE = $HIPFORT_INCLUDE" +echo "HIPFORT_LIB = $HIPFORT_LIB" +echo "AOMPHIP = $AOMPHIP" + +setaompgpu + +if [ ! -d "$HIPFORT_INCLUDE" ] ; then + echo "hipfort include directory not found at $HIPFORT_INCLUDE" + exit 1 fi -currdir=$(pwd) -export FFTW_DIR=$AOMP_SUPP/fftw -if [ -d "$FFTW_DIR" ] ; then - echo "FFTW exists." -else - echo "$FFTW_DIR does not exist. Install FFTW using ./build_supp.sh fftw and try again." - exit +if [ ! -f "$HIPFORT_LIB/libhipfort-amdgcn.a" ] ; then + echo "missing $HIPFORT_LIB/libhipfort-amdgcn.a" + exit 1 fi -export OPENMPI_DIR=$AOMP_SUPP/openmpi -if [ -d "$OPENMPI_DIR" ] ; then - echo "OpenMPI-4.1.1 exists." -else - echo "$OPENMPI_DIR does not exist. Install openmpi using ./build_supp.sh ompi and try again." - exit +if [ ! -d "$FFTW_DIR" ] ; then + echo "$FFTW_DIR does not exist." + exit 1 +fi +if [ ! -d "$OPENMPI_DIR" ] ; then + echo "$OPENMPI_DIR does not exist." + exit 1 +fi +if [ ! -f "$AOMP/bin/flang-new" ] ; then + echo "flang-new not found in $AOMP/bin" + exit 1 +fi +if [ ! -f "$REPO_DIR/files/input" ] ; then + echo "missing $REPO_DIR/files/input" + exit 1 +fi +if [ ! -f "$ROCFFT_LIB/librocfft.so" ] ; then + echo "librocfft.so not found in $ROCFFT_LIB" + exit 1 fi -export LD_LIBRARY_PATH=$HOME/rocm/aomp/lib:$OPENMPI_DIR/lib:$FFTW_DIR/lib:$LD_LIBRARY_PATH -export ROCM_DIR=$AOMP +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$AOMP/lib:$ROCM/lib:$ROCFFT_LIB:$OPENMPI_DIR/lib:$FFTW_DIR/lib +export PYTHONPATH=${PYTHONPATH:-$ROCM/share/amd_smi} export HIP_CLANG_PATH=$AOMP/bin -cd $REPO_DIR + +GESTS_NP=${GESTS_NP:-1} +GESTS_JOBS=${GESTS_JOBS:-1} +export OMP_NUM_THREADS=${OMP_NUM_THREADS:-2} + if [ "$1" != "runonly" ] ; then - echo "================= STARTING MAKE in $PWD ========" + echo "================= STARTING MAKE in $REPO_DIR ========" + cd "$REPO_DIR" || exit 1 make clean - make + make srcmake + + cppflags="-P -DOPENMP -DFFTW -DREVERSE_DIMS -DUSE_EVEN -DTIMERS -DDETAIL_MPI -DUSE_AMD -DUSE_HIPHOSTMALLOC -DPHYMAX -DMODEL_SPECTRUM" + fflags="-cpp -I$HIPFORT_INCLUDE -I. -Ibin -Jbin -O1 --offload-arch=$AOMP_GPU -fopenmp" + fft_inc="-I$FFTW_DIR/include" + other_inc="-I$ROCM/include -I$HIPFORT_INCLUDE -I$OPENMPI_DIR/include -I$OPENMPI_DIR/lib" + fft_lib="-L$FFTW_DIR/lib -lfftw3 -lfftw3f" + other_libs="-L$OPENMPI_DIR/lib -lmpi -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -L$ROCM/lib -L$ROCFFT_LIB -L$HIPFORT_LIB -lrocfft -lamdhip64 -lhipfort-amdgcn -lhsa-runtime64 -lstdc++ -lm" + + make -j "$GESTS_JOBS" MACHINE=local \ + CPPFLAGS="$cppflags" \ + FF="$AOMP/bin/flang-new" \ + CPP=/lib/cpp \ + FFLAGS="$fflags" \ + FFLAGS90="$fflags" \ + FFT_INC="$fft_inc" \ + OTHER_INC="$other_inc" \ + FFT_LIB="$fft_lib" \ + OTHER_LIBS="$other_libs" rc=$? if [ $rc != 0 ] ; then - echo "ERROR in Make" - exit + echo "ERROR in Make" + exit $rc + fi + if [ ! -f "$REPO_DIR/$EXECUTABLE" ] ; then + echo "ERROR: $EXECUTABLE was not built" + exit 1 fi echo echo "================= BUILD IS DONE ========" fi if [ "$1" != "buildonly" ] ; then - cd $REPO_DIR + if [ ! -f "$REPO_DIR/$EXECUTABLE" ] ; then + echo "ERROR: $REPO_DIR/$EXECUTABLE not found" + exit 1 + fi + mkdir -p "$RUN_DIR/MPI_timings" + cp "$REPO_DIR/$EXECUTABLE" "$RUN_DIR/" + cp "$REPO_DIR/files/input" "$RUN_DIR/input" + rm -f "$RUN_DIR/dims" + + cd "$RUN_DIR" || exit 1 echo echo "================= attempting mpirun ========" - _cmd="$OPENMPI_DIR/bin/mpirun -np 1 $AOMP/bin/gpurun ./PSDNS_fft.x" - echo $_cmd + _cmd="$OPENMPI_DIR/bin/mpirun -np $GESTS_NP $AOMP/bin/gpurun -rocmsmi ./$EXECUTABLE" + echo "$_cmd" + echo "np=$GESTS_NP threads=$OMP_NUM_THREADS" time $_cmd + rc=$? + if [ $rc != 0 ] ; then + echo "ERROR: GESTS run failed" + exit $rc + fi echo echo "================= DONE with mpirun ========" fi