From d77c77a38949fa475537f58928e5d75a8a3af298 Mon Sep 17 00:00:00 2001
From: Raghu Maddhipatla <Raghu.Maddhipatla@amd.com>
Date: Fri, 16 Oct 2026 15:16:30 -0500
Subject: [PATCH 1/2] Update GESTS script to use flang-new instead of hipfc and
 its equivalent flags

---
 bin/run_gests.sh | 164 +++++++++++++++++++++++++++++++++--------------
 1 file changed, 116 insertions(+), 48 deletions(-)

diff --git a/bin/run_gests.sh b/bin/run_gests.sh
index 62cd6bb2ab..7ee4b9855e 100755
--- a/bin/run_gests.sh
+++ b/bin/run_gests.sh
@@ -1,88 +1,156 @@
 #!/bin/bash
-# 
-#  run_nekbone.sh: 
+#
+#  run_gests.sh
+#
+#    ./run_gests.sh [buildonly|runonly]
+#
+#  GESTS_NP, GESTS_JOBS, OMP_NUM_THREADS can be set.
+#  The run input is $REPO_DIR/files/input.
 #
 
 # --- Start standard header to set AOMP environment variables ----
 realpath=`realpath $0`
 thisdir=`dirname $realpath`
 export AOMP_USE_CCACHE=0
+AOMP=${AOMP:-$HOME/rocm/aomp/llvm}
+AOMP_SUPP=${AOMP_SUPP:-$HOME/local}
+ROCM=${ROCM:-/opt/rocm}
+HIPFORT_INCLUDE=${HIPFORT_INCLUDE:-$HOME/rocm/aomp}
+HIPFORT_LIB=${HIPFORT_LIB:-$HOME/rocm/aomp/lib}
+AOMPHIP=${AOMPHIP:-$(realpath -m $(realpath -m $AOMP)/../..)}
 
 . $thisdir/aomp_common_vars
 # --- end standard header ----
 
-# Setup AOMP variables
-AOMP=${AOMP:-/usr/lib/aomp}
-AOMP_SUPP=${AOMP_SUPP:-$HOME/local}
-HIPFORT=${HIPFORT:-$HOME/rocm/aomp/hipfort/}
+if [ -d $REPO_DIR ] ; then
+  echo $REPO_DIR " exists."
+else
+  echo "Please contact Raghu Maddhipatla to get a copy of GESTS.tar and place the tar file at $HOME"
+  exit
+fi
 
-# Use function to set and test AOMP_GPU
-setaompgpu
+# rocFFT is not in $ROCM. Use it from the system ROCm if needed.
+if [ -f $ROCM/lib/librocfft.so ] ; then
+  ROCFFT_LIB=$ROCM/lib
+else
+  ROCFFT_LIB=${ROCFFT_LIB:-/opt/rocm/lib}
+fi
 
 REPO_DIR=$AOMP_REPOS_TEST/GESTS
+RUN_DIR=$AOMP_REPOS_TEST/run/GESTS
+EXECUTABLE=DNS_PEN_GPU_p4.x
 
-if [ -f $HOME/GESTS.tar ] && [ ! -d $REPO_DIR ] ; then
-    cp $HOME/GESTS.tar $AOMP_REPOS_TEST
-    cd $AOMP_REPOS_TEST
-    tar xvf $HOME/GESTS.tar
-fi
+export FFTW_DIR=$AOMP_SUPP/fftw
+export OPENMPI_DIR=$AOMP_SUPP/openmpi
 
-if [ -d $REPO_DIR ] ; then
-    echo $REPO_DIR " exists."
-else
-    echo "Please contact Raghu Maddhipatla to get a copy of GESTS.tar and place the tar file at $HOME"
-    exit
-fi
+echo "AOMP            = $AOMP"
+echo "ROCM            = $ROCM"
+echo "AOMP_SUPP       = $AOMP_SUPP"
+echo "HIPFORT_INCLUDE = $HIPFORT_INCLUDE"
+echo "HIPFORT_LIB     = $HIPFORT_LIB"
+echo "AOMPHIP         = $AOMPHIP"
 
-cd $REPO_DIR
-#patchrepo $REPO_DIR
+setaompgpu
 
-if [ -d $HIPFORT ] ; then
-    echo $HIPFORT " exists."
-else
-    echo "Install latest hipfort and try again!"
-    exit
+if [ ! -d $REPO_DIR ] ; then
+  echo "$REPO_DIR does not exist."
+  exit 1
 fi
-currdir=$(pwd)
-export FFTW_DIR=$AOMP_SUPP/fftw
-if [ -d "$FFTW_DIR" ] ; then
-    echo "FFTW exists."
-else
-    echo "$FFTW_DIR does not exist. Install FFTW using ./build_supp.sh fftw and try again."
-    exit
+if [ ! -d $HIPFORT_INCLUDE ] ; then
+  echo "hipfort include directory not found at $HIPFORT_INCLUDE"
+  exit 1
 fi
-export OPENMPI_DIR=$AOMP_SUPP/openmpi
-if [ -d "$OPENMPI_DIR" ] ; then
-    echo "OpenMPI-4.1.1 exists."
-else
-    echo "$OPENMPI_DIR does not exist. Install openmpi using ./build_supp.sh ompi and try again."
-    exit
+if [ ! -f $HIPFORT_LIB/libhipfort-amdgcn.a ] ; then
+  echo "missing $HIPFORT_LIB/libhipfort-amdgcn.a"
+  exit 1
+fi
+if [ ! -d $FFTW_DIR ] ; then
+  echo "$FFTW_DIR does not exist."
+  exit 1
+fi
+if [ ! -d $OPENMPI_DIR ] ; then
+  echo "$OPENMPI_DIR does not exist."
+  exit 1
+fi
+if [ ! -f $AOMP/bin/flang-new ] ; then
+  echo "flang-new not found in $AOMP/bin"
+  exit 1
+fi
+if [ ! -f $REPO_DIR/files/input ] ; then
+  echo "missing $REPO_DIR/files/input"
+  exit 1
+fi
+if [ ! -f $ROCFFT_LIB/librocfft.so ] ; then
+  echo "librocfft.so not found in $ROCFFT_LIB"
+  exit 1
 fi
 
-export LD_LIBRARY_PATH=$HOME/rocm/aomp/lib:$OPENMPI_DIR/lib:$FFTW_DIR/lib:$LD_LIBRARY_PATH
-export ROCM_DIR=$AOMP
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$AOMP/lib:$ROCM/lib:$ROCFFT_LIB:$OPENMPI_DIR/lib:$FFTW_DIR/lib
+export PYTHONPATH=${PYTHONPATH:-$ROCM/share/amd_smi}
 export HIP_CLANG_PATH=$AOMP/bin
-cd $REPO_DIR
+
+GESTS_NP=${GESTS_NP:-1}
+GESTS_JOBS=${GESTS_JOBS:-1}
+export OMP_NUM_THREADS=${OMP_NUM_THREADS:-2}
+
 if [ "$1" != "runonly" ] ; then
-  echo "=================  STARTING MAKE in $PWD ========"
+  echo "=================  STARTING MAKE in $REPO_DIR ========"
+  cd $REPO_DIR || exit 1
   make clean
-  make
+  make srcmake
+
+  cppflags="-P -DOPENMP -DFFTW -DREVERSE_DIMS -DUSE_EVEN -DTIMERS -DDETAIL_MPI -DUSE_AMD -DUSE_HIPHOSTMALLOC -DPHYMAX -DMODEL_SPECTRUM"
+  fflags="-cpp -I$HIPFORT_INCLUDE -I. -Ibin -Jbin -O1 --offload-arch=$AOMP_GPU -fopenmp"
+  fft_inc="-I$FFTW_DIR/include"
+  other_inc="-I$ROCM/include -I$HIPFORT_INCLUDE -I$OPENMPI_DIR/include -I$OPENMPI_DIR/lib"
+  fft_lib="-L$FFTW_DIR/lib -lfftw3 -lfftw3f"
+  other_libs="-L$OPENMPI_DIR/lib -lmpi -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -L$ROCM/lib -L$ROCFFT_LIB -L$HIPFORT_LIB -lrocfft -lamdhip64 -lhipfort-amdgcn -lhsa-runtime64 -lstdc++ -lm"
+
+  make -j $GESTS_JOBS MACHINE=local \
+      CPPFLAGS="$cppflags" \
+      FF=$AOMP/bin/flang-new \
+      CPP=/lib/cpp \
+      FFLAGS="$fflags" \
+      FFLAGS90="$fflags" \
+      FFT_INC="$fft_inc" \
+      OTHER_INC="$other_inc" \
+      FFT_LIB="$fft_lib" \
+      OTHER_LIBS="$other_libs"
   rc=$?
   if [ $rc != 0 ] ; then
-     echo "ERROR in Make"
-     exit
+    echo "ERROR in Make"
+    exit $rc
+  fi
+  if [ ! -f $REPO_DIR/$EXECUTABLE ] ; then
+    echo "ERROR: $EXECUTABLE was not built"
+    exit 1
   fi
   echo
   echo "=================  BUILD IS DONE ========"
 fi
 
 if [ "$1" != "buildonly" ] ; then
-  cd $REPO_DIR
+  if [ ! -f $REPO_DIR/$EXECUTABLE ] ; then
+    echo "ERROR: $REPO_DIR/$EXECUTABLE not found"
+    exit 1
+  fi
+  mkdir -p $RUN_DIR/MPI_timings
+  cp $REPO_DIR/$EXECUTABLE $RUN_DIR/
+  cp $REPO_DIR/files/input $RUN_DIR/input
+  rm -f $RUN_DIR/dims
+
+  cd $RUN_DIR || exit 1
   echo
   echo "=================  attempting mpirun  ========"
-  _cmd="$OPENMPI_DIR/bin/mpirun -np 1 $AOMP/bin/gpurun ./PSDNS_fft.x"
+  _cmd="$OPENMPI_DIR/bin/mpirun -np $GESTS_NP $AOMP/bin/gpurun -rocmsmi ./$EXECUTABLE"
   echo $_cmd
+  echo "np=$GESTS_NP threads=$OMP_NUM_THREADS"
   time $_cmd
+  rc=$?
+  if [ $rc != 0 ] ; then
+    echo "ERROR: GESTS run failed"
+    exit $rc
+  fi
   echo
   echo "=================  DONE with mpirun  ========"
 fi

From ae21c69faa5a7f3290c299a08a440afedab461d8 Mon Sep 17 00:00:00 2001
From: Raghu Maddhipatla <Raghu.Maddhipatla@amd.com>
Date: Fri, 16 Oct 2026 16:15:58 -0500
Subject: [PATCH 2/2] Fix shellcheck warnings on this script.

---
 bin/run_gests.sh | 63 +++++++++++++++++++++++-------------------------
 1 file changed, 30 insertions(+), 33 deletions(-)

diff --git a/bin/run_gests.sh b/bin/run_gests.sh
index 7ee4b9855e..29a6888a3f 100755
--- a/bin/run_gests.sh
+++ b/bin/run_gests.sh
@@ -9,37 +9,38 @@
 #
 
 # --- Start standard header to set AOMP environment variables ----
-realpath=`realpath $0`
-thisdir=`dirname $realpath`
+realpath=$(realpath "$0")
+thisdir=$(dirname "$realpath")
 export AOMP_USE_CCACHE=0
 AOMP=${AOMP:-$HOME/rocm/aomp/llvm}
 AOMP_SUPP=${AOMP_SUPP:-$HOME/local}
 ROCM=${ROCM:-/opt/rocm}
 HIPFORT_INCLUDE=${HIPFORT_INCLUDE:-$HOME/rocm/aomp}
 HIPFORT_LIB=${HIPFORT_LIB:-$HOME/rocm/aomp/lib}
-AOMPHIP=${AOMPHIP:-$(realpath -m $(realpath -m $AOMP)/../..)}
+AOMPHIP=${AOMPHIP:-$(realpath -m "$(realpath -m "$AOMP")/../..")}
 
-. $thisdir/aomp_common_vars
+# shellcheck disable=SC1091
+. "$thisdir/aomp_common_vars"
 # --- end standard header ----
 
-if [ -d $REPO_DIR ] ; then
-  echo $REPO_DIR " exists."
+REPO_DIR=$AOMP_REPOS_TEST/GESTS
+RUN_DIR=$AOMP_REPOS_TEST/run/GESTS
+EXECUTABLE=DNS_PEN_GPU_p4.x
+
+if [ -d "$REPO_DIR" ] ; then
+  echo "$REPO_DIR exists."
 else
   echo "Please contact Raghu Maddhipatla to get a copy of GESTS.tar and place the tar file at $HOME"
   exit
 fi
 
 # rocFFT is not in $ROCM. Use it from the system ROCm if needed.
-if [ -f $ROCM/lib/librocfft.so ] ; then
+if [ -f "$ROCM/lib/librocfft.so" ] ; then
   ROCFFT_LIB=$ROCM/lib
 else
   ROCFFT_LIB=${ROCFFT_LIB:-/opt/rocm/lib}
 fi
 
-REPO_DIR=$AOMP_REPOS_TEST/GESTS
-RUN_DIR=$AOMP_REPOS_TEST/run/GESTS
-EXECUTABLE=DNS_PEN_GPU_p4.x
-
 export FFTW_DIR=$AOMP_SUPP/fftw
 export OPENMPI_DIR=$AOMP_SUPP/openmpi
 
@@ -52,35 +53,31 @@ echo "AOMPHIP         = $AOMPHIP"
 
 setaompgpu
 
-if [ ! -d $REPO_DIR ] ; then
-  echo "$REPO_DIR does not exist."
-  exit 1
-fi
-if [ ! -d $HIPFORT_INCLUDE ] ; then
+if [ ! -d "$HIPFORT_INCLUDE" ] ; then
   echo "hipfort include directory not found at $HIPFORT_INCLUDE"
   exit 1
 fi
-if [ ! -f $HIPFORT_LIB/libhipfort-amdgcn.a ] ; then
+if [ ! -f "$HIPFORT_LIB/libhipfort-amdgcn.a" ] ; then
   echo "missing $HIPFORT_LIB/libhipfort-amdgcn.a"
   exit 1
 fi
-if [ ! -d $FFTW_DIR ] ; then
+if [ ! -d "$FFTW_DIR" ] ; then
   echo "$FFTW_DIR does not exist."
   exit 1
 fi
-if [ ! -d $OPENMPI_DIR ] ; then
+if [ ! -d "$OPENMPI_DIR" ] ; then
   echo "$OPENMPI_DIR does not exist."
   exit 1
 fi
-if [ ! -f $AOMP/bin/flang-new ] ; then
+if [ ! -f "$AOMP/bin/flang-new" ] ; then
   echo "flang-new not found in $AOMP/bin"
   exit 1
 fi
-if [ ! -f $REPO_DIR/files/input ] ; then
+if [ ! -f "$REPO_DIR/files/input" ] ; then
   echo "missing $REPO_DIR/files/input"
   exit 1
 fi
-if [ ! -f $ROCFFT_LIB/librocfft.so ] ; then
+if [ ! -f "$ROCFFT_LIB/librocfft.so" ] ; then
   echo "librocfft.so not found in $ROCFFT_LIB"
   exit 1
 fi
@@ -95,7 +92,7 @@ export OMP_NUM_THREADS=${OMP_NUM_THREADS:-2}
 
 if [ "$1" != "runonly" ] ; then
   echo "=================  STARTING MAKE in $REPO_DIR ========"
-  cd $REPO_DIR || exit 1
+  cd "$REPO_DIR" || exit 1
   make clean
   make srcmake
 
@@ -106,9 +103,9 @@ if [ "$1" != "runonly" ] ; then
   fft_lib="-L$FFTW_DIR/lib -lfftw3 -lfftw3f"
   other_libs="-L$OPENMPI_DIR/lib -lmpi -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -L$ROCM/lib -L$ROCFFT_LIB -L$HIPFORT_LIB -lrocfft -lamdhip64 -lhipfort-amdgcn -lhsa-runtime64 -lstdc++ -lm"
 
-  make -j $GESTS_JOBS MACHINE=local \
+  make -j "$GESTS_JOBS" MACHINE=local \
       CPPFLAGS="$cppflags" \
-      FF=$AOMP/bin/flang-new \
+      FF="$AOMP/bin/flang-new" \
       CPP=/lib/cpp \
       FFLAGS="$fflags" \
       FFLAGS90="$fflags" \
@@ -121,7 +118,7 @@ if [ "$1" != "runonly" ] ; then
     echo "ERROR in Make"
     exit $rc
   fi
-  if [ ! -f $REPO_DIR/$EXECUTABLE ] ; then
+  if [ ! -f "$REPO_DIR/$EXECUTABLE" ] ; then
     echo "ERROR: $EXECUTABLE was not built"
     exit 1
   fi
@@ -130,20 +127,20 @@ if [ "$1" != "runonly" ] ; then
 fi
 
 if [ "$1" != "buildonly" ] ; then
-  if [ ! -f $REPO_DIR/$EXECUTABLE ] ; then
+  if [ ! -f "$REPO_DIR/$EXECUTABLE" ] ; then
     echo "ERROR: $REPO_DIR/$EXECUTABLE not found"
     exit 1
   fi
-  mkdir -p $RUN_DIR/MPI_timings
-  cp $REPO_DIR/$EXECUTABLE $RUN_DIR/
-  cp $REPO_DIR/files/input $RUN_DIR/input
-  rm -f $RUN_DIR/dims
+  mkdir -p "$RUN_DIR/MPI_timings"
+  cp "$REPO_DIR/$EXECUTABLE" "$RUN_DIR/"
+  cp "$REPO_DIR/files/input" "$RUN_DIR/input"
+  rm -f "$RUN_DIR/dims"
 
-  cd $RUN_DIR || exit 1
+  cd "$RUN_DIR" || exit 1
   echo
   echo "=================  attempting mpirun  ========"
   _cmd="$OPENMPI_DIR/bin/mpirun -np $GESTS_NP $AOMP/bin/gpurun -rocmsmi ./$EXECUTABLE"
-  echo $_cmd
+  echo "$_cmd"
   echo "np=$GESTS_NP threads=$OMP_NUM_THREADS"
   time $_cmd
   rc=$?