diff --git a/bin/run_gests.sh b/bin/run_gests.sh index 62cd6bb2ab..29a6888a3f 100755 --- a/bin/run_gests.sh +++ b/bin/run_gests.sh @@ -1,88 +1,153 @@ #!/bin/bash -# -# run_nekbone.sh: +# +# run_gests.sh +# +# ./run_gests.sh [buildonly|runonly] +# +# GESTS_NP, GESTS_JOBS, OMP_NUM_THREADS can be set. +# The run input is $REPO_DIR/files/input. # # --- Start standard header to set AOMP environment variables ---- -realpath=`realpath $0` -thisdir=`dirname $realpath` +realpath=$(realpath "$0") +thisdir=$(dirname "$realpath") export AOMP_USE_CCACHE=0 - -. $thisdir/aomp_common_vars -# --- end standard header ---- - -# Setup AOMP variables -AOMP=${AOMP:-/usr/lib/aomp} +AOMP=${AOMP:-$HOME/rocm/aomp/llvm} AOMP_SUPP=${AOMP_SUPP:-$HOME/local} -HIPFORT=${HIPFORT:-$HOME/rocm/aomp/hipfort/} +ROCM=${ROCM:-/opt/rocm} +HIPFORT_INCLUDE=${HIPFORT_INCLUDE:-$HOME/rocm/aomp} +HIPFORT_LIB=${HIPFORT_LIB:-$HOME/rocm/aomp/lib} +AOMPHIP=${AOMPHIP:-$(realpath -m "$(realpath -m "$AOMP")/../..")} -# Use function to set and test AOMP_GPU -setaompgpu +# shellcheck disable=SC1091 +. "$thisdir/aomp_common_vars" +# --- end standard header ---- REPO_DIR=$AOMP_REPOS_TEST/GESTS +RUN_DIR=$AOMP_REPOS_TEST/run/GESTS +EXECUTABLE=DNS_PEN_GPU_p4.x -if [ -f $HOME/GESTS.tar ] && [ ! -d $REPO_DIR ] ; then - cp $HOME/GESTS.tar $AOMP_REPOS_TEST - cd $AOMP_REPOS_TEST - tar xvf $HOME/GESTS.tar +if [ -d "$REPO_DIR" ] ; then + echo "$REPO_DIR exists." +else + echo "Please contact Raghu Maddhipatla to get a copy of GESTS.tar and place the tar file at $HOME" + exit fi -if [ -d $REPO_DIR ] ; then - echo $REPO_DIR " exists." +# rocFFT is not in $ROCM. Use it from the system ROCm if needed. +if [ -f "$ROCM/lib/librocfft.so" ] ; then + ROCFFT_LIB=$ROCM/lib else - echo "Please contact Raghu Maddhipatla to get a copy of GESTS.tar and place the tar file at $HOME" - exit + ROCFFT_LIB=${ROCFFT_LIB:-/opt/rocm/lib} fi -cd $REPO_DIR -#patchrepo $REPO_DIR +export FFTW_DIR=$AOMP_SUPP/fftw +export OPENMPI_DIR=$AOMP_SUPP/openmpi -if [ -d $HIPFORT ] ; then - echo $HIPFORT " exists." -else - echo "Install latest hipfort and try again!" - exit +echo "AOMP = $AOMP" +echo "ROCM = $ROCM" +echo "AOMP_SUPP = $AOMP_SUPP" +echo "HIPFORT_INCLUDE = $HIPFORT_INCLUDE" +echo "HIPFORT_LIB = $HIPFORT_LIB" +echo "AOMPHIP = $AOMPHIP" + +setaompgpu + +if [ ! -d "$HIPFORT_INCLUDE" ] ; then + echo "hipfort include directory not found at $HIPFORT_INCLUDE" + exit 1 fi -currdir=$(pwd) -export FFTW_DIR=$AOMP_SUPP/fftw -if [ -d "$FFTW_DIR" ] ; then - echo "FFTW exists." -else - echo "$FFTW_DIR does not exist. Install FFTW using ./build_supp.sh fftw and try again." - exit +if [ ! -f "$HIPFORT_LIB/libhipfort-amdgcn.a" ] ; then + echo "missing $HIPFORT_LIB/libhipfort-amdgcn.a" + exit 1 fi -export OPENMPI_DIR=$AOMP_SUPP/openmpi -if [ -d "$OPENMPI_DIR" ] ; then - echo "OpenMPI-4.1.1 exists." -else - echo "$OPENMPI_DIR does not exist. Install openmpi using ./build_supp.sh ompi and try again." - exit +if [ ! -d "$FFTW_DIR" ] ; then + echo "$FFTW_DIR does not exist." + exit 1 +fi +if [ ! -d "$OPENMPI_DIR" ] ; then + echo "$OPENMPI_DIR does not exist." + exit 1 +fi +if [ ! -f "$AOMP/bin/flang-new" ] ; then + echo "flang-new not found in $AOMP/bin" + exit 1 +fi +if [ ! -f "$REPO_DIR/files/input" ] ; then + echo "missing $REPO_DIR/files/input" + exit 1 +fi +if [ ! -f "$ROCFFT_LIB/librocfft.so" ] ; then + echo "librocfft.so not found in $ROCFFT_LIB" + exit 1 fi -export LD_LIBRARY_PATH=$HOME/rocm/aomp/lib:$OPENMPI_DIR/lib:$FFTW_DIR/lib:$LD_LIBRARY_PATH -export ROCM_DIR=$AOMP +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$AOMP/lib:$ROCM/lib:$ROCFFT_LIB:$OPENMPI_DIR/lib:$FFTW_DIR/lib +export PYTHONPATH=${PYTHONPATH:-$ROCM/share/amd_smi} export HIP_CLANG_PATH=$AOMP/bin -cd $REPO_DIR + +GESTS_NP=${GESTS_NP:-1} +GESTS_JOBS=${GESTS_JOBS:-1} +export OMP_NUM_THREADS=${OMP_NUM_THREADS:-2} + if [ "$1" != "runonly" ] ; then - echo "================= STARTING MAKE in $PWD ========" + echo "================= STARTING MAKE in $REPO_DIR ========" + cd "$REPO_DIR" || exit 1 make clean - make + make srcmake + + cppflags="-P -DOPENMP -DFFTW -DREVERSE_DIMS -DUSE_EVEN -DTIMERS -DDETAIL_MPI -DUSE_AMD -DUSE_HIPHOSTMALLOC -DPHYMAX -DMODEL_SPECTRUM" + fflags="-cpp -I$HIPFORT_INCLUDE -I. -Ibin -Jbin -O1 --offload-arch=$AOMP_GPU -fopenmp" + fft_inc="-I$FFTW_DIR/include" + other_inc="-I$ROCM/include -I$HIPFORT_INCLUDE -I$OPENMPI_DIR/include -I$OPENMPI_DIR/lib" + fft_lib="-L$FFTW_DIR/lib -lfftw3 -lfftw3f" + other_libs="-L$OPENMPI_DIR/lib -lmpi -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -L$ROCM/lib -L$ROCFFT_LIB -L$HIPFORT_LIB -lrocfft -lamdhip64 -lhipfort-amdgcn -lhsa-runtime64 -lstdc++ -lm" + + make -j "$GESTS_JOBS" MACHINE=local \ + CPPFLAGS="$cppflags" \ + FF="$AOMP/bin/flang-new" \ + CPP=/lib/cpp \ + FFLAGS="$fflags" \ + FFLAGS90="$fflags" \ + FFT_INC="$fft_inc" \ + OTHER_INC="$other_inc" \ + FFT_LIB="$fft_lib" \ + OTHER_LIBS="$other_libs" rc=$? if [ $rc != 0 ] ; then - echo "ERROR in Make" - exit + echo "ERROR in Make" + exit $rc + fi + if [ ! -f "$REPO_DIR/$EXECUTABLE" ] ; then + echo "ERROR: $EXECUTABLE was not built" + exit 1 fi echo echo "================= BUILD IS DONE ========" fi if [ "$1" != "buildonly" ] ; then - cd $REPO_DIR + if [ ! -f "$REPO_DIR/$EXECUTABLE" ] ; then + echo "ERROR: $REPO_DIR/$EXECUTABLE not found" + exit 1 + fi + mkdir -p "$RUN_DIR/MPI_timings" + cp "$REPO_DIR/$EXECUTABLE" "$RUN_DIR/" + cp "$REPO_DIR/files/input" "$RUN_DIR/input" + rm -f "$RUN_DIR/dims" + + cd "$RUN_DIR" || exit 1 echo echo "================= attempting mpirun ========" - _cmd="$OPENMPI_DIR/bin/mpirun -np 1 $AOMP/bin/gpurun ./PSDNS_fft.x" - echo $_cmd + _cmd="$OPENMPI_DIR/bin/mpirun -np $GESTS_NP $AOMP/bin/gpurun -rocmsmi ./$EXECUTABLE" + echo "$_cmd" + echo "np=$GESTS_NP threads=$OMP_NUM_THREADS" time $_cmd + rc=$? + if [ $rc != 0 ] ; then + echo "ERROR: GESTS run failed" + exit $rc + fi echo echo "================= DONE with mpirun ========" fi