v0.13

CHANGELOG.md

@@ -1,3 +1,32 @@
+## Version 0.13.0
+
+### New environment variables for loop jobs
+
+- Loop jobs now expose three additional environment variables:
+    - `QQ_LOOP_NEXT`, which specifies the index of the next loop cycle.
+    - `QQ_ARCHIVE_CURRENT` and `QQ_ARCHIVE_NEXT`, which specify strings expected in files archived by qq for the current and the next loop cycles, respectively. If the archive format is not a printf pattern, the values of these variables are empty strings.
+
+### `--include` and `--exclude` revisited
+
+- Glob patterns are now supported in `--include` and `--exclude` options.
+- If you explicitly include a file in the working directory using the `--include` submission option, it will no longer be archived even if it matches the archive pattern.
+- If you explicitly exclude a file from the working directory using the `--exclude` submission option, it will no longer be fetched from the archive, even if it matches the archive pattern for this loop job cycle.
+
+### `--ignore` option
+
+- As a complement to `--include` and `--exclude`, `--ignore` allows you to specify files that should be completely ignored by qq's transfer operations. Ignored files will never be copied to the working directory; if they are created in the working directory, they will never be transferred to the input directory. If the job is a loop job, ignored files will also never be archived nor fetched from the archive.
+
+### Bug fixes and other changes
+
+- Directories can now be properly archived.
+- qq commands operating on multiple job IDs no longer fail on the first occurrence of a job that does not exist. They report an error and continue processing the remaining jobs.
+- The archive directory created in the working directory is no longer merged with the actual archive directory in the input directory.
+- The number of free GPUs is no longer relevant for determining node state in `qq nodes`. Nodes with exhausted CPUs will always be marked as busy even if they have free GPUs.
+- Clarified in `qq submit -h` that files and directories that the job creates in the working directory **are** copied back to the input directory **even if they are excluded** from being copied to the working directory.
+- Error messages reformatted to be easier to compose.
+
+---
+
 ## Version 0.12.3
 
 - More robust installation script for Metacentrum.

scripts/qq_scripts/gmx-eta

@@ -5,7 +5,7 @@
 
 """
 Get the estimated time of a Gromacs simulation finishing.
-Version qq 0.12.1.
+Version qq 0.13.0.
 Requires `uv`: https://docs.astral.sh/uv
 """
 
@@ -16,7 +16,7 @@ Requires `uv`: https://docs.astral.sh/uv
 # ]
 #
 # [tool.uv.sources]
-# qq = { git = "https://github.com/VachaLab/qq.git", tag = "v0.12.1" }
+# qq = { git = "https://github.com/VachaLab/qq.git", tag = "v0.13.0" }
 # ///
 
 import argparse

scripts/run_scripts/qq_flex_md

@@ -1,8 +1,9 @@
 #!/usr/bin/env -S qq run
 ########################################
 #      Script for running Gromacs      #
-#  flexible-length loop jobs using qq  #
-#         script version: 0.11         #
+#       flexible-length loop jobs      #
+#         script version: 0.12         #
+#     requires qq version: >=0.13      #
 #      support: ladmeb@gmail.com       #
 ########################################
 
@@ -122,11 +123,16 @@ if [[ -z "${MAX_TIME}" ]]; then
 fi
 echo "[QQ_FLEX_MD]    INFO    Setting the maximum Gromacs runtime to ${MAX_TIME} hours."
 
-# create STAGE strings
-# CURR - prefix for the data produced in this run
-# NEXT - prefix for the checkpoint file for the next run
-printf -v CURR "${QQ_ARCHIVE_FORMAT}" "${QQ_LOOP_CURRENT}"
-printf -v NEXT "${QQ_ARCHIVE_FORMAT}" "$((QQ_LOOP_CURRENT + 1))"
+
+# set up stage strings
+if [ -z "${QQ_ARCHIVE_CURRENT:-}" ] || [ -z "${QQ_ARCHIVE_NEXT:-}" ]; then
+    echo "[QQ_FLEX_MD]    ERROR     QQ_ARCHIVE_CURRENT and QQ_ARCHIVE_NEXT are not set." >&2
+    echo "[QQ_FLEX_MD]    ERROR     You are either not using qq version 0.13 or higher, or your archive format is not a printf pattern." >&2
+    exit 9
+fi
+
+CURR=${QQ_ARCHIVE_CURRENT}
+NEXT=${QQ_ARCHIVE_NEXT}
 
 # determine gromacs version to set the appropriate appending method
 GMX_VER="$(gmx_mpi --version | awk 'BEGIN {i = 0} tolower($0) ~ /gromacs version/ {split($NF, N, "."); if (N[1] > 6) i = 1} END {print i}')"
@@ -193,7 +199,7 @@ if [[ "${QQ_LOOP_CURRENT}" -eq "${QQ_LOOP_START}" ]]; then
         GEN_VEL="$(awk 'BEGIN {i = 0} /gen[-_]vel/ {if (toupper($3) == "YES") i = 1} END {print i}' "${MDP}")"
         if [[ "${GEN_VEL}" -eq 0 ]]; then MAXWARN=0; else MAXWARN=1; fi
     fi
-    
+
     # compile the tpr file
     gmx_mpi grompp -f "${MDP}" -c "${GRO}" ${REF} ${CPT} -p "${TOP}" -n "${NDX}" -o "${CURR}.tpr" -quiet -maxwarn "${MAXWARN}"
 
@@ -225,7 +231,7 @@ if [[ "${QQ_BATCH_SYSTEM}" == *"Slurm"* ]]; then
     --cpus-per-task=${NTOMP} \
     gmx_mpi mdrun -v -deffnm ${CURR} ${CPI} \
     -cpo ${NEXT} -cpt 1 -pf ${CURR}_pf.xvg -px ${CURR}_px.xvg \
-    ${PLUMED} -ntomp ${NTOMP} ${APPEND} -nb ${NB} -pin on -maxh ${MAX_TIME} 
+    ${PLUMED} -ntomp ${NTOMP} ${APPEND} -nb ${NB} -pin on -maxh ${MAX_TIME}
 else
     mpirun \
     -display-allocation \

scripts/run_scripts/qq_flex_re

@@ -1,9 +1,9 @@
 #!/usr/bin/env -S qq run
 ########################################
 #     Script for running Gromacs       #
-#  flexible-length replica exchange    #
-#              using qq                #
-#         script version: 0.7          #
+#   flexible-length replica exchange   #
+#         script version: 0.8          #
+#     requires qq version: >=0.13      #
 #      support: ladmeb@gmail.com       #
 ########################################
 
@@ -152,11 +152,16 @@ if [[ -z "${MAX_TIME}" ]]; then
 fi
 echo "[QQ_FLEX_RE]    INFO    Setting the maximum Gromacs runtime to ${MAX_TIME} hours."
 
-# create STAGE strings
-# CURR - prefix for the data produced in this run
-# NEXT - prefix for the checkpoint file for the next run
-printf -v CURR "${QQ_ARCHIVE_FORMAT}" "${QQ_LOOP_CURRENT}"
-printf -v NEXT "${QQ_ARCHIVE_FORMAT}" "$((QQ_LOOP_CURRENT + 1))"
+
+# set up stage strings
+if [ -z "${QQ_ARCHIVE_CURRENT:-}" ] || [ -z "${QQ_ARCHIVE_NEXT:-}" ]; then
+    echo "[QQ_FLEX_RE]    ERROR     QQ_ARCHIVE_CURRENT and QQ_ARCHIVE_NEXT are not set." >&2
+    echo "[QQ_FLEX_RE]    ERROR     You are either not using qq version 0.13 or higher, or your archive format is not a printf pattern." >&2
+    exit 9
+fi
+
+CURR=${QQ_ARCHIVE_CURRENT}
+NEXT=${QQ_ARCHIVE_NEXT}
 
 # determine gromacs version to set the appropriate appending method
 GMX_VER="$(gmx_mpi --version | awk 'BEGIN {i = 0} tolower($0) ~ /gromacs version/ {split($NF, N, "."); if (N[1] > 6) i = 1} END {print i}')"
@@ -303,7 +308,7 @@ for ((I=0; I<${#CLIENTS[@]}; I++)); do
     SUFFIX=${CLI_SUFFIXES[I]}
 
     cd ${CLI}
-        
+
     # rename output files from gromacs 2016 and later
     for PART in *part*; do
         [[ -e "${PART}" ]] || break
@@ -349,4 +354,4 @@ echo "[QQ_FLEX_RE]    INFO    Last step: ${LAST_STEP} and target step: ${TARGET_
 if [[ "${LAST_STEP}" -ge "${TARGET_STEP}" ]]; then
     # signal to qq that the job should not be resubmitted
     exit "${QQ_NO_RESUBMIT}"
-fi
+fi

scripts/run_scripts/qq_loop_md

@@ -1,8 +1,9 @@
 #!/usr/bin/env -S qq run
 ########################################
 #     Script for running Gromacs       #
-#         loop jobs using qq           #
-#         script version: 0.11         #
+#        fixed length loop jobs        #
+#         script version: 0.12         #
+#     requires qq version: >=0.13      #
 #      support: ladmeb@gmail.com       #
 ########################################
 
@@ -113,11 +114,15 @@ else
     NB="gpu"
 fi
 
-# create STAGE strings
-# CURR - prefix for the data produced in this run
-# NEXT - prefix for the checkpoint file for the next run
-printf -v CURR "${QQ_ARCHIVE_FORMAT}" "${QQ_LOOP_CURRENT}"
-printf -v NEXT "${QQ_ARCHIVE_FORMAT}" "$((QQ_LOOP_CURRENT + 1))"
+# set up stage strings
+if [ -z "${QQ_ARCHIVE_CURRENT:-}" ] || [ -z "${QQ_ARCHIVE_NEXT:-}" ]; then
+    echo "[QQ_LOOP_MD]    ERROR     QQ_ARCHIVE_CURRENT and QQ_ARCHIVE_NEXT are not set." >&2
+    echo "[QQ_LOOP_MD]    ERROR     You are either not using qq version 0.13 or higher, or your archive format is not a printf pattern." >&2
+    exit 9
+fi
+
+CURR=${QQ_ARCHIVE_CURRENT}
+NEXT=${QQ_ARCHIVE_NEXT}
 
 # determine gromacs version to set the appropriate appending method
 GMX_VER="$(gmx_mpi --version | awk 'BEGIN {i = 0} tolower($0) ~ /gromacs version/ {split($NF, N, "."); if (N[1] > 6) i = 1} END {print i}')"
@@ -184,7 +189,7 @@ if [[ "${QQ_LOOP_CURRENT}" -eq "${QQ_LOOP_START}" ]]; then
         GEN_VEL="$(awk 'BEGIN {i = 0} /gen[-_]vel/ {if (toupper($3) == "YES") i = 1} END {print i}' "${MDP}")"
         if [[ "${GEN_VEL}" -eq 0 ]]; then MAXWARN=0; else MAXWARN=1; fi
     fi
-    
+
     # compile the tpr file
     gmx_mpi grompp -f "${MDP}" -c "${GRO}" ${REF} ${CPT} -p "${TOP}" -n "${NDX}" -o "${CURR}.tpr" -quiet -maxwarn "${MAXWARN}"
 

scripts/run_scripts/qq_loop_re

@@ -1,8 +1,9 @@
 #!/usr/bin/env -S qq run
 ########################################
 #     Script for running Gromacs       #
-#     multidir loop jobs using qq      #
-#         script version: 0.7          #
+#    multidir fixed length loop jobs   #
+#         script version: 0.8          #
+#     requires qq version: >=0.13      #
 #      support: ladmeb@gmail.com       #
 ########################################
 
@@ -142,11 +143,15 @@ if [[ -n "${RE_FREQ}" ]]; then
     fi
 fi
 
-# create STAGE strings
-# CURR - prefix for the data produced in this run
-# NEXT - prefix for the checkpoint file for the next run
-printf -v CURR "${QQ_ARCHIVE_FORMAT}" "${QQ_LOOP_CURRENT}"
-printf -v NEXT "${QQ_ARCHIVE_FORMAT}" "$((QQ_LOOP_CURRENT + 1))"
+# set up stage strings
+if [ -z "${QQ_ARCHIVE_CURRENT:-}" ] || [ -z "${QQ_ARCHIVE_NEXT:-}" ]; then
+    echo "[QQ_LOOP_RE]    ERROR     QQ_ARCHIVE_CURRENT and QQ_ARCHIVE_NEXT are not set." >&2
+    echo "[QQ_LOOP_RE]    ERROR     You are either not using qq version 0.13 or higher, or your archive format is not a printf pattern." >&2
+    exit 9
+fi
+
+CURR=${QQ_ARCHIVE_CURRENT}
+NEXT=${QQ_ARCHIVE_NEXT}
 
 # determine gromacs version to set the appropriate appending method
 GMX_VER="$(gmx_mpi --version | awk 'BEGIN {i = 0} tolower($0) ~ /gromacs version/ {split($NF, N, "."); if (N[1] > 6) i = 1} END {print i}')"
@@ -293,7 +298,7 @@ for ((I=0; I<${#CLIENTS[@]}; I++)); do
     SUFFIX=${CLI_SUFFIXES[I]}
 
     cd ${CLI}
-        
+
     # rename output files from gromacs 2016 and later
     for PART in *part*; do
         [[ -e "${PART}" ]] || break
@@ -330,4 +335,4 @@ for ((I=0; I<${#CLIENTS[@]}; I++)); do
     done
 
     cd ..
-done
+done