From 5e22b602241563360cab01b46cfba0e9fbb58839 Mon Sep 17 00:00:00 2001
From: Chuck Atkins <chuck.atkins@kitware.com>
Date: Mon, 30 Oct 2017 11:26:10 -0400
Subject: [PATCH] Submit tests as a job array instead of a single job
---
.../nightly/summitdev-spectrum-tests.lsf | 34 +++++++++++--------
scripts/dashboard/nightly/summitdev.sh | 27 ++++++++++-----
2 files changed, 38 insertions(+), 23 deletions(-)
diff --git a/scripts/dashboard/nightly/summitdev-spectrum-tests.lsf b/scripts/dashboard/nightly/summitdev-spectrum-tests.lsf
index c900985ef..62f9d485a 100755
--- a/scripts/dashboard/nightly/summitdev-spectrum-tests.lsf
+++ b/scripts/dashboard/nightly/summitdev-spectrum-tests.lsf
@@ -1,25 +1,31 @@
#!/bin/bash
#BSUB -P CSC143SUMMITDEV
-#BSUB -W 01:00
+#BSUB -W 30
#BSUB -nnodes 2
+#BSUB -o adios2_nightly.%I.out
+#BSUB -e adios2_nightly.%I.err
+
+case ${LSB_JOBINDEX} in
+ 1) CTEST_COMPILER=gcc
+ ;;
+ 2) CTEST_COMPILER=xl
+ ;;
+ 3) CTEST_COMPILER=pgi
+ ;;
+ *) echo "Error: Unsupported LSB_JOBINDEX: ${LSB_JOBINDEX}"
+ exit
+ ;;
+esac
+
+echo "LSB_JOBINDEX: ${LSB_JOBINDEX}"
+echo "CTEST_COMPILER: ${CTEST_COMPILER}"
module purge
module load git cmake
CTEST=$(which ctest)
SCRIPT_DIR=${PWD}/source/scripts/dashboard/nightly
-
-${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-gcc-spectrum.cmake \
- -Ddashboard_full=OFF \
- -Ddashboard_do_test=ON \
- -Ddashboard_do_submit=OFF 2>&1 1>>summitdev-gcc-spectrum.log
-
-${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-xl-spectrum.cmake \
- -Ddashboard_full=OFF \
- -Ddashboard_do_test=ON \
- -Ddashboard_do_submit=OFF 2>&1 1>>summitdev-xl-spectrum.log
-
-${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-pgi-spectrum.cmake \
+${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-${CTEST_COMPILER}-spectrum.cmake \
-Ddashboard_full=OFF \
-Ddashboard_do_test=ON \
- -Ddashboard_do_submit=OFF 2>&1 1>>summitdev-pgi-spectrum.log
+ -Ddashboard_do_submit=OFF 2>&1 1>>summitdev-${CTEST_COMPILER}-spectrum.log
diff --git a/scripts/dashboard/nightly/summitdev.sh b/scripts/dashboard/nightly/summitdev.sh
index bcf522921..e8661ab32 100755
--- a/scripts/dashboard/nightly/summitdev.sh
+++ b/scripts/dashboard/nightly/summitdev.sh
@@ -37,7 +37,7 @@ log "Running Serial XL"
${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-pgi-nompi.cmake 2>&1 1>summitdev-pgi-nompi.log
# Now run the configure and build steps for the MPI tests
-log "Running Parallel GCC Phase 1"
+log "Running Parallel GCC Build"
${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-gcc-spectrum.cmake \
-Ddashboard_full=OFF \
-Ddashboard_fresh=ON \
@@ -46,7 +46,7 @@ ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-gcc-spectrum.cmake \
-Ddashboard_do_configure=ON \
-Ddashboard_do_build=ON 2>&1 1>summitdev-gcc-spectrum.log
-log "Running Parallel XL Phase 1"
+log "Running Parallel XL Build"
${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-xl-spectrum.cmake \
-Ddashboard_full=OFF \
-Ddashboard_fresh=ON \
@@ -55,7 +55,7 @@ ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-xl-spectrum.cmake \
-Ddashboard_do_configure=ON \
-Ddashboard_do_build=ON 2>&1 1>summitdev-xl-spectrum.log
-log "Running Parallel PGI Phase 1"
+log "Running Parallel PGI Build"
${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-pgi-spectrum.cmake \
-Ddashboard_full=OFF \
-Ddashboard_fresh=ON \
@@ -65,24 +65,33 @@ ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-pgi-spectrum.cmake \
-Ddashboard_do_build=ON 2>&1 1>summitdev-pgi-spectrum.log
# Now run the MPI tests in a batch job
-log "Running Parallel Phase 2"
-bsub -P CSC143SUMMITDEV -W 00:30 -nnodes 2 -I \
- ${SCRIPT_DIR}/summitdev-spectrum-tests.lsf
+log "Submitting Parallel Tests"
+JOBID=$(bsub -J "adios2_nightly[1-3]" ${SCRIPT_DIR}/summitdev-spectrum-tests.lsf | awk '{print $2}' | sed 's|<\([0-9]*\)>|\1|')
+while true
+do
+ NJOBS=$(bjobs 2>/dev/null | grep "^${JOBID} " | wc -l)
+ log "Test jobs active in queue for job array ${JOBID}: ${NJOBS}"
+ if [ ${NJOBS} -eq 0 ]
+ then
+ break
+ fi
+ sleep 30
+done
# Finaly submit the test results from the batch job
-log "Running Parallel GCC Phase 3"
+log "Submitting Parallel GCC Test Results"
${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-gcc-spectrum.cmake \
-Ddashboard_full=OFF \
-Ddashboard_do_test=ON \
-Ddashboard_do_submit_only=ON 2>&1 1>>summitdev-gcc-spectrum.log
-log "Running Parallel XL Phase 3"
+log "Submitting Parallel XL Test Results"
${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-xl-spectrum.cmake \
-Ddashboard_full=OFF \
-Ddashboard_do_test=ON \
-Ddashboard_do_submit_only=ON 2>&1 1>>summitdev-xl-spectrum.log
-log "Running Parallel PGI Phase 3"
+log "Submitting Parallel PGI Test Results"
${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-pgi-spectrum.cmake \
-Ddashboard_full=OFF \
-Ddashboard_do_test=ON \
--
GitLab