Submit tests as a job array instead of a single job

scripts/dashboard/nightly/summitdev-spectrum-tests.lsf

 #!/bin/bash
 #BSUB -P CSC143SUMMITDEV
-#BSUB -W 01:00
+#BSUB -W 30
 #BSUB -nnodes 2
+#BSUB -o adios2_nightly.%I.out
+#BSUB -e adios2_nightly.%I.err
+
+case ${LSB_JOBINDEX} in
+  1) CTEST_COMPILER=gcc
+     ;;
+  2) CTEST_COMPILER=xl
+     ;;
+  3) CTEST_COMPILER=pgi
+     ;;
+  *) echo "Error: Unsupported LSB_JOBINDEX: ${LSB_JOBINDEX}"
+     exit
+     ;;
+esac
+
+echo "LSB_JOBINDEX: ${LSB_JOBINDEX}"
+echo "CTEST_COMPILER: ${CTEST_COMPILER}"
 
 module purge
 module load git cmake
 
 CTEST=$(which ctest)
 SCRIPT_DIR=${PWD}/source/scripts/dashboard/nightly
-
-${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-gcc-spectrum.cmake \
-  -Ddashboard_full=OFF \
-  -Ddashboard_do_test=ON \
-  -Ddashboard_do_submit=OFF 2>&1 1>>summitdev-gcc-spectrum.log
-
-${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-xl-spectrum.cmake \
-  -Ddashboard_full=OFF \
-  -Ddashboard_do_test=ON \
-  -Ddashboard_do_submit=OFF 2>&1 1>>summitdev-xl-spectrum.log
-
-${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-pgi-spectrum.cmake \
+${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-${CTEST_COMPILER}-spectrum.cmake \
   -Ddashboard_full=OFF \
   -Ddashboard_do_test=ON \
-  -Ddashboard_do_submit=OFF 2>&1 1>>summitdev-pgi-spectrum.log
+  -Ddashboard_do_submit=OFF 2>&1 1>>summitdev-${CTEST_COMPILER}-spectrum.log

scripts/dashboard/nightly/summitdev.sh

@@ -37,7 +37,7 @@ log "Running Serial XL"
 ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-pgi-nompi.cmake 2>&1 1>summitdev-pgi-nompi.log
 
 # Now run the configure and build steps for the MPI tests
-log "Running Parallel GCC Phase 1"
+log "Running Parallel GCC Build"
 ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-gcc-spectrum.cmake \
   -Ddashboard_full=OFF \
   -Ddashboard_fresh=ON \
@@ -46,7 +46,7 @@ ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-gcc-spectrum.cmake \
   -Ddashboard_do_configure=ON \
   -Ddashboard_do_build=ON 2>&1 1>summitdev-gcc-spectrum.log
 
-log "Running Parallel XL Phase 1"
+log "Running Parallel XL Build"
 ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-xl-spectrum.cmake \
   -Ddashboard_full=OFF \
   -Ddashboard_fresh=ON \
@@ -55,7 +55,7 @@ ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-xl-spectrum.cmake \
   -Ddashboard_do_configure=ON \
   -Ddashboard_do_build=ON 2>&1 1>summitdev-xl-spectrum.log
 
-log "Running Parallel PGI Phase 1"
+log "Running Parallel PGI Build"
 ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-pgi-spectrum.cmake \
   -Ddashboard_full=OFF \
   -Ddashboard_fresh=ON \
@@ -65,24 +65,33 @@ ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-pgi-spectrum.cmake \
   -Ddashboard_do_build=ON 2>&1 1>summitdev-pgi-spectrum.log
 
 # Now run the MPI tests in a batch job
-log "Running Parallel Phase 2"
-bsub -P CSC143SUMMITDEV -W 00:30 -nnodes 2 -I \
-  ${SCRIPT_DIR}/summitdev-spectrum-tests.lsf
+log "Submitting Parallel Tests"
+JOBID=$(bsub -J "adios2_nightly[1-3]" ${SCRIPT_DIR}/summitdev-spectrum-tests.lsf | awk '{print $2}' | sed 's|<\([0-9]*\)>|\1|')
+while true
+do
+  NJOBS=$(bjobs 2>/dev/null | grep "^${JOBID} " | wc -l) 
+  log "Test jobs active in queue for job array ${JOBID}: ${NJOBS}"
+  if [ ${NJOBS} -eq 0 ]
+  then
+    break
+  fi
+  sleep 30
+done
 
 # Finaly submit the test results from the batch job
-log "Running Parallel GCC Phase 3"
+log "Submitting Parallel GCC Test Results"
 ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-gcc-spectrum.cmake \
   -Ddashboard_full=OFF \
   -Ddashboard_do_test=ON \
   -Ddashboard_do_submit_only=ON 2>&1 1>>summitdev-gcc-spectrum.log
 
-log "Running Parallel XL Phase 3"
+log "Submitting Parallel XL Test Results"
 ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-xl-spectrum.cmake \
   -Ddashboard_full=OFF \
   -Ddashboard_do_test=ON \
   -Ddashboard_do_submit_only=ON 2>&1 1>>summitdev-xl-spectrum.log
 
-log "Running Parallel PGI Phase 3"
+log "Submitting Parallel PGI Test Results"
 ${CTEST} -VV -S ${SCRIPT_DIR}/summitdev-pgi-spectrum.cmake \
   -Ddashboard_full=OFF \
   -Ddashboard_do_test=ON \