Merge branch 'rajeev_multi_jobs'

CADES/POLYSWIFT/submit_multi_4.sh

+#!/bin/bash
+#PBS -S /bin/bash
+#PBS -q high_mem
+#PBS -l nodes=4:ppn=36
+#PBS -l walltime=04:00:00
+#PBS -j oe
+#PBS -N test_PS
+#PBS -A cnms
+#PBS -W group_list=cades-cnms
+#PBS -l naccesspolicy=singlejob
+#PBS -l qos=std
+
+################################################################################################
+#
+# Common PBS setup commands
+#
+################################################################################################
+
+cd $PBS_O_WORKDIR
+
+
+################################################################################################
+#
+# Set environment needed to run PolySwift++
+#
+################################################################################################
+
+# Load CADES modules
+module unload
+module load gcc/4.9.4
+module load openmpi/1.10.3
+
+# Check loaded modules
+module list
+
+# Setup PolySwift++ specific environment
+PSPATH="/lustre/or-hydra/cades-bellerophon/proj-shared/psim/volatile"
+PSEXE_SER=$PSPATH/polyswift/bin/polyswiftser
+PSEXE_PAR=$PSPATH/polyswift/bin/polyswift
+
+source $PSPATH/psall.sh
+echo "PSPATH=$PSPATH"
+echo "PSEXE_SER=$PSEXE_SER"
+echo "PSEXE_PAR=$PSEXE_PAR"
+
+## ------------------------------------------------------------------
+## RESTORE PARAMETERS
+## ------------------------------------------------------------------
+
+let RESTORE_INDEX=0
+
+if [ $RESTORE_INDEX = 0 ]; then
+    echo "Turning restore option OFF"
+    RESTORE_OPTION=
+else
+    echo "Turning restore option ON"
+    RESTORE_OPTION="-r $RESTORE_INDEX"
+fi
+echo "restore option = $RESTORE_OPTION"
+
+## ------------------------------------------------------------------
+
+
+CORES_PER_RUN=36
+
+#the easiest and most configuration proof way to split up resources at the moment is this
+cat $PBS_NODEFILE >pbs_nodefile
+#there are other ways if you have the patience
+#try experimenting with mpirun in an interactive job
+
+declare -a rundirs
+rundirs=(LX_*)
+rdlen=${#rundirs[@]}
+#here we build rank files for each mpi application
+#We have to do this because mpirun is not aware of other mpirun invocations
+#in the job shell
+#if your LX_* directories is greater than you number of nodes you'll run out of contents
+#for the pbs_nodefiles
+for run in $(seq 1 $rdlen)
+do
+    rundir=${rundirs[$((run-1))]}
+    start=$(expr 1 + $(expr $(($run-1)) \* $CORES_PER_RUN ))
+    end=$(expr $run \* $CORES_PER_RUN) 
+    echo "$run $rundir $start $end"
+    awk -v start=$start -v end=$end 'NR >= start && NR <= end'  pbs_nodefile > ${rundir}/pbs_nodefile
+done
+
+
+################################################################################################
+#
+# Run code, collect output and perform any post-processing steps
+#
+################################################################################################
+for rundir in ${rundirs[@]}
+do
+  cd $rundir
+  mpiexec -hostfile pbs_nodefile $PSEXE_PAR -i onlywall.pre -r 0 -o graft_blend_film > STDOUT &
+  cd ../
+done
+
+wait
+