Commit 82d9ca1b authored by Doak, Peter W's avatar Doak, Peter W
Browse files

Merge branch 'rajeev_multi_jobs'

parents 0e8a4b1c 505eb2c5
#!/bin/bash
#PBS -S /bin/bash
#PBS -q high_mem
#PBS -l nodes=4:ppn=36
#PBS -l walltime=04:00:00
#PBS -j oe
#PBS -N test_PS
#PBS -A cnms
#PBS -W group_list=cades-cnms
#PBS -l naccesspolicy=singlejob
#PBS -l qos=std
################################################################################################
#
# Common PBS setup commands
#
################################################################################################
cd $PBS_O_WORKDIR
################################################################################################
#
# Set environment needed to run PolySwift++
#
################################################################################################
# Load CADES modules
module unload
module load gcc/4.9.4
module load openmpi/1.10.3
# Check loaded modules
module list
# Setup PolySwift++ specific environment
PSPATH="/lustre/or-hydra/cades-bellerophon/proj-shared/psim/volatile"
PSEXE_SER=$PSPATH/polyswift/bin/polyswiftser
PSEXE_PAR=$PSPATH/polyswift/bin/polyswift
source $PSPATH/psall.sh
echo "PSPATH=$PSPATH"
echo "PSEXE_SER=$PSEXE_SER"
echo "PSEXE_PAR=$PSEXE_PAR"
## ------------------------------------------------------------------
## RESTORE PARAMETERS
## ------------------------------------------------------------------
let RESTORE_INDEX=0
if [ $RESTORE_INDEX = 0 ]; then
echo "Turning restore option OFF"
RESTORE_OPTION=
else
echo "Turning restore option ON"
RESTORE_OPTION="-r $RESTORE_INDEX"
fi
echo "restore option = $RESTORE_OPTION"
## ------------------------------------------------------------------
CORES_PER_RUN=36
#the easiest and most configuration proof way to split up resources at the moment is this
cat $PBS_NODEFILE >pbs_nodefile
#there are other ways if you have the patience
#try experimenting with mpirun in an interactive job
declare -a rundirs
rundirs=(LX_*)
rdlen=${#rundirs[@]}
#here we build rank files for each mpi application
#We have to do this because mpirun is not aware of other mpirun invocations
#in the job shell
#if your LX_* directories is greater than you number of nodes you'll run out of contents
#for the pbs_nodefiles
for run in $(seq 1 $rdlen)
do
rundir=${rundirs[$((run-1))]}
start=$(expr 1 + $(expr $(($run-1)) \* $CORES_PER_RUN ))
end=$(expr $run \* $CORES_PER_RUN)
echo "$run $rundir $start $end"
awk -v start=$start -v end=$end 'NR >= start && NR <= end' pbs_nodefile > ${rundir}/pbs_nodefile
done
################################################################################################
#
# Run code, collect output and perform any post-processing steps
#
################################################################################################
for rundir in ${rundirs[@]}
do
cd $rundir
mpiexec -hostfile pbs_nodefile $PSEXE_PAR -i onlywall.pre -r 0 -o graft_blend_film > STDOUT &
cd ../
done
wait
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment