Commit ef22c04f authored by Doak, Peter W's avatar Doak, Peter W
Browse files

explicit binding for arb mpi code

parent e0d91647
this illustrates explicitly binding mpi processes to cores
useful when other methods fail to spread processes properly
many...sh is the pbs example
mpi_hello.cpp is a minimal mpi app to stand in for your own.
interactive.sh can be run from an interactive job to allow you to
workon and modify the method.
cat $PBS_NODEFILE >pbs_nodefile
cores_per_node=32
wcr=($(wc pbs_nodefile))
count=${wcr[0]}
cores=2
jobs=16
pos=1
for j in $(seq 1 $jobs)
do
awk -v cores=$cores -v pos=$pos -v count=$count -v cpn=$cores_per_node 'BEGIN { a= pos; r=0; }; { if (a >= FNR && r < cores) { print "rank " r "=" $1 " slot=" (( a - 1 )); r = r + 1; a = a + 1 } }' pbs_nodefile > rankfile_${j}
(( pos = pos + cores ))
mpirun -n 2 -v --report-bindings -rf rankfile_${j} a.out 2>&1 > a_out_${j}&
done
wait
#!/bin/bash #!/bin/bash
##PBS -S /bin/bash ##PBS -S /bin/bash
#PBS -m be #PBS -m be
#PBS -N h2so4 #PBS -q batch
#PBS -q cnms14q #PBS -N many_small_explicit_binding
#PBS -l nodes=2:ppn=32 #PBS -A cnms
#PBS -l walltime=24:00:00 #PBS -W group_list=cades-cnms
#PBS -l qos=devel
#PBS -l nodes=1:ppn=32
#PBS -l walltime=00:02:00
source /opt/modules/3.1.6/init/bash source /opt/modules/3.1.6/init/bash
#cd $GLOBAL_SCRATCH #cd $GLOBAL_SCRATCH
cd $PBS_O_WORKDIR cd $PBS_O_WORKDIR
export MODULEPATH=$MODULEPATH:/projects/cnms/modules export MODULEPATH=$MODULEPATH:/projects/cnms/modules
module load whatever module load compilers/intel/2017.0
module load intel/openmpi/1.10.4
mpicxx mpi_hello.cpp
export OMP_NUM_THREADS=1 export OMP_NUM_THREADS=1
cd $PBS_O_WORKDIR cd $PBS_O_WORKDIR
#this can be useful to see from outside shells
cat $PBS_NODEFILE >pbs_nodefile cat $PBS_NODEFILE >pbs_nodefile
cores_per_node=32 cores_per_node=32
wcr=($(wc pbs_nodefile)) wcr=($(wc pbs_nodefile))
count=${wcr[0]} count=${wcr[0]}
awk -v count=$count -v cpn=$cores_per_node 'BEGIN { a= 0; r=0; cores=count/2; prev_node="" }; { if (a < cpn) { print "rank " r "=" $1 " slot=" a; r = r + 1; a = a + 2; prev_node=$1 } else { if (r < cores && prev_node != $1 ) {a = 0; print "rank " r "=" $1 " slot=" a ; r = r + 1; a = a + 2} }}' pbs_nodefile > rankfile
#VASP runs including significant communication especially for SCALAPACK run cores=2
#slower than they should. Refrain from using more than ~1 cpu per atom jobs=16
#past 64 or 128 cores LSCALAPACK=.FALSE. pos=1
for j in $(seq 1 $jobs)
do
awk -v cores=$cores -v pos=$pos -v count=$count -v cpn=$cores_per_node 'BEGIN { a= pos; r=0; }; { if (a >= FNR && r < cores) { print "rank " r "=" $1 " slot=" (( a - 1 )); r = r + 1; a = a + 1 } }' pbs_nodefile > rankfile_${j}
(( pos = pos + cores ))
mpirun -n 2 -v --report-bindings -rf rankfile_${j} a.out 2>&1 > a_out_${j}&
done
wait
mpirun -n $((count / 2)) -v --report-bindings -rf rankfile ${VASP_MPI_FLAGS} $VASP 2>&1 >vasp_out
#include <chrono>
#include <thread>
#include <mpi.h>
#include <stdio.h>
int main(int argc, char** argv) {
// Initialize the MPI environment
MPI_Init(NULL, NULL);
// Get the number of processes
int world_size;
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
// Get the rank of the process
int world_rank;
MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
// Get the name of the processor
char processor_name[MPI_MAX_PROCESSOR_NAME];
int name_len;
MPI_Get_processor_name(processor_name, &name_len);
std::this_thread::sleep_for(std::chrono::seconds(5));
// Print off a hello world message
printf("Hello world from processor %s, rank %d"
" out of %d processors\n",
processor_name, world_rank, world_size);
// Finalize the MPI environment.
MPI_Finalize();
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment