explicit binding for arb mpi code

CADES/ARB_CODE/README

+this illustrates explicitly binding mpi processes to cores
+useful when other methods fail to spread processes properly
+
+many...sh is the pbs example
+mpi_hello.cpp is a minimal mpi app to stand in for your own.
+interactive.sh can be run from an interactive job to allow you to
+workon and modify the method.

CADES/ARB_CODE/interactive.sh

+cat $PBS_NODEFILE >pbs_nodefile
+cores_per_node=32
+wcr=($(wc pbs_nodefile))
+count=${wcr[0]}
+
+cores=2
+jobs=16
+pos=1
+
+for j in $(seq 1 $jobs)
+do
+    awk -v cores=$cores -v pos=$pos -v count=$count -v cpn=$cores_per_node 'BEGIN { a= pos; r=0; }; { if (a >= FNR && r < cores) { print "rank " r "=" $1 " slot=" (( a  - 1 )); r = r + 1; a = a + 1 } }' pbs_nodefile > rankfile_${j}
+    (( pos = pos + cores ))
+
+    mpirun -n 2 -v --report-bindings -rf rankfile_${j} a.out 2>&1 > a_out_${j}&
+done
+wait

CADES/ARB_CODE/many_small_mpiruns.sh

 #!/bin/bash
 ##PBS -S /bin/bash
 #PBS -m be
-#PBS -N h2so4
-#PBS -q cnms14q
-#PBS -l nodes=2:ppn=32
-#PBS -l walltime=24:00:00
+#PBS -q batch
+#PBS -N many_small_explicit_binding
+#PBS -A cnms
+#PBS -W group_list=cades-cnms
+#PBS -l qos=devel
+#PBS -l nodes=1:ppn=32
+#PBS -l walltime=00:02:00
 
 source /opt/modules/3.1.6/init/bash
 #cd $GLOBAL_SCRATCH
 cd $PBS_O_WORKDIR
 export MODULEPATH=$MODULEPATH:/projects/cnms/modules
 
-module load whatever
+module load compilers/intel/2017.0
+module load intel/openmpi/1.10.4
+
+mpicxx mpi_hello.cpp
 
 export OMP_NUM_THREADS=1
 
 cd $PBS_O_WORKDIR
 
-#this can be useful to see from outside shells
 cat $PBS_NODEFILE >pbs_nodefile
 cores_per_node=32
 wcr=($(wc pbs_nodefile))
 count=${wcr[0]}
-awk -v count=$count -v cpn=$cores_per_node 'BEGIN { a= 0; r=0; cores=count/2; prev_node="" }; { if (a < cpn) { print "rank " r "=" $1 " slot=" a; r = r + 1; a = a + 2; prev_node=$1 } else { if (r < cores && prev_node != $1 ) {a = 0; print "rank " r "=" $1 " slot=" a ; r = r + 1; a = a + 2} }}' pbs_nodefile > rankfile
 
-#VASP runs including significant communication especially for SCALAPACK run
-#slower than they should.  Refrain from using more than ~1 cpu per atom
-#past 64 or 128 cores LSCALAPACK=.FALSE.
+cores=2
+jobs=16
+pos=1
+
+for j in $(seq 1 $jobs)
+do
+    awk -v cores=$cores -v pos=$pos -v count=$count -v cpn=$cores_per_node 'BEGIN { a= pos; r=0; }; { if (a >= FNR && r < cores) { print "rank " r "=" $1 " slot=" (( a  - 1 )); r = r + 1; a = a + 1 } }' pbs_nodefile > rankfile_${j}
+    (( pos = pos + cores ))
+
+    mpirun -n 2 -v --report-bindings -rf rankfile_${j} a.out 2>&1 > a_out_${j}&
+done
+wait
 
-mpirun -n $((count / 2)) -v --report-bindings -rf rankfile ${VASP_MPI_FLAGS} $VASP 2>&1 >vasp_out

CADES/ARB_CODE/mpi_hello.cpp

+#include <chrono>
+#include <thread>
+#include <mpi.h>
+#include <stdio.h>
+
+int main(int argc, char** argv) {
+    // Initialize the MPI environment
+    MPI_Init(NULL, NULL);
+
+    // Get the number of processes
+    int world_size;
+    MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+    // Get the rank of the process
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+
+    // Get the name of the processor
+    char processor_name[MPI_MAX_PROCESSOR_NAME];
+    int name_len;
+    MPI_Get_processor_name(processor_name, &name_len);
+
+    std::this_thread::sleep_for(std::chrono::seconds(5));
+    // Print off a hello world message
+    printf("Hello world from processor %s, rank %d"
+           " out of %d processors\n",
+           processor_name, world_rank, world_size);
+
+    // Finalize the MPI environment.
+    MPI_Finalize();
+}