Commit 2d70e0ea authored by Asa Rentschler's avatar Asa Rentschler
Browse files

A few final touches

parent 96b2497e

mpiimage-centos-cuda/Dockerfile

+3 −4
Original line number Diff line number Diff line 
# specify cuda version

FROM code.ornl.gov:4567/olcfcontainers/olcfbaseimages/baseimage-centos-cuda:latest 

 

# Accept mpi_root environment variable. Should come from host $MPI_ROOT. Should be pointing to GNU instead of XL, etc.
@@ -39,7 +38,7 @@ RUN rm -rf /var/cache/dnf \ 
    && dnf clean all

 

# OSU benchmarks

RUN wget https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.3.tar.gz && tar -xzf osu-micro-benchmarks-7.3.tar.gz --no-same-owner

RUN cd osu-micro-benchmarks-7.3 && ./configure CC=$mpi_root/bin/mpicc CXX=$mpi_root/bin/mpicxx --enable-cuda \

    --with-cuda-include=/usr/local/cuda/include --with-cuda-libpath=/usr/local/cuda/lib64 && make && rm ../osu-micro-benchmarks-7.3.tar.gz

RUN curl -o osubenchmarks.5.7.tar.gz https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.7.tar.gz && tar -xzf osubenchmarks.5.7.tar.gz --no-same-owner

RUN cd osu-micro-benchmarks-5.7 && ./configure CC=$mpi_root/bin/mpicc CXX=$mpi_root/bin/mpicxx --enable-cuda \

    --with-cuda-include=/usr/local/cuda/include --with-cuda-libpath=/usr/local/cuda/lib64 && make && rm ../osubenchmarks.5.7.tar.gz

mpiimage-centos-cuda/example_submit.lsf

+2 −2
Original line number Diff line number Diff line 
#BSUB -P STF007

#BSUB -P STF243

#BSUB -W 0:30

#BSUB -nnodes 4

#BSUB -alloc_flags gpumps
@@ -13,6 +13,6 @@ module load DefApps 
module unload xl

module load cuda/11.7.1



source /gpfs/alpine/stf007/world-shared/containers/utils/requiredmpilibs.source

source /gpfs/alpine/stf243/world-shared/containers/utils/requiredmpilibs.source



jsrun -n 8 -r 2 -c 1 -a 1 singularity exec --bind $MPI_ROOT:$MPI_ROOT,/autofs/nccs-svm1_home1,/autofs/nccs-svm1_home1:/ccs/home osu_benchmarks_container_cuda.sif /osu-micro-benchmarks-5.7/mpi/collective/osu_allgather

mpiimage-centos-cuda/requiredmpilibs.source

+1 −7
Original line number Diff line number Diff line
@@ -2,12 +2,8 @@ 
## library loading script

mpilibs=("${MPI_ROOT}/lib" \

         "${MPI_ROOT}/lib/spectrum_mpi" \

         "${MPI_ROOT}/lib/pami_port") # \

         #"/opt/mellanox/hcoll/lib" \

	 #"/opt/mellanox/mxm/lib")

         "${MPI_ROOT}/lib/pami_port")



#mpilibs=("/opt/mellanox/hcoll/lib" \

#         "/opt/mellanox/mxm/lib")

deplibs=("libucp.so" "libuct.so" "libucs.so" \

         "libucm.so" "libnl-route-3.so" \

         "librdmacm.so" "libibumad.so" \
@@ -35,8 +31,6 @@ for libdir in ${mpilibs[@]}; do 
  done

done



#export APPTAINERENV_LD_PRELOAD=/ccs/home/xjv/libpami_cudahook.so 

export APPTAINER_CONTAINLIBS

#export SINGULARITYENV_PREPEND_PATH=/opt/anaconda2/bin

export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH

export MPICH_RDMA_ENABLED_CUDA=1