Commit 96b2497e authored by Asa Rentschler's avatar Asa Rentschler
Browse files

Formatting

parent 00d2f533

baseimage-centos-cuda/Dockerfile

+1 −0
Original line number Diff line number Diff line 
FROM quay.io/centos/centos:stream8



# install cuda 11.7.1

RUN dnf -y install epel-release && dnf -y install fakeroot 

RUN dnf clean all && rm -r /var/cache/dnf  && fakeroot dnf upgrade -y && fakeroot dnf update -y  

RUN fakeroot dnf install -y wget dnf-plugins-core \

baseimage-centos-cuda/README.md

+1 −1
Original line number Diff line number Diff line 
This image needs to be built somewhere where you have sudo or subuid/subgid mappings present



Currently built with CUDA 11.7.1 libraries for Summit
 
Currently built with CUDA 11.7.1 libraries for Summit because SMPI does not support CUDA >= 12.

mpiimage-centos-cuda/Dockerfile

+7 −9
Original line number Diff line number Diff line 
# specify cuda version

FROM code.ornl.gov:4567/olcfcontainers/olcfbaseimages/baseimage-centos-cuda:latest 

 

# Accept mpi_root environment variable. Should come from host $MPI_ROOT. Should be pointing to GNU instead of XL, etc.
@@ -12,17 +13,15 @@ ENV C_INCLUDE_PATH=$mpi_root/include:$C_INCLUDE_PATH 
ENV CPLUS_INCLUDE_PATH=$mpi_root/include:$CPLUS_INCLUDE_PATH

 

# MOFED is sufficient, but is it necessary?

# Set MOFED version, OS version and platform

# Set MOFED version, OS version and platform (updated to match Summit 1/30/2024)

ENV MOFED_VER 4.9-6.0.6.1

ENV OS_VER rhel8.6

ENV PLATFORM ppc64le

ENV MOFED_DIR /mlnx



# MLNX_OFED

RUN mkdir ${MOFED_DIR} 

 

# Switched to pulling from web

RUN wget https://content.mellanox.com/ofed/MLNX_OFED-${MOFED_VER}/MLNX_OFED_LINUX-${MOFED_VER}-${OS_VER}-${PLATFORM}.tgz -P ${MOFED_DIR}

#COPY MLNX_OFED_LINUX-${MOFED_VER}-${OS_VER}-${PLATFORM}.tgz ${MOFED_DIR}



RUN rm -rf /var/cache/dnf \

    && fakeroot dnf install -y perl lsof numactl-libs pciutils tk libnl3 python36 tcsh gcc-gfortran tcl libmnl ethtool fuse-libs \
@@ -35,13 +34,12 @@ RUN cd ${MOFED_DIR} && \ 
    rm -rf ${MOFED_DIR}

 

# Compiler and dependencies

RUN fakeroot dnf config-manager --set-disabled powertools

RUN rm -rf /var/cache/dnf \

    && dnf install -y gcc gcc-c++ libevent libevent-devel tar libtool-ltdl \

    && dnf clean all

 

# OSU benchmarks

#RUN curl -o osubenchmarks.5.7.tar.gz https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.7.tar.gz && tar -xzf osubenchmarks.5.7.tar.gz --no-same-owner

#RUN cd osu-micro-benchmarks-5.7 && ./configure CC=$mpi_root/bin/mpicc CXX=$mpi_root/bin/mpicxx --enable-cuda \

#    --with-cuda-include=/usr/local/cuda/include --with-cuda-libpath=/usr/local/cuda/lib64 && make && rm ../osubenchmarks.5.7.tar.gz

RUN wget https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.3.tar.gz && tar -xzf osu-micro-benchmarks-7.3.tar.gz --no-same-owner

RUN cd osu-micro-benchmarks-7.3 && ./configure CC=$mpi_root/bin/mpicc CXX=$mpi_root/bin/mpicxx --enable-cuda \

    --with-cuda-include=/usr/local/cuda/include --with-cuda-libpath=/usr/local/cuda/lib64 && make && rm ../osu-micro-benchmarks-7.3.tar.gz

mpiimage-centos-cuda/example_submit.lsf

+2 −1
Original line number Diff line number Diff line
@@ -10,7 +10,8 @@ echo $LD_LIBRARY_PATH 


module purge

module load DefApps

module load  gcc/9.1.0

module unload xl

module load cuda/11.7.1



source /gpfs/alpine/stf007/world-shared/containers/utils/requiredmpilibs.source

mpiimage-centos-cuda/requiredmpilibs.source

+15 −12
Original line number Diff line number Diff line 


## library loading script

mpilibs=("/opt/ibm/spectrum_mpi/lib" \

         "/opt/ibm/spectrum_mpi/lib/spectrum_mpi" \

         "/opt/ibm/spectrum_mpi/lib/pami_port" \

         "/opt/mellanox/hcoll/lib")

mpilibs=("${MPI_ROOT}/lib" \

         "${MPI_ROOT}/lib/spectrum_mpi" \

         "${MPI_ROOT}/lib/pami_port") # \

         #"/opt/mellanox/hcoll/lib" \

	 #"/opt/mellanox/mxm/lib")



#mpilibs=("/opt/mellanox/hcoll/lib" \

#         "/opt/mellanox/mxm/lib")
@@ -16,24 +17,26 @@ deplibs=("libucp.so" "libuct.so" "libucs.so" \ 


for libs in ${deplibs[@]}; do

  for lib in /usr/lib64/${libs}*; do

     if [ -z "${SINGULARITY_CONTAINLIBS:-}" ]; then

            SINGULARITY_CONTAINLIBS="$lib"

     if [ -z "${APPTAINER_CONTAINLIBS:-}" ]; then

            APPTAINER_CONTAINLIBS="$lib"

        else

            SINGULARITY_CONTAINLIBS="$SINGULARITY_CONTAINLIBS,$lib"

            APPTAINER_CONTAINLIBS="$APPTAINER_CONTAINLIBS,$lib"

    fi

  done

done



for libdir in ${mpilibs[@]}; do

  for lib in $libdir/*.so*; do

    if [ -z "${SINGULARITY_CONTAINLIBS:-}" ]; then

            SINGULARITY_CONTAINLIBS="$lib"

    if [ -z "${APPTAINER_CONTAINLIBS:-}" ]; then

            APPTAINER_CONTAINLIBS="$lib"

        else

            SINGULARITY_CONTAINLIBS="$SINGULARITY_CONTAINLIBS,$lib"

            APPTAINER_CONTAINLIBS="$APPTAINER_CONTAINLIBS,$lib"

    fi

  done

done



export SINGULARITYENV_LD_PRELOAD=/gpfs/alpine/stf007/world-shared/containers/utils/libpami_cudahook.so 

export SINGULARITY_CONTAINLIBS

#export APPTAINERENV_LD_PRELOAD=/ccs/home/xjv/libpami_cudahook.so 

export APPTAINER_CONTAINLIBS

#export SINGULARITYENV_PREPEND_PATH=/opt/anaconda2/bin

export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH

export MPICH_RDMA_ENABLED_CUDA=1