Commit 00d2f533 authored by Asa Rentschler's avatar Asa Rentschler
Browse files

Switched to CUDA 11.7.1 and changed LD_LIBRARY_PATH

parent fb929f96

baseimage-centos-cuda/Dockerfile

+7 −12
Original line number Diff line number Diff line 
FROM quay.io/centos/centos:stream8

#FROM quay.io/nvidia/cuda-ppc64le:11.0.3-cudnn8-devel-centos8



# updated cuda libs to 12.2

RUN dnf -y install epel-release && dnf -y install fakeroot 

RUN dnf clean all && rm -r /var/cache/dnf  && fakeroot dnf upgrade -y && fakeroot dnf update -y  

RUN fakeroot dnf install -y wget dnf-plugins-core \

	&& fakeroot dnf config-manager --set-enabled powertools \

	&& wget https://developer.download.nvidia.com/compute/cuda/12.2.0/local_installers/cuda-repo-rhel8-12-2-local-12.2.0_535.54.03-1.ppc64le.rpm \

	&& fakeroot rpm -i cuda-repo-rhel8-12-2-local-12.2.0_535.54.03-1.ppc64le.rpm \

	&& wget https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda-repo-rhel8-11-7-local-11.7.1_515.65.01-1.ppc64le.rpm \

	&& fakeroot rpm -i cuda-repo-rhel8-11-7-local-11.7.1_515.65.01-1.ppc64le.rpm \

	&& fakeroot dnf -y module install nvidia-driver:latest-dkms \

	&& fakeroot dnf -y install cuda \

	&& fakeroot dnf -y install  kernel-devel \

		cuda-command-line-tools-12-2 \

		cuda-command-line-tools-11-7-11.7.1-1.ppc64le \

		java-11-openjdk-devel \

		zip unzip \

		python3-devel \

		tzdata \

		gpg \

		glibc-devel \

	&& rm cuda-repo-rhel8-12-2-local-12.2.0_535.54.03-1.ppc64le.rpm \

	&& rm cuda-repo-rhel8-11-7-local-11.7.1_515.65.01-1.ppc64le.rpm \

	&& dnf -y clean all && rm -r /var/cache/dnf



# replaced old rpm installation with repo

RUN fakeroot dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/ppc64le/cuda-rhel8.repo

RUN fakeroot dnf -y install cuda-nvcc-12-2.ppc64le libcudnn8-8.9.7.29-1.cuda12.2.ppc64le libcudnn8-devel-8.9.7.29-1.cuda12.2.ppc64le && dnf clean all && rm -r /var/cache/dnf

RUN fakeroot dnf -y install cuda-nvcc-11-7-11.7.99-1.ppc64le libcudnn8-8.5.0.96-1.cuda11.7.ppc64le libcudnn8-devel-8.5.0.96-1.cuda11.7.ppc64le && dnf clean all && rm -r /var/cache/dnf



#	&& dnf -y install tar wget git openssh 



ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda/compat:/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH

ENV PATH=/usr/local/cuda-12.2/bin:$PATH

ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH

ENV PATH=/usr/local/cuda-11.7/bin:$PATH

ENV CUDA_HOME="/usr/local/cuda"

ENV CUDA_ROOT="/usr/local/cuda"

ENV CPATH="/usr/local/cuda/include:$CPATH"

baseimage-centos-cuda/README.md

+1 −1
Original line number Diff line number Diff line 
This image needs to be built somewhere where you have sudo or subuid/subgid mappings present



Currently built with CUDA 12.2 libraries
 
Currently built with CUDA 11.7.1 libraries for Summit

baseimage-centos-cuda/build

+7 −0
Original line number Diff line number Diff line 
#!/bin/bash



module purge

module load DefApps

module unload xl

module load cuda/11.7.1



module list



podman build --net=host -f Dockerfile -t code.ornl.gov:4567/olcfcontainers/olcfbaseimages/baseimage-centos-cuda:latest .;

#podman push code.ornl.gov:4567/olcfcontainers/olcfbaseimages/baseimage-centos-cuda:latest;

#podman save baseimage-centos-cuda.tar code.ornl.gov:4567/olcfcontainers/olcfbaseimages/baseimage-centos-cuda:latest;

mpiimage-centos-cuda/Dockerfile

+6 −5
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ ENV CPLUS_INCLUDE_PATH=$mpi_root/include:$CPLUS_INCLUDE_PATH 
 

# MOFED is sufficient, but is it necessary?

# Set MOFED version, OS version and platform

ENV MOFED_VER 4.9-7.1.0.0

ENV MOFED_VER 4.9-6.0.6.1

ENV OS_VER rhel8.6

ENV PLATFORM ppc64le

ENV MOFED_DIR /mlnx
@@ -25,7 +25,7 @@ RUN wget https://content.mellanox.com/ofed/MLNX_OFED-${MOFED_VER}/MLNX_OFED_LINU 
#COPY MLNX_OFED_LINUX-${MOFED_VER}-${OS_VER}-${PLATFORM}.tgz ${MOFED_DIR}



RUN rm -rf /var/cache/dnf \

    && fakeroot dnf install -y perl lsof numactl-libs pciutils tk libnl3 python36 tcsh gcc-gfortran tcl libmnl ethtool \

    && fakeroot dnf install -y perl lsof numactl-libs pciutils tk libnl3 python36 tcsh gcc-gfortran tcl libmnl ethtool fuse-libs \

	&& fakeroot dnf -y install tar wget git openssh \

	&& dnf clean all

RUN cd ${MOFED_DIR} && \
@@ -35,12 +35,13 @@ RUN cd ${MOFED_DIR} && \ 
    rm -rf ${MOFED_DIR}

 

# Compiler and dependencies

RUN fakeroot dnf config-manager --set-disabled powertools

RUN rm -rf /var/cache/dnf \

    && dnf install -y gcc gcc-c++ libevent libevent-devel tar libtool-ltdl \

    && dnf clean all

 

# OSU benchmarks

RUN curl -o osubenchmarks.5.7.tar.gz https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.7.tar.gz && tar -xzf osubenchmarks.5.7.tar.gz --no-same-owner

RUN cd osu-micro-benchmarks-5.7 && ./configure CC=$mpi_root/bin/mpicc CXX=$mpi_root/bin/mpicxx --enable-cuda \

    --with-cuda-include=/usr/local/cuda/include --with-cuda-libpath=/usr/local/cuda/lib64 && make && rm ../osubenchmarks.5.7.tar.gz

#RUN curl -o osubenchmarks.5.7.tar.gz https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.7.tar.gz && tar -xzf osubenchmarks.5.7.tar.gz --no-same-owner

#RUN cd osu-micro-benchmarks-5.7 && ./configure CC=$mpi_root/bin/mpicc CXX=$mpi_root/bin/mpicxx --enable-cuda \

#    --with-cuda-include=/usr/local/cuda/include --with-cuda-libpath=/usr/local/cuda/lib64 && make && rm ../osubenchmarks.5.7.tar.gz

mpiimage-centos-cuda/build

+3 −2
Original line number Diff line number Diff line
@@ -4,9 +4,10 @@ set -e 


module purge

module load DefApps

module load  gcc/9.1.0

module unload xl

module load cuda/11.7.1

 

module -t list

module list

 

podman build --net=host --build-arg mpi_root=$MPI_ROOT -v $MPI_ROOT:$MPI_ROOT -f Dockerfile -t code.ornl.gov:4567/olcfcontainers/olcfbaseimages/mpiimage-centos-cuda:latest .;

#export SINGULARITY_CACHEDIR=/tmp/containers/subil/singularitycache