Commit 60a2388d authored by Abraham, Subil's avatar Abraham, Subil
Browse files

pytorch and tensorflow updates

parent a2337e3d

baseimage-centos-cuda/Dockerfile

+1 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ RUN dnf -y install fakeroot \ 
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda/compat:$LD_LIBRARY_PATH

ENV CUDA_HOME="/usr/local/cuda"

ENV CUDA_ROOT="/usr/local/cuda"

ENV CPATH="/usr/local/cuda/include:$CPATH"

ENV CUDNN_ROOT="/usr"

ENV CUDNN_INCLUDE_DIR="${CUDNN_ROOT}/include"

ENV CUDNN_LIB_DIR="${CUDNN_ROOT}/lib"

pytorchimage-centos-cuda/Dockerfile

+4 −4
Original line number Diff line number Diff line
@@ -17,19 +17,16 @@ RUN dnf -y install \ 
	glibc-devel \

	&& dnf -y clean all



ENV CUDA_HOME="/usr/local/cuda"

ENV CPATH="/usr/local/cuda/include:$CPATH"

ENV CONDA_HOME="/opt/conda"

RUN mkdir /code

WORKDIR /code

ENV _CONDA_INSTALLER="Miniconda3-latest-Linux-ppc64le.sh"

ENV _CONDA_INSTALLER="Miniconda3-py39_4.9.2-Linux-ppc64le.sh"

RUN wget -q -P /code \

  https://repo.anaconda.com/miniconda/${_CONDA_INSTALLER} \

    && bash /code/${_CONDA_INSTALLER} -b -p ${CONDA_HOME} \

    && rm /code/${_CONDA_INSTALLER}



ENV PATH="/opt/conda/bin:$PATH"

ENV CUDA_HOME="/usr/local/cuda"

RUN conda install -y conda-build conda-verify



ENV INSTALL_ROOT=/code
@@ -168,6 +165,9 @@ RUN cd ${INSTALL_ROOT} \ 
	&& rm -rf opencv



# Install horovod

RUN pip install horovod





## Install DALI (not working)

#COPY ./cmake_patch /code 

#RUN cd ${INSTALL_ROOT} \

pytorchimage-centos-cuda/build.sh

+1 −1
Original line number Diff line number Diff line
@@ -4,4 +4,4 @@ set -e 
env TMPDIR=/projects/tmp podman build -v /projects/tmp:/tmp --ulimit nofile=262144:262144 -f Dockerfile -t code.olcf.ornl.gov:4567/nccs-peak/olcfbaseimages/pytorchimage-centos-cuda:latest . ;





podman push code.olcf.ornl.gov:4567/nccs-peak/olcfbaseimages/pytorchimage-centos-cuda:latest;
 
podman push code.ornl.gov:4567/76a/olcfbaseimages/pytorchimage-centos-cuda:latest;

tensorflowimage-centos-cuda/Dockerfile

+15 −7
Original line number Diff line number Diff line
@@ -2,6 +2,8 @@ 
FROM code.ornl.gov:4567/76a/olcfbaseimages/mpiimage-centos-cuda:latest





RUN dnf install -y dnf-plugins-core \

	&& dnf config-manager --set-enabled powertools

RUN dnf -y install \

	kernel-devel \

	cuda-command-line-tools-11-0 \
@@ -22,13 +24,18 @@ RUN wget -q -P /tmp https://cmake.org/files/v3.15/cmake-3.15.7.tar.gz && \ 
    rm -rf /tmp/cmake-3.15.7





ARG CONDA_VER=py37_4.9.2

RUN wget -q -P /tmp \

  https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VER}-Linux-$(uname -m).sh \

    && bash /tmp/Miniconda3-${CONDA_VER}-Linux-$(uname -m).sh -b -p /opt/conda \

    && rm /tmp/Miniconda3-${CONDA_VER}-Linux-$(uname -m).sh

ENV CONDA_HOME="/opt/conda"

RUN mkdir /code

WORKDIR /code

ENV _CONDA_INSTALLER="Miniconda3-py39_4.9.2-Linux-ppc64le.sh"



RUN wget -q -P /code \

  https://repo.anaconda.com/miniconda/${_CONDA_INSTALLER} \

    && bash /code/${_CONDA_INSTALLER} -b -p ${CONDA_HOME} \

    && rm /code/${_CONDA_INSTALLER}



ENV PATH="/opt/conda/bin:$PATH"



ARG CONDA_REPO=https://public.dhe.ibm.com/ibmdl/export/pub/software/server/ibm-ai/conda/

RUN conda config --prepend channels $CONDA_REPO \

    && conda install -y -c conda-forge numpy==1.19.5 scipy==1.7.0 six wheel pip \
@@ -75,7 +82,7 @@ RUN cd /tmp && \ 


## need this to skip the interactive ./configure for tensorflow

RUN echo $'build --action_env PYTHON_BIN_PATH="/opt/conda/bin/python3"\n\

build --action_env PYTHON_LIB_PATH="/opt/conda/lib/python3.7/site-packages"\n\

build --action_env PYTHON_LIB_PATH="/opt/conda/lib/python3.9/site-packages"\n\

build --python_path="/opt/conda/bin/python3"\n\

build:xla --define with_xla_support=true\n\

build --config=xla\n\
@@ -100,5 +107,6 @@ test:v2 --build_tag_filters=-benchmark-test,-no_oss,-no_gpu,-v1only' > /tmp/tens 


RUN cd /tmp/tensorflow && /usr/bin/bazel build --config=v2 //tensorflow/tools/pip_package:build_pip_package 

RUN cd /tmp/tensorflow && ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg  

RUN pip install /tmp/tensorflow_pkg/tensorflow-2.5.0-cp37-cp37m-linux_ppc64le.whl

RUN pip install /tmp/tensorflow_pkg/tensorflow-2.5.0-cp39-cp39-linux_ppc64le.whl



RUN pip install horovod