Commit 8a25c977 authored by Abraham, Subil's avatar Abraham, Subil
Browse files

dali build attempts

parent 7d25d855
Loading
Loading
Loading
Loading
+25 −16
Original line number Diff line number Diff line
@@ -53,7 +53,6 @@ ENV NCCL_ROOT=${INSTALL_ROOT}/nccl


# install magma
## Install PyTorch
ENV TORCH_CUDA_ARCH_LIST="7.0"
RUN conda install -y pip astunparse numpy ninja pyyaml setuptools cmake cffi typing_extensions future six requests dataclasses
RUN conda install -y -c conda-forge lapack==3.9.0
@@ -70,6 +69,7 @@ RUN cd /code \
		control/magma_internal.h /usr/include 

#conda install -y magma-cuda110
## Install PyTorch
RUN git config --global user.email "you@example.com" \
	&& git config --global user.name "Your Name"
RUN cd ${INSTALL_ROOT} \
@@ -154,23 +154,32 @@ RUN cd ${INSTALL_ROOT} \
	&& python setup.py install
	#&& dnf install -y libjpeg-turbo-devel \

# build opencv
# build opencv (need this to build libquirc.a and libade.a)
RUN cd ${INSTALL_ROOT} \ 
	&& git clone https://github.com/opencv/opencv --branch 4.5.3 \
	&& cd opencv \
	&& mkdir build \
	&& cd build \
	&& cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D INSTALL_PYTHON_EXAMPLES=ON -D INSTALL_C_EXAMPLES=ON .. \
	&& make -j32 \
	&& cp /code/opencv/build/3rdparty/lib/libquirc.a /usr/local/lib64/opencv4/3rdparty/ \
	&& cp /code/opencv/build/3rdparty/lib/libade.a /usr/local/lib64/opencv4/3rdparty/ \
	&& cd ${INSTALL_ROOT} \
	&& rm -rf opencv

#then
#cp /code/opencv/build/3rdparty/lib/libquirc.a /usr/local/lib64/opencv4/3rdparty/
#cp /code/opencv/build/3rdparty/lib/libade.a /usr/local/lib64/opencv4/3rdparty/
#then delete the opencv directory
## Install DALI (not working)
#RUN cd ${INSTALL_ROOT} \
#	&& git clone --recurse-submodules --shallow-submodules https://github.com/NVIDIA/DALI --depth 1 --branch v1.6.0 \
#	&& cd DALI \ 
#	&& mkdir build \
#	&& cd build \
#	&& ln -s /usr/lib64/libclang.so /usr/lib64/libclang-11.so \
#	&&  cmake  -DCMAKE_BUILD_TYPE=Release -DBUILD_NVJPEG2K=OFF -DFFMPEG_ROOT_DIR=${CONDA_HOME} -DJPEG_INCLUDE_DIR=/usr/local/include -DJPEG_LIBRARY=/usr/local/lib/libturbojpeg.so -DCUDA_TARGET_ARCHS=70 -DBUILD_PROTOBUF=ON -DProtobuf_LIBRARY=/usr/local/lib/libprotobuf.so -DFFmpeg_Libavcodec=/usr/local/lib/libavcodec.so -DFFmpeg_Libavfilter=/usr/local/lib/libavfilter.so -DFFmpeg_Libavformat=/usr/local/lib/libavformat.so -DFFmpeg_Libavutil=/usr/local/lib/libavutil.so .. \
#	&& make-j4 \
#	&& make install \
#	&& pip install dali/python
COPY ./cmake_patch /code 
RUN cd ${INSTALL_ROOT} \
	&& git clone --recurse-submodules --shallow-submodules https://github.com/NVIDIA/DALI --depth 1 --branch v1.6.0 \
	&& cd DALI \ 
	&& git apply /code/cmake_patch \
	&& mkdir build \
	&& cd build \
	&& ln -s /usr/lib64/libclang.so /usr/lib64/libclang-11.so \
	&& cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_NVJPEG2K=OFF -DFFMPEG_ROOT_DIR=${CONDA_HOME} -DJPEG_INCLUDE_DIR=/usr/local/include -DJPEG_LIBRARY=/usr/local/lib/libturbojpeg.so -DCUDA_TARGET_ARCHS=70 -DBUILD_PROTOBUF=ON -DProtobuf_LIBRARY=/usr/local/lib/libprotobuf.a -DFFmpeg_Libavcodec=/usr/local/lib/libavcodec.so -DFFmpeg_Libavfilter=/usr/local/lib/libavfilter.so -DFFmpeg_Libavformat=/usr/local/lib/libavformat.so -DFFmpeg_Libavutil=/usr/local/lib/libavutil.so .. \
	&& make -j1 \
	&& make install \
	&& pip install dali/python

### failed attempt 1
#RUN cd ${INSTALL_ROOT} \
+34 −0
Original line number Diff line number Diff line
From 5486ad529027f20cb4d60de1ddc80b7b1470df9d Mon Sep 17 00:00:00 2001
From: Nishidha Panpaliya <npanpa23@in.ibm.com>
Date: Tue, 22 Dec 2020 10:14:08 -0500
Subject: [PATCH] Fixed cmake error

---
 cmake/Utils.cmake | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake
index c08e70c..34f9be6 100644
--- a/cmake/Utils.cmake
+++ b/cmake/Utils.cmake
@@ -342,14 +342,14 @@ function(build_per_python_lib)
                 COMMAND python${PYVER}-config --extension-suffix
                 OUTPUT_VARIABLE PYTHON_SUFIX)
             # remove newline and the extension
-            string(REPLACE ".so\n" "" PYTHON_SUFIX ${PYTHON_SUFIX})
+            string(REPLACE ".so\n" "" PYTHON_SUFIX "${PYTHON_SUFIX}")
 
             execute_process(
                 COMMAND python${PYVER}-config --includes
                 OUTPUT_VARIABLE PYTHON_INCLUDES)
             # split and make it a list
-            string(REPLACE "-I" "" PYTHON_INCLUDES ${PYTHON_INCLUDES})
-            string(REPLACE "\n" "" PYTHON_INCLUDES ${PYTHON_INCLUDES})
+            string(REPLACE "-I" "" PYTHON_INCLUDES "${PYTHON_INCLUDES}")
+            string(REPLACE "\n" "" PYTHON_INCLUDES "${PYTHON_INCLUDES}")
             separate_arguments(PYTHON_INCLUDES)
 
             add_library(${PYTHON_LIB_TARGET_FOR_PYVER} SHARED)
-- 
1.8.3.1
+0 −1095

File deleted.

Preview size limit exceeded, changes collapsed.

+670 −0

File added.

Preview size limit exceeded, changes collapsed.

+950 −0

File added.

Preview size limit exceeded, changes collapsed.