Commit 452f61e2 authored by Sujin Philip's avatar Sujin Philip Committed by Maynard, Robert
Browse files

Add Kokkos backend

parent 9f4ae20a
......@@ -49,45 +49,54 @@
GIT_CLONE_PATH: $CI_BUILDS_DIR/gitlab-kitware-sciviz-ci
.centos7: &centos7
image: "kitware/vtkm:ci-centos7_cuda10.2-20200601"
image: "kitware/vtkm:ci-centos7_cuda10.2-20200729"
extends:
- .docker_image
.centos8: &centos8
image: "kitware/vtkm:ci-centos8-20200601"
image: "kitware/vtkm:ci-centos8-20200729"
extends:
- .docker_image
.rhel8: &rhel8
image: "kitware/vtkm:ci-rhel8_cuda10.2-20200601"
image: "kitware/vtkm:ci-rhel8_cuda10.2-20200729"
extends:
- .docker_image
.rhel8_kokkos: &rhel8_kokkos
image: "kitware/vtkm:ci-rhel8_kokkos-20200729"
extends:
- .docker_image
.ubuntu1604: &ubuntu1604
image: "kitware/vtkm:ci-ubuntu1604-20200601"
image: "kitware/vtkm:ci-ubuntu1604-20200729"
extends:
- .docker_image
.ubuntu1604_cuda: &ubuntu1604_cuda
image: "kitware/vtkm:ci-ubuntu1604_cuda9.2-20200601"
image: "kitware/vtkm:ci-ubuntu1604_cuda9.2-20200729"
extends:
- .docker_image
.ubuntu1804: &ubuntu1804
image: "kitware/vtkm:ci-ubuntu1804-20200601"
image: "kitware/vtkm:ci-ubuntu1804-20200729"
extends:
- .docker_image
.ubuntu1804_cuda: &ubuntu1804_cuda
image: "kitware/vtkm:ci-ubuntu1804_cuda10.1-20200601"
image: "kitware/vtkm:ci-ubuntu1804_cuda10.1-20200729"
extends:
- .docker_image
.ubuntu2004_doxygen: &ubuntu2004_doxygen
image: "kitware/vtkm:ci-doxygen-20200601"
image: "kitware/vtkm:ci-doxygen-20200729"
extends:
- .docker_image
.ubuntu2004_kokkos: &ubuntu2004_kokkos
image: "kitware/vtkm:ci-ubuntu2004_kokkos-20200729"
extends:
- .docker_image
.only-default: &only-default
only:
......@@ -178,4 +187,5 @@ include:
- local: '/.gitlab/ci/rhel8.yml'
- local: '/.gitlab/ci/ubuntu1604.yml'
- local: '/.gitlab/ci/ubuntu1804.yml'
- local: '/.gitlab/ci/ubuntu2004.yml'
- local: '/.gitlab/ci/windows10.yml'
......@@ -20,7 +20,6 @@ endif ()
string(REPLACE "+" ";" options "$ENV{VTKM_SETTINGS}")
foreach(option IN LISTS options)
if(static STREQUAL option)
set(BUILD_SHARED_LIBS "OFF" CACHE STRING "")
......@@ -71,6 +70,9 @@ foreach(option IN LISTS options)
elseif(cuda STREQUAL option)
set(VTKm_ENABLE_CUDA "ON" CACHE STRING "")
elseif(kokkos STREQUAL option)
set(VTKm_ENABLE_KOKKOS "ON" CACHE STRING "")
elseif(maxwell STREQUAL option)
set(VTKm_CUDA_Architecture "maxwell" CACHE STRING "")
......
FROM nvidia/cuda:10.2-devel-ubi8
LABEL maintainer "Sujin Philip<sujin.philip@kitware.com>"
RUN yum install make gcc gcc-c++ curl -y
RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | bash
RUN yum install git git-lfs -y
# kokkos backend requires cmake 3.18
RUN mkdir /opt/cmake/ && \
curl -L https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0-Linux-x86_64.sh > cmake-3.18.0-Linux-x86_64.sh && \
sh cmake-3.18.0-Linux-x86_64.sh --prefix=/opt/cmake/ --exclude-subdir --skip-license && \
rm cmake-3.18.0-Linux-x86_64.sh && \
ln -s /opt/cmake/bin/ctest /opt/cmake/bin/ctest-latest
ENV PATH "/opt/cmake/bin:${PATH}"
# Build and install Kokkos
RUN mkdir -p /opt/kokkos/build && \
cd /opt/kokkos/build && \
curl -L https://github.com/kokkos/kokkos/archive/3.1.01.tar.gz > kokkos-3.1.01.tar.gz && \
tar -xf kokkos-3.1.01.tar.gz && \
mkdir bld && cd bld && \
CXX=/opt/kokkos/build/kokkos-3.1.01/bin/nvcc_wrapper \
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/kokkos -DCMAKE_CXX_FLAGS=-fPIC \
-DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_CUDA_CONSTEXPR=ON \
-DKokkos_ENABLE_CUDA_LAMBDA=ON -DKokkos_ENABLE_CUDA_LDG_INTRINSIC=ON \
-DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON -DKokkos_ENABLE_CUDA_UVM=ON \
-DKokkos_ARCH_TURING75=ON ../kokkos-3.1.01 && \
make -j all && \
make install
FROM ubuntu:20.04
LABEL maintainer "Sujin Philip<sujin.philip@kitware.com>"
# Base dependencies for building VTK-m projects
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
cmake \
curl \
g++ \
git \
git-lfs \
libmpich-dev \
libomp-dev \
mpich \
ninja-build \
rsync \
ssh \
software-properties-common
# Need to run git-lfs install manually on ubuntu based images when using the
# system packaged version
RUN git-lfs install
# Provide CMake 3.17 so we can re-run tests easily
# This will be used when we run just the tests
RUN mkdir /opt/cmake/ && \
curl -L https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh > cmake-3.17.3-Linux-x86_64.sh && \
sh cmake-3.17.3-Linux-x86_64.sh --prefix=/opt/cmake/ --exclude-subdir --skip-license && \
rm cmake-3.17.3-Linux-x86_64.sh && \
ln -s /opt/cmake/bin/ctest /opt/cmake/bin/ctest-latest
ENV PATH "${PATH}:/opt/cmake/bin"
# Build and install Kokkos
RUN mkdir -p /opt/kokkos/build && \
cd /opt/kokkos/build && \
curl -L https://github.com/kokkos/kokkos/archive/3.1.01.tar.gz > kokkos-3.1.01.tar.gz && \
tar -xf kokkos-3.1.01.tar.gz && \
mkdir bld && cd bld && \
cmake -GNinja -DCMAKE_INSTALL_PREFIX=/opt/kokkos -DCMAKE_CXX_FLAGS=-fPIC -DKokkos_ENABLE_SERIAL=ON ../kokkos-3.1.01 &&\
ninja all && \
ninja install
......@@ -18,6 +18,10 @@ cd rhel8/cuda10.2
sudo docker build -t kitware/vtkm:ci-rhel8_cuda10.2-$date .
cd ../..
cd rhel8/kokkos
sudo docker build -t kitware/vtkm:ci-rhel8_kokkos-$date .
cd ../..
cd ubuntu1604/base
sudo docker build -t kitware/vtkm:ci-ubuntu1604-$date .
cd ../..
......@@ -38,6 +42,10 @@ cd ubuntu2004/doxygen/
sudo docker build -t kitware/vtkm:ci-doxygen-$date .
cd ../..
cd ubuntu2004/kokkos
sudo docker build -t kitware/vtkm:ci-ubuntu2004_kokkos-$date .
cd ../..
# sudo docker login --username=<docker_hub_name>
sudo docker push kitware/vtkm
sudo docker system prune
......@@ -60,3 +60,40 @@ test:rhel8_vtk_types:
- build:rhel8_vtk_types
needs:
- build:rhel8_vtk_types
# Build on rhel8 with kokkos and test on rhel8
# Uses gcc 8.2.1
build:rhel8_kokkos:
tags:
- build
- vtkm
- docker
- linux
- large-memory
extends:
- .rhel8_kokkos
- .cmake_build_linux
- .only-default
variables:
CMAKE_GENERATOR: "Unix Makefiles"
CMAKE_BUILD_TYPE: Release
VTKM_SETTINGS: "kokkos+static+64bit_floats"
test:rhel8_kokkos:
tags:
- test
- cuda-rt
- turing
- vtkm
- docker
- linux
extends:
- .rhel8_kokkos
- .cmake_test_linux
- .only-default
dependencies:
- build:rhel8_kokkos
needs:
- build:rhel8_kokkos
variables:
CUDA_LAUNCH_BLOCKING: "1"
build:ubuntu2004_kokkos:
tags:
- build
- vtkm
- docker
- linux
extends:
- .ubuntu2004_kokkos
- .cmake_build_linux
- .only-default
variables:
CMAKE_BUILD_TYPE: RelWithDebInfo
VTKM_SETTINGS: "kokkos+shared+64bit_floats"
test:ubuntu2004_kokkos:
tags:
- test
- vtkm
- docker
- linux
extends:
- .ubuntu2004_kokkos
- .cmake_test_linux
- .only-default
dependencies:
- build:ubuntu2004_kokkos
needs:
- build:ubuntu2004_kokkos
......@@ -161,17 +161,21 @@ elseif(VTKM_COMPILER_IS_GNU OR VTKM_COMPILER_IS_CLANG)
endif()
endif()
#common warnings for all platforms when building cuda
if(TARGET vtkm::cuda)
function(setup_cuda_flags)
if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
#nvcc 9 introduced specific controls to disable the stack size warning
#otherwise we let the warning occur. We have to set this in CMAKE_CUDA_FLAGS
#as it is passed to the device link step, unlike compile_options
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xnvlink=--suppress-stack-size-warning")
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xnvlink=--suppress-stack-size-warning" PARENT_SCOPE)
endif()
set(display_error_nums -Xcudafe=--display_error_number)
target_compile_options(vtkm_developer_flags INTERFACE $<$<COMPILE_LANGUAGE:CUDA>:${display_error_nums}>)
endfunction()
#common warnings for all platforms when building cuda
if ((TARGET vtkm::cuda) OR (TARGET vtkm::kokkos_cuda))
setup_cuda_flags()
endif()
if(NOT VTKm_INSTALL_ONLY_LIBRARIES)
......
......@@ -39,6 +39,7 @@
# VTKm_ENABLE_CUDA Will be enabled if VTK-m was built with CUDA support
# VTKm_ENABLE_TBB Will be enabled if VTK-m was built with TBB support
# VTKm_ENABLE_OPENMP Will be enabled if VTK-m was built with OpenMP support
# VTKm_ENABLE_KOKKOS Will be enabled if VTK-m was built with Kokkos support
# VTKm_ENABLE_LOGGING Will be enabled if VTK-m was built with logging support
# VTKm_ENABLE_MPI Will be enabled if VTK-m was built with MPI support
# VTKm_ENABLE_RENDERING Will be enabled if VTK-m was built with rendering support
......@@ -69,6 +70,7 @@ set(VTKm_BUILD_SHARED_LIBS "@VTKm_BUILD_SHARED_LIBS@")
set(VTKm_ENABLE_CUDA "@VTKm_ENABLE_CUDA@")
set(VTKm_ENABLE_TBB "@VTKm_ENABLE_TBB@")
set(VTKm_ENABLE_OPENMP "@VTKm_ENABLE_OPENMP@")
set(VTKm_ENABLE_KOKKOS "@VTKm_ENABLE_KOKKOS@")
set(VTKm_ENABLE_LOGGING "@VTKm_ENABLE_LOGGING@")
set(VTKm_ENABLE_RENDERING "@VTKm_ENABLE_RENDERING@")
set(VTKm_ENABLE_GL_CONTEXT "@VTKm_ENABLE_GL_CONTEXT@")
......
......@@ -251,6 +251,76 @@ if(VTKm_ENABLE_CUDA)
endif()
endif()
#-----------------------------------------------------------------------------
# Kokkos with its Cuda backend enabled, expects everything to be compiled using its
# `nvcc-wrapper` as the CXX compiler. As the name suggests, nvcc-wrapper is a wrapper around
# Cuda's nvcc compiler. Kokkos targets have all of the flags meant for the nvcc compiler set as the
# CXX compiler flags. This function changes all such flags to be CUDA flags so that we can use
# CMake and vtk-m's existing infrastructure to compile for Cuda and Host separately. Without this
# all of the files will be compiled using nvcc which can be very time consuming. It can also have
# issues with calling host functions from device functions when compiling code for other backends.
function(kokkos_fix_compile_options)
set(targets Kokkos::kokkos)
set(seen_targets)
set(cuda_arch)
while(targets)
list(GET targets 0 target_name)
list(REMOVE_AT targets 0)
get_target_property(link_libraries ${target_name} INTERFACE_LINK_LIBRARIES)
foreach(lib_target IN LISTS link_libraries)
if (TARGET ${lib_target})
if (lib_target IN_LIST seen_targets)
continue()
endif()
list(APPEND seen_targets ${lib_target})
list(APPEND targets ${lib_target})
get_target_property(compile_options ${lib_target} INTERFACE_COMPILE_OPTIONS)
if (compile_options)
string(REGEX MATCH "[$]<[$]<COMPILE_LANGUAGE:CXX>:-Xcompiler;.*>" cxx_compile_options "${compile_options}")
string(REGEX MATCH "-arch=sm_[0-9][0-9]" cuda_arch "${compile_options}")
string(REPLACE "-Xcompiler;" "" cxx_compile_options "${cxx_compile_options}")
list(TRANSFORM compile_options REPLACE "COMPILE_LANGUAGE:CXX" "COMPILE_LANGUAGE:CUDA")
list(APPEND compile_options "${cxx_compile_options}")
set_property(TARGET ${lib_target} PROPERTY INTERFACE_COMPILE_OPTIONS ${compile_options})
endif()
set_property(TARGET ${lib_target} PROPERTY INTERFACE_LINK_OPTIONS "")
endif()
endforeach()
endwhile()
set_property(TARGET vtkm::kokkos PROPERTY INTERFACE_LINK_OPTIONS "$<DEVICE_LINK:--relocatable-device-code=true;${cuda_arch}>")
if (OPENMP IN_LIST Kokkos_DEVICES)
set_property(TARGET vtkm::kokkos PROPERTY INTERFACE_LINK_OPTIONS "$<HOST_LINK:-fopenmp>")
endif()
endfunction()
if(VTKm_ENABLE_KOKKOS AND NOT TARGET vtkm::kokkos)
cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
find_package(Kokkos REQUIRED)
if (CUDA IN_LIST Kokkos_DEVICES)
cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
enable_language(CUDA)
string(REGEX MATCH "[0-9][0-9]$" cuda_arch ${Kokkos_ARCH})
set(CMAKE_CUDA_ARCHITECTURES ${cuda_arch})
message(STATUS "Detected Cuda arch from Kokkos: ${cuda_arch}")
add_library(vtkm::kokkos_cuda INTERFACE IMPORTED GLOBAL)
endif()
add_library(vtkm::kokkos INTERFACE IMPORTED GLOBAL)
set_target_properties(vtkm::kokkos PROPERTIES INTERFACE_LINK_LIBRARIES "Kokkos::kokkos")
if (TARGET vtkm::kokkos_cuda)
kokkos_fix_compile_options()
endif()
endif()
if(NOT TARGET Threads::Threads)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
......
......@@ -309,11 +309,16 @@ function(vtkm_add_target_information uses_vtkm_target)
#
# This is required as CUDA currently doesn't support device side calls across
# dynamic library boundaries.
if(TARGET vtkm::cuda)
if((TARGET vtkm::cuda) OR (TARGET vtkm::kokkos_cuda))
set_source_files_properties(${VTKm_TI_DEVICE_SOURCES} PROPERTIES LANGUAGE "CUDA")
foreach(target IN LISTS targets)
get_target_property(lib_type ${target} TYPE)
get_target_property(requires_static vtkm::cuda requires_static_builds)
if (TARGET vtkm::cuda)
get_target_property(requires_static vtkm::cuda requires_static_builds)
endif()
if (TARGET vtkm::kokkos)
get_target_property(requires_static vtkm::kokkos requires_static_builds)
endif()
if(requires_static AND ${lib_type} STREQUAL "SHARED_LIBRARY" AND VTKm_TI_EXTENDS_VTKM)
#We provide different error messages based on if we are building VTK-m
......
......@@ -110,6 +110,10 @@ function(vtkm_test_against_install dir)
)
endif()
if(TARGET vtkm::kokkos)
list(APPEND args "-DKokkos_DIR=${Kokkos_DIR}")
endif()
#determine if the test is expected to compile or fail to build. We use
#this information to built the test name to make it clear to the user
#what a 'passing' test means
......
......@@ -49,7 +49,7 @@ function(vtkm_create_test_executable
#if all backends are enabled, we can use cuda compiler to handle all possible backends.
set(device_sources)
if(TARGET vtkm::cuda AND enable_all_backends)
if(((TARGET vtkm::cuda) OR (TARGET vtkm::kokkos_cuda)) AND enable_all_backends)
set(device_sources ${sources})
endif()
vtkm_add_target_information(${prog} DEVICE_SOURCES ${device_sources})
......@@ -152,6 +152,13 @@ function(vtkm_unit_tests)
#serially
list(APPEND per_device_serial TRUE)
endif()
if (VTKm_ENABLE_KOKKOS)
list(APPEND per_device_command_line_arguments --device=kokkos)
list(APPEND per_device_suffix "KOKKOS")
#may require more time because of kernel generation.
list(APPEND per_device_timeout 1500)
list(APPEND per_device_serial FALSE)
endif()
endif()
set(test_prog)
......
......@@ -81,6 +81,7 @@ endmacro ()
vtkm_option(VTKm_ENABLE_CUDA "Enable Cuda support" OFF)
vtkm_option(VTKm_ENABLE_TBB "Enable TBB support" OFF)
vtkm_option(VTKm_ENABLE_OPENMP "Enable OpenMP support" OFF)
vtkm_option(VTKm_ENABLE_KOKKOS "Enable Kokkos support" OFF)
vtkm_option(VTKm_ENABLE_RENDERING "Enable rendering library" ON)
vtkm_option(VTKm_ENABLE_BENCHMARKS "Enable VTKm Benchmarking" OFF)
vtkm_option(VTKm_ENABLE_MPI "Enable MPI support" OFF)
......
# Add Kokkos backend
Adds a new device backend `Kokkos` which uses the kokkos library for parallelism.
User must provide the kokkos build and Vtk-m will use the default configured execution
space.
......@@ -665,22 +665,18 @@ private:
} // namespace vtkm
#ifdef VTKM_CUDA
// Cuda seems to have a bug where it expects the template class VirtualObjectTransfer
// to be instantiated in a consistent order among all the translation units of an
// executable. Failing to do so results in random crashes and incorrect results.
// We workaroud this issue by explicitly instantiating VirtualObjectTransfer for
// all the implicit functions here.
#include <vtkm/cont/cuda/internal/VirtualObjectTransferCuda.h>
#ifdef VTKM_CUDA
#include <vtkm/cont/internal/VirtualObjectTransferInstantiate.h>
VTKM_EXPLICITLY_INSTANTIATE_TRANSFER(vtkm::Box);
VTKM_EXPLICITLY_INSTANTIATE_TRANSFER(vtkm::Cylinder);
VTKM_EXPLICITLY_INSTANTIATE_TRANSFER(vtkm::Frustum);
VTKM_EXPLICITLY_INSTANTIATE_TRANSFER(vtkm::Plane);
VTKM_EXPLICITLY_INSTANTIATE_TRANSFER(vtkm::Sphere);
#endif
#endif //vtk_m_ImplicitFunction_h
......@@ -51,6 +51,7 @@ public:
using ValueType = typename ArrayHandleImplicitTraits<FunctorType_>::ValueType;
using FunctorType = FunctorType_;
VTKM_SUPPRESS_EXEC_WARNINGS
VTKM_EXEC_CONT
ArrayPortalImplicit()
: Functor()
......@@ -58,6 +59,7 @@ public:
{
}
VTKM_SUPPRESS_EXEC_WARNINGS
VTKM_EXEC_CONT
ArrayPortalImplicit(FunctorType f, vtkm::Id numValues)
: Functor(f)
......
......@@ -239,6 +239,7 @@ add_subdirectory(serial)
add_subdirectory(tbb)
add_subdirectory(openmp)
add_subdirectory(cuda)
add_subdirectory(kokkos)
set(backends )
if(TARGET vtkm::tbb)
......@@ -250,6 +251,9 @@ endif()
if(TARGET vtkm::openmp)
list(APPEND backends vtkm::openmp)
endif()
if(TARGET vtkm::kokkos)
list(APPEND backends vtkm::kokkos)
endif()
target_link_libraries(vtkm_cont PUBLIC vtkm_compiler_flags ${backends})
target_link_libraries(vtkm_cont PUBLIC Threads::Threads)
......
......@@ -16,6 +16,7 @@
// clang-format off
#include <vtkm/cont/cuda/DeviceAdapterCuda.h>
#include <vtkm/cont/kokkos/DeviceAdapterKokkos.h>
#include <vtkm/cont/openmp/DeviceAdapterOpenMP.h>
#include <vtkm/cont/serial/DeviceAdapterSerial.h>
#include <vtkm/cont/tbb/DeviceAdapterTBB.h>
......@@ -46,6 +47,8 @@ namespace cont
/// helpful for debugging.
/// \li \c vtkm::cont::DeviceAdapterTagCuda Dispatches and runs algorithms on a GPU
/// using CUDA. Must be compiling with a CUDA compiler (nvcc).
/// \li \c vtkm::cont::DeviceAdapterTagKokkos Dispatches and runs algorithms using
/// the Kokkos library.
/// \li \c vtkm::cont::DeviceAdapterTagOpenMP Dispatches an algorithm over multiple
/// CPU cores using OpenMP compiler directives. Must be compiling with an
/// OpenMP-compliant compiler with OpenMP pragmas enabled.
......
......@@ -17,6 +17,7 @@
#include <vtkm/List.h>
#include <vtkm/cont/cuda/internal/DeviceAdapterTagCuda.h>
#include <vtkm/cont/kokkos/internal/DeviceAdapterTagKokkos.h>
#include <vtkm/cont/openmp/internal/DeviceAdapterTagOpenMP.h>
#include <vtkm/cont/serial/internal/DeviceAdapterTagSerial.h>
#include <vtkm/cont/tbb/internal/DeviceAdapterTagTBB.h>
......@@ -29,6 +30,7 @@ namespace cont
using DeviceAdapterListCommon = vtkm::List<vtkm::cont::DeviceAdapterTagCuda,
vtkm::cont::DeviceAdapterTagTBB,
vtkm::cont::DeviceAdapterTagOpenMP,
vtkm::cont::DeviceAdapterTagKokkos,
vtkm::cont::DeviceAdapterTagSerial>;
}
} // namespace vtkm::cont
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment