Merge pull request #3621 from anagainaru/sycl-backend (2aace5f2) · Commits · ecpcitest / ADIOS2

CMakeLists.txt

+1 −1

Original line number	Diff line number	Diff line
		@@ -237,7 +237,7 @@ endif()
		set(ADIOS2_CONFIG_OPTS
		BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python Blosc2
		BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ
		Profiling Endian_Reverse AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP
		Profiling Endian_Reverse AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP Kokkos_SYCL
		)

		GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS})

bindings/CXX11/adios2/cxx11/KokkosView.h

+3 −42

Original line number	Diff line number	Diff line
		@@ -9,55 +9,16 @@ namespace detail
		{

		template <typename T>
		struct memspace_kokkos_to_adios2;

		template <>
		struct memspace_kokkos_to_adios2<Kokkos::HostSpace>
		{
		static constexpr adios2::MemorySpace value = adios2::MemorySpace::Host;
		};

		#if defined(KOKKOS_ENABLE_CUDA) && defined(ADIOS2_HAVE_GPU_SUPPORT)

		template <>
		struct memspace_kokkos_to_adios2<Kokkos::CudaSpace>
		struct memspace_kokkos_to_adios2
		{
		static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
		};

		template <>
		struct memspace_kokkos_to_adios2<Kokkos::CudaUVMSpace>
		{
		static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
		};

		template <>
		struct memspace_kokkos_to_adios2<Kokkos::CudaHostPinnedSpace>
		{
		static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
		};

		#endif

		#if defined(KOKKOS_ENABLE_HIP) && defined(ADIOS2_HAVE_GPU_SUPPORT)
		template <>
		struct memspace_kokkos_to_adios2<Kokkos::Experimental::HIPSpace>
		{
		static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
		};

		template <>
		struct memspace_kokkos_to_adios2<Kokkos::Experimental::HIPHostPinnedSpace>
		{
		static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
		};

		template <>
		struct memspace_kokkos_to_adios2<Kokkos::Experimental::HIPManagedSpace>
		struct memspace_kokkos_to_adios2<Kokkos::HostSpace>
		{
		static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
		static constexpr adios2::MemorySpace value = adios2::MemorySpace::Host;
		};
		#endif

		} // namespace detail

cmake/DetectOptions.cmake

+3 −0

Original line number	Diff line number	Diff line
		@@ -192,6 +192,9 @@ if(ADIOS2_USE_Kokkos)
		set(ADIOS2_HAVE_Kokkos_HIP TRUE)
		enable_language(HIP)
		endif()
		if(Kokkos_ENABLE_SYCL)
		set(ADIOS2_HAVE_Kokkos_SYCL TRUE)
		endif()
		set(ADIOS2_HAVE_GPU_Support TRUE)
		endif()
		endif()

scripts/build_scripts/build-adios2-sycl-polaris.sh

0 → 100644

+65 −0

Original line number	Diff line number	Diff line
		#!/bin/bash
		# shellcheck disable=SC2191

		module load oneapi
		module load cmake/3.23.2
		module refresh

		######## User Configurations ########
		Kokkos_HOME=$HOME/kokkos/kokkos
		ADIOS2_HOME=$(pwd)
		BUILD_DIR=${ADIOS2_HOME}/build-kokkos-polaris
		INSTALL_DIR=${ADIOS2_HOME}/install-kokkos-polaris

		num_build_procs=4

		######## Kokkos ########
		mkdir -p "${BUILD_DIR}/kokkos"
		rm -f "${BUILD_DIR}/kokkos/CMakeCache.txt"
		rm -rf "${BUILD_DIR}/kokkos/CMakeFiles"

		ARGS=(
		-D CMAKE_BUILD_TYPE=RelWithDebInfo
		-D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
		-D CMAKE_CXX_COMPILER=clang++

		-D Kokkos_ENABLE_SERIAL=ON
		-D Kokkos_ARCH_ZEN3=ON
		-D Kokkos_ENABLE_SYCL=ON
		-D Kokkos_ARCH_AMPERE80=ON
		-D Kokkos_ENABLE_UNSUPPORTED_ARCHS=ON

		-D CMAKE_CXX_STANDARD=17
		-D CMAKE_CXX_EXTENSIONS=OFF
		-D CMAKE_POSITION_INDEPENDENT_CODE=TRUE
		-D BUILD_SHARED_LIBS=ON
		)
		cmake "${ARGS[@]}" -S "${Kokkos_HOME}" -B "${BUILD_DIR}/kokkos"
		cmake --build "${BUILD_DIR}/kokkos" -j${num_build_procs}
		cmake --install "${BUILD_DIR}/kokkos"

		######## ADIOS2 ########
		mkdir -p "${BUILD_DIR}/adios2"
		rm -f "${BUILD_DIR}/adios2/CMakeCache.txt"
		rm -rf "${BUILD_DIR}/adios2/CMakeFiles"

		ARGS_ADIOS=(
		-D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
		-D BUILD_TESTING=OFF
		#-D ADIOS2_BUILD_EXAMPLES=OFF
		-D CMAKE_CXX_COMPILER=g++
		-D CMAKE_C_COMPILER=gcc

		-D ADIOS2_USE_SST=OFF
		-D ADIOS2_USE_Kokkos=ON
		-D Kokkos_ROOT="${INSTALL_DIR}"

		-D CMAKE_CXX_STANDARD=17
		-D CMAKE_CXX_EXTENSIONS=OFF
		-D CMAKE_POSITION_INDEPENDENT_CODE=TRUE
		-D BUILD_SHARED_LIBS=ON
		-D ADIOS2_USE_Fortran=OFF
		)
		cmake "${ARGS_ADIOS[@]}" -S "${ADIOS2_HOME}" -B "${BUILD_DIR}"/adios2
		cmake --build "${BUILD_DIR}/adios2" -j${num_build_procs}
		cmake --install "${BUILD_DIR}/adios2"

source/adios2/helper/kokkos/adiosKokkos.cpp

+18 −14

Original line number	Diff line number	Diff line
		@@ -13,10 +13,10 @@

		namespace
		{
		template <class MemSpace>
		void KokkosDeepCopy(const char src, char dst, size_t byteCount)
		{
		Kokkos::View<const char *, MemSpace,
		using mem_space = Kokkos::DefaultExecutionSpace::memory_space;
		Kokkos::View<const char *, mem_space,
		Kokkos::MemoryTraits<Kokkos::Unmanaged>>
		srcView(src, byteCount);
		Kokkos::View<char *, Kokkos::HostSpace,
		@@ -43,6 +43,10 @@ void KokkosMinMaxImpl(const char * /values/, const size_t /size/,
		char & /min/, char & /max/)
		{
		}
		void KokkosMinMaxImpl(const long double * /values/, const size_t /size/,
		long double & /min/, long double & /max/)
		{
		}
		void KokkosMinMaxImpl(const std::complex<float> * /values/,
		const size_t /size/, std::complex<float> & /min/,
		std::complex<float> & /max/)
		@@ -62,22 +66,12 @@ namespace helper
		{
		void MemcpyGPUToBuffer(char dst, const char GPUbuffer, size_t byteCount)
		{
		#ifdef ADIOS2_HAVE_KOKKOS_CUDA
		KokkosDeepCopy<Kokkos::CudaSpace>(GPUbuffer, dst, byteCount);
		#endif
		#ifdef ADIOS2_HAVE_KOKKOS_HIP
		KokkosDeepCopy<Kokkos::Experimental::HIPSpace>(GPUbuffer, dst, byteCount);
		#endif
		KokkosDeepCopy(GPUbuffer, dst, byteCount);
		}

		void MemcpyBufferToGPU(char GPUbuffer, const char src, size_t byteCount)
		{
		#ifdef ADIOS2_HAVE_KOKKOS_CUDA
		KokkosDeepCopy<Kokkos::CudaSpace>(src, GPUbuffer, byteCount);
		#endif
		#ifdef ADIOS2_HAVE_KOKKOS_HIP
		KokkosDeepCopy<Kokkos::Experimental::HIPSpace>(src, GPUbuffer, byteCount);
		#endif
		KokkosDeepCopy(src, GPUbuffer, byteCount);
		}

		bool IsGPUbuffer(const void *ptr)
		@@ -98,6 +92,15 @@ bool IsGPUbuffer(const void *ptr)
		{
		return true;
		}
		#endif
		#ifdef ADIOS2_HAVE_KOKKOS_SYCL
		auto ret =
		sycl::address_space_cast<sycl::access::address_space::global_space,
		sycl::access::decorated::no>(ptr);
		if (ret != nullptr)
		{
		return true;
		}
		#endif
		return false;
		}
		@@ -121,6 +124,7 @@ void KokkosInit()
		settings.set_device_id(device_id);
		}
		#endif
		// GetDevice not supported for SYCL, use the default device
		Kokkos::initialize(settings);
		}

Admin message