Commit fbdb51b4 authored by Doak, Peter W.'s avatar Doak, Peter W.
Browse files

adding helper cmake for cades no gpu build, fixing no gpu build

parent e53030d2

build-aux/cades_no_gpu.cmake

0 → 100644
+43 −0
Original line number Diff line number Diff line 
# Initial cache list for cades

#

# Building on this cluster is very brittle due to slurm and bad system level modules?

# Centos 7 in general?

#

# Spack generated hdf5 and magma seemed problematic so both are hand built.

#

# Don't expect this to work at all without sourcing

# build-aux/cades_load_modules.sh

#

# Usage: cmake -C /path/to/this/file /path/to/DCA/source -D<option>=<value> -GNinja ...



# Use srun for executing the tests.

set(TEST_RUNNER "srun" CACHE STRING "Command for executing (MPI) programs.")

set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING

  "Flag used by TEST_RUNNER to specify the number of processes.")

# Use 1 GPU and 64G memory per test process passing all tests will require running on 4 P100 nodes

# i.e.

# salloc -A ccsd -p gpu_p100 --nodes=4 --mem=180G --exclusive --gres=gpu:2 -t 00:30:00

set(MPIEXEC_PREFLAGS "--mem=64G --gpus-per-task=1" CACHE STRING

  "Flags to pass to TEST_RUNNER directly before the executable to run.")



# these aren't needed on cades.

set(SMPIARGS_FLAG_NOMPI "" CACHE STRING

  "Spectrum MPI argument list flag for serial tests.")

# Let's keep this option in case we need it again in the future.

set(SMPIARGS_FLAG_MPI "" CACHE STRING "Spectrum MPI argument list flag for MPI tests.")



# Enable the GPU support.

option(DCA_WITH_CUDA "Enable GPU support." OFF)

option(DCA_WITH_GPU_AWARE_MPI "Enable GPU aware MPI." OFF)



# FFTW paths.

set(FFTW_INCLUDE_DIR $ENV{FFTW_DIR}/include CACHE PATH "Path to fftw3.h.")

set(FFTW_LIBRARY $ENV{FFTW_DIR}/lib/libfftw3.so CACHE FILEPATH "The FFTW3(-compatible) library.")



# HDF5 paths

set(HDF5_ROOT $ENV{HDF5_DIR})



option(DCA_WITH_TESTS_FAST "Fast minimal tests" ON)



#required by dependencies but not picked up by cmake for whatever reason.

set(CMAKE_EXE_LINKER_FLAGS "-ldl -fopenmp" CACHE STRING "additional linking arguments needed")

include/dca/linalg/util/handle_functions.hpp

+7 −0
Original line number Diff line number Diff line
@@ -16,12 +16,18 @@ 
#define DCA_LINALG_UTIL_HANDLE_FUNCTIONS_HPP



#include <vector>



#ifdef DCA_HAVE_GPU

#include "dca/platform/dca_gpu.h"

#include "dca/platform/dca_gpu_blas.h"

#endif



#include "dca/linalg/util/stream_functions.hpp"



#ifdef DCA_HAVE_GPU

#include "dca/linalg/util/gpuBLAS_handles.hpp"

#include "dca/linalg/util/gpu_stream.hpp"

#endif



namespace dca {

namespace linalg {
@@ -31,6 +37,7 @@ namespace util { 
#ifdef DCA_HAVE_GPU







// Global handle container.

inline std::vector<GpuBLASHandle>& getHandleContainer() {

  static std::vector<GpuBLASHandle> handle_container(1);

include/dca/platform/gpu_definitions.h

+6 −1
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 


#ifdef DCA_HAVE_GPU

#include "dca/platform/dca_gpu.h"

#endif



// clang-format off

#if defined(DCA_HAVE_CUDA)
@@ -40,8 +41,12 @@ 
    #define __DEVICE__

    #define __CONSTANT__

  #endif

#else // no GPU type defined

    #define __HOST__

    #define __DEVICE__

    #define __CONSTANT__

#endif

// clang-format on

#endif





#endif  // DCA_UTIL_CUDA_DEFINITIONS_HPP

test/unit/function/CMakeLists.txt

+6 −1
Original line number Diff line number Diff line
@@ -14,6 +14,11 @@ endif() 


dca_add_gtest(set_to_zero_test GTEST_MAIN)



set(FUNCTION_CONSTRUCTION_TEST_LIBS ${DCA_LIBS};parallel_mpi_concurrency;function)

if(DCA_HAVE_GPU)

  set(FUNCTION_CONSTRUCTION_TEST_LIBS "${FUNCTION_CONSTRUCTION_TEST_LIBS} gpu_utils")

endif(DCA_HAVE_GPU)



dca_add_gtest(function_construction_test

  MPI MPI_NUMPROC 4

  LIBS ${DCA_LIBS};parallel_mpi_concurrency;function;gpu_utils)
 
  LIBS ${FUNCTION_CONSTRUCTION_TEST_LIBS})

test/unit/function/function_construction_test.cpp

+2 −1
Original line number Diff line number Diff line
@@ -164,10 +164,11 @@ TYPED_TEST(BlockDistributedFunctionTest, CheckSplitIndexBlocking) { 
int main(int argc, char** argv) {

  int result = 0;



#ifdef DCA_HAVE_GPU

  dca::linalg::util::printInfoDevices();



  dca::linalg::util::initializeMagma();



#endif

  concurrency_ptr = new dca::parallel::MPIConcurrency(argc, argv);



  ::testing::InitGoogleTest(&argc, argv);