Merge remote-tracking branch 'origin/master' into optimize_sp_accum

  cluster_domains

  quantum_domains

  time_and_frequency_domains

  signals

  symmetrization

  coarsegraining

  ${DCA_CONCURRENCY_LIB}

  ${DCA_THREADING_LIBS}

  lapack

  cuda_utils

)

if (DCA_HAVE_CUDA)

  list(APPEND DCA_CUDA_LIBS

    cuda_utils)

  list(APPEND DCA_LIBS

    blas_kernels

    lapack_kernels

set(gtest_dir "${PROJECT_SOURCE_DIR}/libs/googletest-1.8.0" CACHE PATH "Path to Google Test.")

mark_as_advanced(gtest_dir)

set(benchmark_dir "${PROJECT_SOURCE_DIR}/libs/benchmark" CACHE PATH "Path to Google Benchmark.")

mark_as_advanced(benchmark_dir)

if (DCA_WITH_TESTS_FAST OR DCA_WITH_TESTS_EXTENSIVE OR  DCA_WITH_TESTS_STOCHASTIC OR

    DCA_WITH_TESTS_PERFORMANCE)

    message(WARNING "Tests are built with NDEBUG defined!")

  endif()

  if (DCA_WITH_TESTS_PERFORMANCE AND NOT (CMAKE_BUILD_TYPE STREQUAL "Release"))

  set(BENCHMARK_ENABLE_TESTING OFF)

  add_subdirectory(${gtest_dir})

  add_subdirectory(${PROJECT_SOURCE_DIR}/test)

  if (DCA_WITH_TESTS_PERFORMANCE)

    if(NOT (CMAKE_BUILD_TYPE STREQUAL "Release"))

      message(WARNING "Performance tests are only built in release mode.")

    endif()

    add_subdirectory(${benchmark_dir})

  endif()

  if (DCA_HAVE_MPI AND (TEST_RUNNER STREQUAL ""))

    message(FATAL_ERROR "TEST_RUNNER needs to be set to the command for executing MPI programs, e.g. mpiexec, mpirun, aprun or srun.")

  endif()

  add_subdirectory(${gtest_dir})

  add_subdirectory(${PROJECT_SOURCE_DIR}/test)

endif()

################################################################################

set(CTEST_PROJECT_NAME "DCA")

set(CTEST_NIGHTLY_START_TIME "00:00:00 GMT")

set(CTEST_DROP_METHOD "http")

set(CTEST_DROP_METHOD "https")

set(CTEST_DROP_SITE "cdash.cscs.ch")

set(CTEST_DROP_LOCATION "/submit.php?project=DCA")

set(CTEST_DROP_SITE_CDASH TRUE)

#include <string>

#include <iostream>

#include "dca/application/dca_loop_dispatch.hpp"

#include "dca/config/cmake_options.hpp"

// Defines Concurrency, Threading, ParametersType, DcaData, DcaLoop, and Profiler.

#include "dca/config/dca.hpp"

#include "dca/io/json/json_reader.hpp"

#include "dca/util/git_version.hpp"

#include "dca/util/modules.hpp"

#include "dca/util/signal_handler.hpp"

int main(int argc, char** argv) {

  if (argc < 2) {

  Concurrency concurrency(argc, argv);

  try {

    dca::util::SignalHandler::init(concurrency.id() == concurrency.first());

    std::string input_file(argv[1]);

    Profiler::start();

    DcaDataType dca_data(parameters);

    dca_data.initialize();

    DcaLoopType dca_loop(parameters, dca_data, concurrency);

    {

      Profiler profiler(__FUNCTION__, __FILE__, __LINE__);

      dca_loop.initialize();

      dca_loop.execute();

      dca_loop.finalize();

    }

    Profiler::stop(concurrency, parameters.get_filename_profiling());

    if (concurrency.id() == concurrency.first()) {

      std::cout << "\nProcessor " << concurrency.id() << " is writing data." << std::endl;

      dca_loop.write();

      std::cout << "\nFinish time: " << dca::util::print_time() << "\n" << std::endl;

    dca::DistType distribution = parameters.get_g4_distribution();

    switch (distribution) {

#ifdef DCA_HAVE_MPI

      case dca::DistType::MPI: {

        DCALoopDispatch<dca::DistType::MPI> dca_loop_dispatch;

        dca_loop_dispatch(parameters, dca_data, concurrency);

      } break;

#else

      case dca::DistType::MPI: {

        throw std::runtime_error(

            "Input calls for function MPI distribution but DCA is not built with MPI.");

      } break;

#endif

      case dca::DistType::NONE: {

        DCALoopDispatch<dca::DistType::NONE> dca_loop_dispatch;

        dca_loop_dispatch(parameters, dca_data, concurrency);

      } break;

    }

  }

  catch (const std::exception& err) {

    std::cout << "Unhandled exception in main function:\n\t" << err.what();

    std::cout << "Unhandled exception in main function:\n\t" << err.what() << std::endl;

    concurrency.abort();

  }

# Initial cache list for cades

#

# Building on this cluster is very brittle due to slurm and bad system level modules?

# Centos 7 in general?

#

# Spack generated hdf5 and magma seemed problematic so both are hand built.

#

# Don't expect this to work at all without sourcing

# build-aux/cades_load_modules.sh

#

# Usage: cmake -C /path/to/this/file /path/to/DCA/source -D<option>=<value> -GNinja ...

# Use srun for executing the tests.

set(TEST_RUNNER "srun" CACHE STRING "Command for executing (MPI) programs.")

set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING

  "Flag used by TEST_RUNNER to specify the number of processes.")

# Use 1 GPU and 64G memory per test process passing all tests will require running on 4 P100 nodes

# i.e.

# salloc -A ccsd -p gpu_p100 --nodes=4 --mem=180G --exclusive --gres=gpu:2 -t 00:30:00

set(MPIEXEC_PREFLAGS "--mem=64G --gpus-per-task=1" CACHE STRING

  "Flags to pass to TEST_RUNNER directly before the executable to run.")

# these aren't needed on cades.

set(SMPIARGS_FLAG_NOMPI "" CACHE STRING

  "Spectrum MPI argument list flag for serial tests.")

# Let's keep this option in case we need it again in the future.

set(SMPIARGS_FLAG_MPI "" CACHE STRING "Spectrum MPI argument list flag for MPI tests.")

# Enable the GPU support.

option(DCA_WITH_CUDA "Enable GPU support." ON)

option(DCA_WITH_CUDA_AWARE_MPI "Enable CUDA aware MPI." OFF)

set(CUDA_TOOLKIT_ROOT_DIR $ENV{CUDA_DIR} CACHE PATH "path to CUDA toolkit")

# Compile for Volta compute architecture.

set(CUDA_GPU_ARCH "sm_60" CACHE STRING "Name of the *real* architecture to build for.")

# Summit's static CUDA runtime is bugged.

option(CUDA_USE_STATIC_CUDA_RUNTIME OFF)

# For the GPU support we also need MAGMA.

set(MAGMA_DIR $ENV{MAGMA_DIR} CACHE PATH

  "Path to the MAGMA installation directory. Hint for CMake to find MAGMA.")

# FFTW paths.

set(FFTW_INCLUDE_DIR $ENV{FFTW_DIR}/include CACHE PATH "Path to fftw3.h.")

set(FFTW_LIBRARY $ENV{FFTW_DIR}/lib/libfftw3.so CACHE FILEPATH "The FFTW3(-compatible) library.")

# HDF5 paths

set(HDF5_ROOT $ENV{HD5_DIR})

set(HDF5_INCLUDE_DIRS $ENV{HDF5_DIR}/include CACHE PATH "Path to hdf5 includes")

set(HDF5_LIBRARIES "$ENV{HDF5_DIR}/lib/libhdf5_cpp.a;$ENV{HDF5_DIR}/lib/libhdf5.a" CACHE FILEPATH "The hdf5 libraries")

option(DCA_WITH_TESTS_FAST "Fast minimal tests" ON)

#required by dependencies but not picked up by cmake for whatever reason.

set(CMAKE_EXE_LINKER_FLAGS "-ldl -fopenmp" CACHE STRING "additional linking arguments needed")

# Spack modules to get DCA build on Cades

# If you aren't using the suggested CNMS environment you need to uncomment the following two lines.

# module load env/cades-cnms

#. $SOFTWARECNMS/spack/share/spack/setup-env.sh

module load PE-gnu/3.0

spack load emacs@26.3

spack load git

spack load gcc@8.2.0

spack load openmpi/qnfab5m

spack load fftw%gcc@8.2.0

spack load ninja/v2bqky4

spack load cmake/g4ybxxf

spack load openblas@0.3.9

export HDF5_DIR=/software/user_tools/current/cades-cnms/for_nti/hdf5

export MAGMA_DIR=/lustre/or-hydra/cades-cnms/epd/dev/magma

export CUDA_DIR=/software/dev_tools/swtree/cs400_centos7.2_pe2016-08/cuda/11.0/centos7.8_binary

export CUDADIR=/software/dev_tools/swtree/cs400_centos7.2_pe2016-08/cuda/11.0/centos7.8_binary

export CMAKE_PREFIX_PATH=${HDF5_DIR}:${MAGMA_DIR}:$CMAKE_PREFIX_PATH

export FFTW_DIR=`spack find --loaded -p fftw | awk -e '/fftw/ {print $2}'`

export CC=$(which mpicc)

export CXX=$(which mpic++)

# cmake like this can work if you don't want to use cades.cmake

#rm -rf *; CXX=$(which mpic++) CC=$(which mpicc) cmake -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_DIR} -DMAGMA_DIR=${MAGMA_DIR} -DDCA_WITH_CUDA=True -DCUDA_GPU_ARCH=sm_60 -DHDF5_ROOT=${HDF5_DIR} -DHDF5_INCLUDE_DIRS=${HDF5_DIR}/include -DHDF5_LIBRARIES="${HDF5_DIR}/lib/libhdf5_cpp.a;${HDF5_DIR}/lib/libhdf5.a" -DDCA_WITH_TESTS_FAST=True -DTEST_RUNNER=srun -DMPIEXEC_NUMPROC_FLAG="-n" -DMPIEXEC_PREFLAGS="--mem=64G --gpus-per-task=1" -DCMAKE_EXE_LINKER_FLAGS="-ldl -fopenmp" -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DFFTW_DIR=${FFTW_DIR} -DFFTW_INCLUDE_DIR=${FFTW_DIR}/include -DFFTW_LIBRARY="${FFTW_DIR}/lib/libfftw3.a;${FFTW_DIR}/lib/libfftw3f.a" -GNinja ..