Merge branch 'master' into const_matrix_view

  cluster_domains

  quantum_domains

  time_and_frequency_domains

  signals

  symmetrization

  coarsegraining

  stdc++fs # std::filesystem

  ${DCA_CONCURRENCY_LIB}

  ${DCA_THREADING_LIBS}

  lapack

#include <string>

#include <iostream>

#include "dca/application/dca_loop_dispatch.hpp"

#include "dca/config/cmake_options.hpp"

// Defines Concurrency, Threading, ParametersType, DcaData, DcaLoop, and Profiler.

#include "dca/config/dca.hpp"

#include "dca/io/json/json_reader.hpp"

#include "dca/util/git_version.hpp"

#include "dca/util/modules.hpp"

#include "dca/application/dca_loop_dispatch.hpp"

#include "dca/util/signal_handler.hpp"

int main(int argc, char** argv) {

  if (argc < 2) {

  Concurrency concurrency(argc, argv);

  try {

    dca::util::SignalHandler::init(concurrency.id() == concurrency.first());

    std::string input_file(argv[1]);

    Profiler::start();

# Initial cache list for cades

#

# Building on this cluster is very brittle due to slurm and bad system level modules?

# Centos 7 in general?

#

# Spack generated hdf5 and magma seemed problematic so both are hand built.

#

# Don't expect this to work at all without sourcing

# build-aux/cades_load_modules.sh

#

# Usage: cmake -C /path/to/this/file /path/to/DCA/source -D<option>=<value> -GNinja ...

# Use srun for executing the tests.

set(TEST_RUNNER "srun" CACHE STRING "Command for executing (MPI) programs.")

set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING

  "Flag used by TEST_RUNNER to specify the number of processes.")

# Use 1 GPU and 64G memory per test process passing all tests will require running on 4 P100 nodes

# i.e.

# salloc -A ccsd -p gpu_p100 --nodes=4 --mem=180G --exclusive --gres=gpu:2 -t 00:30:00

set(MPIEXEC_PREFLAGS "--mem=64G --gpus-per-task=1" CACHE STRING

  "Flags to pass to TEST_RUNNER directly before the executable to run.")

# these aren't needed on cades.

set(SMPIARGS_FLAG_NOMPI "" CACHE STRING

  "Spectrum MPI argument list flag for serial tests.")

# Let's keep this option in case we need it again in the future.

set(SMPIARGS_FLAG_MPI "" CACHE STRING "Spectrum MPI argument list flag for MPI tests.")

# Enable the GPU support.

option(DCA_WITH_CUDA "Enable GPU support." ON)

option(DCA_WITH_CUDA_AWARE_MPI "Enable CUDA aware MPI." OFF)

set(CUDA_TOOLKIT_ROOT_DIR $ENV{CUDA_DIR} CACHE PATH "path to CUDA toolkit")

# Compile for Volta compute architecture.

set(CUDA_GPU_ARCH "sm_60" CACHE STRING "Name of the *real* architecture to build for.")

# Summit's static CUDA runtime is bugged.

option(CUDA_USE_STATIC_CUDA_RUNTIME OFF)

# For the GPU support we also need MAGMA.

set(MAGMA_DIR $ENV{MAGMA_DIR} CACHE PATH

  "Path to the MAGMA installation directory. Hint for CMake to find MAGMA.")

# FFTW paths.

set(FFTW_INCLUDE_DIR $ENV{FFTW_DIR}/include CACHE PATH "Path to fftw3.h.")

set(FFTW_LIBRARY $ENV{FFTW_DIR}/lib/libfftw3.so CACHE FILEPATH "The FFTW3(-compatible) library.")

# HDF5 paths

set(HDF5_ROOT $ENV{HD5_DIR})

set(HDF5_INCLUDE_DIRS $ENV{HDF5_DIR}/include CACHE PATH "Path to hdf5 includes")

set(HDF5_LIBRARIES "$ENV{HDF5_DIR}/lib/libhdf5_cpp.a;$ENV{HDF5_DIR}/lib/libhdf5.a" CACHE FILEPATH "The hdf5 libraries")

option(DCA_WITH_TESTS_FAST "Fast minimal tests" ON)

#required by dependencies but not picked up by cmake for whatever reason.

set(CMAKE_EXE_LINKER_FLAGS "-ldl -fopenmp" CACHE STRING "additional linking arguments needed")

# Spack modules to get DCA build on Cades

# If you aren't using the suggested CNMS environment you need to uncomment the following two lines.

# module load env/cades-cnms

#. $SOFTWARECNMS/spack/share/spack/setup-env.sh

module load PE-gnu/3.0

spack load emacs@26.3

spack load git

spack load gcc@8.2.0

spack load openmpi/qnfab5m

spack load fftw%gcc@8.2.0

spack load ninja/v2bqky4

spack load cmake/g4ybxxf

spack load openblas@0.3.9

export HDF5_DIR=/software/user_tools/current/cades-cnms/for_nti/hdf5

export MAGMA_DIR=/lustre/or-hydra/cades-cnms/epd/dev/magma

export CUDA_DIR=/software/dev_tools/swtree/cs400_centos7.2_pe2016-08/cuda/11.0/centos7.8_binary

export CUDADIR=/software/dev_tools/swtree/cs400_centos7.2_pe2016-08/cuda/11.0/centos7.8_binary

export CMAKE_PREFIX_PATH=${HDF5_DIR}:${MAGMA_DIR}:$CMAKE_PREFIX_PATH

export FFTW_DIR=`spack find --loaded -p fftw | awk -e '/fftw/ {print $2}'`

export CC=$(which mpicc)

export CXX=$(which mpic++)

# cmake like this can work if you don't want to use cades.cmake

#rm -rf *; CXX=$(which mpic++) CC=$(which mpicc) cmake -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_DIR} -DMAGMA_DIR=${MAGMA_DIR} -DDCA_WITH_CUDA=True -DCUDA_GPU_ARCH=sm_60 -DHDF5_ROOT=${HDF5_DIR} -DHDF5_INCLUDE_DIRS=${HDF5_DIR}/include -DHDF5_LIBRARIES="${HDF5_DIR}/lib/libhdf5_cpp.a;${HDF5_DIR}/lib/libhdf5.a" -DDCA_WITH_TESTS_FAST=True -DTEST_RUNNER=srun -DMPIEXEC_NUMPROC_FLAG="-n" -DMPIEXEC_PREFLAGS="--mem=64G --gpus-per-task=1" -DCMAKE_EXE_LINKER_FLAGS="-ldl -fopenmp" -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DFFTW_DIR=${FFTW_DIR} -DFFTW_INCLUDE_DIR=${FFTW_DIR}/include -DFFTW_LIBRARY="${FFTW_DIR}/lib/libfftw3.a;${FFTW_DIR}/lib/libfftw3f.a" -GNinja ..

################################################################################

# Select the profiler type and enable auto-tuning.

set(DCA_PROFILER "None" CACHE STRING "Profiler type, options are: None | Counting | PAPI.")

set_property(CACHE DCA_PROFILER PROPERTY STRINGS None Counting PAPI)

set(DCA_PROFILER "None" CACHE STRING "Profiler type, options are: None | Counting | PAPI | Cuda.")

set_property(CACHE DCA_PROFILER PROPERTY STRINGS None Counting PAPI Cuda)

if (DCA_PROFILER STREQUAL "Counting")

  set(DCA_PROFILING_EVENT_TYPE dca::profiling::time_event<std::size_t>)

  set(DCA_PROFILER_TYPE dca::profiling::CountingProfiler<Event>)

  set(DCA_PROFILER_INCLUDE "dca/profiling/counting_profiler.hpp")

# Note: this profiler requires using the PTHREAD library and CUDA_TOOLS_EXT_LIBRARY

elseif (DCA_PROFILER STREQUAL "Cuda")

  set(DCA_PROFILING_EVENT_INCLUDE "dca/profiling/events/time.hpp")

  set(DCA_PROFILING_EVENT_TYPE "void")

  set(DCA_PROFILER_TYPE dca::profiling::CudaProfiler)

  set(DCA_PROFILER_INCLUDE "dca/profiling/cuda_profiler.hpp")

  link_libraries(${CUDA_nvToolsExt_LIBRARY})

else()  # DCA_PROFILER = None

  # The NullProfiler doesn't have an event type.

  set(DCA_PROFILING_EVENT_TYPE void)