Commit f950d9e1 authored by Doak, Peter W.'s avatar Doak, Peter W.
Browse files

extensive tests working on summit

parent 4f867de7

build-aux/summit_load_modules.sh

+3 −1
Original line number Diff line number Diff line
@@ -11,9 +11,11 @@ module load cuda/11.1.1 # ldd shows magma is built with this cuda 
module load magma/2.6.1

module load hdf5

module load fftw

module load cmake/3.20.2 # at least 3.20 is required

module load cmake/3.21.3

module load netlib-lapack

module load essl

module load adios2

module load ninja



export CC=mpicc

export CXX=mpic++

include/dca/io/reader.hpp

+21 −14
Original line number Diff line number Diff line
@@ -31,16 +31,8 @@ class Reader { 
public:

  // In: format. output format, HDF5 or JSON.

  // In: verbose. If true, the reader outputs a short log whenever it is executed.

  Reader(

#ifdef DCA_HAVE_ADIOS2

      const adios2::ADIOS& adios,

#endif

      const Concurrency& concurrency, const std::string& format, bool verbose = true)

      :

#ifdef DCA_HAVE_ADIOS2

        adios_(adios),

#endif

        concurrency_(concurrency) {

  Reader(const Concurrency& concurrency, const std::string& format, bool verbose = true)

      : concurrency_(concurrency) {

    if (format == "HDF5") {

      reader_.template emplace<io::HDF5Reader>(verbose);

    }
@@ -49,7 +41,25 @@ public: 
    }

#ifdef DCA_HAVE_ADIOS2

    else if (format == "ADIOS2") {

      reader_.template emplace<io::ADIOS2Reader<Concurrency>>(&concurrency, verbose);

      throw(std::logic_error("ADIOS2 reader requires an adios reference"));

    }

#endif

    else {

      throw(std::logic_error("Invalid input format"));

    }

  }

  

  Reader(adios2::ADIOS& adios, const Concurrency& concurrency, const std::string& format, bool verbose = true)

      : concurrency_(concurrency) {

    if (format == "HDF5") {

      throw(std::logic_error("ADIOS2 reference not an argument for hdf5 reader"));

    }

    else if (format == "JSON") {

      throw(std::logic_error("ADIOS2 reference not an argument for json reader"));

    }

#ifdef DCA_HAVE_ADIOS2

    else if (format == "ADIOS2") {

      reader_.template emplace<io::ADIOS2Reader<Concurrency>>(adios, &concurrency, verbose);

    }

#endif

    else {
@@ -92,9 +102,6 @@ private: 
#endif

               >

      reader_;

#ifdef DCA_HAVE_ADIOS2

  const adios2::ADIOS& adios_;

#endif

  const Concurrency& concurrency_;

};

include/dca/phys/dca_data/dca_data.hpp

+2 −19
Original line number Diff line number Diff line
@@ -111,7 +111,7 @@ public: 


  void read(const std::string& filename);

#ifdef DCA_HAVE_ADIOS2

  void read(const adios2::ADIOS& adios, std::string filename);

  void read(adios2::ADIOS& adios, std::string filename);

#endif



  /** prefer this as it allows for more sensible handling of appendable files like bp4.
@@ -128,11 +128,6 @@ public: 
  void initialize();

  void initializeH0_and_H_i();

  void initialize_G0();

#ifdef DCA_HAVE_ADIOS2

  /** read initializeSigma from adios file, it is probably already open.

   */

  void initializeSigma(const adios2::ADIOS& adios, const std::string& filename);

#endif

  void initializeSigma(const std::string& filename);

  void readSigmaFile(io::Reader<Concurrency>& reader);
@@ -361,7 +356,7 @@ void DcaData<Parameters, DT>::read(const std::string& filename) { 


#ifdef DCA_HAVE_ADIOS2

template <class Parameters, DistType DT>

void DcaData<Parameters, DT>::read(const adios2::ADIOS& adios, std::string filename) {

void DcaData<Parameters, DT>::read(adios2::ADIOS& adios, std::string filename) {

  if (concurrency_.id() == concurrency_.first())

    std::cout << "\n\n\t starts reading \n\n";
@@ -597,18 +592,6 @@ void DcaData<Parameters, DT>::initialize_G0() { 
  G0_r_t_cluster_excluded = G0_r_t;

}



#ifdef DCA_HAVE_ADIOS2

template <class Parameters, DistType DT>

void DcaData<Parameters, DT>::initializeSigma(const adios2::ADIOS& adios,

                                              const std::string& filename) {

  if (concurrency_.id() == concurrency_.first()) {

    io::Reader reader(adios, concurrency_, parameters_.get_output_format());

    reader.open_file(filename);

    readSigmaFile(reader);

  }

}

#endif



template <class Parameters, DistType DT>

void DcaData<Parameters, DT>::initializeSigma(const std::string& filename) {

  if (concurrency_.id() == concurrency_.first()) {

include/dca/phys/dca_loop/dca_loop.hpp

+17 −15
Original line number Diff line number Diff line 
// Copyright (C) 2018 ETH Zurich

// Copyright (C) 2018 UT-Battelle, LLC

// Copyright (C) 2022 ETH Zurich

// Copyright (C) 2022 UT-Battelle, LLC

// All rights reserved.

//

// See LICENSE for terms of usage.
@@ -7,6 +7,7 @@ 
//

// Author: Peter Staar (taa@zurich.ibm.com)

//         Andrei Plamada (plamada@itp.phys.ethz.ch)

//         Peter Doak (doakpw@ornl.gov)

//

// This class executes the DCA(+) loop.
@@ -23,6 +24,7 @@ 
#include "dca/function/domains.hpp"

#include "dca/io/filesystem.hpp"

#include "dca/io/writer.hpp"

#include "dca/io/io_types.hpp"

#include "dca/phys/dca_algorithms/compute_greens_function.hpp"

#include "dca/phys/dca_data/dca_data.hpp"

#include "dca/phys/dca_loop/dca_loop_data.hpp"
@@ -141,9 +143,6 @@ DcaLoop<ParametersType, DcaDataType, MCIntegratorType, DIST>::DcaLoop( 
      concurrency(concurrency_ref),

#ifdef DCA_HAVE_ADIOS2

      adios_("", concurrency_ref.get()),

      DCA_info_struct(adios_),

#else

      DCA_info_struct(),

#endif

      cluster_exclusion_obj(parameters, MOMS),

      double_counting_correction_obj(parameters, MOMS),
@@ -208,8 +207,19 @@ void DcaLoop<ParametersType, DDT, MCIntegratorType, DIST>::initialize() { 
  int last_completed = -1;



  if (parameters.autoresume()) {  // Try to read state of previous run.

    auto& dca_filename = parameters.get_filename_dca();

    std::size_t extension_start = dca_filename.rfind('.');

    std::string extension{dca_filename.substr(extension_start, dca_filename.size())};

    std::cout << extension << '\n';

    // This is not complete, how do we want to decide what format we are reading?  Just from output format?

#ifdef DCA_HAVE_ADIOS2

    if (io::stringToIOType(parameters.get_output_format()) == io::IOType::ADIOS2)

      last_completed =

          DCA_info_struct.readData(file_name_, parameters.get_output_format(), concurrency, adios_);

    else

#endif

      last_completed =

        DCA_info_struct.tryToRead(file_name_ + ".tmp", parameters.get_output_format(), concurrency);

          DCA_info_struct.readData(file_name_ + ".tmp", parameters.get_output_format(), concurrency);

  }

  if (last_completed >= 0) {

    if (concurrency.id() == concurrency.first())
@@ -217,19 +227,11 @@ void DcaLoop<ParametersType, DDT, MCIntegratorType, DIST>::initialize() { 
                << std::endl;



    dca_iteration_ = std::min(last_completed + 1, parameters.get_dca_iterations() - 1);

    #ifdef DCA_HAVE_ADIOS2

    MOMS.initializeSigma(adios_, file_name_ + ".tmp");

    #else

    MOMS.initializeSigma(file_name_ + ".tmp");

    #endif

    perform_lattice_mapping();

  }

  else if (parameters.get_initial_self_energy() != "zero") {

    #ifdef DCA_HAVE_ADIOS2

    MOMS.initializeSigma(adios_, parameters.get_initial_self_energy());

    #else

    MOMS.initializeSigma(parameters.get_initial_self_energy());

    #endif 

    perform_lattice_mapping();

  }

include/dca/phys/dca_loop/dca_loop_data.hpp

+51 −44
Original line number Diff line number Diff line
@@ -44,24 +44,21 @@ public: 
  using k_DCA =

      func::dmn_0<domains::cluster_domain<double, ParametersType::lattice_type::DIMENSION, domains::CLUSTER,

                                          domains::MOMENTUM_SPACE, domains::BRILLOUIN_ZONE>>;



#ifdef DCA_HAVE_ADIOS2

  DcaLoopData(adios2::ADIOS& adios);

#else

  DcaLoopData();

#endif

  template <typename Writer>

  void write(Writer& writer);



  template <typename WRITER>

  void write(WRITER& writer);



  // Attempts to read the loop functions from 'filename'. If successful returns

  // the last completed iteration from the input file, otherwise it returns -1.

  int tryToRead(const std::string& filename, const std::string& format,

  int readData(const std::string& filename, const std::string& format,

                const Concurrency& concurrency);



#ifdef DCA_HAVE_ADIOS2

  const adios2::ADIOS& adios_;

  int readData(const std::string& filename, const std::string& format,

	       const Concurrency& concurrency, adios2::ADIOS& adios);

#endif



  template<class READER>

  void readLoopDataCommon(READER& reader, const std::string& filename, const std::string& format);

  func::function<double, DCA_iteration_domain_type> Gflop_per_mpi_task;

  func::function<double, DCA_iteration_domain_type> times_per_mpi_task;
@@ -93,15 +90,8 @@ public: 
};



template <typename ParametersType>

#ifdef DCA_HAVE_ADIOS2

DcaLoopData<ParametersType>::DcaLoopData(adios2::ADIOS& adios)

#else

DcaLoopData<ParametersType>::DcaLoopData()

#endif

    :

#ifdef DCA_HAVE_ADIOS2

      adios_(adios),

#endif

      Gflop_per_mpi_task("Gflop_per_mpi_task"),

      times_per_mpi_task("times_per_mpi_task"),
@@ -160,14 +150,35 @@ void DcaLoopData<ParametersType>::write(Writer& writer) { 
}



template <typename ParametersType>

int DcaLoopData<ParametersType>::tryToRead(const std::string& filename, const std::string& format,

int DcaLoopData<ParametersType>::readData(const std::string& filename, const std::string& format,

                                              const Concurrency& concurrency) {

  if (concurrency.id() == concurrency.first() && filesystem::exists(filename)) {

#ifdef DCA_HAVE_ADIOS2

    io::Reader reader(adios_, concurrency, format, false);

#else

    io::Reader reader(concurrency, format, false);

    readLoopDataCommon(reader, filename, format);

  }  

  concurrency.broadcast(last_completed_iteration);

  return last_completed_iteration;

}



#ifdef DCA_HAVE_ADIOS2

template <typename ParametersType>

int DcaLoopData<ParametersType>::readData(const std::string& filename, const std::string& format,

                                              const Concurrency& concurrency,

                                              adios2::ADIOS& adios) {

  if (concurrency.id() == concurrency.first() && filesystem::exists(filename)) {

    io::Reader reader(adios, concurrency, format, false);

    readLoopDataCommon(reader, filename, format);

  }

  concurrency.broadcast(last_completed_iteration);

  return last_completed_iteration;

}

#endif



template <typename ParametersType>

template <class READER>

void DcaLoopData<ParametersType>::readLoopDataCommon(READER& reader,

						     const std::string& filename,

						     const std::string& format) {

  reader.open_file(filename);

  reader.open_group("DCA-loop-functions");
@@ -196,10 +207,6 @@ int DcaLoopData<ParametersType>::tryToRead(const std::string& filename, const st 
  reader.close_file();

}



  concurrency.broadcast(last_completed_iteration);

  return last_completed_iteration;

}



}  // namespace phys

}  // namespace dca