Commit 29177b01 authored by Brad King's avatar Brad King
Browse files

engine: Construct with Comm encapsulation

parent 8b8b797c
......@@ -12,6 +12,7 @@
#include "Engine.tcc"
#include <stdexcept>
#include <utility>
#include "adios2/core/IO.h"
......@@ -21,9 +22,9 @@ namespace core
{
Engine::Engine(const std::string engineType, IO &io, const std::string &name,
const Mode openMode, MPI_Comm mpiComm)
const Mode openMode, helper::Comm comm)
: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode),
m_Comm(mpiComm), m_DebugMode(io.m_DebugMode)
m_Comm(std::move(comm)), m_DebugMode(io.m_DebugMode)
{
}
......@@ -69,9 +70,7 @@ void Engine::Close(const int transportIndex)
if (transportIndex == -1)
{
helper::CheckMPIReturn(SMPI_Comm_free(&m_Comm),
"freeing comm in Engine " + m_Name +
", in call to Close");
m_Comm.Free("freeing comm in Engine " + m_Name + ", in call to Close");
m_IsClosed = true;
}
}
......
......@@ -29,6 +29,7 @@
#include "adios2/core/IO.h"
#include "adios2/core/Variable.h"
#include "adios2/core/VariableCompound.h"
#include "adios2/helper/adiosComm.h"
namespace adios2
{
......@@ -62,10 +63,10 @@ public:
* @param io object that generates this Engine
* @param name unique engine name within IO class object
* @param mode open mode from ADIOSTypes.h Mode
* @param mpiComm new communicator passed at Open or from ADIOS class
* @param comm communicator passed at Open or from ADIOS class
*/
Engine(const std::string engineType, IO &io, const std::string &name,
const Mode mode, MPI_Comm mpiComm);
const Mode mode, helper::Comm comm);
virtual ~Engine();
......@@ -446,8 +447,8 @@ public:
protected:
/** from ADIOS class passed to Engine created with Open
* if no new communicator is passed */
MPI_Comm m_Comm;
* if no communicator is passed */
helper::Comm m_Comm;
/** true: additional exceptions */
const bool m_DebugMode = false;
......
......@@ -27,6 +27,7 @@
#include "adios2/engine/skeleton/SkeletonReader.h"
#include "adios2/engine/skeleton/SkeletonWriter.h"
#include "adios2/helper/adiosComm.h"
#include "adios2/helper/adiosFunctions.h" //BuildParametersMap
#include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
#include <adios2sys/SystemTools.hxx> // FileIsDirectory()
......@@ -424,8 +425,7 @@ size_t IO::AddOperation(Operator &op, const Params &parameters) noexcept
return m_Operations.size() - 1;
}
Engine &IO::Open(const std::string &name, const Mode mode,
MPI_Comm mpiComm_orig)
Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm)
{
TAU_SCOPED_TIMER("IO::Open");
auto itEngineFound = m_Engines.find(name);
......@@ -457,8 +457,7 @@ Engine &IO::Open(const std::string &name, const Mode mode,
}
}
MPI_Comm mpiComm;
SMPI_Comm_dup(mpiComm_orig, &mpiComm);
auto comm = helper::Comm::Duplicate(mpiComm);
std::shared_ptr<Engine> engine;
const bool isDefaultEngine = m_EngineType.empty() ? true : false;
std::string engineTypeLC = m_EngineType;
......@@ -493,26 +492,26 @@ Engine &IO::Open(const std::string &name, const Mode mode,
{
if (mode == Mode::Read)
{
engine =
std::make_shared<engine::BP3Reader>(*this, name, mode, mpiComm);
engine = std::make_shared<engine::BP3Reader>(*this, name, mode,
std::move(comm));
}
else
{
engine =
std::make_shared<engine::BP3Writer>(*this, name, mode, mpiComm);
engine = std::make_shared<engine::BP3Writer>(*this, name, mode,
std::move(comm));
}
}
else if (engineTypeLC == "bp4")
{
if (mode == Mode::Read)
{
engine =
std::make_shared<engine::BP4Reader>(*this, name, mode, mpiComm);
engine = std::make_shared<engine::BP4Reader>(*this, name, mode,
std::move(comm));
}
else
{
engine =
std::make_shared<engine::BP4Writer>(*this, name, mode, mpiComm);
engine = std::make_shared<engine::BP4Writer>(*this, name, mode,
std::move(comm));
}
}
else if (engineTypeLC == "hdfmixer")
......@@ -521,10 +520,10 @@ Engine &IO::Open(const std::string &name, const Mode mode,
#if H5_VERSION_GE(1, 11, 0)
if (mode == Mode::Read)
engine = std::make_shared<engine::HDF5ReaderP>(*this, name, mode,
mpiComm);
std::move(comm));
else
engine =
std::make_shared<engine::HDFMixer>(*this, name, mode, mpiComm);
engine = std::make_shared<engine::HDFMixer>(*this, name, mode,
std::move(comm));
#else
throw std::invalid_argument(
"ERROR: update HDF5 >= 1.11 to support VDS.");
......@@ -539,10 +538,10 @@ Engine &IO::Open(const std::string &name, const Mode mode,
#ifdef ADIOS2_HAVE_DATAMAN
if (mode == Mode::Read)
engine = std::make_shared<engine::DataManReader>(*this, name, mode,
mpiComm);
std::move(comm));
else
engine = std::make_shared<engine::DataManWriter>(*this, name, mode,
mpiComm);
std::move(comm));
#else
throw std::invalid_argument(
"ERROR: this version didn't compile with "
......@@ -553,11 +552,11 @@ Engine &IO::Open(const std::string &name, const Mode mode,
{
#ifdef ADIOS2_HAVE_SSC
if (mode == Mode::Read)
engine =
std::make_shared<engine::SscReader>(*this, name, mode, mpiComm);
engine = std::make_shared<engine::SscReader>(*this, name, mode,
std::move(comm));
else
engine =
std::make_shared<engine::SscWriter>(*this, name, mode, mpiComm);
engine = std::make_shared<engine::SscWriter>(*this, name, mode,
std::move(comm));
#else
throw std::invalid_argument("ERROR: this version didn't compile with "
"SSC library, can't use SSC engine\n");
......@@ -568,7 +567,7 @@ Engine &IO::Open(const std::string &name, const Mode mode,
#ifdef ADIOS2_HAVE_TABLE
if (mode == Mode::Write)
engine = std::make_shared<engine::TableWriter>(*this, name, mode,
mpiComm);
std::move(comm));
else
throw std::invalid_argument(
"ERROR: Table engine only supports Write. It uses other "
......@@ -583,11 +582,11 @@ Engine &IO::Open(const std::string &name, const Mode mode,
{
#ifdef ADIOS2_HAVE_SST
if (mode == Mode::Read)
engine =
std::make_shared<engine::SstReader>(*this, name, mode, mpiComm);
engine = std::make_shared<engine::SstReader>(*this, name, mode,
std::move(comm));
else
engine =
std::make_shared<engine::SstWriter>(*this, name, mode, mpiComm);
engine = std::make_shared<engine::SstWriter>(*this, name, mode,
std::move(comm));
#else
throw std::invalid_argument("ERROR: this version didn't compile with "
"Sst library, can't use Sst engine\n");
......@@ -597,11 +596,11 @@ Engine &IO::Open(const std::string &name, const Mode mode,
{
#ifdef ADIOS2_HAVE_DATASPACES
if (mode == Mode::Read)
engine = std::make_shared<engine::DataSpacesReader>(*this, name,
mode, mpiComm);
engine = std::make_shared<engine::DataSpacesReader>(
*this, name, mode, std::move(comm));
else
engine = std::make_shared<engine::DataSpacesWriter>(*this, name,
mode, mpiComm);
engine = std::make_shared<engine::DataSpacesWriter>(
*this, name, mode, std::move(comm));
#else
throw std::invalid_argument(
"ERROR: this version didn't compile with "
......@@ -613,10 +612,10 @@ Engine &IO::Open(const std::string &name, const Mode mode,
#ifdef ADIOS2_HAVE_HDF5
if (mode == Mode::Read)
engine = std::make_shared<engine::HDF5ReaderP>(*this, name, mode,
mpiComm);
std::move(comm));
else
engine = std::make_shared<engine::HDF5WriterP>(*this, name, mode,
mpiComm);
std::move(comm));
#else
throw std::invalid_argument("ERROR: this version didn't compile with "
"HDF5 library, can't use HDF5 engine\n");
......@@ -626,11 +625,11 @@ Engine &IO::Open(const std::string &name, const Mode mode,
{
#ifdef ADIOS2_HAVE_MPI
if (mode == Mode::Read)
engine = std::make_shared<engine::InSituMPIReader>(*this, name,
mode, mpiComm);
engine = std::make_shared<engine::InSituMPIReader>(
*this, name, mode, std::move(comm));
else
engine = std::make_shared<engine::InSituMPIWriter>(*this, name,
mode, mpiComm);
engine = std::make_shared<engine::InSituMPIWriter>(
*this, name, mode, std::move(comm));
#else
throw std::invalid_argument("ERROR: this version didn't compile with "
"MPI, can't use InSituMPI engine\n");
......@@ -640,24 +639,24 @@ Engine &IO::Open(const std::string &name, const Mode mode,
{
if (mode == Mode::Read)
engine = std::make_shared<engine::SkeletonReader>(*this, name, mode,
mpiComm);
std::move(comm));
else
engine = std::make_shared<engine::SkeletonWriter>(*this, name, mode,
mpiComm);
std::move(comm));
}
else if (engineTypeLC == "inline")
{
if (mode == Mode::Read)
engine = std::make_shared<engine::InlineReader>(*this, name, mode,
mpiComm);
std::move(comm));
else
engine = std::make_shared<engine::InlineWriter>(*this, name, mode,
mpiComm);
std::move(comm));
}
else if (engineTypeLC == "null")
{
engine =
std::make_shared<engine::NullEngine>(*this, name, mode, mpiComm);
engine = std::make_shared<engine::NullEngine>(*this, name, mode,
std::move(comm));
}
else if (engineTypeLC == "nullcore")
{
......@@ -666,7 +665,7 @@ Engine &IO::Open(const std::string &name, const Mode mode,
"ERROR: nullcore engine does not support read mode");
else
engine = std::make_shared<engine::NullCoreWriter>(*this, name, mode,
mpiComm);
std::move(comm));
}
else
{
......
......@@ -11,6 +11,7 @@
#include "BP3Reader.h"
#include "BP3Reader.tcc"
#include "adios2/helper/adiosComm.h"
#include "adios2/helper/adiosFunctions.h" // MPI BroadcastVector
#include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
......@@ -22,10 +23,10 @@ namespace engine
{
BP3Reader::BP3Reader(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm)
: Engine("BP3", io, name, mode, mpiComm),
m_BP3Deserializer(mpiComm, m_DebugMode), m_FileManager(mpiComm, m_DebugMode),
m_SubFileManager(mpiComm, m_DebugMode)
helper::Comm comm)
: Engine("BP3", io, name, mode, std::move(comm)),
m_BP3Deserializer(m_Comm, m_DebugMode), m_FileManager(m_Comm, m_DebugMode),
m_SubFileManager(m_Comm, m_DebugMode)
{
TAU_SCOPED_TIMER("BP3Reader::Open");
Init();
......
......@@ -13,6 +13,7 @@
#include "adios2/common/ADIOSConfig.h"
#include "adios2/core/Engine.h"
#include "adios2/helper/adiosComm.h"
#include "adios2/toolkit/format/bp3/BP3.h" //format::BP3Deserializer
#include "adios2/toolkit/transportman/TransportMan.h"
......@@ -32,10 +33,10 @@ public:
* @param io
* @param name
* @param openMode only read
* @param mpiComm
* @param comm
*/
BP3Reader(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm);
helper::Comm comm);
~BP3Reader() = default;
......
......@@ -26,10 +26,10 @@ namespace engine
{
BP3Writer::BP3Writer(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm)
: Engine("BP3", io, name, mode, mpiComm), m_BP3Serializer(mpiComm, m_DebugMode),
m_FileDataManager(mpiComm, m_DebugMode),
m_FileMetadataManager(mpiComm, m_DebugMode)
helper::Comm comm)
: Engine("BP3", io, name, mode, std::move(comm)),
m_BP3Serializer(m_Comm, m_DebugMode), m_FileDataManager(m_Comm, m_DebugMode),
m_FileMetadataManager(m_Comm, m_DebugMode)
{
TAU_SCOPED_TIMER("BP3Writer::Open");
m_IO.m_ReadStreaming = false;
......
......@@ -13,6 +13,7 @@
#include "adios2/common/ADIOSConfig.h"
#include "adios2/core/Engine.h"
#include "adios2/helper/adiosComm.h"
#include "adios2/toolkit/format/bp3/BP3.h"
#include "adios2/toolkit/transportman/TransportMan.h" //transport::TransportsMan
......@@ -31,10 +32,10 @@ public:
* Constructor for file Writer in BP format
* @param name unique name given to the engine
* @param openMode w (supported), r, a from OpenMode in ADIOSTypes.h
* @param mpiComm MPI communicator
* @param comm multi-process communicator
*/
BP3Writer(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm);
helper::Comm comm);
~BP3Writer() = default;
......
......@@ -25,12 +25,11 @@ namespace engine
{
BP4Reader::BP4Reader(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm)
: Engine("BP4Reader", io, name, mode, mpiComm),
m_BP4Deserializer(mpiComm, m_DebugMode),
m_MDFileManager(mpiComm, m_DebugMode),
m_DataFileManager(mpiComm, m_DebugMode),
m_MDIndexFileManager(mpiComm, m_DebugMode)
helper::Comm comm)
: Engine("BP4Reader", io, name, mode, std::move(comm)),
m_BP4Deserializer(m_Comm, m_DebugMode), m_MDFileManager(m_Comm, m_DebugMode),
m_DataFileManager(m_Comm, m_DebugMode),
m_MDIndexFileManager(m_Comm, m_DebugMode)
{
TAU_SCOPED_TIMER("BP4Reader::Open");
Init();
......
......@@ -13,6 +13,7 @@
#include "adios2/common/ADIOSConfig.h"
#include "adios2/core/Engine.h"
#include "adios2/helper/adiosComm.h"
#include "adios2/toolkit/format/bp4/BP4.h" //format::BP4Deserializer
#include "adios2/toolkit/transportman/TransportMan.h"
......@@ -32,10 +33,10 @@ public:
* @param io
* @param name
* @param openMode only read
* @param mpiComm
* @param comm
*/
BP4Reader(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm);
helper::Comm comm);
virtual ~BP4Reader() = default;
......
......@@ -29,12 +29,11 @@ namespace engine
{
BP4Writer::BP4Writer(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm)
: Engine("BP4Writer", io, name, mode, mpiComm),
m_BP4Serializer(mpiComm, m_DebugMode),
m_FileDataManager(mpiComm, m_DebugMode),
m_FileMetadataManager(mpiComm, m_DebugMode),
m_FileMetadataIndexManager(mpiComm, m_DebugMode)
helper::Comm comm)
: Engine("BP4Writer", io, name, mode, std::move(comm)),
m_BP4Serializer(m_Comm, m_DebugMode), m_FileDataManager(m_Comm, m_DebugMode),
m_FileMetadataManager(m_Comm, m_DebugMode),
m_FileMetadataIndexManager(m_Comm, m_DebugMode)
{
TAU_SCOPED_TIMER("BP4Writer::Open");
m_IO.m_ReadStreaming = false;
......
......@@ -13,6 +13,7 @@
#include "adios2/common/ADIOSConfig.h"
#include "adios2/core/Engine.h"
#include "adios2/helper/adiosComm.h"
#include "adios2/toolkit/format/bp4/BP4.h"
#include "adios2/toolkit/transportman/TransportMan.h" //transport::TransportsMan
......@@ -31,10 +32,10 @@ public:
* Constructor for file Writer in BP4 format
* @param name unique name given to the engine
* @param openMode w (supported), r, a from OpenMode in ADIOSTypes.h
* @param mpiComm MPI communicator
* @param comm multi-process communicator
*/
BP4Writer(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm);
helper::Comm comm);
~BP4Writer();
......
......@@ -19,13 +19,13 @@ namespace engine
DataManCommon::DataManCommon(const std::string engineType, IO &io,
const std::string &name, const Mode mode,
MPI_Comm mpiComm)
: Engine(engineType, io, name, mode, mpiComm),
helper::Comm comm)
: Engine(engineType, io, name, mode, std::move(comm)),
m_IsRowMajor(helper::IsRowMajor(io.m_HostLanguage)),
m_DataManSerializer(mpiComm, m_IsRowMajor)
m_DataManSerializer(m_Comm, m_IsRowMajor)
{
MPI_Comm_rank(mpiComm, &m_MpiRank);
MPI_Comm_size(mpiComm, &m_MpiSize);
MPI_Comm_rank(m_Comm, &m_MpiRank);
MPI_Comm_size(m_Comm, &m_MpiSize);
GetParameter(m_IO.m_Parameters, "IPAddress", m_IPAddress);
GetParameter(m_IO.m_Parameters, "Port", m_Port);
GetParameter(m_IO.m_Parameters, "StagingMode", m_StagingMode);
......
......@@ -14,6 +14,7 @@
#include "adios2/common/ADIOSConfig.h"
#include "adios2/common/ADIOSMacros.h"
#include "adios2/core/Engine.h"
#include "adios2/helper/adiosComm.h"
#include "adios2/helper/adiosSystem.h"
#include "adios2/toolkit/format/dataman/DataManSerializer.h"
#include "adios2/toolkit/format/dataman/DataManSerializer.tcc"
......@@ -32,7 +33,7 @@ class DataManCommon : public Engine
public:
DataManCommon(const std::string engineType, IO &io, const std::string &name,
const Mode mode, MPI_Comm mpiComm);
const Mode mode, helper::Comm comm);
virtual ~DataManCommon();
protected:
......
......@@ -23,8 +23,8 @@ namespace engine
{
DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm)
: DataManCommon("DataManReader", io, name, mode, mpiComm)
helper::Comm comm)
: DataManCommon("DataManReader", io, name, mode, std::move(comm))
{
GetParameter(m_IO.m_Parameters, "AlwaysProvideLatestTimestep",
m_ProvideLatest);
......
......@@ -14,6 +14,8 @@
#include "DataManCommon.h"
#include "adios2/helper/adiosComm.h"
namespace adios2
{
namespace core
......@@ -26,7 +28,7 @@ class DataManReader : public DataManCommon
public:
DataManReader(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm);
helper::Comm comm);
virtual ~DataManReader();
StepStatus BeginStep(StepMode stepMode, const float timeoutSeconds) final;
size_t CurrentStep() const final;
......
......@@ -24,8 +24,8 @@ namespace engine
{
DataManWriter::DataManWriter(IO &io, const std::string &name,
const Mode openMode, MPI_Comm mpiComm)
: DataManCommon("DataManWriter", io, name, openMode, mpiComm)
const Mode openMode, helper::Comm comm)
: DataManCommon("DataManWriter", io, name, openMode, std::move(comm))
{
if (m_StagingMode == "wide")
{
......
......@@ -13,6 +13,7 @@
#define ADIOS2_ENGINE_DATAMAN_DATAMAN_WRITER_H_
#include "DataManCommon.h"
#include "adios2/helper/adiosComm.h"
namespace adios2
{
......@@ -26,7 +27,7 @@ class DataManWriter : public DataManCommon
public:
DataManWriter(IO &io, const std::string &name, const Mode mode,
MPI_Comm mpiComm);
helper::Comm comm);
virtual ~DataManWriter();
StepStatus BeginStep(StepMode mode,
......
......@@ -24,8 +24,8 @@ namespace engine
{
DataSpacesReader::DataSpacesReader(IO &io, const std::string &name,
const Mode mode, MPI_Comm mpiComm)
: Engine("DataSpacesReader", io, name, mode, mpiComm)
const Mode mode, helper::Comm comm)
: Engine("DataSpacesReader", io, name, mode, std::move(comm))
{
f_Name = name;
......@@ -52,6 +52,7 @@ DataSpacesReader::DataSpacesReader(IO &io, const std::string &name,
{
m_ProvideLatest = false;
}
MPI_Comm mpiComm = m_Comm;
ret = adios_read_dataspaces_init(&mpiComm, &m_data);
if (ret < 0)
{
......
......@@ -15,6 +15,7 @@
#include "adios2/common/ADIOSConfig.h"
#include "adios2/core/Engine.h"
#include "adios2/core/IO.h"
#include "adios2/helper/adiosComm.h"
#include "adios2/toolkit/dataspaces/ds.h"
#include "mpi.h"
......@@ -30,7 +31,7 @@ class DataSpacesReader : public Engine
public:
DataSpacesReader(IO &adios, const std::string &name, const Mode openMode,
MPI_Comm mpiComm);
helper::Comm comm);
~DataSpacesReader();
StepStatus BeginStep();
......
......@@ -25,8 +25,8 @@ namespace engine
{
DataSpacesWriter::DataSpacesWriter(IO &io, const std::string &name,
const Mode mode, MPI_Comm mpiComm)
: Engine("DataSpacesWriter", io, name, mode, mpiComm)
const Mode mode, helper::Comm comm)
: Engine("DataSpacesWriter", io, name, mode, std::move(comm))
{
f_Name = name;
......@@ -40,6 +40,7 @@ DataSpacesWriter::DataSpacesWriter(IO &io, const std::string &name,
{
m_data.appid = 0;
}
MPI_Comm mpiComm = m_Comm;
ret = adios_dataspaces_init(&mpiComm, &m_data);