Commit 715ebb10 authored by Ruonan Wang's avatar Ruonan Wang
Browse files

one way xgc test worked

parent 3b8fdc31
......@@ -92,6 +92,8 @@ void MpiHandshake::Test()
bool MpiHandshake::Check(const std::string &filename)
{
int m_Verbosity = 1;
Test();
// check if RendezvousAppCount reached
......@@ -99,6 +101,15 @@ bool MpiHandshake::Check(const std::string &filename)
if (m_WritersMap[filename].size() + m_ReadersMap[filename].size() !=
m_RendezvousAppCounts[filename])
{
if (m_Verbosity >= 10)
{
std::cout << "MpiHandshake Rank " << m_WorldRank << " Stream "
<< filename << ": " << m_WritersMap[filename].size()
<< " writers and " << m_ReadersMap[filename].size()
<< " readers found out of "
<< m_RendezvousAppCounts[filename]
<< " total rendezvous apps" << std::endl;
}
return false;
}
......@@ -206,8 +217,8 @@ void MpiHandshake::Handshake(const std::string &filename, const char mode,
nowTime - startTime);
if (duration.count() > timeoutSeconds)
{
PrintMaps(0);
throw(std::runtime_error("Mpi handshake timeout on Rank" + std::to_string(m_WorldRank)));
throw(std::runtime_error("Mpi handshake timeout on Rank" +
std::to_string(m_WorldRank)));
}
}
......@@ -242,39 +253,35 @@ MpiHandshake::GetReaderMap(const std::string &filename)
return m_ReadersMap[filename];
}
void MpiHandshake::PrintMaps(const int printRank)
void MpiHandshake::PrintMaps(const int printRank, const std::string &filename)
{
if (m_WorldRank == printRank)
{
std::cout << "Writers: " << std::endl;
for (const auto &stream : m_WritersMap)
std::cout << "Printing MPI handshake map for Stream " << filename
<< " from Rank " << printRank << std::endl;
std::cout << " Writers: " << std::endl;
for (const auto &app : m_WritersMap[filename])
{
std::cout << " Stream " << stream.first << std::endl;
for (const auto &app : stream.second)
std::cout << " App Master Rank " << app.first << std::endl;
std::cout << " ";
for (const auto &rank : app.second)
{
std::cout << " App Master Rank " << app.first << std::endl;
std::cout << " ";
for (const auto &rank : app.second)
{
std::cout << rank << ", ";
}
std::cout << std::endl;
std::cout << rank << ", ";
}
std::cout << std::endl;
}
std::cout << "Readers: " << std::endl;
for (const auto &stream : m_ReadersMap)
std::cout << " Readers: " << std::endl;
for (const auto &app : m_ReadersMap[filename])
{
std::cout << " Stream " << stream.first << std::endl;
for (const auto &app : stream.second)
std::cout << " App Master Rank " << app.first << std::endl;
std::cout << " ";
for (const auto &rank : app.second)
{
std::cout << " App Master Rank " << app.first << std::endl;
std::cout << " ";
for (const auto &rank : app.second)
{
std::cout << rank << ", ";
}
std::cout << std::endl;
std::cout << rank << ", ";
}
std::cout << std::endl;
}
}
}
......
......@@ -100,7 +100,7 @@ private:
static bool Check(const std::string &filename);
static size_t PlaceInBuffer(const size_t stream, const int rank);
static void PrintMaps();
static void PrintMaps(const int printRank);
static void PrintMaps(const int printRank, const std::string &filename);
static std::vector<char> m_Buffer;
static std::vector<std::vector<MPI_Request>> m_SendRequests;
......
......@@ -48,7 +48,10 @@ if(ADIOS2_HAVE_MPI)
gtest_add_tests_helper(MultiApp TRUE Ssc Engine.SSC. "")
SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscMultiApp "" TRUE)
gtest_add_tests_helper(XgcPattern TRUE Ssc Engine.SSC. "")
SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscXgcPattern "" TRUE)
gtest_add_tests_helper(Xgc2Way TRUE Ssc Engine.SSC. "")
SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscXgc2Way "" TRUE)
gtest_add_tests_helper(Xgc3Way TRUE Ssc Engine.SSC. "")
SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscXgc3Way "" TRUE)
endif()
using namespace adios2;
int print_lines = 0;
template <class T>
void PrintData(const T *data, const size_t step, const Dims &start,
const Dims &count, const int rank)
{
size_t size = std::accumulate(count.begin(), count.end(), 1,
std::multiplies<size_t>());
std::cout << "Rank: " << rank << " Step: " << step << " Size:" << size
<< "\n";
size_t printsize = 128;
if (size < printsize)
{
printsize = size;
}
int s = 0;
for (size_t i = 0; i < printsize; ++i)
{
++s;
std::cout << data[i] << " ";
if (s == count[1])
{
std::cout << std::endl;
s = 0;
}
}
std::cout << "]" << std::endl;
}
template <class T>
void GenDataRecursive(std::vector<size_t> start, std::vector<size_t> count,
std::vector<size_t> shape, size_t n0, size_t y,
std::vector<T> &vec)
{
for (size_t i = 0; i < count[0]; i++)
{
size_t i0 = n0 * count[0] + i;
size_t z = y * shape[0] + (i + start[0]);
auto start_next = start;
auto count_next = count;
auto shape_next = shape;
start_next.erase(start_next.begin());
count_next.erase(count_next.begin());
shape_next.erase(shape_next.begin());
if (start_next.size() == 1)
{
for (size_t j = 0; j < count_next[0]; j++)
{
vec[i0 * count_next[0] + j] =
z * shape_next[0] + (j + start_next[0]);
}
}
else
{
GenDataRecursive(start_next, count_next, shape_next, i0, z, vec);
}
}
}
template <class T>
void GenData(std::vector<T> &vec, const size_t step,
const std::vector<size_t> &start, const std::vector<size_t> &count,
const std::vector<size_t> &shape)
{
size_t total_size = std::accumulate(count.begin(), count.end(), 1,
std::multiplies<size_t>());
vec.resize(total_size);
GenDataRecursive(start, count, shape, 0, 0, vec);
}
template <class T>
void VerifyData(const std::complex<T> *data, size_t step, const Dims &start,
const Dims &count, const Dims &shape)
{
size_t size = std::accumulate(count.begin(), count.end(), 1,
std::multiplies<size_t>());
std::vector<std::complex<T>> tmpdata(size);
GenData(tmpdata, step, start, count, shape);
for (size_t i = 0; i < size; ++i)
{
ASSERT_EQ(data[i], tmpdata[i]);
}
if (print_lines < 32)
{
PrintData(data, step, start, count);
++print_lines;
}
}
template <class T>
void VerifyData(const T *data, size_t step, const Dims &start,
const Dims &count, const Dims &shape, const int rank)
{
size_t size = std::accumulate(count.begin(), count.end(), 1,
std::multiplies<size_t>());
bool compressed = false;
std::vector<T> tmpdata(size);
if (print_lines < 32)
{
PrintData(data, step, start, count, rank);
++print_lines;
}
GenData(tmpdata, step, start, count, shape);
for (size_t i = 0; i < size; ++i)
{
if (!compressed)
{
ASSERT_EQ(data[i], tmpdata[i]);
}
}
}
/*
* Distributed under the OSI-approved Apache License, Version 2.0. See
* accompanying file Copyright.txt for details.
*/
#include <adios2.h>
#include <gtest/gtest.h>
#ifdef ADIOS2_HAVE_MPI
#include <mpi.h>
#endif
#include "TestSscCommon.h"
#include <numeric>
#include <thread>
using namespace adios2;
int mpiRank = 0;
int mpiSize = 1;
int worldRank, worldSize;
int mpiGroup;
MPI_Comm mpiComm;
char runMode;
class SscEngineTest : public ::testing::Test
{
public:
SscEngineTest() = default;
};
void xgc(const Dims &shape, const Dims &start, const Dims &count,
const size_t steps, const adios2::Params &engineParams)
{
size_t datasize = std::accumulate(count.begin(), count.end(), 1,
std::multiplies<size_t>());
std::vector<float> x_to_g_data(datasize);
std::vector<float> g_to_x_data;
adios2::ADIOS adios(mpiComm, adios2::DebugON);
adios2::IO x_to_g_io = adios.DeclareIO("x_to_g");
x_to_g_io.SetEngine("ssc");
x_to_g_io.SetParameters(engineParams);
adios2::IO g_to_x_io = adios.DeclareIO("g_to_x");
g_to_x_io.SetEngine("ssc");
g_to_x_io.SetParameters(engineParams);
auto x_to_g_var =
x_to_g_io.DefineVariable<float>("x_to_g", shape, start, count);
std::cout << " ======================== " << std::endl;
std::cout << " before handshake on xgc " << worldRank << std::endl;
adios2::Engine x_to_g_engine =
x_to_g_io.Open("x_to_g", adios2::Mode::Write);
std::cout << " ======================== " << std::endl;
std::cout << " x_to_g handshake finished on xgc rank " << worldRank
<< std::endl;
/*
adios2::Engine g_to_x_engine = g_to_x_io.Open("g_to_x", adios2::Mode::Read);
std::cout << " ======================== " << std::endl;
std::cout << " g_to_x handshake finished on xgc rank " << worldRank <<
std::endl;
*/
for (int i = 0; i < steps; ++i)
{
x_to_g_engine.BeginStep();
GenData(x_to_g_data, i, start, count, shape);
x_to_g_engine.Put(x_to_g_var, x_to_g_data.data(), adios2::Mode::Sync);
x_to_g_engine.EndStep();
/*
g_to_x_engine.BeginStep();
auto g_to_x_var = x_to_g_io.InquireVariable<float>("g_to_x");
g_to_x_data.resize(std::accumulate(g_to_x_var.Shape().begin(),
g_to_x_var.Shape().end(), 1, std::multiplies<size_t>()));
g_to_x_engine.Get(g_to_x_var, g_to_x_data.data(), adios2::Mode::Sync);
VerifyData(g_to_x_data.data(), i, Dims(g_to_x_var.Shape().size(), 0),
g_to_x_var.Shape(), g_to_x_var.Shape(), mpiRank);
g_to_x_engine.EndStep();
*/
}
x_to_g_engine.Close();
/*
g_to_x_engine.BeginStep();
g_to_x_engine.Close();
*/
}
void gene(const Dims &shape, const Dims &start, const Dims &count,
const size_t steps, const adios2::Params &engineParams)
{
adios2::ADIOS adios(mpiComm, adios2::DebugON);
adios2::IO x_to_g_io = adios.DeclareIO("x_to_g");
x_to_g_io.SetEngine("ssc");
x_to_g_io.SetParameters(engineParams);
adios2::IO g_to_x_io = adios.DeclareIO("g_to_x");
g_to_x_io.SetEngine("ssc");
g_to_x_io.SetParameters(engineParams);
auto g_to_x_var =
g_to_x_io.DefineVariable<float>("g_to_x", shape, start, count);
std::cout << " ======================== " << std::endl;
std::cout << " before handshake on gene " << worldRank << std::endl;
adios2::Engine x_to_g_engine = x_to_g_io.Open("x_to_g", adios2::Mode::Read);
std::cout << " ======================== " << std::endl;
std::cout << " x_to_g handshake finished on gene rank " << worldRank
<< std::endl;
/*
adios2::Engine g_to_x_engine = g_to_x_io.Open("g_to_x",
adios2::Mode::Write); std::cout << " ======================== " <<
std::endl; std::cout << " g_to_x handshake finished on gene rank " <<
worldRank << std::endl;
*/
size_t datasize = std::accumulate(shape.begin(), shape.end(), 1,
std::multiplies<size_t>());
std::vector<float> x_to_g_data;
std::vector<float> g_to_x_data(datasize);
for (int i = 0; i < steps; ++i)
{
x_to_g_engine.BeginStep(StepMode::Read, 5);
auto x_to_g_var = x_to_g_io.InquireVariable<float>("x_to_g");
auto readShape = x_to_g_var.Shape();
x_to_g_data.resize(std::accumulate(readShape.begin(), readShape.end(),
1, std::multiplies<size_t>()));
x_to_g_engine.Get(x_to_g_var, x_to_g_data.data(), adios2::Mode::Sync);
VerifyData(x_to_g_data.data(), i, Dims(readShape.size(), 0), readShape,
readShape, mpiRank);
x_to_g_engine.EndStep();
/*
g_to_x_engine.BeginStep();
GenData(g_to_x_data, i, start, count, shape);
g_to_x_engine.Put(g_to_x_var, g_to_x_data.data(), adios2::Mode::Sync);
g_to_x_engine.EndStep();
*/
}
x_to_g_engine.BeginStep();
x_to_g_engine.Close();
// g_to_x_engine.Close();
}
TEST_F(SscEngineTest, TestSscXgc2Way)
{
Dims start, count, shape;
MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
if (worldSize < 6)
{
return;
}
if (worldRank < 2)
{
mpiGroup = 0;
}
else
{
mpiGroup = 1;
}
MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm);
MPI_Comm_rank(mpiComm, &mpiRank);
MPI_Comm_size(mpiComm, &mpiSize);
size_t steps = 20;
if (mpiGroup == 0)
{
shape = {(size_t)mpiSize, 10};
start = {(size_t)mpiRank, 0};
count = {1, 10};
adios2::Params engineParams = {{"RendezvousAppCount", "2"},
{"MaxStreamsPerApp", "2"},
{"OpenTimeoutSecs", "3"},
{"Verbose", "0"}};
std::cout << "Rank " << worldRank << " launched xgc" << std::endl;
xgc(shape, start, count, steps, engineParams);
}
if (mpiGroup == 1)
{
shape = {(size_t)mpiSize, 10};
start = {(size_t)mpiRank, 0};
count = {1, 10};
adios2::Params engineParams = {{"RendezvousAppCount", "2"},
{"MaxStreamsPerApp", "2"},
{"OpenTimeoutSecs", "3"},
{"Verbose", "0"}};
std::cout << "Rank " << worldRank << " launched gene" << std::endl;
gene(shape, start, shape, steps, engineParams);
}
MPI_Barrier(MPI_COMM_WORLD);
}
int main(int argc, char **argv)
{
MPI_Init(&argc, &argv);
int worldRank, worldSize;
MPI_Comm_rank(MPI_COMM_WORLD, &worldRank);
MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
::testing::InitGoogleTest(&argc, argv);
int result = RUN_ALL_TESTS();
MPI_Finalize();
return result;
}
......@@ -3,20 +3,22 @@
* accompanying file Copyright.txt for details.
*/
#include <adios2.h>
#include <gtest/gtest.h>
#ifdef ADIOS2_HAVE_MPI
#include <mpi.h>
#endif
#include "TestSscCommon.h"
#include <adios2.h>
#include <gtest/gtest.h>
#include <numeric>
#include <thread>
using namespace adios2;
int mpiRank = 0;
int mpiSize = 1;
int worldRank, worldSize;
int mpiGroup;
MPI_Comm mpiComm;
int print_lines = 0;
char runMode;
......@@ -26,123 +28,11 @@ public:
SscEngineTest() = default;
};
template <class T>
void PrintData(const T *data, const size_t step, const Dims &start,
const Dims &count)
{
size_t size = std::accumulate(count.begin(), count.end(), 1,
std::multiplies<size_t>());
std::cout << "Rank: " << mpiRank << " Step: " << step << " Size:" << size
<< "\n";
size_t printsize = 128;
if (size < printsize)
{
printsize = size;
}
int s = 0;
for (size_t i = 0; i < printsize; ++i)
{
++s;
std::cout << data[i] << " ";
if (s == count[1])
{
std::cout << std::endl;
s = 0;
}
}
std::cout << "]" << std::endl;
}
template <class T>
void GenDataRecursive(std::vector<size_t> start, std::vector<size_t> count,
std::vector<size_t> shape, size_t n0, size_t y,
std::vector<T> &vec)
{
for (size_t i = 0; i < count[0]; i++)
{
size_t i0 = n0 * count[0] + i;
size_t z = y * shape[0] + (i + start[0]);
auto start_next = start;
auto count_next = count;
auto shape_next = shape;
start_next.erase(start_next.begin());
count_next.erase(count_next.begin());
shape_next.erase(shape_next.begin());
if (start_next.size() == 1)
{
for (size_t j = 0; j < count_next[0]; j++)
{
vec[i0 * count_next[0] + j] =
z * shape_next[0] + (j + start_next[0]);
}
}
else
{
GenDataRecursive(start_next, count_next, shape_next, i0, z, vec);
}
}
}
template <class T>
void GenData(std::vector<T> &vec, const size_t step,
const std::vector<size_t> &start, const std::vector<size_t> &count,
const std::vector<size_t> &shape)
{
size_t total_size = std::accumulate(count.begin(), count.end(), 1,
std::multiplies<size_t>());
vec.resize(total_size);
GenDataRecursive(start, count, shape, 0, 0, vec);
}
template <class T>
void VerifyData(const std::complex<T> *data, size_t step, const Dims &start,
const Dims &count, const Dims &shape)
void coupler(const Dims &shape, const Dims &start, const Dims &count,
const size_t steps, const adios2::Params &engineParams)
{
size_t size = std::accumulate(count.begin(), count.end(), 1,
std::multiplies<size_t>());
std::vector<std::complex<T>> tmpdata(size);
GenData(tmpdata, step, start, count, shape);
for (size_t i = 0; i < size; ++i)
{
ASSERT_EQ(data[i], tmpdata[i]);
}
if (print_lines < 32)
{
PrintData(data, step, start, count);
++print_lines;
}
}
template <class T>
void VerifyData(const T *data, size_t step, const Dims &start,
const Dims &count, const Dims &shape)
{
size_t size = std::accumulate(count.begin(), count.end(), 1,
std::multiplies<size_t>());
bool compressed = false;