Commit 46b917b7 authored by Ruonan Wang's avatar Ruonan Wang Committed by Wang, Ruonan
Browse files

moved some json processing stuff into SscHelper

parent b000d331
......@@ -10,6 +10,7 @@
#include "SscHelper.h"
#include "adios2/common/ADIOSMacros.h"
#include "adios2/helper/adiosJSONcomplex.h"
#include "adios2/helper/adiosType.h"
#include <iostream>
#include <numeric>
......@@ -132,6 +133,84 @@ RankPosMap AllOverlapRanks(const BlockVecVec &bvv)
return ret;
}
void BlockVecToJson(const BlockVec &input, nlohmann::json &output)
{
for (const auto &b : input)
{
output["Variables"].emplace_back();
auto &jref = output["Variables"].back();
jref["Name"] = b.name;
jref["Type"] = b.type;
jref["Shape"] = b.shape;
jref["Start"] = b.start;
jref["Count"] = b.count;
jref["BufferStart"] = b.bufferStart;
jref["BufferCount"] = b.bufferCount;
}
}
void AttributeMapToJson(IO &input, nlohmann::json &output)
{
const auto &attributeMap = input.GetAttributesDataMap();
auto &attributesJson = output["Attributes"];
for (const auto &attributePair : attributeMap)
{
const std::string name(attributePair.first);
const std::string type(attributePair.second.first);
if (type.empty())
{
}
#define declare_type(T) \
else if (type == helper::GetType<T>()) \
{ \
const auto &attribute = input.InquireAttribute<T>(name); \
nlohmann::json attributeJson; \
attributeJson["Name"] = attribute->m_Name; \
attributeJson["Type"] = attribute->m_Type; \
attributeJson["IsSingleValue"] = attribute->m_IsSingleValue; \
if (attribute->m_IsSingleValue) \
{ \
attributeJson["Value"] = attribute->m_DataSingleValue; \
} \
else \
{ \
attributeJson["Array"] = attribute->m_DataArray; \
} \
output["Attributes"].emplace_back(std::move(attributeJson)); \
}
ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type)
#undef declare_type
}
}
void LocalJsonToGlobalJson(const std::vector<char> &input,
const size_t maxLocalSize, const int streamSize,
nlohmann::json &output)
{
try
{
for (size_t i = 0; i < streamSize; ++i)
{
if (input[i * maxLocalSize] == '\0')
{
output[i] = nullptr;
}
else
{
output[i] = nlohmann::json::parse(
input.begin() + i * maxLocalSize,
input.begin() + (i + 1) * maxLocalSize);
}
}
}
catch (std::exception &e)
{
throw(std::runtime_error(
std::string("corrupted global write pattern metadata, ") +
std::string(e.what())));
}
}
void JsonToBlockVecVec(const nlohmann::json &input, BlockVecVec &output)
{
for (int i = 0; i < output.size(); ++i)
......
......@@ -12,6 +12,7 @@
#define ADIOS2_ENGINE_SSCHELPER_H_
#include "adios2/common/ADIOSTypes.h"
#include "adios2/core/IO.h"
#include "nlohmann/json.hpp"
#include <map>
#include <vector>
......@@ -60,6 +61,12 @@ void CalculateOverlap(BlockVecVec &globalPattern, const BlockVec &localPattern);
RankPosMap AllOverlapRanks(const BlockVecVec &mapVec);
void BlockVecToJson(const BlockVec &input, nlohmann::json &output);
void AttributeMapToJson(IO &input, nlohmann::json &output);
void LocalJsonToGlobalJson(const std::vector<char> &input,
const size_t maxLocalSize, const int streamSize,
nlohmann::json &output);
void JsonToBlockVecVec(const nlohmann::json &input, BlockVecVec &output);
void JsonToBlockVecVec(const std::vector<char> &input, BlockVecVec &output);
void JsonToBlockVecVec(const std::string &input, BlockVecVec &output);
......
......@@ -250,28 +250,8 @@ void SscReader::SyncWritePattern()
maxLocalSize, MPI_CHAR, m_StreamComm);
// deserialize global metadata Json
try
{
for (size_t i = 0; i < m_StreamSize; ++i)
{
if (globalVec[i * maxLocalSize] == '\0')
{
m_GlobalWritePatternJson[i] = nullptr;
}
else
{
m_GlobalWritePatternJson[i] = nlohmann::json::parse(
globalVec.begin() + i * maxLocalSize,
globalVec.begin() + (i + 1) * maxLocalSize);
}
}
}
catch (std::exception &e)
{
throw(std::runtime_error(
std::string("corrupted global write pattern metadata, ") +
std::string(e.what())));
}
ssc::LocalJsonToGlobalJson(globalVec, maxLocalSize, m_StreamSize,
m_GlobalWritePatternJson);
// deserialize variables metadata
ssc::JsonToBlockVecVec(m_GlobalWritePatternJson, m_GlobalWritePattern);
......
......@@ -13,6 +13,7 @@
#include "SscReader.h"
#include "adios2/helper/adiosMemory.h"
#include "adios2/helper/adiosSystem.h"
#include <iostream>
namespace adios2
......
......@@ -11,8 +11,6 @@
#include "SscWriter.tcc"
#include "adios2/helper/adiosComm.h"
#include "adios2/helper/adiosCommMPI.h"
#include "adios2/helper/adiosJSONcomplex.h"
#include "nlohmann/json.hpp"
namespace adios2
{
......@@ -238,51 +236,13 @@ void SscWriter::SyncWritePattern()
<< ", Writer Rank " << m_WriterRank << std::endl;
}
// serialize local writer rank variables metadata
nlohmann::json localRankMetaJ;
for (const auto &b : m_GlobalWritePattern[m_StreamRank])
{
localRankMetaJ["Variables"].emplace_back();
auto &jref = localRankMetaJ["Variables"].back();
jref["Name"] = b.name;
jref["Type"] = b.type;
jref["Shape"] = b.shape;
jref["Start"] = b.start;
jref["Count"] = b.count;
jref["BufferStart"] = b.bufferStart;
jref["BufferCount"] = b.bufferCount;
}
// serialize local writer rank attributes metadata
auto &attributesJson = localRankMetaJ["Attributes"];
const auto &attributeMap = m_IO.GetAttributesDataMap();
for (const auto &attributePair : attributeMap)
ssc::BlockVecToJson(m_GlobalWritePattern[m_StreamRank], localRankMetaJ);
if (m_WriterRank == 0)
{
const std::string name(attributePair.first);
const std::string type(attributePair.second.first);
if (type.empty())
{
}
#define declare_type(T) \
else if (type == helper::GetType<T>()) \
{ \
const auto &attribute = m_IO.InquireAttribute<T>(name); \
nlohmann::json attributeJson; \
attributeJson["Name"] = attribute->m_Name; \
attributeJson["Type"] = attribute->m_Type; \
attributeJson["IsSingleValue"] = attribute->m_IsSingleValue; \
if (attribute->m_IsSingleValue) \
{ \
attributeJson["Value"] = attribute->m_DataSingleValue; \
} \
else \
{ \
attributeJson["Array"] = attribute->m_DataArray; \
} \
attributesJson.emplace_back(std::move(attributeJson)); \
}
ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type)
#undef declare_type
ssc::AttributeMapToJson(m_IO, localRankMetaJ);
}
std::string localStr = localRankMetaJ.dump();
......@@ -300,28 +260,8 @@ void SscWriter::SyncWritePattern()
// deserialize global metadata Json
nlohmann::json globalJson;
try
{
for (size_t i = 0; i < m_StreamSize; ++i)
{
if (globalVec[i * maxLocalSize] == '\0')
{
globalJson[i] = nullptr;
}
else
{
globalJson[i] = nlohmann::json::parse(
globalVec.begin() + i * maxLocalSize,
globalVec.begin() + (i + 1) * maxLocalSize);
}
}
}
catch (std::exception &e)
{
throw(std::runtime_error(
std::string("corrupted global write pattern metadata, ") +
std::string(e.what())));
}
ssc::LocalJsonToGlobalJson(globalVec, maxLocalSize, m_StreamSize,
globalJson);
// deserialize variables metadata
ssc::JsonToBlockVecVec(globalJson, m_GlobalWritePattern);
......
......@@ -236,7 +236,7 @@ if(ADIOS2_HAVE_MPI)
endif()
if(ADIOS2_HAVE_SSC)
set (SSC_TESTS "1x1;1x1.Attrs;TimeoutOnOpen;2x1;1x2;3x5;5x3;3x5LockGeometry;2x1ZeroDataVar;2x1ZeroDataR64")
set (SSC_TESTS "1x1;1x1.Attrs;2x1;1x2;2x1ZeroDataVar;2x1ZeroDataR64;3x5;5x3;3x5LockGeometry;TimeoutOnOpen")
foreach(test ${SSC_TESTS})
add_common_test(${test} SSC)
endforeach()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment