Merge pull request #984 from JasonRuonanWang/dataman-attribute (79e162f1) · Commits · Podhorszki, Norbert / ADIOS2

source/adios2/engine/dataman/DataManReader.cpp

+10 −1

Original line number	Diff line number	Diff line
		@@ -100,7 +100,16 @@ StepStatus DataManReader::BeginStep(StepMode stepMode,
		"[DataManReader::BeginStep] Step mode is not supported!"));
		}

		vars = m_MetaDataMap[m_CurrentStep];
		auto currentStepIt = m_MetaDataMap.find(m_CurrentStep);
		if (currentStepIt != m_MetaDataMap.end())
		{
		vars = currentStepIt->second;
		}
		}

		if (m_CurrentStep == 0)
		{
		m_DataManDeserializer.GetAttributes(m_IO);
		}

		for (const auto &i : *vars)

source/adios2/engine/dataman/DataManWriter.cpp

+4 −0

Original line number	Diff line number	Diff line
		@@ -62,6 +62,10 @@ void DataManWriter::EndStep()
		{
		for (size_t i = 0; i < m_TransportChannels; ++i)
		{
		if (m_CurrentStep == 0)
		{
		m_DataManSerializer[i]->PutAttributes(m_IO, m_MPIRank);
		}
		const std::shared_ptr<std::vector<char>> buf =
		m_DataManSerializer[i]->Get();
		m_BufferSize = buf->size() * 2;

source/adios2/toolkit/format/dataman/DataManDeserializer.cpp

+41 −1

Original line number	Diff line number	Diff line
		@@ -60,6 +60,18 @@ int DataManDeserializer::Put(

		for (auto stepMapIt = metaJ.begin(); stepMapIt != metaJ.end(); ++stepMapIt)
		{
		if (stepMapIt.key() == "G" \|\| stepMapIt.key() == "A")
		{
		for (const auto &rankVec : stepMapIt.value())
		{
		for (const auto &gVar : rankVec)
		{
		m_GlobalVars[gVar["N"].get<std::string>()] = gVar;
		}
		}
		continue;
		}

		for (auto rankMapIt = stepMapIt.value().begin();
		rankMapIt != stepMapIt.value().end(); ++rankMapIt)
		{
		@@ -69,10 +81,10 @@ int DataManDeserializer::Put(
		try
		{
		// compulsory properties
		var.step = stoull(stepMapIt.key());
		var.name = varBlock["N"].get<std::string>();
		var.start = varBlock["O"].get<Dims>();
		var.count = varBlock["C"].get<Dims>();
		var.step = varBlock["T"].get<size_t>();
		var.size = varBlock["I"].get<size_t>();

		// optional properties
		@@ -246,5 +258,33 @@ bool DataManDeserializer::HasOverlap(Dims in_start, Dims in_count,
		return true;
		}

		void DataManDeserializer::GetAttributes(core::IO &io)
		{
		std::lock_guard<std::mutex> l(m_Mutex);
		for (const auto &j : m_GlobalVars)
		{
		const std::string type(j["Y"].get<std::string>());
		if (type == "unknown")
		{
		}
		#define declare_type(T) \
		else if (type == helper::GetType<T>()) \
		{ \
		if (j["V"].get<bool>()) \
		{ \
		io.DefineAttribute<T>(j["N"].get<std::string>(), j["G"].get<T>()); \
		} \
		else \
		{ \
		io.DefineAttribute<T>(j["N"].get<std::string>(), \
		j["G"].get<std::vector<T>>().data(), \
		j["G"].get<std::vector<T>>().size()); \
		} \
		}
		ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(declare_type)
		#undef declare_type
		}
		}

		} // namespace format
		} // namespace adios2

source/adios2/toolkit/format/dataman/DataManDeserializer.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -14,6 +14,7 @@
		#include <nlohmann/json.hpp>

		#include "adios2/ADIOSTypes.h"
		#include "adios2/core/IO.h"
		#include "adios2/core/Variable.h"

		#include <mutex>
		@@ -56,6 +57,7 @@ public:
		GetMetaData(const size_t step);
		const std::unordered_map<size_t, std::shared_ptr<std::vector<DataManVar>>>
		GetMetaData();
		void GetAttributes(core::IO &io);

		private:
		bool HasOverlap(Dims in_start, Dims in_count, Dims out_start,
		@@ -69,6 +71,7 @@ private:
		size_t m_MinStep = std::numeric_limits<size_t>::max();
		bool m_IsRowMajor;
		bool m_IsLittleEndian;
		nlohmann::json m_GlobalVars;

		std::mutex m_Mutex;
		};

source/adios2/toolkit/format/dataman/DataManSerializer.cpp

+22 −1

Original line number	Diff line number	Diff line
		@@ -39,7 +39,7 @@ void DataManSerializer::New(size_t size)

		const std::shared_ptr<std::vector<char>> DataManSerializer::Get()
		{
		std::vector<uint8_t> metacbor = nlohmann::json::to_msgpack(m_Metadata);
		std::vector<uint8_t> metacbor(nlohmann::json::to_msgpack(m_Metadata));
		size_t metasize = metacbor.size();
		m_Buffer->resize(m_Position + metasize);
		(reinterpret_cast<uint64_t *>(m_Buffer->data()))[0] = m_Position;
		@@ -97,5 +97,26 @@ bool DataManSerializer::IsCompressionAvailable(const std::string &method,
		return false;
		}

		void DataManSerializer::PutAttributes(core::IO &io, const int rank)
		{
		const auto attributesDataMap = io.GetAttributesDataMap();
		for (const auto &attributePair : attributesDataMap)
		{
		const std::string name(attributePair.first);
		const std::string type(attributePair.second.first);
		if (type == "unknown")
		{
		}
		#define declare_type(T) \
		else if (type == helper::GetType<T>()) \
		{ \
		core::Attribute<T> &attribute = *io.InquireAttribute<T>(name); \
		PutAttribute(attribute, rank); \
		}
		ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(declare_type)
		#undef declare_type
		}
		}

		} // namespace format
		} // namespace adios2