From e44b43edea4fdf2b82234bab99b6895b3d15d4f9 Mon Sep 17 00:00:00 2001
From: William F Godoy <williamfgc@yahoo.com>
Date: Fri, 17 Nov 2017 16:05:53 -0500
Subject: [PATCH 1/2] Reading attributes
Need to add examples and tests
Fixed some bugs in Attribute Read
---
.../helloBPAttributeWriter.cpp | 20 +++
source/adios2/ADIOSMacros.h | 13 ++
source/adios2/core/IO.cpp | 77 ++++++-----
source/adios2/core/IO.h | 6 +
source/adios2/engine/bp/BPFileReader.cpp | 2 -
source/adios2/engine/bp/BPFileWriter.tcc | 2 -
source/adios2/helper/adiosType.cpp | 1 -
source/adios2/helper/adiosType.h | 6 +
source/adios2/helper/adiosType.inl | 68 +++++++++-
source/adios2/toolkit/format/bp3/BP3Base.tcc | 19 ++-
.../toolkit/format/bp3/BP3Deserializer.cpp | 125 +++++++++++++++++-
.../toolkit/format/bp3/BP3Deserializer.h | 5 +
.../toolkit/format/bp3/BP3Deserializer.tcc | 35 ++++-
source/utils/bpls2/BPLS2.cpp | 29 ++--
14 files changed, 354 insertions(+), 54 deletions(-)
diff --git a/examples/hello/bpAttributeWriter/helloBPAttributeWriter.cpp b/examples/hello/bpAttributeWriter/helloBPAttributeWriter.cpp
index cd2298891..ad72bfe56 100644
--- a/examples/hello/bpAttributeWriter/helloBPAttributeWriter.cpp
+++ b/examples/hello/bpAttributeWriter/helloBPAttributeWriter.cpp
@@ -64,6 +64,26 @@ int main(int argc, char *argv[])
/** Create bp file, engine becomes unreachable after this*/
bpWriter.Close();
+
+ adios2::IO &bpReader = adios.DeclareIO("BPReader");
+
+ adios2::Engine &bpReaderEngine =
+ bpReader.Open("fileAttributes.bp", adios2::Mode::Read);
+
+ const auto attributesInfo = bpReader.GetAvailableAttributes();
+
+ for (const auto &attributeInfoPair : attributesInfo)
+ {
+ std::cout << "Attribute: " << attributeInfoPair.first;
+ for (const auto &attributePair : attributeInfoPair.second)
+ {
+ std::cout << "\tKey: " << attributePair.first
+ << "\tValue: " << attributePair.second << "\n";
+ }
+ std::cout << "\n";
+ }
+
+ bpReaderEngine.Close();
}
catch (std::invalid_argument &e)
{
diff --git a/source/adios2/ADIOSMacros.h b/source/adios2/ADIOSMacros.h
index 0d002d332..6aad52b51 100644
--- a/source/adios2/ADIOSMacros.h
+++ b/source/adios2/ADIOSMacros.h
@@ -114,6 +114,19 @@
MACRO(double) \
MACRO(long double)
+#define ADIOS2_FOREACH_NUMERIC_ATTRIBUTE_TYPE_1ARG(MACRO) \
+ MACRO(short) \
+ MACRO(unsigned short) \
+ MACRO(int) \
+ MACRO(unsigned int) \
+ MACRO(long int) \
+ MACRO(long long int) \
+ MACRO(unsigned long int) \
+ MACRO(unsigned long long int) \
+ MACRO(float) \
+ MACRO(double) \
+ MACRO(long double)
+
/**
<pre>
The ADIOS_FOREACH_TYPE_2ARGS macro assumes the given argument is a macro
diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp
index b4b0abf45..f0f9bd40f 100644
--- a/source/adios2/core/IO.cpp
+++ b/source/adios2/core/IO.cpp
@@ -173,46 +173,61 @@ std::map<std::string, Params> IO::GetAvailableVariables() noexcept
if (type == "compound")
{
}
-// TODO : enable string, add dimensions
-#define declare_template_instantiation(C) \
- else if (type == GetType<C>()) \
+#define declare_template_instantiation(T) \
+ else if (type == GetType<T>()) \
{ \
- Variable<C> &variable = *InquireVariable<C>(name); \
- \
- const int min = static_cast<int>(variable.m_Min); \
- variablesInfo[name]["Min"] = std::to_string(min); \
- \
- const int max = static_cast<int>(variable.m_Max); \
- variablesInfo[name]["Max"] = std::to_string(max); \
- \
- variablesInfo[name]["StepsStart"] = \
- std::to_string(variable.m_AvailableStepsStart); \
- variablesInfo[name]["StepsCount"] = \
- std::to_string(variable.m_AvailableStepsCount); \
+ Variable<T> &variable = *InquireVariable<T>(name); \
+ variablesInfo[name]["Min"] = ValueToString(variable.m_Min); \
+ variablesInfo[name]["Max"] = ValueToString(variable.m_Max); \
+ variablesInfo[name]["Value"] = ValueToString(variable.m_Value); \
+ variablesInfo[name]["AvailableStepsStart"] = \
+ ValueToString(variable.m_AvailableStepsStart); \
+ variablesInfo[name]["AvailableStepsCount"] = \
+ ValueToString(variable.m_AvailableStepsCount); \
+ variablesInfo[name]["Shape"] = VectorToCSV(variable.m_Shape); \
+ variablesInfo[name]["Start"] = VectorToCSV(variable.m_Start); \
+ variablesInfo[name]["Count"] = VectorToCSV(variable.m_Count); \
}
- ADIOS2_FOREACH_CHAR_TYPE_1ARG(declare_template_instantiation)
+ ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation)
#undef declare_template_instantiation
+ }
+
+ return variablesInfo;
+}
+std::map<std::string, Params> IO::GetAvailableAttributes() noexcept
+{
+ std::map<std::string, Params> attributesInfo;
+ for (const auto &attributePair : m_Attributes)
+ {
+ const std::string name(attributePair.first);
+ const std::string type(attributePair.second.first);
+ attributesInfo[name]["Type"] = type;
+
+ if (type == "compound")
+ {
+ }
#define declare_template_instantiation(T) \
else if (type == GetType<T>()) \
{ \
- Variable<T> &variable = *InquireVariable<T>(name); \
- std::ostringstream minSS; \
- minSS << variable.m_Min; \
- variablesInfo[name]["Min"] = minSS.str(); \
- std::ostringstream maxSS; \
- maxSS << variable.m_Max; \
- variablesInfo[name]["Max"] = maxSS.str(); \
- variablesInfo[name]["StepsStart"] = \
- std::to_string(variable.m_AvailableStepsStart); \
- variablesInfo[name]["StepsCount"] = \
- std::to_string(variable.m_AvailableStepsCount); \
+ Attribute<T> &attribute = *InquireAttribute<T>(name); \
+ \
+ if (attribute.m_IsSingleValue) \
+ { \
+ attributesInfo[name]["Value"] = \
+ ValueToString(attribute.m_DataSingleValue); \
+ } \
+ else \
+ { \
+ attributesInfo[name]["Value"] = \
+ "{ " + VectorToCSV(attribute.m_DataArray) + " }"; \
+ } \
}
- ADIOS2_FOREACH_NUMERIC_TYPE_1ARG(declare_template_instantiation)
+ ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(declare_template_instantiation)
#undef declare_template_instantiation
- }
- return variablesInfo;
+ } // end for
+ return attributesInfo;
}
std::string IO::InquireVariableType(const std::string &name) const noexcept
@@ -427,4 +442,4 @@ ADIOS2_FOREACH_TYPE_1ARG(define_template_instantiation)
ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(declare_template_instantiation)
#undef declare_template_instantiation
-} // end namespace adios
+} // end namespace adios2
diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h
index 34e945590..209cbec33 100644
--- a/source/adios2/core/IO.h
+++ b/source/adios2/core/IO.h
@@ -224,6 +224,12 @@ public:
template <class T>
Attribute<T> *InquireAttribute(const std::string &name) noexcept;
+ /**
+ * Map with variables info: key: name, value: type
+ * @return populate map with current variables
+ */
+ std::map<std::string, Params> GetAvailableAttributes() noexcept;
+
/**
* Check existence in config file passed to ADIOS class
* @return true: defined in config file
diff --git a/source/adios2/engine/bp/BPFileReader.cpp b/source/adios2/engine/bp/BPFileReader.cpp
index 6900ccd06..5249d3ca7 100644
--- a/source/adios2/engine/bp/BPFileReader.cpp
+++ b/source/adios2/engine/bp/BPFileReader.cpp
@@ -8,8 +8,6 @@
* Author: William F Godoy godoywf@ornl.gov
*/
-#include <iostream> //TODO will go away
-
#include "BPFileReader.h"
#include "BPFileReader.tcc"
diff --git a/source/adios2/engine/bp/BPFileWriter.tcc b/source/adios2/engine/bp/BPFileWriter.tcc
index 0956fb97a..82cd2c984 100644
--- a/source/adios2/engine/bp/BPFileWriter.tcc
+++ b/source/adios2/engine/bp/BPFileWriter.tcc
@@ -54,8 +54,6 @@ void BPFileWriter::PutSyncCommon(Variable<T> &variable, const T *values)
// WRITE INDEX to data buffer and metadata structure (in memory)//
m_BP3Serializer.PutVariableMetadata(variable);
m_BP3Serializer.PutVariablePayload(variable);
-
- // variable.SetData(nullptr); // not needed after PutSync?
}
template <class T>
diff --git a/source/adios2/helper/adiosType.cpp b/source/adios2/helper/adiosType.cpp
index 280046147..8c8ee76b2 100644
--- a/source/adios2/helper/adiosType.cpp
+++ b/source/adios2/helper/adiosType.cpp
@@ -9,7 +9,6 @@
*/
#include "adiosType.h"
-#include "adiosType.inl"
/// \cond EXCLUDE_FROM_DOXYGEN
#include <algorithm> //std::transform, std::count
diff --git a/source/adios2/helper/adiosType.h b/source/adios2/helper/adiosType.h
index d5eec186b..8392a9672 100644
--- a/source/adios2/helper/adiosType.h
+++ b/source/adios2/helper/adiosType.h
@@ -157,6 +157,12 @@ U *InquireKey(const T &key, std::map<T, U> &input) noexcept;
template <class T, class U>
U *InquireKey(const T &key, std::unordered_map<T, U> &input) noexcept;
+template <class T>
+std::string VectorToCSV(const std::vector<T> &input) noexcept;
+
+template <class T>
+std::string ValueToString(const T value) noexcept;
+
} // end namespace adios2
#include "adiosType.inl"
diff --git a/source/adios2/helper/adiosType.inl b/source/adios2/helper/adiosType.inl
index c7d29243a..b74574019 100644
--- a/source/adios2/helper/adiosType.inl
+++ b/source/adios2/helper/adiosType.inl
@@ -14,7 +14,10 @@
#error "Inline file should only be included from it's header, never on it's own"
#endif
-#include <algorithm>
+#include <algorithm> //std::transform
+#include <sstream> //std::ostringstream
+
+#include "adios2/ADIOSMacros.h"
namespace adios2
{
@@ -184,6 +187,69 @@ U *InquireKey(const T &key, std::unordered_map<T, U> &input) noexcept
return &itKey->second;
}
+#define declare_template_instantiation(C) \
+ template <> \
+ inline std::string ValueToString(const C value) noexcept \
+ { \
+ const int valueInt = static_cast<int>(value); \
+ return std::to_string(valueInt); \
+ }
+ADIOS2_FOREACH_CHAR_TYPE_1ARG(declare_template_instantiation)
+#undef declare_template_instantiation
+
+template <class T>
+inline std::string ValueToString(const T value) noexcept
+{
+ std::ostringstream valueSS;
+ valueSS << value;
+ const std::string valueStr(valueSS.str());
+ return valueStr;
+}
+
+#define declare_template_instantiation(C) \
+ template <> \
+ inline std::string VectorToCSV(const std::vector<C> &input) noexcept \
+ { \
+ if (input.empty()) \
+ { \
+ return std::string(); \
+ } \
+ \
+ std::ostringstream valueSS; \
+ for (const auto value : input) \
+ { \
+ const int valueInt = static_cast<int>(value); \
+ valueSS << valueInt << ", "; \
+ } \
+ std::string csv(valueSS.str()); \
+ csv.pop_back(); \
+ csv.pop_back(); \
+ \
+ return csv; \
+ }
+ADIOS2_FOREACH_CHAR_TYPE_1ARG(declare_template_instantiation)
+#undef declare_template_instantiation
+
+template <class T>
+inline std::string VectorToCSV(const std::vector<T> &input) noexcept
+{
+ if (input.empty())
+ {
+ return std::string();
+ }
+
+ std::ostringstream valueSS;
+ for (const auto value : input)
+ {
+ valueSS << value << ", ";
+ }
+ std::string csv(valueSS.str());
+ csv.pop_back();
+ csv.pop_back();
+
+ return csv;
+}
+
} // end namespace adios2
#endif /* ADIOS2_HELPER_ADIOSTYPE_INL_ */
diff --git a/source/adios2/toolkit/format/bp3/BP3Base.tcc b/source/adios2/toolkit/format/bp3/BP3Base.tcc
index b2e121c7e..5c7fcbdcd 100644
--- a/source/adios2/toolkit/format/bp3/BP3Base.tcc
+++ b/source/adios2/toolkit/format/bp3/BP3Base.tcc
@@ -201,28 +201,35 @@ inline void BP3Base::ParseCharacteristics(
if (dataType == type_string)
{
// first get the length of the string
- const size_t size =
+ const size_t length =
static_cast<size_t>(ReadValue<uint16_t>(buffer, position));
characteristics.Statistics.Value =
- std::string(&buffer[position], size);
+ std::string(&buffer[position], length);
characteristics.Statistics.IsValue = true;
+ position += length;
}
else if (dataType == type_string_array)
{
- const size_t elements =
- static_cast<size_t>(ReadValue<uint32_t>(buffer, position));
+ if (characteristics.Count.size() != 1)
+ {
+ // TODO: add exception here?
+ break;
+ }
+ const size_t elements = characteristics.Count.front();
characteristics.Statistics.Values.reserve(elements);
for (size_t e = 0; e < elements; ++e)
{
- const size_t size = static_cast<size_t>(
+ const size_t length = static_cast<size_t>(
ReadValue<uint16_t>(buffer, position));
characteristics.Statistics.Values.push_back(
- std::string(&buffer[position], size));
+ std::string(&buffer[position], length));
+
+ position += length;
}
}
diff --git a/source/adios2/toolkit/format/bp3/BP3Deserializer.cpp b/source/adios2/toolkit/format/bp3/BP3Deserializer.cpp
index f614d90e3..317e0770c 100644
--- a/source/adios2/toolkit/format/bp3/BP3Deserializer.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Deserializer.cpp
@@ -37,7 +37,7 @@ void BP3Deserializer::ParseMetadata(IO &io)
ParseMinifooter();
ParsePGIndex();
ParseVariablesIndex(io);
- // ParseAttributesIndex(io);
+ ParseAttributesIndex(io);
}
void BP3Deserializer::ClipContiguousMemory(
@@ -238,6 +238,7 @@ void BP3Deserializer::ParseVariablesIndex(IO &io)
} // end switch
};
+ // STARTS HERE
const auto &buffer = m_Metadata.m_Buffer;
size_t position = m_Minifooter.VarsIndexStart;
@@ -288,7 +289,127 @@ void BP3Deserializer::ParseVariablesIndex(IO &io)
}
}
-void BP3Deserializer::ParseAttributesIndex(IO &io) {}
+void BP3Deserializer::ParseAttributesIndex(IO &io)
+{
+ auto lf_ReadElementIndex = [&](IO &io, const std::vector<char> &buffer,
+ size_t position) {
+
+ const ElementIndexHeader header =
+ ReadElementIndexHeader(buffer, position);
+
+ switch (header.DataType)
+ {
+
+ case (type_string):
+ {
+ DefineAttributeInIO<std::string>(header, io, buffer, position);
+ break;
+ }
+
+ case (type_string_array):
+ {
+ DefineAttributeInIO<std::string>(header, io, buffer, position);
+ break;
+ }
+
+ case (type_byte):
+ {
+ DefineAttributeInIO<signed char>(header, io, buffer, position);
+ break;
+ }
+
+ case (type_short):
+ {
+ DefineAttributeInIO<short>(header, io, buffer, position);
+ break;
+ }
+
+ case (type_integer):
+ {
+ DefineAttributeInIO<int>(header, io, buffer, position);
+ break;
+ }
+
+ case (type_long):
+ {
+#ifdef _WIN32
+ DefineAttributeInIO<long long int>(header, io, buffer, position);
+#else
+ DefineAttributeInIO<long int>(header, io, buffer, position);
+#endif
+ break;
+ }
+
+ case (type_unsigned_byte):
+ {
+ DefineAttributeInIO<unsigned char>(header, io, buffer, position);
+ break;
+ }
+
+ case (type_unsigned_short):
+ {
+ DefineAttributeInIO<unsigned short>(header, io, buffer, position);
+ break;
+ }
+
+ case (type_unsigned_integer):
+ {
+ DefineAttributeInIO<unsigned int>(header, io, buffer, position);
+ break;
+ }
+
+ case (type_unsigned_long):
+ {
+#ifdef _WIN32
+ DefineAttributeInIO<unsigned long long int>(header, io, buffer,
+ position);
+#else
+ DefineAttributeInIO<unsigned long int>(header, io, buffer,
+ position);
+#endif
+ break;
+ }
+
+ case (type_real):
+ {
+ DefineAttributeInIO<float>(header, io, buffer, position);
+ break;
+ }
+
+ case (type_double):
+ {
+ DefineAttributeInIO<double>(header, io, buffer, position);
+ break;
+ }
+
+ case (type_long_double):
+ {
+ DefineAttributeInIO<long double>(header, io, buffer, position);
+ break;
+ }
+
+ } // end switch
+ };
+
+ const auto &buffer = m_Metadata.m_Buffer;
+ size_t position = m_Minifooter.AttributesIndexStart;
+
+ const uint32_t count = ReadValue<uint32_t>(buffer, position);
+ const uint64_t length = ReadValue<uint64_t>(buffer, position);
+
+ const size_t startPosition = position;
+ size_t localPosition = 0;
+
+ // Read sequentially
+ while (localPosition < length)
+ {
+ lf_ReadElementIndex(io, buffer, position);
+ const size_t elementIndexSize =
+ static_cast<size_t>(ReadValue<uint32_t>(buffer, position));
+ position += elementIndexSize;
+ localPosition = position - startPosition;
+ }
+}
std::map<std::string, SubFileInfoMap>
BP3Deserializer::PerformGetsVariablesSubFileInfo(IO &io)
diff --git a/source/adios2/toolkit/format/bp3/BP3Deserializer.h b/source/adios2/toolkit/format/bp3/BP3Deserializer.h
index b81107c04..eb24da84f 100644
--- a/source/adios2/toolkit/format/bp3/BP3Deserializer.h
+++ b/source/adios2/toolkit/format/bp3/BP3Deserializer.h
@@ -82,6 +82,11 @@ private:
const std::vector<char> &buffer,
size_t position) const;
+ template <class T>
+ void DefineAttributeInIO(const ElementIndexHeader &header, IO &io,
+ const std::vector<char> &buffer,
+ size_t position) const;
+
template <class T>
SubFileInfoMap GetSubFileInfo(const Variable<T> &variable) const;
diff --git a/source/adios2/toolkit/format/bp3/BP3Deserializer.tcc b/source/adios2/toolkit/format/bp3/BP3Deserializer.tcc
index 316006292..469bea855 100644
--- a/source/adios2/toolkit/format/bp3/BP3Deserializer.tcc
+++ b/source/adios2/toolkit/format/bp3/BP3Deserializer.tcc
@@ -118,6 +118,38 @@ void BP3Deserializer::DefineVariableInIO(const ElementIndexHeader &header,
}
}
+template <class T>
+void BP3Deserializer::DefineAttributeInIO(const ElementIndexHeader &header,
+ IO &io,
+ const std::vector<char> &buffer,
+ size_t position) const
+{
+ const size_t initialPosition = position;
+
+ const Characteristics<T> characteristics =
+ ReadElementIndexCharacteristics<T>(
+ buffer, position, static_cast<DataTypes>(header.DataType));
+
+ std::string attributeName(header.Name);
+ if (!header.Path.empty())
+ {
+ attributeName = header.Path + PathSeparator + header.Name;
+ }
+
+ Attribute<T> *attribute = nullptr;
+ if (characteristics.Statistics.IsValue)
+ {
+ attribute = &io.DefineAttribute<T>(attributeName,
+ characteristics.Statistics.Value);
+ }
+ else
+ {
+ attribute = &io.DefineAttribute<T>(
+ attributeName, characteristics.Statistics.Values.data(),
+ characteristics.Statistics.Values.size());
+ }
+}
+
template <class T>
SubFileInfoMap
BP3Deserializer::GetSubFileInfo(const Variable<T> &variable) const
@@ -163,7 +195,8 @@ BP3Deserializer::GetSubFileInfo(const Variable<T> &variable) const
{
continue;
}
- // if they intersect get info Seeks (first: start, second: count)
+ // if they intersect get info Seeks (first: start, second:
+ // count)
info.Seeks.first =
blockCharacteristics.Statistics.PayloadOffset +
LinearIndex(info.BlockBox, info.IntersectionBox.first,
diff --git a/source/utils/bpls2/BPLS2.cpp b/source/utils/bpls2/BPLS2.cpp
index 3efc63443..4620e6d15 100644
--- a/source/utils/bpls2/BPLS2.cpp
+++ b/source/utils/bpls2/BPLS2.cpp
@@ -185,9 +185,9 @@ void BPLS2::SetParameters(const std::string argument, const bool isLong)
void BPLS2::ProcessTransport() const
{
- auto lf_PrintVerboseHeader = [](const BPFileReader &bpFileReader,
- const size_t variablesSize,
- const size_t attributesSize) {
+ auto lf_PrintVerboseHeader = [](
+ const BPFileReader &bpFileReader, const size_t variablesSize,
+ const size_t attributesSize, const size_t stepsCount) {
const auto &metadataSet = bpFileReader.m_BP3Deserializer.m_MetadataSet;
std::cout << "File info:\n";
@@ -195,7 +195,7 @@ void BPLS2::ProcessTransport() const
std::cout << " variables: " << variablesSize << "\n";
std::cout << " attributes: " << attributesSize << "\n";
std::cout << " meshes: TODO\n";
- std::cout << " steps: " << metadataSet.TimeStep << "\n";
+ std::cout << " steps: " << stepsCount << "\n";
std::cout << " file size: "
<< bpFileReader.m_FileManager.GetFileSize(0) << " bytes\n";
@@ -259,16 +259,29 @@ void BPLS2::ProcessTransport() const
ADIOS adios(true);
IO &io = adios.DeclareIO("bpls2");
BPFileReader bpFileReader(io, m_FileName, Mode::Read, io.m_MPIComm);
- const std::map<std::string, Params> variablesMap =
+ const std::map<std::string, Params> variablesInfo =
io.GetAvailableVariables();
- // const auto attributesMap = io.GetAvailableAttributes();
if (m_Parameters.count("verbose") == 1)
{
- lf_PrintVerboseHeader(bpFileReader, variablesMap.size(), 0);
+ size_t stepsCount = 1;
+
+ for (const auto variableInfo : variablesInfo)
+ {
+ const size_t variableStepsCount = static_cast<size_t>(
+ std::stoul(variableInfo.second.at("AvailableStepsCount")));
+
+ if (variableStepsCount > stepsCount)
+ {
+ stepsCount = variableStepsCount;
+ }
+ }
+
+ lf_PrintVerboseHeader(bpFileReader, variablesInfo.size(), 0,
+ stepsCount);
}
- lf_PrintVariables(variablesMap);
+ lf_PrintVariables(variablesInfo);
}
} // end namespace utils
--
GitLab
From cd92e069c3fabdf57b53d142c4596b68d0e3eead Mon Sep 17 00:00:00 2001
From: William F Godoy <williamfgc@yahoo.com>
Date: Sun, 19 Nov 2017 09:31:49 -0500
Subject: [PATCH 2/2] Added Attribute value and array tests
Add Path separator for Windows tests
And long long (Windows) vs long (Linux) type
---
testing/adios2/engine/bp/CMakeLists.txt | 7 +
.../bp/TestBPWriteReadAttributesADIOS2.cpp | 389 ++++++++++++++++++
testing/adios2/xml/TestXMLConfig.cpp | 8 +-
3 files changed, 400 insertions(+), 4 deletions(-)
create mode 100644 testing/adios2/engine/bp/TestBPWriteReadAttributesADIOS2.cpp
diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt
index 55e8f14ea..736278bf6 100644
--- a/testing/adios2/engine/bp/CMakeLists.txt
+++ b/testing/adios2/engine/bp/CMakeLists.txt
@@ -6,12 +6,19 @@
add_executable(TestBPWriteReadADIOS2 TestBPWriteReadADIOS2.cpp)
target_link_libraries(TestBPWriteReadADIOS2 adios2 gtest gtest_main)
+add_executable(TestBPWriteReadAttributesADIOS2 TestBPWriteReadAttributesADIOS2.cpp)
+target_link_libraries(TestBPWriteReadAttributesADIOS2 adios2 gtest gtest_main)
+
+
if(ADIOS2_HAVE_MPI)
target_link_libraries(TestBPWriteReadADIOS2 MPI::MPI_C)
+ target_link_libraries(TestBPWriteReadAttributesADIOS2 MPI::MPI_C)
+
set(extra_test_args EXEC_WRAPPER ${MPIEXEC_COMMAND})
endif()
gtest_add_tests(TARGET TestBPWriteReadADIOS2 ${extra_test_args})
+gtest_add_tests(TARGET TestBPWriteReadAttributesADIOS2 ${extra_test_args})
if (ADIOS2_HAVE_ADIOS1)
add_executable(TestBPWriteRead TestBPWriteRead.cpp)
diff --git a/testing/adios2/engine/bp/TestBPWriteReadAttributesADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteReadAttributesADIOS2.cpp
new file mode 100644
index 000000000..db949a31b
--- /dev/null
+++ b/testing/adios2/engine/bp/TestBPWriteReadAttributesADIOS2.cpp
@@ -0,0 +1,389 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ */
+#include <cstdint>
+#include <string>
+
+#include <iostream>
+#include <stdexcept>
+
+#include <adios2.h>
+
+#include <gtest/gtest.h>
+
+#include "../SmallTestData.h"
+
+class BPWriteReadAttributeTestADIOS2 : public ::testing::Test
+{
+public:
+ BPWriteReadAttributeTestADIOS2() = default;
+
+ SmallTestData m_TestData;
+};
+
+// ADIOS2 write, read for single value attributes
+TEST_F(BPWriteReadAttributeTestADIOS2, ADIOS2BPWriteReadSingleTypes)
+{
+ const std::string fName = "foo" + std::string(&adios2::PathSeparator, 1) +
+ "ADIOS2BPWriteAttributeReadSingleTypes.bp";
+
+ int mpiRank = 0, mpiSize = 1;
+#ifdef ADIOS2_HAVE_MPI
+ MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+ MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+ const std::string mpiRankString = std::to_string(mpiRank);
+ const std::string s1_Single = std::string("s1_Single_") + mpiRankString;
+ const std::string i8_Single = std::string("i8_Single_") + mpiRankString;
+ const std::string i16_Single = std::string("i16_Single_") + mpiRankString;
+ const std::string i32_Single = std::string("i32_Single_") + mpiRankString;
+ const std::string i64_Single = std::string("i64_Single_") + mpiRankString;
+ const std::string u8_Single = std::string("u8_Single_") + mpiRankString;
+ const std::string u16_Single = std::string("u16_Single_") + mpiRankString;
+ const std::string u32_Single = std::string("u32_Single_") + mpiRankString;
+ const std::string u64_Single = std::string("u64_Single_") + mpiRankString;
+ const std::string r32_Single = std::string("r32_Single_") + mpiRankString;
+ const std::string r64_Single = std::string("r64_Single_") + mpiRankString;
+
+ // When collective meta generation has landed, use
+ // generateNewSmallTestData(m_TestData, 0, mpiRank, mpiSize);
+ // Generate current testing data
+ SmallTestData currentTestData =
+ generateNewSmallTestData(m_TestData, 0, 0, 0);
+
+// Write test data using BP
+#ifdef ADIOS2_HAVE_MPI
+ adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON);
+#else
+ adios2::ADIOS adios(true);
+#endif
+ {
+ adios2::IO &io = adios.DeclareIO("TestIO");
+
+ // Declare Single Value Attributes
+ io.DefineAttribute<std::string>(s1_Single, currentTestData.S1);
+ io.DefineAttribute<int8_t>(i8_Single, currentTestData.I8.front());
+ io.DefineAttribute<int16_t>(i16_Single, currentTestData.I16.front());
+ io.DefineAttribute<int32_t>(i32_Single, currentTestData.I32.front());
+ io.DefineAttribute<int64_t>(i64_Single, currentTestData.I64.front());
+
+ io.DefineAttribute<uint8_t>(u8_Single, currentTestData.U8.front());
+ io.DefineAttribute<uint16_t>(u16_Single, currentTestData.U16.front());
+ io.DefineAttribute<uint32_t>(u32_Single, currentTestData.U32.front());
+ io.DefineAttribute<uint64_t>(u64_Single, currentTestData.U64.front());
+
+ io.DefineAttribute<float>(r32_Single, currentTestData.R32.front());
+ io.DefineAttribute<double>(r64_Single, currentTestData.R64.front());
+
+ io.SetEngine("BPFileWriter");
+ io.AddTransport("file");
+
+ adios2::Engine &engine = io.Open(fName, adios2::Mode::Write);
+ // only attributes are written
+ engine.Close();
+ }
+
+ {
+ adios2::IO &ioRead = adios.DeclareIO("ioRead");
+
+ adios2::Engine &bpRead = ioRead.Open(fName, adios2::Mode::Read);
+
+ auto attr_s1 = ioRead.InquireAttribute<std::string>(s1_Single);
+ auto attr_i8 = ioRead.InquireAttribute<int8_t>(i8_Single);
+ auto attr_i16 = ioRead.InquireAttribute<int16_t>(i16_Single);
+ auto attr_i32 = ioRead.InquireAttribute<int32_t>(i32_Single);
+ auto attr_i64 = ioRead.InquireAttribute<int64_t>(i64_Single);
+
+ auto attr_u8 = ioRead.InquireAttribute<uint8_t>(u8_Single);
+ auto attr_u16 = ioRead.InquireAttribute<uint16_t>(u16_Single);
+ auto attr_u32 = ioRead.InquireAttribute<uint32_t>(u32_Single);
+ auto attr_u64 = ioRead.InquireAttribute<uint64_t>(u64_Single);
+
+ auto attr_r32 = ioRead.InquireAttribute<float>(r32_Single);
+ auto attr_r64 = ioRead.InquireAttribute<double>(r64_Single);
+
+ ASSERT_NE(attr_s1, nullptr);
+ ASSERT_EQ(attr_s1->m_Name, s1_Single);
+ ASSERT_EQ(attr_s1->m_IsSingleValue, true);
+ ASSERT_EQ(attr_s1->m_Type, "string");
+ ASSERT_EQ(attr_s1->m_DataSingleValue, currentTestData.S1);
+
+ ASSERT_NE(attr_i8, nullptr);
+ ASSERT_EQ(attr_i8->m_Name, i8_Single);
+ ASSERT_EQ(attr_i8->m_IsSingleValue, true);
+ ASSERT_EQ(attr_i8->m_Type, "signed char");
+ ASSERT_EQ(attr_i8->m_DataSingleValue, currentTestData.I8.front());
+
+ ASSERT_NE(attr_i16, nullptr);
+ ASSERT_EQ(attr_i16->m_Name, i16_Single);
+ ASSERT_EQ(attr_i16->m_IsSingleValue, true);
+ ASSERT_EQ(attr_i16->m_Type, "short");
+ ASSERT_EQ(attr_i16->m_DataSingleValue, currentTestData.I16.front());
+
+ ASSERT_NE(attr_i32, nullptr);
+ ASSERT_EQ(attr_i32->m_Name, i32_Single);
+ ASSERT_EQ(attr_i32->m_IsSingleValue, true);
+ ASSERT_EQ(attr_i32->m_Type, "int");
+ ASSERT_EQ(attr_i32->m_DataSingleValue, currentTestData.I32.front());
+
+ ASSERT_NE(attr_i64, nullptr);
+ ASSERT_EQ(attr_i64->m_Name, i64_Single);
+ ASSERT_EQ(attr_i64->m_IsSingleValue, true);
+#ifdef _WIN32
+ ASSERT_EQ(attr_i64->m_Type, "long long int");
+#else
+ ASSERT_EQ(attr_i64->m_Type, "long int");
+#endif
+ ASSERT_EQ(attr_i64->m_DataSingleValue, currentTestData.I64.front());
+
+ ASSERT_NE(attr_u8, nullptr);
+ ASSERT_EQ(attr_u8->m_Name, u8_Single);
+ ASSERT_EQ(attr_u8->m_IsSingleValue, true);
+ ASSERT_EQ(attr_u8->m_Type, "unsigned char");
+ ASSERT_EQ(attr_u8->m_DataSingleValue, currentTestData.U8.front());
+
+ ASSERT_NE(attr_u16, nullptr);
+ ASSERT_EQ(attr_u16->m_Name, u16_Single);
+ ASSERT_EQ(attr_u16->m_IsSingleValue, true);
+ ASSERT_EQ(attr_u16->m_Type, "unsigned short");
+ ASSERT_EQ(attr_u16->m_DataSingleValue, currentTestData.U16.front());
+
+ ASSERT_NE(attr_u32, nullptr);
+ ASSERT_EQ(attr_u32->m_Name, u32_Single);
+ ASSERT_EQ(attr_u32->m_IsSingleValue, true);
+ ASSERT_EQ(attr_u32->m_Type, "unsigned int");
+ ASSERT_EQ(attr_u32->m_DataSingleValue, currentTestData.U32.front());
+
+ ASSERT_NE(attr_u64, nullptr);
+ ASSERT_EQ(attr_u64->m_Name, u64_Single);
+ ASSERT_EQ(attr_u64->m_IsSingleValue, true);
+
+#ifdef _WIN32
+ ASSERT_EQ(attr_u64->m_Type, "unsigned long long int");
+#else
+ ASSERT_EQ(attr_u64->m_Type, "unsigned long int");
+#endif
+ ASSERT_EQ(attr_u64->m_DataSingleValue, currentTestData.U64.front());
+
+ ASSERT_NE(attr_r32, nullptr);
+ ASSERT_EQ(attr_r32->m_Name, r32_Single);
+ ASSERT_EQ(attr_r32->m_IsSingleValue, true);
+ ASSERT_EQ(attr_r32->m_Type, "float");
+ ASSERT_EQ(attr_r32->m_DataSingleValue, currentTestData.R32.front());
+
+ ASSERT_NE(attr_r64, nullptr);
+ ASSERT_EQ(attr_r64->m_Name, r64_Single);
+ ASSERT_EQ(attr_r64->m_IsSingleValue, true);
+ ASSERT_EQ(attr_r64->m_Type, "double");
+ ASSERT_EQ(attr_r64->m_DataSingleValue, currentTestData.R64.front());
+
+ bpRead.Close();
+ }
+}
+
+// ADIOS2 write read for array attributes
+TEST_F(BPWriteReadAttributeTestADIOS2, ADIOS2BPWriteReadArrayTypes)
+{
+ const std::string fName = "foo" + std::string(&adios2::PathSeparator, 1) +
+ "ADIOS2BPWriteAttributeReadArrayTypes.bp";
+
+ int mpiRank = 0, mpiSize = 1;
+#ifdef ADIOS2_HAVE_MPI
+ MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+ MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+ const std::string mpiRankString = std::to_string(mpiRank);
+ const std::string s1_Array = std::string("s1_Array_") + mpiRankString;
+ const std::string i8_Array = std::string("i8_Array_") + mpiRankString;
+ const std::string i16_Array = std::string("i16_Array_") + mpiRankString;
+ const std::string i32_Array = std::string("i32_Array_") + mpiRankString;
+ const std::string i64_Array = std::string("i64_Array_") + mpiRankString;
+ const std::string u8_Array = std::string("u8_Array_") + mpiRankString;
+ const std::string u16_Array = std::string("u16_Array_") + mpiRankString;
+ const std::string u32_Array = std::string("u32_Array_") + mpiRankString;
+ const std::string u64_Array = std::string("u64_Array_") + mpiRankString;
+ const std::string r32_Array = std::string("r32_Array_") + mpiRankString;
+ const std::string r64_Array = std::string("r64_Array_") + mpiRankString;
+
+ // When collective meta generation has landed, use
+ // generateNewSmallTestData(m_TestData, 0, mpiRank, mpiSize);
+ // Generate current testing data
+ SmallTestData currentTestData =
+ generateNewSmallTestData(m_TestData, 0, 0, 0);
+
+// Write test data using BP
+#ifdef ADIOS2_HAVE_MPI
+ adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON);
+#else
+ adios2::ADIOS adios(true);
+#endif
+ {
+ adios2::IO &io = adios.DeclareIO("TestIO");
+
+ // Declare Single Value Attributes
+ io.DefineAttribute<std::string>(s1_Array, currentTestData.S3.data(),
+ currentTestData.S3.size());
+
+ io.DefineAttribute<int8_t>(i8_Array, currentTestData.I8.data(),
+ currentTestData.I8.size());
+ io.DefineAttribute<int16_t>(i16_Array, currentTestData.I16.data(),
+ currentTestData.I16.size());
+ io.DefineAttribute<int32_t>(i32_Array, currentTestData.I32.data(),
+ currentTestData.I32.size());
+ io.DefineAttribute<int64_t>(i64_Array, currentTestData.I64.data(),
+ currentTestData.I64.size());
+
+ io.DefineAttribute<uint8_t>(u8_Array, currentTestData.U8.data(),
+ currentTestData.U8.size());
+ io.DefineAttribute<uint16_t>(u16_Array, currentTestData.U16.data(),
+ currentTestData.U16.size());
+ io.DefineAttribute<uint32_t>(u32_Array, currentTestData.U32.data(),
+ currentTestData.U32.size());
+ io.DefineAttribute<uint64_t>(u64_Array, currentTestData.U64.data(),
+ currentTestData.U64.size());
+
+ io.DefineAttribute<float>(r32_Array, currentTestData.R32.data(),
+ currentTestData.R32.size());
+ io.DefineAttribute<double>(r64_Array, currentTestData.R64.data(),
+ currentTestData.R64.size());
+
+ io.SetEngine("BPFileWriter");
+ io.AddTransport("file");
+
+ adios2::Engine &engine = io.Open(fName, adios2::Mode::Write);
+ // only attributes are written
+ engine.Close();
+ }
+
+ {
+ adios2::IO &ioRead = adios.DeclareIO("ioRead");
+
+ adios2::Engine &bpRead = ioRead.Open(fName, adios2::Mode::Read);
+
+ auto attr_s1 = ioRead.InquireAttribute<std::string>(s1_Array);
+
+ auto attr_i8 = ioRead.InquireAttribute<int8_t>(i8_Array);
+ auto attr_i16 = ioRead.InquireAttribute<int16_t>(i16_Array);
+ auto attr_i32 = ioRead.InquireAttribute<int32_t>(i32_Array);
+ auto attr_i64 = ioRead.InquireAttribute<int64_t>(i64_Array);
+
+ auto attr_u8 = ioRead.InquireAttribute<uint8_t>(u8_Array);
+ auto attr_u16 = ioRead.InquireAttribute<uint16_t>(u16_Array);
+ auto attr_u32 = ioRead.InquireAttribute<uint32_t>(u32_Array);
+ auto attr_u64 = ioRead.InquireAttribute<uint64_t>(u64_Array);
+
+ auto attr_r32 = ioRead.InquireAttribute<float>(r32_Array);
+ auto attr_r64 = ioRead.InquireAttribute<double>(r64_Array);
+
+ ASSERT_NE(attr_s1, nullptr);
+ ASSERT_EQ(attr_s1->m_Name, s1_Array);
+ ASSERT_EQ(attr_s1->m_IsSingleValue, false);
+ ASSERT_EQ(attr_s1->m_Type, "string");
+
+ ASSERT_NE(attr_i8, nullptr);
+ ASSERT_EQ(attr_i8->m_Name, i8_Array);
+ ASSERT_EQ(attr_i8->m_IsSingleValue, false);
+ ASSERT_EQ(attr_i8->m_Type, "signed char");
+
+ ASSERT_NE(attr_i16, nullptr);
+ ASSERT_EQ(attr_i16->m_Name, i16_Array);
+ ASSERT_EQ(attr_i16->m_IsSingleValue, false);
+ ASSERT_EQ(attr_i16->m_Type, "short");
+
+ ASSERT_NE(attr_i32, nullptr);
+ ASSERT_EQ(attr_i32->m_Name, i32_Array);
+ ASSERT_EQ(attr_i32->m_IsSingleValue, false);
+ ASSERT_EQ(attr_i32->m_Type, "int");
+
+ ASSERT_NE(attr_i64, nullptr);
+ ASSERT_EQ(attr_i64->m_Name, i64_Array);
+ ASSERT_EQ(attr_i64->m_IsSingleValue, false);
+#ifdef _WIN32
+ ASSERT_EQ(attr_i64->m_Type, "long long int");
+#else
+ ASSERT_EQ(attr_i64->m_Type, "long int");
+#endif
+
+ ASSERT_NE(attr_u8, nullptr);
+ ASSERT_EQ(attr_u8->m_Name, u8_Array);
+ ASSERT_EQ(attr_u8->m_IsSingleValue, false);
+ ASSERT_EQ(attr_u8->m_Type, "unsigned char");
+
+ ASSERT_NE(attr_u16, nullptr);
+ ASSERT_EQ(attr_u16->m_Name, u16_Array);
+ ASSERT_EQ(attr_u16->m_IsSingleValue, false);
+ ASSERT_EQ(attr_u16->m_Type, "unsigned short");
+
+ ASSERT_NE(attr_u32, nullptr);
+ ASSERT_EQ(attr_u32->m_Name, u32_Array);
+ ASSERT_EQ(attr_u32->m_IsSingleValue, false);
+ ASSERT_EQ(attr_u32->m_Type, "unsigned int");
+
+ ASSERT_NE(attr_u64, nullptr);
+ ASSERT_EQ(attr_u64->m_Name, u64_Array);
+ ASSERT_EQ(attr_u64->m_IsSingleValue, false);
+#ifdef _WIN32
+ ASSERT_EQ(attr_u64->m_Type, "unsigned long long int");
+#else
+ ASSERT_EQ(attr_u64->m_Type, "unsigned long int");
+#endif
+ ASSERT_NE(attr_r32, nullptr);
+ ASSERT_EQ(attr_r32->m_Name, r32_Array);
+ ASSERT_EQ(attr_r32->m_IsSingleValue, false);
+ ASSERT_EQ(attr_r32->m_Type, "float");
+
+ ASSERT_NE(attr_r64, nullptr);
+ ASSERT_EQ(attr_r64->m_Name, r64_Array);
+ ASSERT_EQ(attr_r64->m_IsSingleValue, false);
+ ASSERT_EQ(attr_r64->m_Type, "double");
+
+ auto &I8 = attr_i8->m_DataArray;
+ auto &I16 = attr_i16->m_DataArray;
+ auto &I32 = attr_i32->m_DataArray;
+ auto &I64 = attr_i64->m_DataArray;
+
+ auto &U8 = attr_u8->m_DataArray;
+ auto &U16 = attr_u16->m_DataArray;
+ auto &U32 = attr_u32->m_DataArray;
+ auto &U64 = attr_u64->m_DataArray;
+
+ const size_t Nx = 10;
+ for (size_t i = 0; i < Nx; ++i)
+ {
+ EXPECT_EQ(I8[i], currentTestData.I8[i]);
+ EXPECT_EQ(I16[i], currentTestData.I16[i]);
+ EXPECT_EQ(I32[i], currentTestData.I32[i]);
+ EXPECT_EQ(I64[i], currentTestData.I64[i]);
+
+ EXPECT_EQ(U8[i], currentTestData.U8[i]);
+ EXPECT_EQ(U16[i], currentTestData.U16[i]);
+ EXPECT_EQ(U32[i], currentTestData.U32[i]);
+ EXPECT_EQ(U64[i], currentTestData.U64[i]);
+ }
+
+ bpRead.Close();
+ }
+}
+
+//******************************************************************************
+// main
+//******************************************************************************
+
+int main(int argc, char **argv)
+{
+#ifdef ADIOS2_HAVE_MPI
+ MPI_Init(nullptr, nullptr);
+#endif
+
+ ::testing::InitGoogleTest(&argc, argv);
+ int result = RUN_ALL_TESTS();
+
+#ifdef ADIOS2_HAVE_MPI
+ MPI_Finalize();
+#endif
+
+ return result;
+}
diff --git a/testing/adios2/xml/TestXMLConfig.cpp b/testing/adios2/xml/TestXMLConfig.cpp
index de6e67b56..ecb2bb6f9 100644
--- a/testing/adios2/xml/TestXMLConfig.cpp
+++ b/testing/adios2/xml/TestXMLConfig.cpp
@@ -24,8 +24,8 @@ protected:
TEST_F(XMLConfigTest, TwoIOs)
{
- const std::string configFile(configDir + adios2::PathSeparator +
- "config1.xml");
+ const std::string configFile(
+ configDir + std::string(&adios2::PathSeparator, 1) + "config1.xml");
#ifdef ADIOS2_HAVE_MPI
adios2::ADIOS adios(configFile, MPI_COMM_WORLD, adios2::DebugON);
@@ -57,8 +57,8 @@ TEST_F(XMLConfigTest, TwoIOs)
TEST_F(XMLConfigTest, TwoEnginesException)
{
- const std::string configFile(configDir + adios2::PathSeparator +
- "config2.xml");
+ const std::string configFile(
+ configDir + std::string(&adios2::PathSeparator, 1) + "config2.xml");
#ifdef ADIOS2_HAVE_MPI
EXPECT_THROW(
--
GitLab