Skip to content
Snippets Groups Projects
Commit 230ef75b authored by Atkins, Charles Vernon's avatar Atkins, Charles Vernon Committed by GitHub
Browse files

Merge pull request #177 from chuckatkins/switch-to-pugixml

Switch to pugixml
parents 89fb784b 2e12b4b2
No related branches found
No related tags found
No related merge requests found
<?xml version="1.0"?>
<adios-config>
<io name="BPFile_N2N">
<engine type="BPFileWriter">
Threads=1; <!-- for vectorized memory operations and asynchronous tasks -->
ProfileUnits=Microseconds; <!-- Microseconds (default), Milliseconds, Seconds, Minutes, Hours -->
MaxBufferSize=20Mb; <!-- XXKb, XXMb, or XXXGb supported, 16Mb (default should depend on system) -->
InitialBufferSize=1Mb; <!-- XXKb, XXMb, or XXXGb supported, 16Kb (default should depend on system) -->
BufferGrowthFactor=2; <!-- exponential growth factor > 1, 1.5 (default, e.g. STL default=2), for this case: 1, 2, 4, 8, 16, 20 Mb-->
<!-- for vectorized memory operations and asynchronous tasks -->
<parameter key="Threads" value="1"/>
<!-- Microseconds (default), Milliseconds, Seconds, Minutes, Hours -->
<parameter key="ProfileUnits" value="Microseconds"/>
<!-- XXKb, XXMb, or XXXGb supported, 16Mb (default should depend on system) -->
<parameter key="MaxBufferSize" value="20Mb"/>
<!-- XXKb, XXMb, or XXXGb supported, 16Kb (default should depend on system) -->
<parameter key="InitialBufferSize" value="1Mb"/>
<!-- exponential growth factor > 1, 1.5 (default, e.g. STL default=2), for this case: 1, 2, 4, 8, 16, 20 Mb-->
<parameter key="BufferGrowthFactor" value="2"/>
</engine>
<transport type="File">
Library=POSIX;
ProfileUnits=Milliseconds;
<parameter key="Library" value="POSIX"/>
<parameter key="ProfileUnits" value="Milliseconds"/>
</transport>
</io>
</adios-config>
\ No newline at end of file
</adios-config>
......@@ -48,7 +48,7 @@ target_include_directories(adios2
$<INSTALL_INTERFACE:include>
PRIVATE ${ADIOS2_SOURCE_DIR}/source
)
target_link_libraries(adios2 PRIVATE adios2sys)
target_link_libraries(adios2 PRIVATE adios2sys pugixml)
find_package(Threads REQUIRED)
target_link_libraries(adios2 PUBLIC ${CMAKE_THREAD_LIBS_INIT})
......
......@@ -79,6 +79,18 @@ IO &ADIOS::DeclareIO(const std::string ioName)
return ioPair.first->second;
}
IO &ADIOS::GetIO(const std::string name)
{
auto itIO = m_IOs.find(name);
if (itIO == m_IOs.end())
{
throw std::invalid_argument(
"ERROR: Unable to find previously defined IO object with name \"" +
name + "\" in call to GetIO.");
}
return itIO->second;
}
// PRIVATE FUNCTIONS
void ADIOS::CheckMPI() const
{
......
......@@ -80,6 +80,11 @@ public:
*/
IO &DeclareIO(const std::string ioName);
/**
* Retrieve an already defined IO object
*/
IO &GetIO(const std::string name);
protected: // no const member to allow default empty and copy constructors
/** XML File to be read containing configuration information */
std::string m_ConfigFile;
......
......@@ -45,6 +45,8 @@ void IO::SetIOMode(const IOMode ioMode) { m_IOMode = ioMode; };
void IO::SetParameters(const Params &parameters) { m_Parameters = parameters; }
const Params &IO::GetParameters() const { return m_Parameters; }
unsigned int IO::AddTransport(const std::string type,
const std::vector<std::string> &parametersVector)
{
......
......@@ -87,6 +87,11 @@ public:
*/
void SetParameters(const Params &parameters = Params());
/**
* Retrieve existing parameter set
*/
const Params &GetParameters() const;
/**
* Adds a transport and its parameters for the method
* @param type must be a supported transport type under /include/transport
......
......@@ -11,473 +11,163 @@
#include "adiosXML.h"
/// \cond EXCLUDE_FROM_DOXYGEN
#include <sstream>
#include <stdexcept> //std::invalid_argument
/// \endcond
#include "adios2/ADIOSMPI.h"
#include "adios2/ADIOSTypes.h"
#include "adios2/helper/adiosString.h"
#include <pugixml.hpp>
namespace adios2
{
void RemoveCommentsXML(std::string &currentContent) noexcept
Params InitParametersXML(pugi::xml_node node, bool debugMode)
{
std::string::size_type startComment(currentContent.find("<!--"));
while (startComment != currentContent.npos)
Params params;
for (pugi::xml_node paramNode : node.children("parameter"))
{
std::string::size_type endComment(currentContent.find("-->"));
currentContent.erase(startComment, endComment - startComment + 3);
startComment = currentContent.find("<!--");
}
}
TagXML GetTagXML(const std::string tagName, const std::string &content,
std::string::size_type &position)
{
auto lf_SetPositions =
[](const std::string input, const std::string &content,
std::string::size_type &position) -> std::string::size_type {
const std::string::size_type inputPosition =
GetStringPositionXML(input, content, position);
if (inputPosition != std::string::npos)
{
position = inputPosition + input.size();
}
return inputPosition;
};
TagXML tagXML;
std::string name(tagName);
if (name.back() == ' ')
{
name.pop_back();
}
auto openingStart = lf_SetPositions("<" + name, content, position);
auto openingEnd = lf_SetPositions(">", content, position);
if (openingStart == std::string::npos || openingEnd == std::string::npos)
{
tagXML.IsFull = false;
return tagXML;
}
tagXML.Header = content.substr(openingStart, openingEnd + 1 - openingStart);
auto closingStart =
GetStringPositionXML("</" + name + ">", content, position);
if (closingStart == std::string::npos)
{
throw std::invalid_argument(
"ERROR: could not find closing tag </" + name +
"> in XML config file, in call to ADIOS constructor\n");
}
tagXML.IsFull = true;
tagXML.Elements =
content.substr(openingEnd + 1, closingStart - (openingEnd + 1));
// std::cout << "START..." << tagXML.Header << "...";
// std::cout << tagXML.Elements << "...END\n";
return tagXML;
}
std::string::size_type
GetStringPositionXML(const std::string input, const std::string &content,
const std::string::size_type &startPosition) noexcept
{
std::string::size_type foundPosition(content.find(input, startPosition));
if (foundPosition == content.npos)
{
return foundPosition;
}
// check if it is not inside " " or ' '
std::string::size_type currentPosition(startPosition);
while (foundPosition != content.npos)
{
const std::string::size_type singleQuotePosition(
content.find('\'', currentPosition));
const std::string::size_type doubleQuotePosition(
content.find('\"', currentPosition));
if ((singleQuotePosition == content.npos &&
doubleQuotePosition == content.npos) ||
(singleQuotePosition == content.npos &&
foundPosition < doubleQuotePosition) ||
(doubleQuotePosition == content.npos &&
foundPosition < singleQuotePosition) ||
(foundPosition < singleQuotePosition &&
foundPosition < doubleQuotePosition))
{
break;
}
// find the closing corresponding quote
std::string::size_type closingQuotePosition;
if (singleQuotePosition != content.npos &&
doubleQuotePosition == content.npos)
{
currentPosition = singleQuotePosition;
closingQuotePosition = content.find('\'', currentPosition + 1);
}
else if (singleQuotePosition == content.npos &&
doubleQuotePosition != content.npos)
{
currentPosition = doubleQuotePosition;
closingQuotePosition = content.find('\"', currentPosition + 1);
}
else
pugi::xml_attribute attrKey = paramNode.attribute("key");
if (!attrKey)
{
if (singleQuotePosition < doubleQuotePosition)
{
currentPosition = singleQuotePosition;
closingQuotePosition = content.find('\'', currentPosition + 1);
}
else
if (debugMode)
{
currentPosition = doubleQuotePosition;
closingQuotePosition = content.find('\"', currentPosition + 1);
throw std::invalid_argument("ERROR: XML: No \"key\" attribute "
"found on <parameter> element.");
}
}
// if can't find closing it's open until the end
if (closingQuotePosition == content.npos)
{
currentPosition == content.npos;
break;
}
currentPosition = closingQuotePosition + 1;
if (closingQuotePosition < foundPosition)
{
continue;
}
else
{
// if this point is reached it means it's a value inside " " or ' ',
// iterate
foundPosition = content.find(input, currentPosition);
currentPosition = foundPosition;
}
}
return foundPosition;
}
Params GetTagAttributesXML(const std::string &tagHeader)
{
auto lf_GetQuotedValue = [](const char quote,
const std::string::size_type &quotePosition,
std::string &currentTag) -> std::string {
currentTag = currentTag.substr(quotePosition + 1);
auto nextQuotePosition = currentTag.find(quote);
if (nextQuotePosition == currentTag.npos)
pugi::xml_attribute attrValue = paramNode.attribute("value");
if (!attrValue)
{
throw std::invalid_argument(
"ERROR: Invalid attribute in..." + currentTag +
"...check XML file, in call to ADIOS constructor\n");
}
const std::string value(currentTag.substr(0, nextQuotePosition));
currentTag = currentTag.substr(nextQuotePosition + 1);
return value;
};
auto lf_GetAttributes = [&](const std::string &tag) -> Params {
Params attributes;
std::string currentTag(tag.substr(tag.find_first_of(" \t\n")));
std::string::size_type currentPosition(0);
while (currentTag.find('=', currentPosition) !=
currentTag.npos) // equalPosition
{
currentTag = currentTag.substr(
currentTag.find_first_not_of(" \t\n", currentPosition));
auto equalPosition = currentTag.find('=');
if (currentTag.size() <= equalPosition + 1)
{
throw std::invalid_argument(
"ERROR: tag " + tag +
" is incomplete, check XML config file, "
"in call to ADIOS constructor\n");
}
std::string key(currentTag.substr(0, equalPosition));
key.erase(key.find_last_not_of(" \t\n") + 1);
std::string value;
auto quotePosition =
currentTag.find_first_not_of(" \t\n", equalPosition + 1);
const char quote = currentTag.at(quotePosition);
if (quote == '\'' || quote == '"')
{
value = lf_GetQuotedValue(quote, quotePosition, currentTag);
}
else
if (debugMode)
{
throw std::invalid_argument(
"ERROR: quote must be \" or ' in XML config tag " + tag +
", in call to ADIOS constructor");
throw std::invalid_argument("ERROR: XML: No \"value\" "
"attribute found on <parameter> "
"element.");
}
attributes.emplace(key, value);
currentPosition = quotePosition + value.size() + 1;
}
return attributes;
};
// BODY of function starts here
Params attributes;
// eliminate < >
std::string openingTag = tagHeader.substr(1, tagHeader.size() - 2);
if (tagHeader.back() == '/') // last char is / --> "XML empty tag"
{
// attributes = lf_GetAttributes(openingTag);
// throw exception here, ADIOS2 doesn't allow XML empty tags
}
else if (tagHeader[0] == '/') // first char is / ---> closing tag
{
attributes = lf_GetAttributes(openingTag);
if (attributes.size() > 0)
{
throw std::invalid_argument(
"ERROR: closing tag " + tagHeader +
" can't have attributes, in call to ADIOS constructor\n");
}
}
else // opening tag
{
attributes = lf_GetAttributes(openingTag);
}
return attributes;
}
void InitXML(const std::string configXML, const MPI_Comm mpiComm,
const bool debugMode,
std::vector<std::shared_ptr<Transform>> &transforms,
std::map<std::string, IO> &ios)
{
// independent IO
std::string fileContents(FileToString(configXML));
if (fileContents.empty())
{
// issue a warning?
return;
}
RemoveCommentsXML(fileContents);
// adios-config
std::string::size_type position(0);
const TagXML adiosConfigXML(
GetTagXML("adios-config", fileContents, position));
// process transforms, not yet implemented
while (position != std::string::npos)
{
const TagXML transformXML(
GetTagXML("transform ", adiosConfigXML.Elements, position));
if (transformXML.Header.empty())
{
break;
continue;
}
// InitTransform(transformTag, debugMode, transforms);
}
position = 0;
// process IOs
while (position != std::string::npos)
{
// io
const TagXML ioXML(GetTagXML("io ", adiosConfigXML.Elements, position));
if (ioXML.Header.empty()) // no more groups to find
{
break;
}
InitIOXML(ioXML, mpiComm, debugMode, transforms, ios);
params.emplace(attrKey.value(), attrValue.value());
}
return params;
}
void InitIOXML(const TagXML &ioXML, const MPI_Comm mpiComm,
void InitIOXML(const pugi::xml_node ioNode, const MPI_Comm mpiComm,
const bool debugMode,
std::vector<std::shared_ptr<Transform>> &transforms,
std::map<std::string, IO> &ios)
{
const Params ioAttributes(GetTagAttributesXML(ioXML.Header));
std::string ioName;
for (const auto &ioAttribute : ioAttributes)
{
if (ioAttribute.first == "name")
{
ioName = ioAttribute.second;
}
}
if (debugMode)
// Extract <io name=""> attribute
pugi::xml_attribute nameAttr = ioNode.attribute("name");
if (!nameAttr)
{
if (ioName.empty())
if (debugMode)
{
throw std::invalid_argument(
"ERROR: io name=\"value\" attribute not found in opening XML "
"tag " +
ioXML.Header +
", check XML config file, in call to ADIOS constructor\n");
}
if (ios.count(ioName) == 1) // io exists
{
throw std::invalid_argument("ERROR: io name " + ioName +
" must be unique in XML config file, "
"in call to ADIOS constructor\n");
"ERROR: XML: No \"name\" attribute found on <io> element.");
}
return;
}
std::string ioName = nameAttr.value();
// emplace io with inConfigFile argument as true
auto itIO = ios.emplace(ioName, IO(ioName, mpiComm, true, debugMode));
// process engine
std::string::size_type position(0);
TagXML engineXML(GetTagXML("engine ", ioXML.Elements, position));
if (!engineXML.Header.empty()) // found first one
{
InitEngineXML(engineXML, debugMode, itIO.first->second);
}
// Build the IO object
auto ioIt = ios.emplace(ioName, IO(ioName, mpiComm, true, debugMode));
IO &io = ioIt.first->second;
if (debugMode)
// Extract <engine> element
pugi::xml_node engineNode = ioNode.child("engine");
if (!engineNode)
{
// try finding a 2nd one from current position
TagXML engineXML(GetTagXML("engine ", ioXML.Elements, position));
if (!engineXML.Header.empty()) // found first one
{
throw std::invalid_argument(
"ERROR: more than one engine found in <io name=" + ioName +
"...>, only one per io tag is allowed in XML "
"config file, in call to "
"ADIOS constructor\n");
}
throw std::invalid_argument(
"ERROR: XML: No <engine> element found in <io> element.");
}
position = 0;
// process transports
while (position != std::string::npos)
pugi::xml_attribute engineTypeAttr = engineNode.attribute("type");
if (!engineTypeAttr)
{
TagXML transportXML(GetTagXML("transport", ioXML.Elements, position));
if (transportXML.Header.empty()) // no more groups to find
{
break;
}
InitTransportXML(transportXML, debugMode, itIO.first->second);
throw std::invalid_argument(
"ERROR: XML: No \"type\" attribute found on <engine> element.");
}
}
io.SetEngine(engineTypeAttr.value());
void InitEngineXML(const TagXML &engineXML, const bool debugMode, IO &io)
{
const Params attributes = GetTagAttributesXML(engineXML.Header);
// Process <engine> parameters
io.SetParameters(InitParametersXML(engineNode, debugMode));
std::string type;
for (const auto &attribute : attributes)
// Extract and process <transport> elements
for (pugi::xml_node transportNode : ioNode.children("transport"))
{
if (attribute.first == "type")
pugi::xml_attribute typeAttr = transportNode.attribute("type");
if (!typeAttr)
{
type = attribute.second;
break;
if (debugMode)
{
throw std::invalid_argument("ERROR: XML: No \"type\" attribute "
"found on <transport> element.");
}
continue;
}
io.AddTransport(typeAttr.value(),
InitParametersXML(transportNode, debugMode));
}
if (!type.empty())
{
io.SetEngine(type);
}
io.SetParameters(ParseParamsXML(engineXML.Elements, debugMode));
}
void InitTransportXML(const TagXML &transportXML, const bool debugMode, IO &io)
void InitXML(const std::string configXML, const MPI_Comm mpiComm,
const bool debugMode,
std::vector<std::shared_ptr<Transform>> &transforms,
std::map<std::string, IO> &ios)
{
const Params attributes = GetTagAttributesXML(transportXML.Header);
int mpiRank;
MPI_Comm_rank(mpiComm, &mpiRank);
std::string fileContents;
unsigned long long len;
std::string type;
for (const auto &attribute : attributes)
// Read the file on rank 0 and broadcast it to everybody else
if (mpiRank == 0)
{
if (attribute.first == "type")
{
type = attribute.second;
break;
}
fileContents = FileToString(configXML);
len = static_cast<unsigned long long>(fileContents.size());
}
if (type.empty())
MPI_Bcast(&len, 1, MPI_UNSIGNED_LONG, 0, mpiComm);
if (mpiRank != 0)
{
throw std::invalid_argument(
"ERROR: missing transport type in " + transportXML.Header +
", in XML config file, in call to ADIOS constructor\n");
fileContents.resize(len);
}
MPI_Bcast(const_cast<char *>(fileContents.data()), len, MPI_CHAR, 0,
mpiComm);
io.AddTransport(type, ParseParamsXML(transportXML.Elements, debugMode));
}
Params ParseParamsXML(const std::string &tagElements, const bool debugMode)
{
auto start = tagElements.find_first_not_of(" \t\n");
auto end = tagElements.find_last_not_of(" \t\n");
std::string parametersString(tagElements.substr(start, end - start + 1));
if (debugMode)
pugi::xml_document doc;
auto parse_result = doc.load_buffer_inplace(
const_cast<char *>(fileContents.data()), fileContents.size());
if (!parse_result)
{
if (parametersString.back() != ';')
if (debugMode)
{
throw std::invalid_argument(
"ERROR: parameters in config XML file must end with a ; " +
tagElements + ", in call to ADIOS constructor\n");
std::string("ERROR: XML: Parse error: ") +
parse_result.description());
}
return;
}
std::istringstream parametersSS(parametersString);
std::string pair;
Params parameters;
while (std::getline(parametersSS, pair, ';'))
pugi::xml_node configNode = doc.child("adios-config");
if (!configNode)
{
pair = pair.substr(pair.find_first_not_of(" \t\n"));
auto equalPosition = pair.find("=");
if (debugMode)
{
if (equalPosition == std::string::npos ||
equalPosition == pair.size())
{
throw std::invalid_argument("ERROR: wrong parameter " + pair +
" format is "
"key=value in XML config file, in "
"call to ADIOS constructor\n");
}
throw std::invalid_argument(
"ERROR: XML: No <adios-config> element found");
}
return;
}
const std::string key(pair.substr(0, equalPosition));
const std::string value(pair.substr(equalPosition + 1));
parameters.emplace(key, value);
ios.clear();
for (pugi::xml_node ioNode : configNode.children("io"))
{
InitIOXML(ioNode, mpiComm, debugMode, transforms, ios);
}
return parameters;
}
} // end namespace adios
......@@ -25,24 +25,6 @@
namespace adios2
{
struct TagXML
{
std::string Header;
std::string Elements;
bool IsFull;
};
void RemoveCommentsXML(std::string &currentContent) noexcept;
TagXML GetTagXML(const std::string tagName, const std::string &content,
std::string::size_type &position);
std::string::size_type
GetStringPositionXML(const std::string input, const std::string &content,
const std::string::size_type &startPosition) noexcept;
Params GetTagAttributesXML(const std::string &tagHeader);
/**
* Called inside the ADIOS XML constructors to get contents from file,
* broadcast and fill transforms and ios
......@@ -56,17 +38,6 @@ void InitXML(const std::string configXML, const MPI_Comm mpiComm,
const bool debugMode,
std::vector<std::shared_ptr<Transform>> &transforms,
std::map<std::string, IO> &ios);
void InitIOXML(const TagXML &ioXML, const MPI_Comm mpiComm,
const bool debugMode,
std::vector<std::shared_ptr<Transform>> &transforms,
std::map<std::string, IO> &ios);
void InitEngineXML(const TagXML &engineXML, const bool debugMode, IO &io);
void InitTransportXML(const TagXML &transportXML, const bool debugMode, IO &io);
Params ParseParamsXML(const std::string &tagContents, const bool debugMode);
}
#endif /* ADIOS2_HELPER_ADIOSXML_H_ */
......@@ -6,3 +6,4 @@
add_subdirectory(interface)
add_subdirectory(engine)
add_subdirectory(bindings)
add_subdirectory(xml)
#------------------------------------------------------------------------------#
# Distributed under the OSI-approved Apache License, Version 2.0. See
# accompanying file Copyright.txt for details.
#------------------------------------------------------------------------------#
add_executable(TestXMLConfig TestXMLConfig.cpp)
target_link_libraries(TestXMLConfig adios2 gtest gtest_main)
target_compile_definitions(TestXMLConfig PRIVATE
"XML_CONFIG_DIR=${CMAKE_CURRENT_SOURCE_DIR}"
)
gtest_add_tests(TARGET TestXMLConfig)
#include <cstdint>
#include <iostream>
#include <stdexcept>
#include <adios2.h>
#include <gtest/gtest.h>
#define str_helper(X) #X
#define str(X) str_helper(X)
class XMLConfigTest : public ::testing::Test
{
public:
XMLConfigTest() : configDir(str(XML_CONFIG_DIR)) {}
protected:
// virtual void SetUp() { }
// virtual void TearDown() { }
std::string configDir;
};
TEST_F(XMLConfigTest, TwoIOs)
{
std::string configFile = configDir + "/config1.xml";
#ifdef ADIOS2_HAVE_MPI
adios2::ADIOS adios(configFile, MPI_COMM_WORLD, adios2::DebugON);
#else
adios2::ADIOS adios(configFile, adios2::DebugON);
#endif
EXPECT_NO_THROW({
adios2::IO &io = adios.GetIO("Test IO 1");
const adios2::Params &params = io.GetParameters();
ASSERT_EQ(params.size(), 5);
EXPECT_THROW(params.at("DoesNotExist"), std::out_of_range);
EXPECT_EQ(params.at("Threads"), "1");
EXPECT_EQ(params.at("ProfileUnits"), "Microseconds");
EXPECT_EQ(params.at("MaxBufferSize"), "20Mb");
EXPECT_EQ(params.at("InitialBufferSize"), "1Mb");
EXPECT_EQ(params.at("BufferGrowthFactor"), "2");
auto engine = io.Open("Test BP Writer 1", adios2::OpenMode::Write);
});
EXPECT_NO_THROW({
adios2::IO &io = adios.GetIO("Test IO 2");
const adios2::Params &params = io.GetParameters();
ASSERT_EQ(params.size(), 0);
});
}
int main(int argc, char **argv)
{
#ifdef ADIOS2_HAVE_MPI
MPI_Init(nullptr, nullptr);
#endif
::testing::InitGoogleTest(&argc, argv);
int result = RUN_ALL_TESTS();
#ifdef ADIOS2_HAVE_MPI
MPI_Finalize();
#endif
return result;
}
<?xml version="1.0"?>
<adios-config>
<io name="Test IO 1">
<engine type="BPFileWriter">
<parameter key="Threads" value="1"/>
<parameter key="ProfileUnits" value="Microseconds"/>
<parameter key="MaxBufferSize" value="20Mb"/>
<parameter key="InitialBufferSize" value="1Mb"/>
<parameter key="BufferGrowthFactor" value="2"/>
</engine>
<transport type="File">
<parameter key="Library" value="POSIX"/>
<parameter key="ProfileUnits" value="Milliseconds"/>
</transport>
</io>
<io name="Test IO 2">
<engine type="BPFileWriter" />
</io>
</adios-config>
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment