From 23f3718794e0ba7b3d5fbcca779a17ec20f68170 Mon Sep 17 00:00:00 2001 From: William F Godoy <williamfgc@yahoo.com> Date: Thu, 15 Jun 2017 18:12:31 -0400 Subject: [PATCH] Started working on xml parser --- .../runtimeconfig/grandSchema.xml | 76 ++++++++++ .../runtimeconfig/multipleIOResolutions.xml | 24 ++++ .../multipleTransformedVariables.xml | 26 ++++ examples/hello/bpWriter/config.xml | 41 ------ examples/hello/bpWriter/helloBPWriter.xml | 15 ++ source/adios2/ADIOSMPI.h | 13 ++ source/adios2/helper/adiosSystem.cpp | 40 ++++++ source/adios2/helper/adiosSystem.h | 12 ++ source/adios2/helper/adiosXML.cpp | 136 ++++++++++++------ source/adios2/helper/adiosXML.h | 38 +++-- 10 files changed, 326 insertions(+), 95 deletions(-) create mode 100644 examples/experimental/runtimeconfig/grandSchema.xml create mode 100644 examples/experimental/runtimeconfig/multipleIOResolutions.xml create mode 100644 examples/experimental/runtimeconfig/multipleTransformedVariables.xml delete mode 100644 examples/hello/bpWriter/config.xml create mode 100644 examples/hello/bpWriter/helloBPWriter.xml diff --git a/examples/experimental/runtimeconfig/grandSchema.xml b/examples/experimental/runtimeconfig/grandSchema.xml new file mode 100644 index 000000000..e069a7764 --- /dev/null +++ b/examples/experimental/runtimeconfig/grandSchema.xml @@ -0,0 +1,76 @@ +<?xml version="1.0"?> +<adios-config> + + <io name="Output"> + + <var name="myArray"> <!-- provide a sequence of transforms --> + <transform type="BZip2"> <!-- not implemented yet --> + BlockSize100K=3; <!-- must be a number between 0 and 9--> + </transform> + <transform type="Zfp"> <!-- not implemented yet, mutually exclusive options --> + Tolerance=1e-3; <!-- must be a number between 0 and 9--> + <!-- Rate=1; --> <!-- must be a number between 0 and 9--> + <!-- Precision=1; --> <!-- must be a number between 0 and 9--> + </transform> + </var> + + <engine type="BPSuperEngine"> + Threads=4; <!-- for memory operations and asynchronous tasks --> + Verbose=4; <!-- 0 (default) to 5, not implemented --> + ProfileUnits=Microseconds; <!-- Microseconds (default), Milliseconds, Seconds, Minutes, Hours --> + MaxBufferSize=20Mb; <!-- XXKb, XXMb, or XXXGb supported, 16Mb (default should depend on system) --> + InitialBufferSize=1Mb; <!-- XXKb, XXMb, or XXXGb supported, 16Kb (default shoudl depende on system) --> + BufferGrowthFactor=2; <!-- exponential growth factor > 1, 1.5 (default), for this case: 1, 2, 4, 8, 16, 20 Mb--> + </engine> + + <transport type="File"> + ProfileUnits=Microseconds; <!-- Microseconds, Milliseconds, Seconds, Minutes, Hours --> + Library=stdio; <!-- lowest level IO library, POSIX (default), stdio (C FILE*), fstream (C++), and interop: hdf5, netcdf? --> + CacheSize=8192; <!-- not to be confused with ADIOS heap buffer, this is the low-level setvbuffer, putsetbuf, default is 8192 bytes in most systems --> + CollectiveMetadata=Yes; <!-- this transport requires a collective/global metadata at the end, default = no --> + </transport> + + <transport type="File"> + ProfileUnits=Microseconds; <!-- Microseconds, Milliseconds, Seconds, Minutes, Hours --> + Library=hdf5; <!-- lowest level IO library, POSIX (default), stdio (C FILE*), fstream (C++), and interop: hdf5, netcdf? --> + Name=data.h5; <!-- Mandatory if more than one File transport to avoid conflict --> + </transport> + + <transport type="SharedMemory"> + ProfileUnits=Microseconds; <!-- Microseconds, Milliseconds, Seconds, Minutes, Hours --> + Library=SystemV; <!-- lowest level shared memory library: SystemV (default), Boost (interprocess), IPC (POSIX), IPCMMap (POSIX) --> + Name=data.shmem; <!-- Mandatory to create process key --> + SetSize=10Mb; <!-- Mandatory and fixed size (can't be extended) --> + </transport> + + <transport type="NVRAM"> + ProfileUnits=Microseconds; <!-- Microseconds, Milliseconds, Seconds, Minutes, Hours --> + Library=POSIX; <!-- lowest level library --> + Name=/bb_location/data.bb; <!-- Mandatory --> + SetSize=10Mb; <!-- Mandatory and fixed size (can't be extended) --> + </transport> + + <transport type="WAN"> + ProfileUnits=Microseconds; <!-- Microseconds, Milliseconds, Seconds, Minutes, Hours --> + IPAddress=128.219.4.85; <!-- Mandatory --> + Port=500; <!-- Default=500? --> + Library=zmq; <!-- lowest level IO library, zmq, UDP, etc. --> + CacheSize=8192; <!-- not to be confused with ADIOS heap buffer, this is transport library cache, always in bytes --> + </transport> + + <transport type="RDMA"> <!-- Staging gets more complex depending on library requirements--> + ProfileUnits=Microseconds; <!-- Microseconds, Milliseconds, Seconds, Minutes, Hours --> + Library=ibverbs; <!-- SST (default), Flexpath, Dataspaces, ibverbs --> + Port=18515; <!-- ibverbs --> + IBPort=1; <!-- ibverbs --> + SetSize=1Mb; <!-- Mandatory and fixed size (can't be extended) --> + TXDepth=100; <!-- ibverbs --> + SocketFD=-1; <!-- ibverbs --> + ServerName=NULL; <!-- ibverbs --> + RemoteConnection=NULL; <!-- ibverbs --> + IBDev=NULL; <!-- ibverbs --> + </transport> + + </io> + +</adios-config> \ No newline at end of file diff --git a/examples/experimental/runtimeconfig/multipleIOResolutions.xml b/examples/experimental/runtimeconfig/multipleIOResolutions.xml new file mode 100644 index 000000000..28fe85408 --- /dev/null +++ b/examples/experimental/runtimeconfig/multipleIOResolutions.xml @@ -0,0 +1,24 @@ +<?xml version="1.0"?> +<adios-config> + + <io name="LowResolution"> + <var name="myArray"> <!-- provide a sequence of transforms --> + <transform type="Zfp"> <!-- not implemented yet, mutually exclusive options --> + Tolerance=1e-2; <!-- must be a number between 0 and 9--> + <!-- Rate=1; --> <!-- must be a number between 0 and 9--> + <!-- Precision=1; --> <!-- must be a number between 0 and 9--> + </transform> + </var> + </io> + + <io name="HighResolution"> + <var name="myArray"> <!-- provide a sequence of transforms --> + <transform type="Zfp"> <!-- not implemented yet, mutually exclusive options --> + Tolerance=1e-5; <!-- must be a number between 0 and 9--> + <!-- Rate=1; --> <!-- must be a number between 0 and 9--> + <!-- Precision=1; --> <!-- must be a number between 0 and 9--> + </transform> + </var> + </io> + +</adios-config> \ No newline at end of file diff --git a/examples/experimental/runtimeconfig/multipleTransformedVariables.xml b/examples/experimental/runtimeconfig/multipleTransformedVariables.xml new file mode 100644 index 000000000..8338320d5 --- /dev/null +++ b/examples/experimental/runtimeconfig/multipleTransformedVariables.xml @@ -0,0 +1,26 @@ +<?xml version="1.0"?> +<adios-config> + + <io name="LowResolution"> + <transform type="Zfp"> <!-- not implemented yet, mutually exclusive options --> + Tolerance=1e-2; <!-- must be a number between 0 and 9 --> + <!-- Rate=1; --> <!-- must be a number between 0 and 9 --> + <!-- Precision=1; --> <!-- must be a number between 0 and 9 --> + <variable name="myArray1"></variable> + <variable name="myArray2"></variable> + <variable name="myArray3"></variable> + </transform> + </io> + + <io name="HighResolution"> + <transform type="Zfp"> <!-- not implemented yet, mutually exclusive options --> + Tolerance=1e-5; <!-- must be a number between 0 and 9 --> + <!-- Rate=1; --> <!-- must be a number between 0 and 9 --> + <!-- Precision=1; --> <!-- must be a number between 0 and 9 --> + <variable name="myArray1"></variable> + <variable name="myArray2"></variable> + <variable name="myArray3"></variable> + </transform> + </io> + +</adios-config> diff --git a/examples/hello/bpWriter/config.xml b/examples/hello/bpWriter/config.xml deleted file mode 100644 index 10dfa830e..000000000 --- a/examples/hello/bpWriter/config.xml +++ /dev/null @@ -1,41 +0,0 @@ -<?xml version="1.0"?> -<adios-config> - - <io name="Output"> - <engine name="BPFileWriter">verbose=4;profile_units=mus</engine> - <transport name= "File"> - profile_units=mus; - abort_on_error; - have_metadata_file - = - no; - </transport> - - <!-- Create a named transform and add variables to it here. - name is optional, required only if it is used outside the definition - options is optional to pass parameters to the transformation - --> - <transform name="LossyCompression" transform="zfp" options="accuracy=0.001"> - <var name="myMatrix"/> - <var name="ThisVarDoesNotExists"/> - </transform> - - <!-- Unnamed transformation --> - <transform transform="bzip2"> - <var name="myMatrix2"/> - </transform> - - <!-- A variable can have its own private transform definition. - Also its own ordered chain of transformations. - Also can refer to a transform defined previously - --> - <var name="myDoubles"> - <transform transform="identity">verbose=DEBUG</transform> - <transform name="LossyCompression"/> - </var> - - <buffer max-size-MB="20"/> - </io> - -</adios-config> - diff --git a/examples/hello/bpWriter/helloBPWriter.xml b/examples/hello/bpWriter/helloBPWriter.xml new file mode 100644 index 000000000..00afc9126 --- /dev/null +++ b/examples/hello/bpWriter/helloBPWriter.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<adios-config> + + <io name="BPFile_N2N"> + <engine type="BPFileWriter"> + Threads=1; <!-- for vectorized memory operations and asynchronous tasks --> + ProfileUnits=Microseconds; <!-- Microseconds (default), Milliseconds, Seconds, Minutes, Hours --> + MaxBufferSize=20Mb; <!-- XXKb, XXMb, or XXXGb supported, 16Mb (default should depend on system) --> + InitialBufferSize=1Mb; <!-- XXKb, XXMb, or XXXGb supported, 16Kb (default shoudl depende on system) --> + BufferGrowthFactor=2; <!-- exponential growth factor > 1, 1.5 (default), for this case: 1, 2, 4, 8, 16, 20 Mb--> + </engine> + </io> + +</adios-config> + diff --git a/source/adios2/ADIOSMPI.h b/source/adios2/ADIOSMPI.h index 3524532d3..95ae7619a 100644 --- a/source/adios2/ADIOSMPI.h +++ b/source/adios2/ADIOSMPI.h @@ -14,4 +14,17 @@ #include "adios2/mpidummy.h" #endif +#include <climits> //UXXX_MAX +#include <cstdint> //SIZE_MAX + +#if SIZE_MAX == UINT_MAX +#define ADIOS2_MPI_SIZE_T MPI_UNSIGNED +#elif SIZE_MAX == ULONG_MAX +#define ADIOS2_MPI_SIZE_T MPI_UNSIGNED_LONG +#elif SIZE_MAX == ULLONG_MAX +#define ADIOS2_MPI_SIZE_T MPI_UNSIGNED_LONG_LONG +#else +#error "size_t could not be mapped to a MPI Type" +#endif + #endif /* ADIOS2_ADIOSMPI_H_ */ diff --git a/source/adios2/helper/adiosSystem.cpp b/source/adios2/helper/adiosSystem.cpp index 983613079..0dd5896d8 100644 --- a/source/adios2/helper/adiosSystem.cpp +++ b/source/adios2/helper/adiosSystem.cpp @@ -16,7 +16,9 @@ #include <chrono> //system_clock, now #include <ctime> //std::ctime +#include "adios2/ADIOSMPI.h" #include "adios2/ADIOSTypes.h" +#include "adios2/helper/adiosString.h" namespace adios { @@ -77,4 +79,42 @@ std::string LocalTimeDate() noexcept return std::string(std::ctime(&now)); } +std::string BroadcastFileContents(const std::string &fileName, + MPI_Comm mpiComm) noexcept +{ + std::string fileContents; + size_t characterCount = 0; + + int rank; + MPI_Comm_rank(mpiComm, &rank); + + if (rank == 0) // sender + { + fileContents = FileToString(fileName); + characterCount = fileContents.size(); + + // broadcast size for allocation + MPI_Bcast(&characterCount, 1, ADIOS2_MPI_SIZE_T, 0, mpiComm); + + // broadcast contents + MPI_Bcast(const_cast<char *>(fileContents.c_str()), + static_cast<int>(characterCount), MPI_CHAR, 0, mpiComm); + } + else // receivers + { + // receive size + MPI_Bcast(&characterCount, 1, ADIOS2_MPI_SIZE_T, 0, mpiComm); + + // allocate receiver + std::vector<char> fileContentsReceiver(characterCount); + MPI_Bcast(fileContentsReceiver.data(), static_cast<int>(characterCount), + MPI_CHAR, 0, mpiComm); + + fileContents.assign(fileContentsReceiver.begin(), + fileContentsReceiver.end()); + } + + return fileContents; +} + } // end namespace adios diff --git a/source/adios2/helper/adiosSystem.h b/source/adios2/helper/adiosSystem.h index 9374d1603..acbae391d 100644 --- a/source/adios2/helper/adiosSystem.h +++ b/source/adios2/helper/adiosSystem.h @@ -17,6 +17,8 @@ #include <vector> /// \endcond +#include "adios2/ADIOSMPICommOnly.h" + namespace adios { @@ -41,6 +43,16 @@ bool IsLittleEndian() noexcept; */ std::string LocalTimeDate() noexcept; +/** + * Rank 0: opens file, dumps contents string and broadcast. + * Others: receive std::vector<char> and copy to a string + * @param fileName + * @param mpiComm + * @return fileContents as a single string + */ +std::string BroadcastFileContents(const std::string &fileName, + MPI_Comm mpiComm) noexcept; + } // end namespace adios #endif /* ADIOS2_HELPER_ADIOSSYSTEM_H_ */ diff --git a/source/adios2/helper/adiosXML.cpp b/source/adios2/helper/adiosXML.cpp index 746a43614..3691d0ce7 100644 --- a/source/adios2/helper/adiosXML.cpp +++ b/source/adios2/helper/adiosXML.cpp @@ -14,12 +14,14 @@ #include <stdexcept> //std::invalid_argument /// \endcond +#include "adios2/helper/adiosSystem.h" + namespace adios { -void GetSubString(const std::string initialTag, const std::string finalTag, - const std::string content, std::string &subString, - std::string::size_type ¤tPosition) +std::string GetSubString(const std::string initialTag, + const std::string finalTag, const std::string &content, + std::string::size_type ¤tPosition) { auto lf_Wipe = [](std::string &subString, std::string::size_type ¤tPosition) { @@ -36,11 +38,13 @@ void GetSubString(const std::string initialTag, const std::string finalTag, }; // BODY OF FUNCTION STARTS HERE + std::string subString; + std::string::size_type start(content.find(initialTag, currentPosition)); if (start == content.npos) { lf_Wipe(subString, currentPosition); - return; + return subString; } currentPosition = start; @@ -48,7 +52,7 @@ void GetSubString(const std::string initialTag, const std::string finalTag, if (end == content.npos) { lf_Wipe(subString, currentPosition); - return; + return subString; } // here make sure the finalTag is not a value surrounded by " " or ' ', if @@ -101,11 +105,11 @@ void GetSubString(const std::string initialTag, const std::string finalTag, } } - if (closingQuotePosition == - content.npos) // if can't find closing it's open until the end + // if can't find closing it's open until the end + if (closingQuotePosition == content.npos) { lf_Wipe(subString, currentPosition); - return; + return subString; } currentPosition = closingQuotePosition + 1; @@ -123,6 +127,7 @@ void GetSubString(const std::string initialTag, const std::string finalTag, subString = content.substr(start, end - start + finalTag.size()); currentPosition = end; + return subString; } void GetQuotedValue(const char quote, @@ -195,6 +200,18 @@ void GetPairsFromTag( } } +void RemoveXMLComments(std::string ¤tContent) const noexcept +{ + std::string::size_type startComment(currentContent.find("<!--")); + + while (startComment != currentContent.npos) + { + std::string::size_type endComment(currentContent.find("-->")); + currentContent.erase(startComment, endComment - startComment + 3); + startComment = currentContent.find("<!--"); + } +} + // void SetMembers( const std::string& fileContent, const MPI_Comm mpiComm, // const bool debugMode, // std::string& hostLanguage, std::vector< @@ -350,47 +367,82 @@ void GetPairsFromTag( // } //} -// void InitXML( const std::string xmlConfigFile, const MPI_Comm mpiComm, const -// bool debugMode, -// std::string& hostLanguage, std::vector< -// std::shared_ptr<Transform> >& transforms, -// std::map< std::string, Group >& groups ) +void InitXML(const std::string configXML, const MPI_Comm mpiComm, + const bool debugMode, + std::vector<std::shared_ptr<Transform>> &transforms, + std::map<std::string, IO> &ios) +{ + // if using collective IO only? + std::string fileContents = BroadcastFileContents(configXML, mpiComm); + + SetMembers(fileContents, debugMode, transforms, ios); +} + +// void SetMembers(const std::string &fileContents, const bool debugMode, +// std::vector<std::shared_ptr<Transform>> &transforms, +// std::map<std::string, IO> &ios) //{ -// int xmlFileContentSize; -// std::string xmlFileContent; // -// int rank; -// MPI_Comm_rank( mpiComm, &rank ); +// // adios-config +// std::string::size_type currentPosition(0); +// std::string currentContent(GetSubString("<adios-config ", +// "</adios-config>", +// fileContents, currentPosition)); +// RemoveXMLComments(currentContent); // -// if( rank == 0 ) //serial part -// { -// DumpFileToString( xmlConfigFile, xmlFileContent ); //in -// ADIOSFunctions.h dumps all XML Config File to xmlFileContent -// xmlFileContentSize = xmlFileContent.size( ) + 1; // add one for the -// null character -// -// MPI_Bcast( &xmlFileContentSize, 1, MPI_INT, 0, mpiComm ); //broadcast -// size for allocation -// MPI_Bcast( (char*)xmlFileContent.c_str(), xmlFileContentSize, -// MPI_CHAR, 0, mpiComm ); //broadcast contents -// } -// else +// while (currentPosition != std::string::npos) // { -// MPI_Bcast( &xmlFileContentSize, 1, MPI_INT, 0, mpiComm ); //receive -// size +// // io +// std::string io( +// GetSubString("<io ", "</io>", currentContent, currentPosition)); // -// char* xmlFileContentMPI = new char[ xmlFileContentSize ]; //allocate -// xml C-char -// MPI_Bcast( xmlFileContentMPI, xmlFileContentSize, MPI_CHAR, 0, mpiComm -// ); //receive xml C-char -// xmlFileContent.assign( xmlFileContentMPI ); //copy to a string +// if (io.empty()) // no more groups to find +// { +// break; +// } // -// delete []( xmlFileContentMPI ); //delete char* needed for MPI, might -// add size is moving to C++14 for optimization, avoid memory leak -// } +// // get io name +// std::string::size_type ioPosition(0); +// const std::string tag(GetSubString("<io ", ">", io, ioPosition)); +// if (debugMode) +// { +// if (tag.size() < 2) +// { +// throw std::invalid_argument("ERROR: wrong XML tag <io " + tag +// + +// ", in call to ADIOS +// constructor\n"); +// } +// } // -// SetMembers( xmlFileContent, mpiComm, debugMode, hostLanguage, transforms, -// groups ); +// tag = tag.substr(1, tag.size() - 2); // eliminate < > +// GetPairsFromTag(io, tag, pairs); +// std::string groupName; +// +// for (auto &pair : pairs) +// { +// if (pair.first == "name") +// groupName = pair.second; +// } +// +// if (debugMode) +// { +// if (groupName.empty()) +// throw std::invalid_argument("ERROR: group name not found. +// \n"); +// +// if (groups.count(groupName) == 1) // group exists +// throw std::invalid_argument( "ERROR: group " + groupName + +// " +// defined twice.\n" ); +// } +// +// groups.emplace(groupName, +// Group(groupName, xmlGroup, transforms, debugMode)); +// +// currentContent.erase(currentContent.find(xmlGroup), xmlGroup.size()); +// currentPosition = 0; +// } //} } // end namespace adios diff --git a/source/adios2/helper/adiosXML.h b/source/adios2/helper/adiosXML.h index f3df29905..9ea5c29f9 100644 --- a/source/adios2/helper/adiosXML.h +++ b/source/adios2/helper/adiosXML.h @@ -12,11 +12,15 @@ #define ADIOS2_HELPER_ADIOSXML_H_ /// \cond EXCLUDE_FROM_DOXYGEN +#include <map> #include <string> #include <utility> //std::pair #include <vector> /// \endcond +#include "adios2/core/IO.h" +#include "adios2/core/Transform.h" + namespace adios { @@ -30,9 +34,9 @@ namespace adios * @param currentPosition to start the search, moved forward to finalTag * position */ -void GetSubString(const std::string initialTag, const std::string finalTag, - const std::string content, std::string &subString, - std::string::size_type ¤tPosition); +std::string GetSubString(const std::string initialTag, + const std::string finalTag, const std::string &content, + std::string::size_type ¤tPosition); /** * Extracts the value inside quotes in a string currentTag ( Example: currentTag @@ -80,11 +84,8 @@ void GetPairsFromTag( * variables with transformations * @param groups passed returns the map of groups defined in fileContent */ -// void SetMembers( const std::string& fileContent, const MPI_Comm mpiComm, -// std::string& hostLanguage, std::vector< -// std::shared_ptr<Transform> >& transforms, -// std::map< std::string, Group >& groups ); +void RemoveXMLComments(std::string ¤tContent) const noexcept; /** * Called inside the ADIOS XML constructors to get contents from file, broadcast * and set hostLanguage and groups from ADIOS class @@ -96,11 +97,24 @@ void GetPairsFromTag( * variables with transformations * @param groups passed returns the map of groups defined in fileContent */ -// void InitXML( const std::string xmlConfigFile, const MPI_Comm mpiComm, const -// bool debugMode, -// std::string& hostLanguage, std::vector< -// std::shared_ptr<Transform> >& transforms, -// std::map< std::string, Group >& groups ); + +void SetMembers(const std::string &fileContents, const bool debugMode, + std::vector<std::shared_ptr<Transform>> &transforms, + std::map<std::string, IO> &ios); + +/** + * Called inside the ADIOS XML constructors to get contents from file, + * broadcast + * @param configXMLFile + * @param mpiComm + * @param debugMode + * @param transforms + * @param ios + */ +void InitXML(const std::string configXML, const MPI_Comm mpiComm, + const bool debugMode, + std::vector<std::shared_ptr<Transform>> &transforms, + std::map<std::string, IO> &ios); } #endif /* ADIOS2_HELPER_ADIOSXML_H_ */ -- GitLab