diff --git a/include/core/CCapsule.h b/include/core/CCapsule.h index b71a9e8fc0eb334247f2c88b9287bc1f4d38ccb1..1de4c12955169d778f1b94eb85676654b2a8f959 100644 --- a/include/core/CCapsule.h +++ b/include/core/CCapsule.h @@ -50,6 +50,7 @@ public: int m_SizeMPI = 1; ///< current MPI processes size std::string m_CurrentGroup; ///< associated group to look for variable information + size_t m_MaxBufferSize = 0; CCapsule( ); ///< Default Empty constructor used with ADIOS class empty constructor @@ -103,7 +104,6 @@ private: std::vector< std::shared_ptr<CTransport> > m_Transports; std::string m_AccessMode; std::vector<char> m_Buffer; - size_t m_MaxBufferSize = 0; const bool m_DebugMode = false; std::string GetName( const std::vector<std::string>& arguments ) const; }; diff --git a/include/core/CGroup.h b/include/core/CGroup.h index 695f094936ef207c4a9021a7fe64ae637eced84e..70b6e32024f2b12392ad81c02e87b4b38aea0f57 100644 --- a/include/core/CGroup.h +++ b/include/core/CGroup.h @@ -71,7 +71,7 @@ public: void DefineVariable( const std::string variableName, const std::string type, const std::string dimensionsCSV, const std::string globalDimensionsCSV, const std::string globalOffsetsCSV, - const int transformIndex = -1, const int compressionLevel = 0 ); + const short transformIndex = -1, const unsigned short int compressionLevel = 0 ); /** * Sets a variable transform contained in ADIOS.m_Transforms (single container for all groups and variables) @@ -154,7 +154,7 @@ private: * @param globalOffsetsCSV comma separated variables defining global offsets (e.g. "oNx,oNY,oNz") * @return -1 if not global --> both inputs are empty, otherwise index in m_GlobalBounds if exist or create a new element in m_GlobalBounds; */ - const int SetGlobalBounds( const std::string globalDimensionsCSV, const std::string globalOffsetsCSV ) noexcept; + const short SetGlobalBounds( const std::string globalDimensionsCSV, const std::string globalOffsetsCSV ) noexcept; /** * Used by SetVariable to check if transform exists in m_Transform diff --git a/include/core/CTransport.h b/include/core/CTransport.h index d98231454318b9fffd3fe9a8c445162e66b76af7..cea0a114c7b0746c580657076874bcbd28563bf2 100644 --- a/include/core/CTransport.h +++ b/include/core/CTransport.h @@ -72,7 +72,7 @@ public: * Sets the buffer and bufferSize for certain transport methods * @param buffer to be set to transport */ - virtual void SetBuffer( const std::vector<char>& buffer ) + virtual void SetBuffer( std::vector<char>& buffer ) { }; virtual void Write( std::vector<char>& buffer ) @@ -89,7 +89,6 @@ protected: virtual void Init( const std::vector<std::string>& arguments ) { }; - }; diff --git a/include/functions/ADIOSFunctions.h b/include/functions/ADIOSFunctions.h index ea077e643e68d7e507b32e8a7095beef2ba3c349..4f8177d9f43f2fc76f22d9d9888f0d3d4c1b0d09 100644 --- a/include/functions/ADIOSFunctions.h +++ b/include/functions/ADIOSFunctions.h @@ -81,9 +81,11 @@ void GetPairsFromTag( const std::string& fileContent, const std::string tag, * @param fileContent file Content in a single string * @param mpiComm MPI Communicator passed from application passed to Transport method if required * @param hostLanguage return the host language from fileContent + * @param transforms return the modified transforms vector if there are variables with transformations * @param groups passed returns the map of groups defined in fileContent */ -void SetMembers( const std::string& fileContent, const MPI_Comm mpiComm, std::string& hostLanguage, +void SetMembers( const std::string& fileContent, const MPI_Comm mpiComm, + std::string& hostLanguage, std::vector< std::shared_ptr<CTransform> >& transforms, std::map< std::string, CGroup >& groups ); @@ -93,11 +95,12 @@ void SetMembers( const std::string& fileContent, const MPI_Comm mpiComm, std::st * @param mpiComm communicator used from broadcasting * @param debugMode from ADIOS m_DebugMode passed to CGroup in groups * @param hostLanguage set from host-language in xml file + * @param transforms return the modified transforms vector if there are variables with transformations * @param groups passed returns the map of groups defined in fileContent - * @param debugMode if true will do more checks, exceptions, warnings, expect slower code */ void InitXML( const std::string xmlConfigFile, const MPI_Comm mpiComm, const bool debugMode, - std::string& hostLanguage, std::map< std::string, CGroup >& groups ); + std::string& hostLanguage, std::vector< std::shared_ptr<CTransform> >& transforms, + std::map< std::string, CGroup >& groups ); /** diff --git a/include/functions/ADIOSTemplates.h b/include/functions/ADIOSTemplates.h index 858eccd95a82fe884bbf408e1460e566ddcd13fa..24859f7743c535ed3911b8b7294a08e7ebcaaa5b 100644 --- a/include/functions/ADIOSTemplates.h +++ b/include/functions/ADIOSTemplates.h @@ -55,7 +55,8 @@ template<class T> void WriteHelper( CCapsule& capsule, CGroup& group, const std::string variableName, const T* values, const int transportIndex, const bool debugMode ) { - auto lf_DebugType = []( const bool debugMode, const std::string type, const std::set<std::string>& typeAliases ) + auto lf_DebugType = []( const bool debugMode, const std::string type, const std::set<std::string>& typeAliases, + const std::string variableName ) { if( debugMode == true ) { @@ -80,67 +81,67 @@ void WriteHelper( CCapsule& capsule, CGroup& group, const std::string variableNa //will need to add a lambda function later and put types in a set if( std::is_same<T,char>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("char") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("char"), variableName ); WriteHelperToCapsule( capsule, group, group.m_Char[index], values, transportIndex ); } else if( std::is_same<T,unsigned char>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("unsigned char") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("unsigned char"), variableName ); WriteHelperToCapsule( capsule, group, group.m_UChar[index], values, transportIndex ); } else if( std::is_same<T,short>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("short") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("short"), variableName ); WriteHelperToCapsule( capsule, group, group.m_Short[index], values, transportIndex ); } else if( std::is_same<T,unsigned short>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("unsigned short") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("unsigned short"), variableName ); WriteHelperToCapsule( capsule, group, group.m_UShort[index], values, transportIndex ); } else if( std::is_same<T,int>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("int") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("int"), variableName ); WriteHelperToCapsule( capsule, group, group.m_Int[index], values, transportIndex ); } else if( std::is_same<T,unsigned int>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("unsigned int") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("unsigned int"), variableName ); WriteHelperToCapsule( capsule, group, group.m_UInt[index], values, transportIndex ); } else if( std::is_same<T,long int>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("long int") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("long int"), variableName ); WriteHelperToCapsule( capsule, group, group.m_LInt[index], values, transportIndex ); } else if( std::is_same<T,unsigned long int>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("unsigned long int") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("unsigned long int"), variableName ); WriteHelperToCapsule( capsule, group, group.m_ULInt[index], values, transportIndex ); } else if( std::is_same<T,long long int>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("long long int") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("long long int"), variableName ); WriteHelperToCapsule( capsule, group, group.m_LLInt[index], values, transportIndex ); } else if( std::is_same<T,unsigned long long int>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("unsigned long long int") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("unsigned long long int"), variableName ); WriteHelperToCapsule( capsule, group, group.m_ULLInt[index], values, transportIndex ); } else if( std::is_same<T,float>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("float") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("float"), variableName ); WriteHelperToCapsule( capsule, group, group.m_Float[index], values, transportIndex ); } else if( std::is_same<T,double>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("double") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("double"), variableName ); WriteHelperToCapsule( capsule, group, group.m_Double[index], values, transportIndex ); } else if( std::is_same<T,long double>::value ) { - lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("long double") ); + lf_DebugType( debugMode, type, SSupport::DatatypesAliases.at("long double"), variableName ); WriteHelperToCapsule( capsule, group, group.m_LDouble[index], values, transportIndex ); } } diff --git a/include/functions/CCapsuleTemplates.h b/include/functions/CCapsuleTemplates.h index fcbb140e1397945e8712ff08b6cfa2d8b92f7d7c..ba84ab239d402c9be16a6e4fc69f9710f780d652 100644 --- a/include/functions/CCapsuleTemplates.h +++ b/include/functions/CCapsuleTemplates.h @@ -35,8 +35,8 @@ void MemcpyThreads( T* destination, const U* source, std::size_t count, const un return; } - const unsigned long long int stride = (unsigned long long int)std::floor( (unsigned long long int)count/cores ); - const unsigned long long int remainder = (unsigned long int) count % cores; + const unsigned long long int stride = count/cores; + const unsigned long long int remainder = count % cores; const unsigned long long int last = stride + remainder; std::vector<std::thread> memcpyThreads; diff --git a/include/transport/CDataMan.h b/include/transport/CDataMan.h index 24272790ca2acd445e0efc152b8fcb2d2c40e4c6..ffee0b7d00273e3bffe2d2f493fe1b88ed21ca30 100644 --- a/include/transport/CDataMan.h +++ b/include/transport/CDataMan.h @@ -36,7 +36,7 @@ public: * You might not need this one since Buffer has all data, we might turn it on for future reference * @param buffer */ - void SetBuffer( const std::vector<char>& buffer ); + void SetBuffer( std::vector<char>& buffer ); /** * Here you will receive a reference (not a copy) to the variable being written as raw data ready to be sent. diff --git a/src/core/CCapsule.cpp b/src/core/CCapsule.cpp index 620fb9cebc572b3f859d3e7c5c67ab42d0a8434c..0bc99b830b47846863d1c71cc940dd891b8d84ac 100644 --- a/src/core/CCapsule.cpp +++ b/src/core/CCapsule.cpp @@ -17,10 +17,15 @@ #include "transform/CBZIP2.h" #endif +#include "transport/CPOSIX.h" +#include "transport/CFStream.h" +#include "transport/CDataMan.h" + namespace adios { + CCapsule::CCapsule( ) { } @@ -30,7 +35,7 @@ CCapsule::CCapsule( const MPI_Comm mpiComm, const bool debugMode, const std::str m_MPIComm{ mpiComm }, m_DebugMode{ debugMode } { - int transportIndex = AddTransport( streamName, accessMode, true, transport, arguments ); + AddTransport( streamName, accessMode, true, transport, arguments ); } @@ -54,7 +59,7 @@ int CCapsule::AddTransport( const std::string streamName, const std::string acce } if( transport == "POSIX" ) - m_Transports.push_back( std::make_shared<CPOSIX>( m_MPIComm, m_DebugMode, arguments ) ); //need to add arguments + m_Transports.push_back( std::make_shared<CPOSIX>( m_MPIComm, m_DebugMode, arguments ) ); else if( transport == "FStream" ) m_Transports.push_back( std::make_shared<CFStream>( m_MPIComm, m_DebugMode, arguments ) ); diff --git a/src/core/CGroup.cpp b/src/core/CGroup.cpp index f9485989bc73d2e2686f46c0f78deab640945835..a15e700ec61c206cc8a90864abbb12188c3bea86 100644 --- a/src/core/CGroup.cpp +++ b/src/core/CGroup.cpp @@ -42,7 +42,7 @@ CGroup::~CGroup( ) void CGroup::DefineVariable( const std::string variableName, const std::string type, const std::string dimensionsCSV, const std::string globalDimensionsCSV, const std::string globalOffsetsCSV, - const int transformIndex, const int compressionLevel ) + const short transformIndex, const unsigned short int compressionLevel ) { if( m_DebugMode == true ) { @@ -54,7 +54,7 @@ void CGroup::DefineVariable( const std::string variableName, const std::string t throw std::invalid_argument( "ERROR: variable " + variableName + " already exists, in call to DefineVariable\n" ); } - int globalBoundsIndex = SetGlobalBounds( globalDimensionsCSV, globalOffsetsCSV ); + short globalBoundsIndex = SetGlobalBounds( globalDimensionsCSV, globalOffsetsCSV ); if( IsTypeAlias( type, SSupport::DatatypesAliases.at("char") ) == true ) { @@ -230,7 +230,7 @@ const unsigned long long int CGroup::GetIntVariableValue( const std::string vari const std::string type( m_Variables.at( variableName ).first ); const unsigned int index = m_Variables.at( variableName ).second; - const long long int value = -1; + long long int value = -1; if( IsTypeAlias( type, SSupport::DatatypesAliases.at("short") ) == true ) value = *( m_Short[index].Values ); @@ -397,7 +397,7 @@ void CGroup::ParseXMLGroup( const std::string& xmlGroup, std::vector< std::share } -const int CGroup::SetGlobalBounds( const std::string globalDimensionsCSV, const std::string globalOffsetsCSV ) noexcept +const short CGroup::SetGlobalBounds( const std::string globalDimensionsCSV, const std::string globalOffsetsCSV ) noexcept { if( globalDimensionsCSV.empty() || globalOffsetsCSV.empty() ) return -1; diff --git a/src/functions/ADIOSFunctions.cpp b/src/functions/ADIOSFunctions.cpp index 8f8fef3ac757e3e7f6517796311f1f5eda05144d..e28ab1721121a79464f29b6e40a7c6de9fd1f221 100644 --- a/src/functions/ADIOSFunctions.cpp +++ b/src/functions/ADIOSFunctions.cpp @@ -12,11 +12,19 @@ #include <iostream> #include <thread> //std::thread #include <cstring> //std::memcpy + +#include <sys/types.h> //CreateDirectory +#include <sys/stat.h> //stat +#include <unistd.h> //CreateDirectory /// \endcond #include "functions/ADIOSFunctions.h" #include "public/SSupport.h" +#ifdef HAVE_BZIP2 +#include "transform/CBZIP2.h" +#endif + namespace adios { @@ -181,7 +189,8 @@ void GetPairsFromTag( const std::string& fileContent, const std::string tag, void SetMembers( const std::string& fileContent, const MPI_Comm mpiComm, const bool debugMode, - std::string& hostLanguage, std::map< std::string, CGroup >& groups ) + std::string& hostLanguage, std::vector< std::shared_ptr<CTransform> >& transforms, + std::map< std::string, CGroup >& groups ) { //adios-config std::string currentContent; @@ -260,7 +269,7 @@ void SetMembers( const std::string& fileContent, const MPI_Comm mpiComm, const b throw std::invalid_argument( "ERROR: group " + groupName + " defined twice.\n" ); } - groups.emplace( groupName, CGroup( hostLanguage, xmlGroup, debugMode ) ); + groups.emplace( groupName, CGroup( hostLanguage, xmlGroup, transforms, debugMode ) ); currentContent.erase( currentContent.find( xmlGroup ), xmlGroup.size() ); currentPosition = 0; @@ -319,7 +328,8 @@ void SetMembers( const std::string& fileContent, const MPI_Comm mpiComm, const b void InitXML( const std::string xmlConfigFile, const MPI_Comm mpiComm, const bool debugMode, - std::string& hostLanguage, std::map< std::string, CGroup >& groups ) + std::string& hostLanguage, std::vector< std::shared_ptr<CTransform> >& transforms, + std::map< std::string, CGroup >& groups ) { int xmlFileContentSize; std::string xmlFileContent; @@ -346,7 +356,7 @@ void InitXML( const std::string xmlConfigFile, const MPI_Comm mpiComm, const boo delete []( xmlFileContentMPI ); //delete char* needed for MPI, might add size is moving to C++14 for optimization, avoid memory leak } - SetMembers( xmlFileContent, mpiComm, debugMode, hostLanguage, groups ); + SetMembers( xmlFileContent, mpiComm, debugMode, hostLanguage, transforms, groups ); } @@ -365,7 +375,7 @@ void CreateDirectory( const std::string fullPath ) noexcept { auto lf_Mkdir = []( const std::string directory, struct stat& st ) { - if ( stat( directory.c_str(), &st) == -1 ) + if ( stat( directory.c_str(), &st ) == -1 ) { mkdir( directory.c_str(), 0777 ); } @@ -429,7 +439,12 @@ void SetTransformHelper( const std::string transform, std::vector< std::shared_p if( transformIndex == -1 ) //not found, then create a new transform { if( transform == "bzip2" ) + { + #ifdef HAVE_BZIP2 transforms.push_back( std::make_shared<CBZIP2>( ) ); + #endif + } + } transformIndex = static_cast<int>( transforms.size() - 1 ); diff --git a/src/public/ADIOS.cpp b/src/public/ADIOS.cpp index bb8027c5b1ebf83151e9ddc8805e8c24ef29f21d..e6f1ba905d1f183d1c83bff368c84e0c06f292a9 100644 --- a/src/public/ADIOS.cpp +++ b/src/public/ADIOS.cpp @@ -32,7 +32,7 @@ ADIOS::ADIOS( const std::string xmlConfigFile, const bool debugMode ): m_XMLConfigFile{ xmlConfigFile }, m_DebugMode{ debugMode } { - InitXML( m_XMLConfigFile, m_MPIComm, m_DebugMode, m_HostLanguage, m_Groups ); + InitXML( m_XMLConfigFile, m_MPIComm, m_DebugMode, m_HostLanguage, m_Transforms, m_Groups ); } @@ -43,7 +43,7 @@ ADIOS::ADIOS( const std::string xmlConfigFile, const MPI_Comm mpiComm, const boo { MPI_Comm_rank( m_MPIComm, &m_RankMPI ); MPI_Comm_size( m_MPIComm, &m_SizeMPI ); - InitXML( m_XMLConfigFile, m_MPIComm, m_DebugMode, m_HostLanguage, m_Groups ); + InitXML( m_XMLConfigFile, m_MPIComm, m_DebugMode, m_HostLanguage, m_Transforms, m_Groups ); } @@ -169,7 +169,7 @@ void ADIOS::CheckGroup( std::map< std::string, CGroup >::const_iterator itGroup, void ADIOS::CheckCapsule( std::map< std::string, CCapsule >::const_iterator itCapsule, const std::string streamName, const std::string hint ) const { - if( itCapsule == m_Groups.end() ) + if( itCapsule == m_Capsules.end() ) throw std::invalid_argument( "ERROR: stream (or file) " + streamName + " not created with Open , " + hint + "\n" ); } diff --git a/src/transport/CDataMan.cpp b/src/transport/CDataMan.cpp index 716badeff9ddcc4621a04ca82da7a1580f47208a..920ede9f3a3b6b4b895ec51d8eacb0c498447aef 100644 --- a/src/transport/CDataMan.cpp +++ b/src/transport/CDataMan.cpp @@ -34,7 +34,7 @@ void CDataMan::Open( const std::string streamName, const std::string accessMode } -void CDataMan::SetBuffer( const std::vector<char>& buffer ) +void CDataMan::SetBuffer( std::vector<char>& buffer ) { //empty for now } diff --git a/src/transport/CFStream.cpp b/src/transport/CFStream.cpp index f9c9f2c6c7ec3563e6a4deddd8f795add32864b6..39f8d985013ca9095df7636422a00078ea254401 100644 --- a/src/transport/CFStream.cpp +++ b/src/transport/CFStream.cpp @@ -14,14 +14,17 @@ /// \endcond #include "transport/CFStream.h" +#include "functions/ADIOSFunctions.h" //CreateDirectory namespace adios { -CFStream::CFStream( MPI_Comm mpiComm, const bool debugMode ): +CFStream::CFStream( MPI_Comm mpiComm, const bool debugMode, const std::vector<std::string>& arguments ): CTransport( "FStream", mpiComm, debugMode ) -{ } +{ + //here do something with arguments +} CFStream::~CFStream( )