diff --git a/Makefile b/Makefile index 75525d2d31b7d550ef922df406a248f5c53d52af..3e048587b8123825737f168bd90d817df6f2a37a 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ ARFLAGS:=rcs HFiles:=$(shell find ./include -type f -name "*.h") CPPFiles:=$(shell find ./src -type f -name "*.cpp") INC:=-I./include -VPATH = ./src/core ./src/functions ./src/public ./src/transform ./src/transport +VPATH = ./src ./src/core ./src/functions ./src/transform ./src/transport #SEPARATE EXTERNAL LIBRARIES HANDLING in Makefile.libs export $(HFiles) $(CPPFiles) $(CFLAGS) $(LIBS) diff --git a/include/ADIOS.h b/include/ADIOS.h index 214274a3955b60bbe454418e66267a6c539f9e65..a5abe2fc294ac632bbbff057b3b6548ed31e96de 100644 --- a/include/ADIOS.h +++ b/include/ADIOS.h @@ -47,6 +47,7 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO int m_RankMPI = 0; ///< current MPI rank process int m_SizeMPI = 1; ///< current MPI processes size + /** * @brief ADIOS empty constructor. Used for non XML config file API calls. */ @@ -82,23 +83,66 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO /** - * @brief Open to Write, Read - * @param name unique stream or file name - * @param accessMode - * @param mpiComm - * @param method corresponding method - * @return + * Creates an empty group + * @param groupName + */ + void DeclareGroup( const std::string groupName ); + + /** + * Creates a new Variable, if debugMode = true, program will throw an exception, else it will overwrite current variable with the same name + * @param groupName corresponding variable group + * @param variableName corresponding variable name + * @param type variable type + * @param dimensionsCSV comma separated (no space) dimensions "Nx,Ny,Nz" defined by other variables + * @param globalDimensionsCSV comma separated (no space) global dimensions "gNx,gNy,gNz" defined by other variables + * @param globalOffsetsCSV comma separated (no space) global offsets "oNx,oNy,oNz" defined by other variables + */ + void DefineVariable( const std::string groupName, const std::string variableName, const std::string type, + const std::string dimensionsCSV = "", const std::string globalDimensionsCSV = "", + const std::string globalOffsetsCSV = "" ); + + + /** + * Sets a transform method to a variable, to be applied when writing + * @param groupName corresponding variable group + * @param variableName corresponding variable name + * @param transform method to be applied for this variable + */ + void SetTransform( const std::string groupName, const std::string variableName, const std::string transform ); + + + /** + * @brief Creates a new Attribute. + * Debug Mode: program will throw an exception if attribute exists (attributeName), + * else: overwrites current attribute using the same name + * @param groupName corresponding variable group + * @param attributeName corresponding attribute name + * @param type string or number + * @param value string contents of the attribute (e.g. "Communication value", "1" ) + */ + void DefineAttribute( const std::string groupName, const std::string attributeName, + const std::string type, const std::string value ); + + /** + * @brief Open to Write, Read. Creates a new engine from previously defined method + * @param streamName unique stream or file name + * @param accessMode "w" or "write", "r" or "read", "a" or "append" + * @param mpiComm option to modify communicator from ADIOS class constructor + * @param method looks for corresponding Method object in ADIOS to initialize the engine + * @return handler to created engine */ - const unsigned int Open( const std::string name, const std::string accessMode, MPI_Comm mpiComm ); + const unsigned int Open( const std::string streamName, const std::string accessMode, MPI_Comm mpiComm, + const std::string methodName ); + /** - * @brief Create an Engine described by a previously defined method - * @param name - * @param accessMode - * @param method - * @return + * @brief Open to Write, Read. Creates a new engine from previously defined method. Reuses MPI communicator from ADIOS class constructor. + * @param streamName unique stream or file name + * @param accessMode "w" or "write", "r" or "read", "a" or "append" + * @param method looks for corresponding Method object in ADIOS to initialize the engine + * @return handler to created engine */ - const unsigned int Open( const std::string name, const std::string accessMode, const std::string method ); + const unsigned int Open( const std::string streamName, const std::string accessMode, const std::string methodName ); /** @@ -106,32 +150,29 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO * @param streamName * @param maxBufferSize */ - void SetMaxBufferSize( const std::string streamName, const size_t maxBufferSize ); + void SetMaxBufferSize( const unsigned int engineHandler, const size_t maxBufferSize ); /** * Sets a default group to be associated with a stream before writing variables with Write function. * @param streamName unique name * @param groupName default group from which variables will be used */ - void SetCurrentGroup( const std::string streamName, const std::string groupName ); + void SetGroup( const unsigned int handler, const std::string groupName ); template<class T> - void Write( const std::string streamName, const std::string groupName, const std::string variableName, const T* values, - const int transportIndex = -1 ) + void Write( const unsigned int handler, const std::string groupName, const std::string variableName, const T* values ) { + auto itEngine = m_Engines.find( handler ); auto itGroup = m_Groups.find( groupName ); if( m_DebugMode == true ) { - CheckCapsule( streamName, " from call to Write variable " + variableName ); + CheckEngine( itEngine, handler, " from call to Write variable " + variableName ); CheckGroup( itGroup, groupName, " from call to Write variable " + variableName ); - if( transportIndex < -1 ) - throw std::invalid_argument( "ERROR: transport index " + std::to_string( transportIndex ) + - " must be >= -1, in call to Write\n" ); } - WriteHelper( itCapsule->second, itGroup->second, variableName, values, transportIndex, m_DebugMode ); + itEngine->second->Write( itGroup->second, variableName, values ); } @@ -144,31 +185,23 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO * @param cores optional parameter for threaded version */ template<class T> - void Write( const int handler, const std::string variableName, const T* values, - const int transportIndex = -1 ) + void Write( const unsigned int handler, const std::string variableName, const T* values ) { + auto itEngine = m_Engines.find( handler ); + if( m_DebugMode == true ) - CheckCapsule( streamName, " from call to Write variable " + variableName ); + CheckEngine( itEngine, handler, " from call to Write variable " + variableName ); - WriteHelper( itCapsule->second, *itCapsule->second.m_CurrentGroup, variableName, values, transportIndex, m_DebugMode ); + itEngine->second->Write( variableName, values ); } - template<class T> - void Write( const int handler, const std::string variableName, const T* values, - const int transportIndex = -1 ) - { - if( m_DebugMode == true ) - CheckCapsule( streamName, " from call to Write variable " + variableName ); - - WriteHelper( itCapsule->second, *itCapsule->second.m_CurrentGroup, variableName, values, transportIndex, m_DebugMode ); - } - /** * Close a particular stream and the corresponding transport * @param streamName stream to be closed with all corresponding transports * @param transportIndex identifier to a particular transport, if == -1 Closes all transports */ - void Close( const std::string streamName, const int transportIndex = -1 ); + void Close( const unsigned int handler, const int transportIndex = -1 ); + /** * @brief Dumps groups information to a file stream or standard output. @@ -177,43 +210,7 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO */ void MonitorGroups( std::ostream& logStream ) const; - /** - * Creates an empty group - * @param groupName - */ - void DeclareGroup( const std::string groupName ); - - /** - * Creates a new Variable, if debugMode = true, program will throw an exception, else it will overwrite current variable with the same name - * @param groupName corresponding variable group - * @param variableName corresponding variable name - * @param type variable type - * @param dimensionsCSV comma separated (no space) dimensions "Nx,Ny,Nz" defined by other variables - * @param globalDimensionsCSV comma separated (no space) global dimensions "gNx,gNy,gNz" defined by other variables - * @param globalOffsetsCSV comma separated (no space) global offsets "oNx,oNy,oNz" defined by other variables - */ - void DefineVariable( const std::string groupName, const std::string variableName, const std::string type, - const std::string dimensionsCSV = "", const std::string globalDimensionsCSV = "", - const std::string globalOffsetsCSV = "" ); - - /** - * Sets a transform method to a variable, to be applied when writing - * @param groupName corresponding variable group - * @param variableName corresponding variable name - * @param transform method to be applied for this variable - */ - void SetTransform( const std::string groupName, const std::string variableName, const std::string transform ); - - /** - * Creates a new Variable, if debugMode = true, program will throw an exception, else it will overwirte current variable with the same name - * @param groupName corresponding variable group - * @param attributeName corresponding attribute name - * @param type string or number - * @param value string contents of the attribute (e.g. "Communication value" ) - */ - void DefineAttribute( const std::string groupName, const std::string attributeName, - const std::string type, const std::string value ); private: @@ -243,7 +240,7 @@ private: * Value: Engine derived class * </pre> */ - std::unordered_map< unsigned int, std::shared_ptr<Capsule> > m_Engines; + std::unordered_map< unsigned int, std::shared_ptr<Engine> > m_Engines; std::set< std::string > m_EngineNames; ///< set used to check Engine name uniqueness in debug mode int m_EngineCounter = -1; ///< used to set the unsigned int key in m_Capsules, helpful is a capsule is removed @@ -261,12 +258,12 @@ private: const std::string groupName, const std::string hint ) const; /** - * @brief Checks for capsule existence in m_Groups, if failed throws std::invalid_argument exception - * @param itCapsule m_Capsule iterator, usually from find function - * @param streamName unique name, passed for thrown exception only + * @brief Checks for engine existence in m_Engines, if failed throws std::invalid_argument exception + * @param itEngine from Open * @param hint adds information to thrown exception */ - void CheckEngine( const std::string streamName, const std::string hint ) const; + void CheckEngine( std::unordered_map< unsigned int, std::shared_ptr<Engine> >::const_iterator itEngine, + const unsigned int handle, const std::string hint ) const; }; diff --git a/include/capsule/Heap.h b/include/capsule/Heap.h new file mode 100644 index 0000000000000000000000000000000000000000..6b5421290f0d7e64862062d437d4be7b6aa18365 --- /dev/null +++ b/include/capsule/Heap.h @@ -0,0 +1,42 @@ +/* + * Heap.h + * + * Created on: Dec 19, 2016 + * Author: wfg + */ + +#ifndef HEAP_H_ +#define HEAP_H_ + + +#include "core/Capsule.h" + + +namespace adios +{ + +/** + * Buffer and Metadata are allocated in the Heap + */ +class Heap : public Capsule +{ + +public: + + std::vector<char> m_Buffer; ///< buffer allocated using the STL + std::vector<char> m_Metadata; ///< metadata buffer allocated using the STL + + Heap( const std::string accessMode ); + + ~Heap( ); + +}; + +} //end namespace + + + + + + +#endif /* HEAP_H_ */ diff --git a/include/core/Capsule.h b/include/core/Capsule.h index d8ab0667f53d26e8d5bc18f6e1b76344745031c1..17ced6e32b040d0c749dd08cbfbdae336210f681 100644 --- a/include/core/Capsule.h +++ b/include/core/Capsule.h @@ -19,19 +19,20 @@ class Capsule public: const std::string m_Type; ///< buffer type - const char m_AccessMode; ///< 'w': write, 'r': read, 'a': append + const std::string m_AccessMode; ///< 'w': write, 'r': read, 'a': append /** * Base class constructor providing type from derived class and accessMode * @param type derived class type * @param accessMode 'w':write, 'r':read, 'a':append */ - Capsule( const std::string type, const char accessMode ); + Capsule( const std::string type, const std::string accessMode ); virtual ~Capsule( ); virtual void Write( const std::vector<char>& data, const std::size_t size ); + }; diff --git a/include/core/Engine.h b/include/core/Engine.h index 4eed322c89652b35fa50b629b7f5a0c0955a2380..1c3b39006bacd7e5ecacefb891db3e729a0d8ea5 100644 --- a/include/core/Engine.h +++ b/include/core/Engine.h @@ -47,45 +47,69 @@ public: MPI_Comm m_MPIComm = 0; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C #endif - Group* m_CurrentGroup = nullptr; ///< associated group to look for variable information + std::string m_EngineType; ///< from derived class + const std::string m_Name; ///< name used for this engine + const std::string m_AccessMode; ///< accessMode for buffers used by this engine + Method* m_Method = nullptr; ///< associated method containing engine metadata + Group* m_Group = nullptr; ///< associated group to look for variable information + int m_RankMPI = 0; ///< current MPI rank process int m_SizeMPI = 1; ///< current MPI processes size /** - * Multiple argument constructor + * Unique constructor based on a method (engine metadata) + * @param engineType given by derived classes + * @param name engine name * @param accessMode * @param mpiComm - * @param debugMode + * @param method */ + Engine( const std::string engineType, const std::string name, const std::string accessMode, const MPI_Comm mpiComm, + const Method& method, const bool debugMode ); - Engine( const std::string streamName, const std::string accessMode, const MPI_Comm mpiComm, - const Method& method ); virtual ~Engine( ); - virtual void Write( Group& group, Variable<char>& variable ); - virtual void Write( Group& group, Variable<unsigned char>& variable ); - virtual void Write( Group& group, Variable<short>& variable ); - virtual void Write( Group& group, Variable<unsigned short>& variable ); - virtual void Write( Group& group, Variable<int>& variable ); - virtual void Write( Group& group, Variable<unsigned int>& variable ); - virtual void Write( Group& group, Variable<long int>& variable ); - virtual void Write( Group& group, Variable<unsigned long int>& variable ); - virtual void Write( Group& group, Variable<long long int>& variable ); - virtual void Write( Group& group, Variable<unsigned long long int>& variable ); - virtual void Write( Group& group, Variable<float>& variable ); - virtual void Write( Group& group, Variable<double>& variable ); - virtual void Write( Group& group, Variable<long double>& variable ); - - virtual void Close( int transportIndex ); ///< Closes a particular transport - -private: + virtual void Write( const Group& group, const std::string variableName, const char* values ); + virtual void Write( const Group& group, const std::string variableName, const unsigned char* values ); + virtual void Write( const Group& group, const std::string variableName, const short* values ); + virtual void Write( const Group& group, const std::string variableName, const unsigned short* values ); + virtual void Write( const Group& group, const std::string variableName, const int* values ); + virtual void Write( const Group& group, const std::string variableName, const unsigned int* values ); + virtual void Write( const Group& group, const std::string variableName, const long int* values ); + virtual void Write( const Group& group, const std::string variableName, const unsigned long int* values ); + virtual void Write( const Group& group, const std::string variableName, const long long int* values ); + virtual void Write( const Group& group, const std::string variableName, const unsigned long long int* values ); + virtual void Write( const Group& group, const std::string variableName, const float* values ); + virtual void Write( const Group& group, const std::string variableName, const double* values ); + virtual void Write( const Group& group, const std::string variableName, const long double* values ); + + virtual void Write( const std::string variableName, const char* values ); + virtual void Write( const std::string variableName, const unsigned char* values ); + virtual void Write( const std::string variableName, const short* values ); + virtual void Write( const std::string variableName, const unsigned short* values ); + virtual void Write( const std::string variableName, const int* values ); + virtual void Write( const std::string variableName, const unsigned int* values ); + virtual void Write( const std::string variableName, const long int* values ); + virtual void Write( const std::string variableName, const unsigned long int* values ); + virtual void Write( const std::string variableName, const long long int* values ); + virtual void Write( const std::string variableName, const unsigned long long int* values ); + virtual void Write( const std::string variableName, const float* values ); + virtual void Write( const std::string variableName, const double* values ); + virtual void Write( const std::string variableName, const long double* values ); + + virtual void Close( int transportIndex = -1 ); ///< Closes a particular transport + + +protected: std::vector< std::shared_ptr<Transport> > m_Transports; ///< transports managed - std::vector< Capsule > m_Buffers; ///< managed Buffers - const bool m_DebugMode = false; + std::vector< std::shared_ptr<Capsule> > m_Capsules; ///< managed Buffers + bool m_DebugMode = false; + + void SetTransports( ); - std::string GetName( const std::vector<std::string>& arguments ) const; + std::string GetName( const std::vector<std::string>& arguments ) const; //might move this to adiosFunctions }; diff --git a/include/core/Group.h b/include/core/Group.h index ae57a850b7b25c280bd6165fd5db248560f819eb..2d2a0b42c967fd27a13041db28c11bb844782356 100644 --- a/include/core/Group.h +++ b/include/core/Group.h @@ -41,21 +41,24 @@ class Group public: /** - * @brief Constructor for XML config file - * @param hostLanguage reference from ADIOS class - * @param xmlGroup contains <adios-group (tag excluded)....</adios-group> single group definition from XML config file - * @param transforms passed from ADIOS.m_Transforms, single look up table for all transforms - * @param debugMode from ADIOS + * Empty constructor */ - Group( const std::string hostLanguage, const std::string& xmlGroup, - std::vector< std::shared_ptr<Transform> >& transforms, const bool debugMode ); + Group( ); /** - * Non-XML empty constructor + * Empty constructor + * @param debugMode true: additional checks throwing exceptions, false: skip checks + */ + Group( const bool debugMode = false ); + + /** + * @brief Constructor for XML config file * @param hostLanguage reference from ADIOS class + * @param xmlGroup contains <adios-group (tag excluded)....</adios-group> single group definition from XML config file + * @param transforms passed from ADIOS.m_Transforms, single look up table for all transforms * @param debugMode */ - Group( const std::string hostLanguage, const bool debugMode ); + Group( const std::string& xmlGroup, std::vector< std::shared_ptr<Transform> >& transforms, const bool debugMode ); ~Group( ); ///< Using STL containers, no deallocation @@ -67,20 +70,21 @@ public: * @param dimensionsCSV comma separated variable local dimensions (e.g. "Nx,Ny,Nz") * @param globalDimensionsCSV comma separated variable global dimensions (e.g. "gNx,gNy,gNz"), if globalOffsetsCSV is also empty variable is local * @param globalOffsetsCSV comma separated variable global dimensions (e.g. "gNx,gNy,gNz"), if globalOffsetsCSV is also empty variable is local + * @param transform pointer reference to a Transform object, default = nullptr + * @param parameter corresponding parameter used by Transform to do operations */ void DefineVariable( const std::string variableName, const std::string type, - const std::string dimensionsCSV, - const std::string globalDimensionsCSV, const std::string globalOffsetsCSV, - const short transformIndex = -1, const unsigned short int compressionLevel = 0 ); + const std::string dimensionsCSV = "", + const std::string globalDimensionsCSV = "", const std::string globalOffsetsCSV = "", + const Transform* transform = nullptr, const unsigned short parameter = 0 ); /** - * Sets a variable transform contained in ADIOS.m_Transforms (single container for all groups and variables) + * Sets a variable transform contained in ADIOS Transforms (single container for all groups and variables) * @param variableName variable to be assigned a transformation - * @param transformIndex index in - * @param compressionLevel from 0 to 9 + * @param transform corresponding transform object + * @param parameter optional parameter interpreted by the corresponding Transform */ - void SetTransform( const std::string variableName, const unsigned int transformIndex, const unsigned int compressionLevel ); - + void SetTransform( const std::string variableName, const Transform& transform, const short parameter = 0 ); /** * Define a new attribute @@ -110,8 +114,7 @@ public: private: - const std::string m_HostLanguage; ///< copy of ADIOS m_HostLanguage - const bool m_DebugMode = false; ///< if true will do more checks, exceptions, warnings, expect slower code, known at compile time + bool m_DebugMode = false; ///< if true will do more checks, exceptions, warnings, expect slower code, known at compile time std::map< std::string, std::pair< std::string, unsigned int > > m_Variables; ///< Makes variable name unique, key: variable name, value: pair.first = type, pair.second = index in corresponding vector of CVariable @@ -140,14 +143,17 @@ private: */ std::map< std::string, Attribute > m_Attributes; + std::vector< std::pair< std::string, std::string > > m_GlobalBounds; ///< if a variable or an attribute is global it fills this container, from global-bounds in XML File, data in global space, pair.first = global dimensions, pair.second = global bounds + /** * Called from XML constructor * @param xmlGroup contains <adios-group....</adios-group> single group definition from XML config file passing by reference as it could be big */ void ParseXMLGroup( const std::string& xmlGroup, std::vector< std::shared_ptr<Transform> >& transforms ); + /** * Used by SetVariable and SetAttribute to check if global bounds exist in m_GlobalBounds * @param globalDimensionsCSV comma separated variables defining global dimensions (e.g. "Nx,NY,Nz") @@ -156,12 +162,6 @@ private: */ const short SetGlobalBounds( const std::string globalDimensionsCSV, const std::string globalOffsetsCSV ) noexcept; - /** - * Used by SetVariable to check if transform exists in m_Transform - * @param transform variable transformation method (e.g. bzip2, szip, zlib ) - * @return -1 variable is not associated with a transform, otherwise index in m_Transforms - */ - const int SetTransforms( const std::string transform ) noexcept; /** * Retrieves the value of a variable representing another's variable dimensions. Set with Write diff --git a/include/core/Method.h b/include/core/Method.h index 4de9f2a15bc2a501a9c6459f4657c04630622141..e97e68b234044e92b59d791a1907f84eb8e04495 100644 --- a/include/core/Method.h +++ b/include/core/Method.h @@ -20,6 +20,7 @@ namespace adios */ struct Method { + std::string Name; ///< Method name std::vector< std::string > Capsules; ///< Capsule type std::map< std::string, std::vector<std::string> > Transports; ///< key: transports, value: arguments to Transport std::string Group; ///< Associated group name to this engine diff --git a/include/core/Support.h b/include/core/Support.h index 493b245cece02edcda4a86c19b903a2f2bfaef52..65f78f2de94a6261eca8fb89ce259e48a1600287 100644 --- a/include/core/Support.h +++ b/include/core/Support.h @@ -18,6 +18,7 @@ namespace adios { + struct Support { static const std::string Version; ///< current ADIOS version @@ -26,6 +27,7 @@ struct Support static const std::set<std::string> Transforms; ///< supported data transform methods static const std::map<std::string, std::set<std::string> > Datatypes; ///< supported data types, key: host language, value: all supported types static const std::map<std::string, std::set<std::string> > DatatypesAliases; ///< all supported int aliases, key: C++ type (e.g. int), value: aliases to type in key (e.g. int, integer) + }; diff --git a/include/core/Variable.h b/include/core/Variable.h index 0d0b9ee00e736ef03a11a4c731a19ffce3886d43..2e411383e20cb9394f45a479f9f7dbf280f620ab 100644 --- a/include/core/Variable.h +++ b/include/core/Variable.h @@ -15,19 +15,21 @@ #include <sstream> /// \endcond +#include "Transform.h" + namespace adios { /** * @param Base (parent) class for template derived (child) class CVariable. Required to put CVariable objects in STL containers. */ template< class T > -struct Variable +class Variable { const std::string DimensionsCSV; ///< comma separated list for variables to search for local dimensions const T* Values; ///< pointer to values passed from ADIOS Write - const short GlobalBoundsIndex; ///< if global > 0, index corresponds to global-bounds in m_GlobalBounds in CGroup, if local then = -1 - short TransformIndex = -1; ///< if no transformation then = -1, otherwise index to m_Transforms container in ADIOS - unsigned short CompressionLevel = 0; ///< if 0, then use default, values from 1 to 9 indicate compression level + const unsigned short GlobalBounds; ///< if global > 0, index corresponds to global-bounds in m_GlobalBounds in CGroup, if local then = -1 + Transform* Transform = nullptr; ///< if no transformation then nullptr, otherwise pointer reference to a Transport object + short Parameter = -1; ///< additional optional parameter understood by a Transform }; diff --git a/include/engine/Single.h b/include/engine/Single.h index f8e97046273f651b8f08cc268010075191adac47..ff60729c1c058fb4f9e2ca0ead69cacf0aa770be 100644 --- a/include/engine/Single.h +++ b/include/engine/Single.h @@ -8,12 +8,35 @@ #ifndef SINGLE_H_ #define SINGLE_H_ +#include "core/Engine.h" namespace adios { +class Single : public Engine +{ + +public: + + /** + * Constructor for single capsule engine + * @param streamName + * @param accessMode + * @param mpiComm + * @param method + * @param debugMode + */ + Single( const std::string streamName, const std::string accessMode, const MPI_Comm mpiComm, const Method& method, + const bool debugMode = false ); + + + ~Single( ); + +}; + + diff --git a/include/functions/adiosTemplates.h b/include/functions/adiosTemplates.h index ae9cc01eac3c585461e1c13992fb9ec4f4415352..92672514140e193cb82e1146bf84c1fa415abf62 100644 --- a/include/functions/adiosTemplates.h +++ b/include/functions/adiosTemplates.h @@ -26,8 +26,8 @@ namespace adios { template<class T> -void WriteHelperToCapsule( Capsule& capsule, Group& group, Variable<T>& variable, const T* values, - const int transportIndex ) noexcept +void WriteHelperToEngine( Capsule& capsule, Group& group, Variable<T>& variable, const T* values, + const int transportIndex ) noexcept { variable.m_Values = values; auto localDimensions = group.GetDimensions( variable.m_DimensionsCSV ); @@ -40,7 +40,7 @@ void WriteHelperToCapsule( Capsule& capsule, Group& group, Variable<T>& variable } else //write local variable { - capsule.Write( variable.m_Values, GetTotalSize( localDimensions ), transportIndex ); + capsule.Write( variable.m_Values, GetTotalSize( localDimensions ) ); } } @@ -80,7 +80,6 @@ void WriteHelper( Capsule& capsule, Group& group, const std::string variableName const unsigned int index = itVariable->second; group.m_SetVariables.insert( variableName ); //should be done before writing to buffer, in case there is a crash? - //will need to add a lambda function later and put types in a set if( std::is_same<T,char>::value ) { lf_DebugType( debugMode, type, Support::DatatypesAliases.at("char"), variableName ); diff --git a/src/ADIOS.cpp b/src/ADIOS.cpp index e721a4fa9322af1165f5c7d8cac665cc5d1bacee..3ed30fe818bbc9b234f7e687a5d60ad7400deed9 100644 --- a/src/ADIOS.cpp +++ b/src/ADIOS.cpp @@ -59,22 +59,39 @@ ADIOS::~ADIOS( ) { } -void ADIOS::SetCurrentGroup( const unsigned int engineHandler, const std::string groupName ) +void ADIOS::DeclareGroup( const std::string groupName ) { - auto itEngine = m_Engine if( m_DebugMode == true ) - CheckCapsule( itCapsule, streamName, " from call to SetCurrentGroup\n" ); + { + if( m_Groups.count( groupName ) == 1 ) + throw std::invalid_argument( "ERROR: group " + groupName + " already exist, from call to DeclareGroup\n" ); + } - itCapsule->second.m_CurrentGroup = groupName; + m_Groups.emplace( groupName, Group( m_HostLanguage, m_DebugMode ) ); } -const unsigned int ADIOS::Open( const std::string streamName, const std::string accessMode, +void ADIOS::SetGroup( const unsigned int handler, const std::string groupName ) +{ + auto itEngine = m_Engines.find( handler ); + auto itGroup = m_Groups.find( groupName ); + + if( m_DebugMode == true ) + { + CheckEngine( itEngine, handler, " in call to SetGroup.\n" ); + CheckGroup( itGroup, groupName, " in call to SetGroup.\n" ); + } + + itEngine->second->m_Group = &( itGroup->second ); +} + + +const unsigned int ADIOS::Open( const std::string name, const std::string accessMode, MPI_Comm mpiComm, const std::string methodName ) { if( m_DebugMode == true ) { - if( m_EngineNames.count( streamName ) == 1 ) //Engine exists + if( m_EngineNames.count( name ) == 1 ) //Engine exists throw std::invalid_argument( "ERROR: method " + methodName + " already created by Open, in call from Open.\n" ); if( m_Methods.count( methodName ) == 0 ) // @@ -83,7 +100,7 @@ const unsigned int ADIOS::Open( const std::string streamName, const std::string ++m_EngineCounter; - if( methodName.empty() ) + if( methodName.empty() ) //default engine with one transport { } @@ -117,17 +134,6 @@ void ADIOS::Close( const unsigned int methodHandler, const int transportIndex ) } -void ADIOS::DeclareGroup( const std::string groupName ) -{ - if( m_DebugMode == true ) - { - if( m_Groups.count( groupName ) == 1 ) - throw std::invalid_argument( "ERROR: group " + groupName + " already exist, from call to CreateGroup\n" ); - } - - m_Groups.emplace( groupName, Group( m_HostLanguage, m_DebugMode ) ); -} - void ADIOS::DefineVariable( const std::string groupName, const std::string variableName, const std::string type, const std::string dimensionsCSV, @@ -192,10 +198,11 @@ void ADIOS::CheckGroup( std::map< std::string, Group >::const_iterator itGroup, throw std::invalid_argument( "ERROR: group " + groupName + " not found " + hint + "\n" ); } -void ADIOS::CheckCapsule( const std::string streamName, const std::string hint ) const +void ADIOS::CheckEngine( std::unordered_map< unsigned int, std::shared_ptr<Engine> >::const_iterator itEngine, + const unsigned int handle, const std::string hint ) const { - if( itCapsule == m_Capsules.end() ) - throw std::invalid_argument( "ERROR: stream (or file) " + streamName + " not created with Open , " + hint + "\n" ); + if( itEngine == m_Engines.end() ) + throw std::invalid_argument( "ERROR: stream (or file) from handle " + handle + " not created with Open , " + hint + "\n" ); } diff --git a/src/core/Capsule.cpp b/src/core/Capsule.cpp index dbc2a3a985fa3017e999973de7591b7527aec22e..1a8011fc5fa835849f30b95429cd81853a62235e 100644 --- a/src/core/Capsule.cpp +++ b/src/core/Capsule.cpp @@ -5,112 +5,23 @@ * Author: wfg */ -/// \cond EXCLUDE_FROM_DOXYGEN -#include <stdexcept> //std::invalid_argument -#include <cstring> //memcpy -/// \endcond - -#ifdef HAVE_BZIP2 -#include "transform/BZIP2.h" -#endif - -#include "../../include/core/Engine.h" -#include "../../include/core/Support.h" - -//transport below -#include "transport/POSIX.h" -#include "transport/FStream.h" +#include "core/Capsule.h" namespace adios { -Capsule::Capsule( ) +Capsule::Capsule( const std::string type, const std::string accessMode ): + m_Type{ type }, + m_AccessMode{ accessMode } { } -Capsule::Capsule( const MPI_Comm mpiComm, const bool debugMode, const std::string streamName, - const std::string accessMode, const std::string transport, const std::vector<std::string>& arguments ): - m_MPIComm{ mpiComm }, - m_DebugMode{ debugMode } -{ - AddTransport( streamName, accessMode, true, transport, arguments ); -} - - Capsule::~Capsule( ) { } -int Capsule::AddTransport( const std::string streamName, const std::string accessMode, const bool isDefault, - const std::string transport, const std::vector<std::string>& arguments ) -{ - - std::string name( streamName ); //default name to transport - if( isDefault == false ) //adding more than default - { - name = GetName( arguments ); - if( m_DebugMode == true ) - { - if( name == streamName ) - throw std::invalid_argument( "ERROR: name= argument can't be the same as stream (or file ) from Open, in call to AddTransport\n" ); - } - } - - if( transport == "POSIX" ) - m_Transports.push_back( std::make_shared<POSIX>( m_MPIComm, m_DebugMode, arguments ) ); - - else if( transport == "FStream" ) - m_Transports.push_back( std::make_shared<FStream>( m_MPIComm, m_DebugMode, arguments ) ); - - int transportIndex = static_cast<int>( m_Transports.size() - 1 ); - m_Transports[ transportIndex ]->Open( name, accessMode ); - - return transportIndex; -} - - -void Capsule::Close( int transportIndex ) -{ - if( transportIndex == -1 ) //close all transports - { - for( auto& transport : m_Transports ) - transport->Close( ); - } - else - { - m_Transports[transportIndex]->Close( ); - } -} - - -//PRIVATE FUNCTIONS -std::string Capsule::GetName( const std::vector<std::string>& arguments ) const -{ - bool isNameFound = false; - std::string name; - - for( const auto argument : arguments ) - { - auto namePosition = argument.find( "name=" ); - if( namePosition != argument.npos ) - { - isNameFound = true; - name = argument.substr( namePosition + 5 ); - break; - } - } - - if( m_DebugMode == true ) - { - if( name.empty() || isNameFound == false ) - std::invalid_argument( "ERROR: argument to name= is empty or not found in call to AddTransport" ); - } - - return name; -} - } //end namespace diff --git a/src/core/Engine.cpp b/src/core/Engine.cpp new file mode 100644 index 0000000000000000000000000000000000000000..460553f6217c0458a17c7e053752b21b60dbfa31 --- /dev/null +++ b/src/core/Engine.cpp @@ -0,0 +1,104 @@ +/* + * Engine.cpp + * + * Created on: Dec 19, 2016 + * Author: wfg + */ + + +#include "core/Engine.h" + + +namespace adios +{ + + +Engine::Engine( const std::string engineType, const std::string name, const std::string accessMode, + const MPI_Comm mpiComm, const Method& method, + const bool debugMode ): + m_EngineType{ engineType }, + m_Name{ name }, + m_AccessMode{ accessMode }, + m_Method{ &method }, + m_MPIComm{ mpiComm } +{ + MPI_Comm_rank( m_MPIComm, &m_RankMPI ); + MPI_Comm_size( m_MPIComm, &m_SizeMPI ); +} + + +Engine::~Engine( ) +{ } + + +void Engine::Close( int transportIndex ) +{ + if( transportIndex == -1 ) //close all transports + { + for( auto& transport : m_Transports ) + transport->Close( ); + } + else + { + m_Transports[transportIndex]->Close( ); + } +} + + +//PROTECTED FUNCTIONS +void Engine::SetTransports( ) +{ + for( const auto& transportPair : m_Method->Transports ) + { + const std::string transport = transportPair.first; + const std::vector<std::string>& arguments = transportPair.second; + + if( transport == "POSIX" ) + m_Transports.push_back( std::make_shared<POSIX>( m_MPIComm, m_DebugMode, arguments ) ); + + else if( transport == "FStream" ) + m_Transports.push_back( std::make_shared<FStream>( m_MPIComm, m_DebugMode, arguments ) ); + + else + { + if( m_DebugMode == true ) + throw std::invalid_argument( "ERROR: transport + " + transport + " not supported, in call from Open.\n" ); + } + + std::string name = GetName( arguments ); + m_Transports.back()->Open( name, m_AccessMode ); + } +} + + + +std::string Engine::GetName( const std::vector<std::string>& arguments ) const +{ + bool isNameFound = false; + std::string name; + + for( const auto argument : arguments ) + { + auto namePosition = argument.find( "name=" ); + if( namePosition != argument.npos ) + { + isNameFound = true; + name = argument.substr( namePosition + 5 ); + break; + } + } + + if( m_DebugMode == true ) + { + if( name.empty() || isNameFound == false ) + std::invalid_argument( "ERROR: argument to name= is empty or not found in call to AddTransport" ); + } + + return name; +} + + +} //end namespace + + + diff --git a/src/core/Group.cpp b/src/core/Group.cpp index fecc0c1a71bae99639484dd5ef5f216afcd53282..da0afe1203414de55ea9302368ee7d6aa561935f 100644 --- a/src/core/Group.cpp +++ b/src/core/Group.cpp @@ -12,9 +12,7 @@ #include "core/Group.h" - -#include "../../include/core/Support.h" -#include "core/Variable.h" +#include "core/Support.h" #include "functions/adiosFunctions.h" @@ -22,15 +20,11 @@ namespace adios { -Group::Group( const std::string hostLanguage, const bool debugMode ): - m_HostLanguage{ hostLanguage }, - m_DebugMode{ debugMode } +Group::Group( ) { } -Group::Group( const std::string hostLanguage, const std::string& xmlGroup, - std::vector< std::shared_ptr<Transform> >& transforms, const bool debugMode ): - m_HostLanguage{ hostLanguage }, +Group::Group( const std::string& xmlGroup, std::vector< std::shared_ptr<Transform> >& transforms, const bool debugMode ): m_DebugMode{ debugMode } { ParseXMLGroup( xmlGroup, transforms ); @@ -42,16 +36,12 @@ Group::~Group( ) void Group::DefineVariable( const std::string variableName, const std::string type, - const std::string dimensionsCSV, - const std::string globalDimensionsCSV, const std::string globalOffsetsCSV, - const short transformIndex, const unsigned short int compressionLevel ) + const std::string dimensionsCSV, + const std::string globalDimensionsCSV, const std::string globalOffsetsCSV, + const Transform* transform, const unsigned short parameter ) { if( m_DebugMode == true ) { - if( Support::Datatypes.at( m_HostLanguage ).count( type ) == 0 ) - throw std::invalid_argument( "ERROR: type " + type + " for variable " + variableName + - " is not supported, in call to DefineVariable\n" ); - if( m_Variables.count( variableName ) == 1 ) throw std::invalid_argument( "ERROR: variable " + variableName + " already exists, in call to DefineVariable\n" ); } @@ -60,75 +50,81 @@ void Group::DefineVariable( const std::string variableName, const std::string ty if( IsTypeAlias( type, Support::DatatypesAliases.at("char") ) == true ) { - m_Char.push_back( Variable<char>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_Char.push_back( Variable<char>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_Char.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("unsigned char") ) == true ) { - m_UChar.push_back( Variable<unsigned char>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_UChar.push_back( Variable<unsigned char>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_UChar.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("short") ) == true ) { - m_Short.push_back( Variable<short>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_Short.push_back( Variable<short>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_Short.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("unsigned short") ) == true ) { - m_UShort.push_back( Variable<unsigned short>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_UShort.push_back( Variable<unsigned short>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_UShort.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("int") ) == true ) { - m_Int.push_back( Variable<int>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_Int.push_back( Variable<int>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_Int.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("unsigned int") ) == true ) { - m_UInt.push_back( Variable<unsigned int>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_UInt.push_back( Variable<unsigned int>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_UInt.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("long int") ) == true ) { - m_LInt.push_back( Variable<long int>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_LInt.push_back( Variable<long int>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_LInt.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("unsigned long int") ) == true ) { - m_ULInt.push_back( Variable<unsigned long int>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_ULInt.push_back( Variable<unsigned long int>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_ULInt.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("long long int") ) == true ) { - m_LLInt.push_back( Variable<long long int>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_LLInt.push_back( Variable<long long int>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_LLInt.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("unsigned long long int") ) == true ) { - m_ULLInt.push_back( Variable<unsigned long long int>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_ULLInt.push_back( Variable<unsigned long long int>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_ULLInt.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("float") ) == true ) { - m_Float.push_back( Variable<float>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_Float.push_back( Variable<float>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_Float.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("double") ) == true ) { - m_Double.push_back( Variable<double>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_Double.push_back( Variable<double>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_Double.size()-1 ); } else if( IsTypeAlias( type, Support::DatatypesAliases.at("long double") ) == true ) { - m_LDouble.push_back( Variable<long double>{ dimensionsCSV, nullptr, globalBoundsIndex, transformIndex, compressionLevel } ); + m_LDouble.push_back( Variable<long double>{ dimensionsCSV, nullptr, globalBoundsIndex, transform, parameter } ); m_Variables[variableName] = std::make_pair( type, m_LDouble.size()-1 ); } + else + { + if( m_DebugMode == true ) + throw std::invalid_argument( "ERROR: type " + type + " not supported, in call to DefineVariable.\n" ); + } m_SerialSize += variableName.size() + type.size() + dimensionsCSV.size() + 4 * sizeof( char ); //4, adding one more for globalBoundsIndex } -void Group::SetTransform( const std::string variableName, const unsigned int transformIndex, const unsigned int compressionLevel ) +void Group::SetTransform( const std::string variableName, const Transform& transform, + const short parameter ) { auto itVariable = m_Variables.find( variableName ); @@ -143,63 +139,63 @@ void Group::SetTransform( const std::string variableName, const unsigned int tra if( IsTypeAlias( type, Support::DatatypesAliases.at("char") ) == true ) { - m_Char[index].TransformIndex = transformIndex; - m_Char[index].CompressionLevel = compressionLevel; + m_Char[index].Transform = &transform; + m_Char[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("unsigned char") ) == true ) { - m_UChar[index].TransformIndex = transformIndex; - m_UChar[index].CompressionLevel = compressionLevel; + m_UChar[index].Transform = &transform; + m_UChar[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("short") ) == true ) { - m_Short[index].TransformIndex = transformIndex; - m_Short[index].CompressionLevel = compressionLevel; + m_Short[index].Transform = &transform; + m_Short[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("unsigned short") ) == true ) { - m_UShort[index].TransformIndex = transformIndex; - m_UShort[index].CompressionLevel = compressionLevel; + m_UShort[index].Transform = &transform; + m_UShort[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("int") ) == true ) { - m_Int[index].TransformIndex = transformIndex; - m_Int[index].CompressionLevel = compressionLevel; + m_Int[index].Transform = &transform; + m_Int[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("unsigned int") ) == true ) { - m_UInt[index].TransformIndex = transformIndex; - m_UInt[index].CompressionLevel = compressionLevel; + m_UInt[index].Transform = &transform; + m_UInt[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("long int") ) == true ) { - m_LInt[index].TransformIndex = transformIndex; - m_LInt[index].CompressionLevel = compressionLevel; + m_LInt[index].Transform = &transform; + m_LInt[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("unsigned long int") ) == true ) { - m_ULInt[index].TransformIndex = transformIndex; - m_ULInt[index].CompressionLevel = compressionLevel; + m_ULInt[index].Transform = &transform; + m_ULInt[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("unsigned long long int") ) == true ) { - m_ULLInt[index].TransformIndex = transformIndex; - m_ULLInt[index].CompressionLevel = compressionLevel; + m_ULLInt[index].Transform = &transform; + m_ULLInt[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("float") ) == true ) { - m_Float[index].TransformIndex = transformIndex; - m_Float[index].CompressionLevel = compressionLevel; + m_Float[index].Transform = &transform; + m_Float[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("double") ) == true ) { - m_Double[index].TransformIndex = transformIndex; - m_Double[index].CompressionLevel = compressionLevel; + m_Double[index].Transform = &transform; + m_Double[index].Parameter = parameter; } else if( IsTypeAlias( type, Support::DatatypesAliases.at("long double") ) == true ) { - m_LDouble[index].TransformIndex = transformIndex; - m_LDouble[index].CompressionLevel = compressionLevel; + m_LDouble[index].Transform = &transform; + m_LDouble[index].Parameter = parameter; } } @@ -276,7 +272,10 @@ const unsigned long long int Group::GetIntVariableValue( const std::string varia value = *( m_ULLInt[index].Values ); else - throw std::invalid_argument( "ERROR: variable " + variableName + " must be of integer type : short, int or associated type (long int, unsigned long int, etc.)\n" ); + { + if( m_DebugMode == true ) + throw std::invalid_argument( "ERROR: variable " + variableName + " must be of integer type : short, int or associated type (long int, unsigned long int, etc.)\n" ); + } if( m_DebugMode == true ) { diff --git a/src/engine/Single.cpp b/src/engine/Single.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0c9eb405cb31e961b57bb542729399bb3ab7dbf5 --- /dev/null +++ b/src/engine/Single.cpp @@ -0,0 +1,31 @@ +/* + * Single.cpp + * + * Created on: Dec 19, 2016 + * Author: wfg + */ + + +#include "engine/Single.h" + +namespace adios +{ + +Single::Single( const std::string streamName, const std::string accessMode, const MPI_Comm mpiComm, const Method& method, + const bool debugMode ): + Engine( "Single Buffer Engine", streamName, accessMode, mpiComm, method, debugMode ) +{ + if( m_DebugMode == true ) + { + if( m_Method->Capsules.size() > 1 ) + throw std::invalid_argument( "ERROR: single buffer engine only allows one buffer, from Single constructor in Open.\n" ); + } + + SetTransports( ); +} + + + + +} +