diff --git a/Makefile.libs b/Makefile.libs index d7eb7b142878a05668cc74c6d9c3c89ab59fbd90..7547da2892940c18430767649bf7db055dcfca79 100644 --- a/Makefile.libs +++ b/Makefile.libs @@ -1,3 +1,14 @@ +#Language API +LANGUAGE_INC:=./include/public +LANGUAGE_SRC:=./src/public + +ifeq ($(C_API),yes) +else + HFiles:=$(filter-out $(LANGUAGE_INC)/ADIOS_C.h,$(HFiles)) + CPPFiles:=$(filter-out $(LANGUAGE_SRC)/ADIOS_C.cpp,$(CPPFiles)) +endif + + #TRANSPORTS WITH EXTERNAL DEPENDENCIES TRANSPORT_INC:=./include/transport TRANSPORT_SRC:=./src/transport diff --git a/include/capsule/CBoostInterprocess.h b/include/capsule/CBoostInterprocess.h deleted file mode 100644 index 24407c9483c203847317b6676ec886be7848aa3a..0000000000000000000000000000000000000000 --- a/include/capsule/CBoostInterprocess.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * CInterprocess.h Capsule that uses boost interprocess for shared memory operations - * - * Created on: Nov 8, 2016 - * Author: wfg - */ - -#ifndef CBOOSTINTERPROCESS_H_ -#define CBOOSTINTERPROCESS_H_ - -#include "core/CCapsule.h" - - -namespace adios -{ - -/** - * Base class for Capsule operations - */ -class CBoostInterprocess : public CCapsule -{ - -public: - - #ifdef HAVE_MPI - MPI_Comm m_MPIComm = NULL; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C - #else - MPI_Comm m_MPIComm = 0; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C - #endif - - std::vector<char> m_Buffer; ///< buffer to be managed, just one type for now - - CBoostInterprocess( ); ///< default empty constructor - - ~CBoostInterprocess( ); - - /** - * This will add to the m_Transports and m_Transforms map - * @param group - */ - void OpenGroupBuffer( const CGroup& group ); - - - template< class T> - void WriteVariableToBuffer( const CGroup& group, const SVariable<T>& variable ) - { -// while -// { -// -// send -// receive status -// if( status ) -// switch Transport -// } - } - - /** - * Closes the buffer and moves it into the - * @param group - */ - void CloseGroupBuffer( const CGroup& group ); - -}; - - -} //end namespace - - - -#endif /* CBOOSTINTERPROCESS_H_ */ diff --git a/include/capsule/CEmptyCapsule.h b/include/capsule/CEmptyCapsule.h deleted file mode 100644 index dadaf9b9a55711c5ee94e4949fd0ef0880926426..0000000000000000000000000000000000000000 --- a/include/capsule/CEmptyCapsule.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * CEmptyCapsule.h - * - * Created on: Nov 9, 2016 - * Author: wfg - */ - -#ifndef CEMPTYCAPSULE_H_ -#define CEMPTYCAPSULE_H_ - -#include "core/CCapsule.h" - - -namespace adios -{ - -/** - * Base class for Capsule operations - */ -class CEmptyCapsule : public CCapsule -{ - -public: - - CEmptyCapsule( MPI_Comm mpiComm ); ///< default empty constructor - - ~CEmptyCapsule( ); - - /** - * This will add to the m_Transports and m_Transforms map - * @param group - */ - void OpenGroupBuffer( const CGroup& group ); - - - /** - * Closes the buffer and moves it into the - * @param group - */ - void CloseGroupBuffer( const CGroup& group ); - -}; - - -} //end namespace - - - -#endif /* CEMPTYCAPSULE_H_ */ diff --git a/include/capsule/CMMap.h b/include/capsule/CMMap.h deleted file mode 100644 index b0707d5ba0efd8e5754fbfa8ce19c7684247a974..0000000000000000000000000000000000000000 --- a/include/capsule/CMMap.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * CMMap.h capsule that uses mapped files for shared memory - * - * Created on: Nov 8, 2016 - * Author: wfg - */ - -#ifndef CMMAP_H_ -#define CMMAP_H_ - -#include "core/CCapsule.h" - - -namespace adios -{ - -/** - * Capsule based on mmap (mapped files) - */ -class CMMap : public CCapsule -{ - -public: - - #ifdef HAVE_MPI - MPI_Comm m_MPIComm = NULL; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C - #else - MPI_Comm m_MPIComm = 0; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C - #endif - - std::vector<char> m_Buffer; ///< buffer to be managed, just one type for now - - CMMap( ); ///< default empty constructor - - ~CMMap( ); - - /** - * This will add to the m_Transports and m_Transforms map - * @param group - */ - void OpenGroupBuffer( const CGroup& group ); - - - template< class T> - void WriteVariableToBuffer( const CGroup& group, const SVariable<T>& variable ); - - /** - * Closes the buffer and moves it into the - * @param group - */ - void CloseGroupBuffer( const CGroup& group ); - -private: - - template< class T> - void SpatialAggregation( const CGroup& group, const SVariable<T>& variable ) - { } - - template< class T> - void TimeAggregation( const CGroup& group, const SVariable<T>& variable ) - { } - -}; - -} //end namespace - -#endif /* CMMAP_H_ */ diff --git a/include/capsule/CShm.h b/include/capsule/CShm.h deleted file mode 100644 index 04123edfff9f4fc9bf83148334da0cb07638eaf7..0000000000000000000000000000000000000000 --- a/include/capsule/CShm.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * CShm.h Capsule that uses POSIX shared-memory (shm) api - * - * Created on: Nov 8, 2016 - * Author: wfg - */ - -#ifndef CSHM_H_ -#define CSHM_H_ - - -#include "core/CCapsule.h" - - -namespace adios -{ - -/** - * Capsule based on POSIX shared memory - */ -class CShm : public CCapsule -{ - -public: - - #ifdef HAVE_MPI - MPI_Comm m_MPIComm = NULL; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C - #else - MPI_Comm m_MPIComm = 0; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C - #endif - - std::vector<char> m_Buffer; ///< buffer to be managed, just one type for now - - CShm( ); ///< default empty constructor - - ~CShm( ); - - /** - * This will add to the m_Transports and m_Transforms map - * @param group - */ - void OpenGroupBuffer( const CGroup& group ); - - - template< class T> - void WriteVariableToBuffer( const CGroup& group, const SVariable<T>& variable ) - { } - - /** - * Closes the buffer and moves it into the - * @param group - */ - void CloseGroupBuffer( const CGroup& group ); - - -private: - - template< class T> - void SpatialAggregation( const CGroup& group, const SVariable<T>& variable ) - { } - - template< class T> - void TimeAggregation( const CGroup& group, const SVariable<T>& variable ) - { } - -}; - - - -} //end namespace - - -#endif /* CSHM_H_ */ diff --git a/include/core/CCapsule.h b/include/core/CCapsule.h index b078b32cb8941a320943d044ab2b43ceb2853d43..0be7624b61048f4cc38b8c5ef244d13a5ca4b43a 100644 --- a/include/core/CCapsule.h +++ b/include/core/CCapsule.h @@ -8,7 +8,13 @@ #ifndef CCAPSULE_H_ #define CCAPSULE_H_ + +/// \cond EXCLUDE_FROM_DOXYGEN #include <vector> +#include <string> +#include <memory> +#include <map> +/// \endcond #ifdef HAVE_MPI #include <mpi.h> @@ -16,7 +22,6 @@ #include "public/mpidummy.h" #endif -#include "core/CGroup.h" #include "core/SVariable.h" #include "core/CTransform.h" #include "core/CTransport.h" @@ -39,58 +44,81 @@ public: MPI_Comm m_MPIComm = 0; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C #endif - std::map< std::string, std::vector<char> > m_Buffer; ///< buffer to be managed, just one type for now + int m_RankMPI = 0; ///< current MPI rank process + int m_SizeMPI = 1; ///< current MPI processes size + + const bool m_DebugMode = false; + + std::map< std::string, std::vector<char> > m_Buffer; ///< buffer to be managed, key is the fileName just one type for now + std::map< std::string, std::shared_ptr<CTransport> > m_Transports; ///< transport associated with ADIOS run + std::map< std::string, std::shared_ptr<CTransform> > m_Transforms; ///< transforms associated with ADIOS run - std::map< std::string, std::shared_ptr<CTransport> > m_Transports; ///< transports associated with ADIOS run + + ///Maybe add a communication class object + + /** + * Empty constructor + */ + CCapsule( ); + + /** + * Debug mode + * @param debugMode + */ + CCapsule( const bool debugMode ); /** * Unique constructor * @param mpiComm communicator passed from ADIOS - * @param bufferSize passed by the user */ - CCapsule( const MPI_Comm mpiComm, const unsigned long int bufferSize ); + CCapsule( MPI_Comm mpiComm, const bool debugMode ); + + ~CCapsule( ); - virtual ~CCapsule( ); + void SetTransform( const std::string transform ); - void CreateTransform( const std::string transform ); + void SetTransport( const std::string streamName, const std::string transport, const bool debugMode ); + + void SetBuffer( const std::string streamName, const unsigned int long maxBufferSize ); - void CreateTransport( const std::string transport ); /** - * This will add to the m_Transports and m_Transforms map - * @param group + * Open a certain stream based on transport method + * @param streamName associated file or stream + * @param accessMode "w": write, "a": append, need more info on this + * @param bufferSize */ - void OpenGroupBuffer( const CGroup& group ) = 0; - - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<char>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<unsigned char>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<short>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<unsigned short>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<int>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<unsigned int>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<long int>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<unsigned long int>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<long long int>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<unsigned long long int>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<float>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<double>& variable ); - virtual void WriteVariableToBuffer( const CGroup& group, const SVariable<long double>& variable ); + void Open( const std::string streamName, const std::string accessMode ); + + void WriteVariableToBuffer( const std::string streamName, const SVariable<char>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<unsigned char>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<short>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<unsigned short>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<int>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<unsigned int>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<long int>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<unsigned long int>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<long long int>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<unsigned long long int>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<float>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<double>& variable ); + void WriteVariableToBuffer( const std::string streamName, const SVariable<long double>& variable ); /** - * Closes the buffer and moves it into the - * @param group + * Closes the buffer + * @param streamName associated streamName to this buffer */ - void CloseGroupBuffer( const CGroup& group ) = 0; - - -protected: + void CloseBuffer( const std::string streamName ); - virtual void SpatialAggregation( const CGroup& group, const SVariable<unsigned int>& variable ); - virtual void SpatialAggregation( const CGroup& group, const SVariable<int>& variable ); - virtual void SpatialAggregation( const CGroup& group, const SVariable<float>& variable ); - virtual void SpatialAggregation( const CGroup& group, const SVariable<double>& variable ); - virtual void TimeAggregation( const CGroup& group, const SVariable<unsigned int>& variable ); +//protected: +// +// virtual void SpatialAggregation( const CGroup& group, const SVariable<unsigned int>& variable ); +// virtual void SpatialAggregation( const CGroup& group, const SVariable<int>& variable ); +// virtual void SpatialAggregation( const CGroup& group, const SVariable<float>& variable ); +// virtual void SpatialAggregation( const CGroup& group, const SVariable<double>& variable ); +// +// virtual void TimeAggregation( const CGroup& group, const SVariable<unsigned int>& variable ); }; diff --git a/include/core/CGroup.h b/include/core/CGroup.h index 061c535a92461137ca4791a1c8244eae7a1a13d0..d4990a349805399f1f4396b460330cc2a079ba05 100644 --- a/include/core/CGroup.h +++ b/include/core/CGroup.h @@ -14,6 +14,7 @@ #include <memory> //for shared_pointer #include <vector> #include <ostream> +#include <set> /// \endcond #ifdef HAVE_MPI @@ -23,7 +24,7 @@ #endif -#include <core/SVariable.h> +#include "core/SVariable.h" #include "core/SAttribute.h" #include "core/CTransport.h" @@ -48,7 +49,7 @@ public: * @param groupName returns the groupName from <adios-group name=" " * @param debugMode from ADIOS */ - CGroup( const std::string& hostLanguage, const std::string& xmlGroup, const bool debugMode = false ); + CGroup( const std::string& hostLanguage, const std::string& xmlGroup, const bool debugMode ); /** @@ -56,19 +57,23 @@ public: * @param hostLanguage reference from ADIOS class * @param debugMode */ - CGroup( const std::string& hostLanguage, const bool debugMode = false ); - - - ~CGroup( ); ///< Using STL containers, no deallocation + CGroup( const std::string& hostLanguage, const bool debugMode ); /** - * Called from ADIOS open, sets group as active and passes associated bufferName and accessMode - * @param bufferName associated buffer for this group - * @param accessMode associated access mode (read, write, append) for bufferName + * Create a Group object associated with a transport, can be modified later + * @param hostLanguage reference from ADIOS class + * @param transport transport method + * @param priority transport priority + * @param iterations iterations for transport + * @param debugMode reference from ADIOS class */ - void Open( const std::string bufferName, const std::string accessMode ); + CGroup( const std::string& hostLanguage, const std::string transport, const unsigned int priority, const unsigned int iterations, + const bool debugMode ); + + ~CGroup( ); ///< Using STL containers, no deallocation + /** * Creates a new variable in the group object * @param name variable name, must be unique. If name exists it removes the current variable. In debug mode program will exit. @@ -94,7 +99,21 @@ public: const std::string globalDimensionsCSV, const std::string globalOffsetsCSV ); - void Close( ); ///< set m_IsOpen to false and sets m_BufferName and m_AccessMode to empty + /** + * Sets transport method super-seeding the existing one + * @param transport method name + */ + void SetTransport( const std::string transport ); + + /** + * Called from ADIOS open, sets group as active and passes associated bufferName and accessMode + * @param streamName associated buffer for this group + */ + void Open( const std::string streamName ); + + + + void Close( ); ///< sets m_IsOpen to false, and m_BufferName and m_AccessMode to empty /** @@ -141,11 +160,10 @@ private: std::vector< std::pair< std::string, std::string > > m_GlobalBounds; ///< if a variable or an attribute is global it fills this container, from global-bounds in XML File, data in global space, pair.first = global dimensions, pair.second = global bounds - std::string m_Transport; ///< current transport method associated with this group - std::string m_BufferName; ///< associated buffer (file, stream, vector, etc.) if the Group is opened. - std::string m_AccessMode; ///< associated access mode for associated buffer from m_BufferName + unsigned long int m_SerialSize = 0; ///< size used for potential serialization of metadata into a std::vector<char>. Counts sizes from m_Variables, m_Attributes, m_GlobalBounds - unsigned long int m_SerialSize; ///< size used for potential serialization of metadata into a std::vector<char>. Counts sizes from m_Variables, m_Attributes, m_GlobalBounds + std::string m_Transport; ///< associated transport method for this group + std::string m_StreamName; ///< associated transport method for this group /** * Called from XML constructor @@ -170,7 +188,6 @@ private: * @return -1 variable is not associated with a transform, otherwise index in m_Transforms */ const int SetTransforms( const std::string transform ) noexcept; - }; diff --git a/include/core/CTransform.h b/include/core/CTransform.h index 7d5e457038cbfb3ec322037630f39f9affecf958..1628e2ff99ae56704aa224c3ebb9887794871a4b 100644 --- a/include/core/CTransform.h +++ b/include/core/CTransform.h @@ -10,10 +10,10 @@ /// \cond EXCLUDE_FROM_DOXYGEN -#include <core/SVariable.h> #include <string> /// \endcond +#include "core/SVariable.h" namespace adios @@ -27,23 +27,43 @@ class CTransform public: - const std::string m_Method; ///< name of the transformation method - const unsigned int m_CompressionLevel; ///< depends on library implementation - /** * Initialize parent method * @param method zlib, bzip2, szip * @param variable */ - CTransform( const unsigned int compressionLevel ): - m_CompressionLevel( compressionLevel ) + CTransform( ) { } virtual ~CTransform( ){ }; - virtual void Compress( ) const = 0; ///< Compress m_Variable data m_Value - - virtual void Decompress( ) const = 0; ///< Ext m_Variable data m_Value + virtual void Compress( const SVariable<char>& variableIn, SVariable<char>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<unsigned char>& variableIn, SVariable<unsigned char>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<short>& variableIn, SVariable<short>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<unsigned short>& variableIn, SVariable<unsigned short>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<int>& variableIn, SVariable<int>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<unsigned int>& variableIn, SVariable<unsigned int>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<long int>& variableIn, SVariable<long int>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<unsigned long int>& variableIn, SVariable<unsigned long int>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<long long int>& variableIn, SVariable<long long int>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<unsigned long long int>& variableIn, SVariable<unsigned long long int>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<float>& variableIn, SVariable<float>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<double>& variableIn, SVariable<double>& variableOut, const unsigned int level ) const; + virtual void Compress( const SVariable<long double>& variableIn, SVariable<long double>& variableOut, const unsigned int level ) const; + + virtual void Decompress( const SVariable<char>& variableIn, SVariable<char>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<unsigned char>& variableIn, SVariable<unsigned char>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<short>& variableIn, SVariable<short>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<unsigned short>& variableIn, SVariable<unsigned short>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<int>& variableIn, SVariable<int>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<unsigned int>& variableIn, SVariable<unsigned int>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<long int>& variableIn, SVariable<long int>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<unsigned long int>& variableIn, SVariable<unsigned long int>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<long long int>& variableIn, SVariable<long long int>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<unsigned long long int>& variableIn, SVariable<unsigned long long int>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<float>& variableIn, SVariable<float>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<double>& variableIn, SVariable<double>& variableOut, const unsigned int level ) const; + virtual void Decompress( const SVariable<long double>& variableIn, SVariable<long double>& variableOut, const unsigned int level ) const; }; diff --git a/include/core/CTransport.h b/include/core/CTransport.h index b125e74ac4d18024a1e6aca2e1583e4bd5d36e7c..21c2227bc83ed2070fb637960ef9752ee8860004 100644 --- a/include/core/CTransport.h +++ b/include/core/CTransport.h @@ -28,30 +28,29 @@ class CTransport public: - const unsigned int m_Priority; ///< writing priority for this transport - const unsigned int m_Iteration; ///< iteration number for this transport - MPI_Comm m_MPIComm; ///< passed MPI communicator + #ifdef HAVE_MPI + MPI_Comm m_MPIComm = NULL; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C + #else + MPI_Comm m_MPIComm = 0; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C + #endif + + //const unsigned int m_Iterations; ///< iteration number for this transport const bool m_DebugMode; ///< if true: additional checks and exceptions - std::string m_StreamName; ///< file or stream name associated with a group that owns the transport - //This part should go to a capsule ?? int m_RankMPI = 0; ///< current MPI rank process int m_SizeMPI = 1; ///< current MPI processes size + /** * Base constructor that all derived classes pass - * @param method passed to m_Method + * @param mpiComm passed to m_MPIComm * @param priority passed to m_Priority * @param iteration passed to m_Iteration - * @param mpiComm passed to m_MPIComm * @param debugMode passed to m_DebugMode */ - CTransport( const std::string method, const unsigned int priority, const unsigned int iteration, - MPI_Comm mpiComm, const bool debugMode ): - m_Priority( priority ), - m_Iteration( iteration ), - m_MPIComm( mpiComm ), - m_DebugMode( debugMode ) + CTransport( MPI_Comm mpiComm, const bool debugMode ): + m_MPIComm{ mpiComm }, + m_DebugMode{ debugMode } { MPI_Comm_rank( m_MPIComm, &m_RankMPI ); MPI_Comm_size( m_MPIComm, &m_SizeMPI ); @@ -63,10 +62,13 @@ public: /** * Open Output file accesing a mode - * @param fileName name of file + * @param streamName name of file * @param accessMode r or read, w or write, a or append */ - virtual void Open( const std::string outputName, const std::string accessMode ) = 0; + virtual void Open( const std::string streamName, const std::string accessMode ) = 0; + + virtual void SetBuffer( std::vector<char>& buffer ) + { }; virtual void Close( ) = 0; //here think what needs to be passed diff --git a/include/core/SVariable.h b/include/core/SVariable.h index f4dc4738598c1884ede07c767ca7fbb959f8fa60..aea69b36c6f746772fdbf03dc1bc968b9615f55d 100644 --- a/include/core/SVariable.h +++ b/include/core/SVariable.h @@ -15,11 +15,8 @@ #include <sstream> /// \endcond -#include "core/CGroup.h" //friend class (manager) - namespace adios { - /** * @param Base (parent) class for template derived (child) class CVariable. Required to put CVariable objects in STL containers. */ diff --git a/include/functions/ADIOSTemplates.h b/include/functions/ADIOSTemplates.h index 4ad576df0daffb1c0468ef320bf7334d912c1183..b1db2016810dd0f7c4956770c71b2573ecda89c6 100644 --- a/include/functions/ADIOSTemplates.h +++ b/include/functions/ADIOSTemplates.h @@ -16,85 +16,101 @@ namespace adios { +/** + * Helper function called from ADIOS Write template function. Checks variable type, adds to group.m_SetVariables, + * and calls corresponding Capsule virtual function. + * @param variableName + * @param values + * @param group + * @param capsule + */ template< class T > -void WriteVariable( const std::string variableName, const T* values, CGroup& group, CCapsule& capsule ) noexcept +void WriteVariable( const std::string variableName, const T* values, CGroup& group, CCapsule& capsule ) { - auto itVariable = group.m_Variables.find( variableName ); - const std::string type( itVariable->second.first ); - const unsigned int index( itVariable->second.second ); + const bool debugMode( group.m_DebugMode ); + const std::string streamName( group.m_StreamName ); + const auto itVariable = group.m_Variables.find( variableName ); + + if( debugMode == true ) + { + if( itVariable == group.m_Variables.end() ) + throw std::invalid_argument( "ERROR: from Write function, variable " + variableName + " doesn't exist\n" ); + } + + const unsigned int index( itVariable->second.second ); //index is second in the pair Value of the m_Variables map group.m_SetVariables.insert( variableName ); - if( std::is_same<T,char> ) + if( std::is_same<T,char>::value ) //maybe use type? { SVariable<char>& variable = group.m_Char[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,unsigned char> ) + else if( std::is_same<T,unsigned char>::value ) { SVariable<unsigned char>& variable = group.m_UChar[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,short> ) + else if( std::is_same<T,short>::value ) { SVariable<short>& variable = group.m_Short[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,unsigned short> ) + else if( std::is_same<T,unsigned short>::value ) { SVariable<unsigned short>& variable = group.m_UShort[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,int> ) + else if( std::is_same<T,int>::value ) { SVariable<int>& variable = group.m_Int[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,unsigned int> ) + else if( std::is_same<T,unsigned int>::value ) { SVariable<unsigned int>& variable = group.m_Int[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,long int> ) + else if( std::is_same<T,long int>::value ) { SVariable<long int>& variable = group.m_LInt[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,long long int> ) + else if( std::is_same<T,long long int>::value ) { SVariable<long int>& variable = group.m_LLInt[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,unsigned long long int> ) + else if( std::is_same<T,unsigned long long int>::value ) { SVariable<unsigned long long int>& variable = group.m_ULLInt[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,float> ) + else if( std::is_same<T,float>::value ) { SVariable<float>& variable = group.m_Float[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,double> ) + else if( std::is_same<T,double>::value ) { SVariable<double>& variable = group.m_Double[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } - else if( std::is_same<T,long double> ) + else if( std::is_same<T,long double>::value ) { - SVariable<double>& variable = group.m_Double[index]; + SVariable<double>& variable = group.m_LDouble[index]; variable.m_Values = values; - capsule.WriteVariableToBuffer( group, variable ); + capsule.WriteVariableToBuffer( streamName, variable ); } } diff --git a/include/public/ADIOS.h b/include/public/ADIOS.h index 52e58ef0c5de9f79dc8cfe44c530c1490f6f7046..277523bc575787752b1139314be65f6baa1b2f32 100644 --- a/include/public/ADIOS.h +++ b/include/public/ADIOS.h @@ -82,8 +82,9 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO * @param groupName should match an existing group from XML file or created through CreateGroup * @param fileName associated file or stream * @param accessMode "w": write, "a": append, need more info on this + * @param maxBufferSize used for transport */ - void Open( const std::string groupName, const std::string fileName, const std::string accessMode = "w" ); + void Open( const std::string groupName, const std::string streamName, const std::string accessMode = "w", unsigned long int maxBufferSize ); /** @@ -106,7 +107,6 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO WriteVariable( variableName, values, itGroup->second, m_Capsule ); } - /** * Close a particular group, group will be out of scope and destroyed * @param groupName group to be closed @@ -121,24 +121,46 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO */ void MonitorGroups( std::ostream& logStream ) const; - /** * @brief Create a new group or replace an existing one * @param groupName unique name - * @param transport transport method - * @param streamFile */ - void CreateGroup( const std::string groupName, const std::string transport = "" ); - + void CreateGroup( const std::string groupName ); + /** + * Creates a new Variable, if debugMode = true, program will throw an exception, else it will overwrite current variable with the same name + * @param groupName corresponding variable group + * @param variableName corresponding variable name + * @param type variable type + * @param dimensionsCSV comma separated (no space) dimensions "Nx,Ny,Nz" defined by other variables + * @param transform transformation method applied to current variable + * @param globalDimensionsCSV comma separated (no space) global dimensions "gNx,gNy,gNz" defined by other variables + * @param globalOffsetsCSV comma separated (no space) global offsets "oNx,oNy,oNz" defined by other variables + */ void CreateVariable( const std::string groupName, const std::string variableName, const std::string type, const std::string dimensionsCSV = "", const std::string transform = "", const std::string globalDimensionsCSV = "", const std::string globalOffsetsCSV = "" ); + /** + * Creates a new Variable, if debugMode = true, program will throw an exception, else it will overwirte current variable with the same name + * @param groupName corresponding variable group + * @param attributeName corresponding attribute name + * @param type string or number + * @param value string contents of the attribute (e.g. "Communication value" ) + * @param globalDimensionsCSV comma separated (no space) global dimensions "gNx,gNy,gNz" defined by other variables + * @param globalOffsetsCSV comma separated (no space) global offsets "oNx,oNy,oNz" defined by other variables + */ void CreateAttribute( const std::string groupName, const std::string attributeName, const std::string type, const std::string value, const std::string globalDimensionsCSV = "", const std::string globalOffsetsCSV = "" ); + /** + * Sets a transport method to be associated with a group + * @param groupName unique name + * @param transport transport method + */ + void SetTransport( const std::string groupName, const std::string transport ); + private: @@ -155,7 +177,7 @@ private: */ std::map< std::string, CGroup > m_Groups; - std::shared_ptr<CCapsule> m_Capsule; ///< manager of data transports, transforms and movement operations + CCapsule m_Capsule; ///< manager of data transports, transforms and movement operations /** * @brief Maximum buffer size in ADIOS write() operation. From buffer max - size - MB in XML file diff --git a/include/transform/CBZIP2.h b/include/transform/CBZIP2.h index 8326796cec000d5f42bac2c6a036f2c2654d9d0b..f71dc8240d354dbcb381471051737a448116e03b 100644 --- a/include/transform/CBZIP2.h +++ b/include/transform/CBZIP2.h @@ -29,14 +29,37 @@ public: * @param compressionLevel * @param variable */ - CBZIP2( const unsigned int compressionLevel, CVariableBase& variable ); + CBZIP2( ); ~CBZIP2( ); - void Compress( ) const; - - void Decompress( ) const; - + void Compress( const SVariable<char>& variableIn, SVariable<char>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<unsigned char>& variableIn, SVariable<unsigned char>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<short>& variableIn, SVariable<short>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<unsigned short>& variableIn, SVariable<unsigned short>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<int>& variableIn, SVariable<int>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<unsigned int>& variableIn, SVariable<unsigned int>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<long int>& variableIn, SVariable<long int>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<unsigned long int>& variableIn, SVariable<unsigned long int>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<long long int>& variableIn, SVariable<long long int>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<unsigned long long int>& variableIn, SVariable<unsigned long long int>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<float>& variableIn, SVariable<float>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<double>& variableIn, SVariable<double>& variableOut, const unsigned int level ) const; + void Compress( const SVariable<long double>& variableIn, SVariable<long double>& variableOut, const unsigned int level ) const; + + void Decompress( const SVariable<char>& variableIn, SVariable<char>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<unsigned char>& variableIn, SVariable<unsigned char>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<short>& variableIn, SVariable<short>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<unsigned short>& variableIn, SVariable<unsigned short>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<int>& variableIn, SVariable<int>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<unsigned int>& variableIn, SVariable<unsigned int>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<long int>& variableIn, SVariable<long int>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<unsigned long int>& variableIn, SVariable<unsigned long int>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<long long int>& variableIn, SVariable<long long int>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<unsigned long long int>& variableIn, SVariable<unsigned long long int>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<float>& variableIn, SVariable<float>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<double>& variableIn, SVariable<double>& variableOut, const unsigned int level ) const; + void Decompress( const SVariable<long double>& variableIn, SVariable<long double>& variableOut, const unsigned int level ) const; }; diff --git a/include/transport/CFStream.h b/include/transport/CFStream.h index 084c88ee818d627b11af86536316703c59307002..c78e2c6039cf10e1d4db7bc2728fa8689743c0f1 100644 --- a/include/transport/CFStream.h +++ b/include/transport/CFStream.h @@ -24,11 +24,13 @@ class CFStream : public CTransport public: - CFStream( const unsigned int priority, const unsigned int iteration, MPI_Comm mpiComm, const bool debugMode ); + CFStream( MPI_Comm mpiComm, const bool debugMode ); ~CFStream( ); - void Open( const std::string fileName, const std::string accessMode ); + void Open( const std::string streamName, const std::string accessMode ); + + void SetBuffer( std::vector<char>& buffer ); void Close( ); diff --git a/include/transport/CPOSIX.h b/include/transport/CPOSIX.h index e1ca3b8ca9ba7ee33ff1645c0af141f147d84a5a..e0256ccfab9411f4b0a16f098b0132f26fe3d9c4 100644 --- a/include/transport/CPOSIX.h +++ b/include/transport/CPOSIX.h @@ -8,6 +8,7 @@ #ifndef CPOSIX_H_ #define CPOSIX_H_ +#include <stdio.h> //FILE* #include "core/CTransport.h" @@ -21,22 +22,25 @@ class CPOSIX : public CTransport public: - CPOSIX( const unsigned int priority, const unsigned int iteration, MPI_Comm mpiComm, const bool debugMode ); + CPOSIX( MPI_Comm mpiComm, const bool debugMode ); ~CPOSIX( ); - void Open( const std::string fileName, const std::string accessMode ); + void Open( const std::string streamName, const std::string accessMode ); - void Close( ); + void SetBuffer( std::vector<char>& buffer ); -}; + void Close( ); -} //end namespace +private: + FILE* m_File; +}; +} //end namespace #endif /* CPOSIX_H_ */ diff --git a/src/core/CCapsule.cpp b/src/core/CCapsule.cpp index 9bcbbeafe9b5c30d23baa58064e6143c4d50596d..9529733c1070f9a16522ca4cae873be7446d9edc 100644 --- a/src/core/CCapsule.cpp +++ b/src/core/CCapsule.cpp @@ -7,21 +7,99 @@ #include "core/CCapsule.h" +#ifdef HAVE_BZIP2 +#include "transform/CBZIP2.h" +#endif + + +//transports +#include "transport/CPOSIX.h" +#include "transport/CFStream.h" + namespace adios { -CCapsule::CCapsule( const MPI_Comm mpiComm, const unsigned long int bufferSize ): - m_MPIComm{ mpiComm }, - m_BufferSize{ bufferSize } +CCapsule::CCapsule( ) { } +CCapsule::CCapsule( const bool debugMode ): + m_DebugMode{ debugMode } +{ } + + +CCapsule::CCapsule( MPI_Comm mpiComm, const bool debugMode ): + m_MPIComm{ mpiComm }, + m_DebugMode{ debugMode } +{ + MPI_Comm_rank( m_MPIComm, &m_RankMPI ); + MPI_Comm_size( m_MPIComm, &m_SizeMPI ); +} + + CCapsule::~CCapsule( ) { } +void CCapsule::SetTransform( const std::string transform ) +{ + std::string method( transform ); + auto colonPosition = transform.find(":"); + + if( colonPosition != transform.npos ) + { + method = transform.substr( 0, colonPosition ); + } + if( m_Transforms.find( method ) != m_Transforms.end() ) //transform method already exists, do nothing + return; + if( method == "bzip2" ) //here must add debug mode exception + { + #ifdef HAVE_BZIP2 + m_Transforms["bzip2"] = std::make_shared<CBZIP2>( ); + #endif + } +} + + +void CCapsule::SetTransport( const std::string streamName, const std::string transport, const bool debugMode ) +{ + if( transport == "POSIX" ) + m_Transports[streamName] = std::make_shared<CPOSIX>( m_MPIComm, debugMode ); + else if( transport == "FStream" ) + m_Transports[streamName] = std::make_shared<CFStream>( m_MPIComm, debugMode ); } + + +void CCapsule::SetBuffer( const std::string streamName, const unsigned int long maxBufferSize ) +{ + auto itBuffer = m_Buffer.find( streamName ); + + if( m_DebugMode == true ) + { + if( itBuffer == m_Buffer.end() ) + throw std::invalid_argument( "ERROR: buffer for stream " + streamName + " not found, not setting size\n" ); + } + + itBuffer->second.reserve( maxBufferSize ); +} + + +void CCapsule::Open( const std::string streamName, const std::string accessMode ) +{ + m_Transports[streamName]->Open( streamName, accessMode ); + m_Transports[streamName]->SetBuffer( m_Buffer[streamName] ); +} + + +void CCapsule::CloseBuffer( const std::string streamName ) +{ + m_Transports[streamName]->Close( ); +} + + + +} //end namespace diff --git a/src/core/CGroup.cpp b/src/core/CGroup.cpp index 0a266f3822595b5bab4253e617c7dd2dc9490ec6..e1f731e2d571e496c2cb14b3140ccc1d5eb37aaf 100644 --- a/src/core/CGroup.cpp +++ b/src/core/CGroup.cpp @@ -21,30 +21,26 @@ namespace adios CGroup::CGroup( const std::string& hostLanguage, const bool debugMode ): m_HostLanguage{ hostLanguage }, - m_DebugMode{ debugMode }, - m_SerialSize{ 0 } + m_DebugMode{ debugMode } { } CGroup::CGroup( const std::string& hostLanguage, const std::string& xmlGroup, const bool debugMode ): m_HostLanguage{ hostLanguage }, - m_DebugMode{ debugMode }, - m_SerialSize{ 0 } + m_DebugMode{ debugMode } { ParseXMLGroup( xmlGroup ); } - -CGroup::~CGroup( ) +CGroup::CGroup( const std::string& hostLanguage, const std::string transport, + const unsigned int priority, const unsigned int iterations, const bool debugMode ): + m_HostLanguage{ hostLanguage }, + m_DebugMode{ debugMode } { } -void CGroup::Open( const std::string bufferName, const std::string accessMode ) -{ - m_IsOpen = true; - m_BufferName = bufferName; - m_AccessMode = accessMode; -} +CGroup::~CGroup( ) +{ } void CGroup::CreateVariable( const std::string name, const std::string type, @@ -103,6 +99,8 @@ void CGroup::CreateVariable( const std::string name, const std::string type, else if( type == "long double" || type == "real*16" ) m_LDouble.push_back( SVariable<long double>{ dimensionsCSV, nullptr, transformIndex, globalBoundsIndex } ); + + m_SerialSize += name.size() + type.size() + dimensionsCSV.size() + transform.size() + 5 * sizeof(char); //5, adding one more for globalBoundsIndex } @@ -124,16 +122,26 @@ void CGroup::CreateAttribute( const std::string name, const std::string type, co throw std::invalid_argument( "ERROR: attribute " + name + " exists, NOT setting a new variable\n" ); } else + { lf_EmplaceVariable( name, type, value, globalDimensionsCSV, globalOffsetsCSV ); + } + + m_SerialSize += name.size() + type.size() + value.size() + 4; //adding one more for globalBoundsIndex } -void CGroup::Close( ) + +void CGroup::Open( const std::string streamName ) { - m_IsOpen = false; - m_BufferName.clear(); - m_AccessMode.clear(); + m_StreamName = streamName; } + +void CGroup::SetTransport( const std::string transport ) +{ + m_Transport = transport; +} + + //PRIVATE FUNCTIONS BELOW void CGroup::Monitor( std::ostream& logStream ) const { @@ -150,9 +158,6 @@ void CGroup::Monitor( std::ostream& logStream ) const logStream << "\t" << attributePair.first << " \t " << attributePair.second.Type << " \t " << attributePair.second.Value << "\n"; } std::cout << "\n"; - - logStream << "\tTransport Method " << m_Transport << "\n"; - std::cout << "\n"; } diff --git a/src/public/ADIOS.cpp b/src/public/ADIOS.cpp index 16c47091a44d22678cd8701bd8936590f63856da..fc9692dc92481ef810044a8e2fedf57019271b0c 100644 --- a/src/public/ADIOS.cpp +++ b/src/public/ADIOS.cpp @@ -27,7 +27,8 @@ ADIOS::ADIOS( ) ADIOS::ADIOS( const std::string xmlConfigFile, const bool debugMode ): m_XMLConfigFile{ xmlConfigFile }, - m_DebugMode{ debugMode } + m_DebugMode{ debugMode }, + m_Capsule{ CCapsule( debugMode ) } { InitXML( m_XMLConfigFile, m_MPIComm, m_DebugMode, m_HostLanguage, m_Groups ); } @@ -36,17 +37,25 @@ ADIOS::ADIOS( const std::string xmlConfigFile, const bool debugMode ): ADIOS::ADIOS( const std::string xmlConfigFile, const MPI_Comm mpiComm, const bool debugMode ): m_MPIComm{ mpiComm }, m_XMLConfigFile{ xmlConfigFile }, - m_DebugMode{ debugMode } + m_DebugMode{ debugMode }, + m_Capsule{ CCapsule( mpiComm, debugMode ) } { InitXML( m_XMLConfigFile, m_MPIComm, m_DebugMode, m_HostLanguage, m_Groups ); } +ADIOS::ADIOS( const MPI_Comm mpiComm, const bool debugMode ): + m_MPIComm{ mpiComm }, + m_DebugMode{ debugMode }, + m_Capsule{ CCapsule( mpiComm, debugMode ) } +{ } + + ADIOS::~ADIOS( ) { } -void ADIOS::CreateGroup( const std::string groupName, const std::string transport ) +void ADIOS::CreateGroup( const std::string groupName ) { if( m_DebugMode == true ) { @@ -58,15 +67,17 @@ void ADIOS::CreateGroup( const std::string groupName, const std::string transpor } - -void ADIOS::Open( const std::string groupName, const std::string fileName, const std::string accessMode ) +void ADIOS::Open( const std::string groupName, const std::string streamName, const std::string accessMode, unsigned long int maxBufferSize ) { auto itGroup = m_Groups.find( groupName ); if( m_DebugMode == true ) - CheckGroup( itGroup, groupName, " from call to Open with file " + fileName ); + CheckGroup( itGroup, groupName, " from call to Open with file " + streamName ); - itGroup->second.Open( fileName, accessMode ); + itGroup->second.Open( streamName ); + m_Capsule.SetTransport( streamName, itGroup->second.m_Transport, m_DebugMode ); //Set Transport + m_Capsule.SetBuffer( streamName, maxBufferSize ); + m_Capsule.Open( streamName, accessMode ); } @@ -81,7 +92,7 @@ void ADIOS::Close( const std::string groupName ) throw std::invalid_argument( "ERROR: group " + groupName + " is not open in Write function.\n" ); } - m_Capsule->CloseGroupBuffer( itGroup->second ); + //m_Capsule->CloseGroupBuffer( itGroup->second ); itGroup->second.Close( ); } @@ -92,16 +103,35 @@ void ADIOS::CreateVariable( const std::string groupName, const std::string varia { auto itGroup = m_Groups.find( groupName ); if( m_DebugMode == true ) - { CheckGroup( itGroup, groupName, " from call to CreateVariable \n" ); - } itGroup->second.CreateVariable( variableName, type, dimensionsCSV, transform, globalDimensionsCSV, globalOffsetsCSV ); +} +void ADIOS::CreateAttribute( const std::string groupName, const std::string attributeName, + const std::string type, const std::string value, + const std::string globalDimensionsCSV, const std::string globalOffsetsCSV ) +{ + auto itGroup = m_Groups.find( groupName ); + if( m_DebugMode == true ) + CheckGroup( itGroup, groupName, " from call to CreateAttribute \n" ); + itGroup->second.CreateAttribute( attributeName, type, value, globalDimensionsCSV, globalOffsetsCSV ); } + +void ADIOS::SetTransport( const std::string groupName, const std::string transport ) +{ + auto itGroup = m_Groups.find( groupName ); + if( m_DebugMode == true ) + CheckGroup( itGroup, groupName, " from call to SetTransport \n" ); + + itGroup->second.SetTransport( transport ); +} + + + void ADIOS::MonitorGroups( std::ostream& logStream ) const { for( auto& groupPair : m_Groups ) @@ -111,7 +141,7 @@ void ADIOS::MonitorGroups( std::ostream& logStream ) const } } - +//PRIVATE FUNCTIONS BELOW void ADIOS::CheckGroup( std::map< std::string, CGroup >::const_iterator itGroup, const std::string groupName, const std::string hint ) { if( itGroup == m_Groups.end() ) diff --git a/src/transform/CBZIP2.cpp b/src/transform/CBZIP2.cpp index 6f73282207997a57ee85c84f8a70d679ac240431..4678fb0a33c3fb74ef95400fd1afffd2e94d5494 100644 --- a/src/transform/CBZIP2.cpp +++ b/src/transform/CBZIP2.cpp @@ -15,19 +15,116 @@ namespace adios { -CBZIP2::CBZIP2( const unsigned int compressionLevel, CVariableBase& variable ): - CTransform( "bzip2", compressionLevel, variable ) +CBZIP2::CBZIP2( ) { } + CBZIP2::~CBZIP2( ) { } -void CBZIP2::Compress( ) const -{ } -void CBZIP2::Compress( ) const +void CBZIP2::Compress( const SVariable<char>& variableIn, SVariable<char>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<unsigned char>& variableIn, SVariable<unsigned char>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<short>& variableIn, SVariable<short>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<unsigned short>& variableIn, SVariable<unsigned short>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<int>& variableIn, SVariable<int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<unsigned int>& variableIn, SVariable<unsigned int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<long int>& variableIn, SVariable<long int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<unsigned long int>& variableIn, SVariable<unsigned long int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<long long int>& variableIn, SVariable<long long int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<unsigned long long int>& variableIn, SVariable<unsigned long long int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<float>& variableIn, SVariable<float>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<double>& variableIn, SVariable<double>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Compress( const SVariable<long double>& variableIn, SVariable<long double>& variableOut, const unsigned int level ) const { } +//DECOMPRESS +void CBZIP2::Decompress( const SVariable<char>& variableIn, SVariable<char>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<unsigned char>& variableIn, SVariable<unsigned char>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<short>& variableIn, SVariable<short>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<unsigned short>& variableIn, SVariable<unsigned short>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<int>& variableIn, SVariable<int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<unsigned int>& variableIn, SVariable<unsigned int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<long int>& variableIn, SVariable<long int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<unsigned long int>& variableIn, SVariable<unsigned long int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<long long int>& variableIn, SVariable<long long int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<unsigned long long int>& variableIn, SVariable<unsigned long long int>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<float>& variableIn, SVariable<float>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<double>& variableIn, SVariable<double>& variableOut, const unsigned int level ) const +{ } + + +void CBZIP2::Decompress( const SVariable<long double>& variableIn, SVariable<long double>& variableOut, const unsigned int level ) const +{ } } //end namespace diff --git a/src/transport/CFStream.cpp b/src/transport/CFStream.cpp index cf5f492413d973d59b5848c255b12c86405ba885..a7f8029a31a656ee8a472fd89f045199a6a59947 100644 --- a/src/transport/CFStream.cpp +++ b/src/transport/CFStream.cpp @@ -6,8 +6,6 @@ */ /// \cond EXCLUDED_FROM_DOXYGEN -#include <core/SVariable.h> -#include <core/SVariable.h> #include <iostream> #include <sstream> #include <cmath> @@ -16,15 +14,13 @@ /// \endcond #include "transport/CFStream.h" -#include "functions/GroupFunctions.h" -#include "functions/Templates.h" namespace adios { -CFStream::CFStream( const unsigned int priority, const unsigned int iteration, MPI_Comm mpiComm, const bool debugMode ): - CTransport( "CFStream", priority, iteration, mpiComm, debugMode ) +CFStream::CFStream( MPI_Comm mpiComm, const bool debugMode ): + CTransport( mpiComm, debugMode ) { } @@ -32,37 +28,41 @@ CFStream::~CFStream( ) { } -void CFStream::Open( const std::string fileName, const std::string accessMode ) +void CFStream::Open( const std::string streamName, const std::string accessMode ) { - m_StreamName = fileName; - if( m_RankMPI == 0 ) { if( accessMode == "w" || accessMode == "write" ) - m_FStream.open( fileName, std::fstream::out ); + m_FStream.open( streamName, std::fstream::out ); else if( accessMode == "a" || accessMode == "append" ) - m_FStream.open( fileName, std::fstream::out | std::fstream::app ); + m_FStream.open( streamName, std::fstream::out | std::fstream::app ); else if( accessMode == "r" || accessMode == "read" ) - m_FStream.open( fileName, std::fstream::in ); + m_FStream.open( streamName, std::fstream::in ); if( m_DebugMode == true ) { if( m_FStream.good() == false ) - throw std::ios_base::failure( "ERROR: couldn't open file " + fileName + " in Open function\n" ); + throw std::ios_base::failure( "ERROR: couldn't open file " + streamName + " in Open function\n" ); } } MPI_Barrier( m_MPIComm ); //all of them must wait until the file is opened } -void CFStream::Write( const CVariable& variable ) +void CFStream::SetBuffer( std::vector<char>& buffer ) { - //local buffer, to be send over MPI - std::vector<char> buffer; - const std::string type( variable.m_Type ); - auto var = GetVariableValues( variable ); + m_FStream.rdbuf()->pubsetbuf( &buffer[0], buffer.size() ); +} + + +//void CFStream::Write( const CVariable& variable ) +//{ +// //local buffer, to be send over MPI +// std::vector<char> buffer; +// const std::string type( variable.m_Type ); +// auto var = GetVariableValues( variable ); // if( type.find( "vector" ) ) //is a vector // { @@ -120,7 +120,7 @@ void CFStream::Write( const CVariable& variable ) // // MPI_Barrier( m_MPIComm ); // } -} +//} void CFStream::Close( ) { diff --git a/src/transport/CPOSIX.cpp b/src/transport/CPOSIX.cpp index 87c4fb114ebab2a50d5b5b7a888a8edcfe0a866c..e5ac7ce2d97a9b239848675b8d0392319105081b 100644 --- a/src/transport/CPOSIX.cpp +++ b/src/transport/CPOSIX.cpp @@ -5,8 +5,7 @@ * Author: wfg */ - -#include <iostream> +#include <stdio.h> #include "transport/CPOSIX.h" @@ -15,8 +14,9 @@ namespace adios { -CPOSIX::CPOSIX( const unsigned int priority, const unsigned int iteration, MPI_Comm mpiComm, const bool debugMode ): - CTransport( "POSIX", priority, iteration, mpiComm, debugMode ) +CPOSIX::CPOSIX( MPI_Comm mpiComm, const bool debugMode ): + CTransport( mpiComm, debugMode ), + m_File( NULL ) { } @@ -24,21 +24,40 @@ CPOSIX::~CPOSIX( ) { } -void CPOSIX::Open( const std::string fileName, const std::string accessMode ) +void CPOSIX::Open( const std::string streamName, const std::string accessMode ) { + if( m_RankMPI == 0 ) + { + if( accessMode == "w" || accessMode == "write" ) + m_File = fopen( streamName.c_str(), "w" ); + + else if( accessMode == "a" || accessMode == "append" ) + m_File = fopen( streamName.c_str(), "a" ); + + else if( accessMode == "r" || accessMode == "read" ) + m_File = fopen( streamName.c_str(), "r" ); + if( m_DebugMode == true ) + { + if( m_File == NULL ) + throw std::ios_base::failure( "ERROR: couldn't open file " + streamName + " in Open function\n" ); + } + } + + MPI_Barrier( m_MPIComm ); //all of them must wait until the file is opened } -void CPOSIX::Write( const CVariableBase& variable ) +void CPOSIX::SetBuffer( std::vector<char>& buffer ) { - + setvbuf( m_File, &buffer[0], _IOFBF, buffer.size() ); } void CPOSIX::Close( ) { - + if( m_RankMPI == 0 ) + fclose( m_File ); }