diff --git a/Makefile b/Makefile index 706583463cd721beca2c2e359721c48806522185..ab0db8f0f1f613d200613c00cc159921b1689222 100644 --- a/Makefile +++ b/Makefile @@ -69,4 +69,4 @@ nompi: $(HFiles) $(OBJNoMPI) $(CC) $(CFLAGS) $(INC) -DADIOS_NOMPI -o $@ $< clean: - rm ./bin/mpi/*.o ./lib/libadios.a ./bin/nompi/*.o ./lib/libadios_nompi.a + rm ./bin/mpi/*.o ./bin/nompi/*.o ./lib/*.a ./lib/*.so diff --git a/bindings/python/Makefile b/bindings/python/Makefile index 13d9aa397d358872c63e2d55a20065175753144c..8a00f5b0ab35ee3c12e01114a20b4e907e7aa393 100644 --- a/bindings/python/Makefile +++ b/bindings/python/Makefile @@ -8,21 +8,21 @@ PYTHON_INC:= /usr/include/python$(PYTHON_VERSION) PYTHON_LIBLOC:= /usr/lib/python2.7/config-x86_64-linux-gnu/ # location of the Boost Python include files and library -BOOST_INC:= /usr/include/boost -BOOST_LIBLOC:= /lib/x86_64-linux-gnu/ +BOOST_INC:= /home/wgodoy/Apps/boost_1_63_0/ +BOOST_LIBLOC:= /home/wgodoy/Apps/boost_1_63_0/stage/lib/ # location of ADIOS include and lib ADIOS_INC:= /home/wgodoy/workspace/ADIOSPP/include ADIOS_LIB:= /home/wgodoy/workspace/ADIOSPP/lib ADIOSPy_INC:=./include -CFLAGS:=-c -Wall -fPIC -O0 -g -std=c++11 +CFLAGS:=-c -Wall -fPIC -O0 -g -std=c++11 -Wno-deprecated-declarations CPPFiles:=$(shell find ./src -type f -name "*.cpp") OBJS:=$(patsubst %.cpp, ./bin/%.o, $(notdir $(CPPFiles)) ) INC:= -I$(PYTHON_INC) -I$(BOOST_INC) -I$(ADIOS_INC) -I$(ADIOSPy_INC) -LIBS:= -L$(ADIOS_LIB) -ladios -L$(BOOST_LIBLOC) -lboost_python-py27 -L$(PYTHON_LIBLOC) -lpython$(PYTHON_VERSION) +LIBS:= -L$(ADIOS_LIB) -ladios -L$(BOOST_LIBLOC) -lboost_python -lboost_numpy -L$(PYTHON_LIBLOC) -lpython$(PYTHON_VERSION) # compile mesh classes diff --git a/bindings/python/include/ADIOSPy.h b/bindings/python/include/ADIOSPy.h index f615c980e2b31ecd00c22010bbb6658a861fc068..278453bf1736e23f24b0e459dc78a26717bd67fc 100644 --- a/bindings/python/include/ADIOSPy.h +++ b/bindings/python/include/ADIOSPy.h @@ -9,14 +9,15 @@ #define ADIOSPY_H_ #include <string> +#include <memory> //std::shared_ptr -#include <boost/python.hpp> +#include "boost/python.hpp" #include "ADIOS.h" #include "adiosPyFunctions.h" //ListToVector, VectorToList #include "VariablePy.h" #include "MethodPy.h" - +#include "EnginePy.h" namespace adios { @@ -48,8 +49,8 @@ public: MethodPy& DeclareMethodPy( const std::string methodName, const std::string type = "" ); - - + EnginePy OpenPy( const std::string name, const std::string accessMode, + const MethodPy& method, boost::python::object py_comm = boost::python::object() ); }; diff --git a/bindings/python/include/EnginePy.h b/bindings/python/include/EnginePy.h new file mode 100644 index 0000000000000000000000000000000000000000..cfe6019d6929fb9c5e42e3d73e118a22f428fa78 --- /dev/null +++ b/bindings/python/include/EnginePy.h @@ -0,0 +1,45 @@ +/* + * EnginePy.h + * + * Created on: Mar 15, 2017 + * Author: wgodoy + */ + +#ifndef ENGINEPY_H_ +#define ENGINEPY_H_ + +#include "boost/python.hpp" +#include "boost/python/numpy.hpp" + +#include "core/Engine.h" +#include "VariablePy.h" + +namespace adios +{ + +namespace np = boost::python::numpy; + +class EnginePy +{ + +public: + + std::shared_ptr<Engine> m_Engine; + + void WritePy( VariablePy<double>& variable, const np::ndarray& array ); + + void WritePy( VariablePy<float>& variable, const np::ndarray& array ); + + void Close( ); + + void GetType( ) const; + +}; + + +} //end namespace + + + + +#endif /* BINDINGS_PYTHON_INCLUDE_ENGINEPY_H_ */ diff --git a/bindings/python/include/adiosPyFunctions.h b/bindings/python/include/adiosPyFunctions.h index cd940ade09f1ff9e6467c7933c74484d6be896c6..ec21e8652781b35e5d550123a157f4927186a67a 100644 --- a/bindings/python/include/adiosPyFunctions.h +++ b/bindings/python/include/adiosPyFunctions.h @@ -11,7 +11,8 @@ #include <vector> #include <map> #include <string> -#include <boost/python.hpp> + +#include "boost/python.hpp" namespace adios @@ -27,8 +28,26 @@ Dims ListToVector( const boost::python::list& list ); std::map<std::string, std::string> DictToMap( const boost::python::dict& dictionary ); + +template< class T > +T* PyObjectToPointer( const boost::python::object& object ) +{ + return reinterpret_cast<T*>( object.ptr() ); + + +// Py_buffer pyBuffer; +// if(PyObject_GetBuffer( object.ptr(), &pyBuffer, PyBUF_SIMPLE)!=-1) +// { +// +// } +// +// return nullptr; } +} //end namespace + + + #endif /* ADIOSPYFUNCTIONS_H_ */ diff --git a/bindings/python/src/ADIOSPy.cpp b/bindings/python/src/ADIOSPy.cpp index d3b6e8526e60308ea57b08c485b9ae57b804431a..2f3bdb46bc3f7c90a76acd206f3d7eb2ef453c47 100644 --- a/bindings/python/src/ADIOSPy.cpp +++ b/bindings/python/src/ADIOSPy.cpp @@ -7,8 +7,12 @@ #include <iostream> +#include <mpi4py/mpi4py.h> + #include "ADIOSPy.h" +#include "core/Engine.h" + namespace adios { @@ -35,6 +39,26 @@ MethodPy& ADIOSPy::DeclareMethodPy( const std::string methodName, const std::str } +EnginePy ADIOSPy::OpenPy( const std::string name, const std::string accessMode, + const MethodPy& method, boost::python::object py_comm ) +{ + EnginePy enginePy; + if( py_comm == boost::python::object() ) //None + { + enginePy.m_Engine = Open( name, accessMode, method ); + } + else + { + if (import_mpi4py() < 0) throw std::logic_error( "ERROR: could not import mpi4py communicator in Open " + name + "\n" ); + MPI_Comm* comm_p = PyMPIComm_Get( py_comm.ptr() ); + if( comm_p == nullptr ) boost::python::throw_error_already_set(); + + enginePy.m_Engine = Open( name, accessMode, *comm_p, method ); + } + //here downcast + return enginePy; +} + diff --git a/bindings/python/src/EnginePy.cpp b/bindings/python/src/EnginePy.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b98b8265acf3a80cdf444d07199b8fc897513b48 --- /dev/null +++ b/bindings/python/src/EnginePy.cpp @@ -0,0 +1,42 @@ +/* + * EnginePy.cpp + * + * Created on: Mar 15, 2017 + * Author: wgodoy + */ + + + +#include "EnginePy.h" + +#include "adiosPyFunctions.h" + +namespace adios +{ + + +void EnginePy::WritePy( VariablePy<double>& variable, const np::ndarray& array ) +{ + const double* values = reinterpret_cast<const double*>( array.get_data() ); + m_Engine->Write( variable, values ); +} + +void EnginePy::WritePy( VariablePy<float>& variable, const np::ndarray& array ) +{ + const float* values = reinterpret_cast<const float*>( array.get_data() ); + m_Engine->Write( variable, values ); +} + +void EnginePy::GetType( ) const +{ + std::cout << "Engine type " << m_Engine->m_EngineType << "\n"; +} + +void EnginePy::Close( ) +{ + m_Engine->Close( -1 ); +} + + + +} //end namespace diff --git a/bindings/python/src/MethodPy.cpp b/bindings/python/src/MethodPy.cpp index 1eedcc531142ea59287d79d4df6b35a22037fa35..231c22e200f63738871acb3cf96a1d25a6659b26 100644 --- a/bindings/python/src/MethodPy.cpp +++ b/bindings/python/src/MethodPy.cpp @@ -51,15 +51,16 @@ void MethodPy::PrintAll( ) const { std::cout << "Method parameters\n"; for( const auto& param : m_Parameters ) - std::cout << "Key: " << param.first << "\t Value: " << param.second << "\n"; + std::cout << "Parameter: " << param.first << "\t Value: " << param.second << "\n"; std::cout << "\n"; - std::cout << "Method transports\n"; - std::cout << "\n"; + std::cout << "Transport Parameters\n"; + for( const auto& transportParameters : m_TransportParameters ) { - for( const auto& param : transportParameters ) - std::cout << "Key: " << param.first << "\t Value: " << param.second << "\n"; + std::cout << "Transport:\n"; + for( const auto& param : transportParameters ) + std::cout << "Parameter: " << param.first << "\t Value: " << param.second << "\n"; std::cout << "\n"; } diff --git a/bindings/python/src/glue.cpp b/bindings/python/src/glue.cpp index ba380128e2c2e4e5da76d4054b3640666a70234a..e716c1bcc53d9e1af3df586663905935699107f8 100644 --- a/bindings/python/src/glue.cpp +++ b/bindings/python/src/glue.cpp @@ -6,24 +6,33 @@ */ #include <mpi4py/mpi4py.h> -#include <boost/python.hpp> -#include <boost/python/suite/indexing/vector_indexing_suite.hpp> -#include <boost/python/raw_function.hpp> + +#include "boost/python.hpp" +#include "boost/python/suite/indexing/vector_indexing_suite.hpp" +#include "boost/python/raw_function.hpp" +#include "boost/python/numpy.hpp" + #include "ADIOSPy.h" #include "adiosPyFunctions.h" -adios::ADIOSPy ADIOSPy( boost::python::object py_comm, const bool debug ) +namespace bpy = boost::python; +namespace np = boost::python::numpy; + + +adios::ADIOSPy ADIOSPy( bpy::object py_comm, const bool debug ) { MPI_Comm* comm_p = PyMPIComm_Get( py_comm.ptr() ); - if (comm_p == NULL) boost::python::throw_error_already_set(); + if (comm_p == NULL) bpy::throw_error_already_set(); return adios::ADIOSPy( *comm_p, debug ); } +using ReturnInternalReference = bpy::return_internal_reference<>; +BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS( open_overloads, adios::ADIOSPy::OpenPy, 3, 4 ) -using ReturnInternalReference = boost::python::return_internal_reference<>; +BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS( write_overload, adios::EnginePy::WritePy, 2, 2 ) @@ -31,31 +40,43 @@ BOOST_PYTHON_MODULE( ADIOSPy ) { if (import_mpi4py() < 0) return; /* Python 2.X */ + Py_Initialize(); + np::initialize(); - boost::python::class_< adios::Dims >("Dims") + + bpy::class_< adios::Dims >("Dims") .def(boost::python::vector_indexing_suite< adios::Dims >() ); //functions - boost::python::def("ADIOSPy", ADIOSPy ); + bpy::def("ADIOSPy", ADIOSPy ); //classes - boost::python::class_<adios::ADIOSPy>("ADIOS", boost::python::no_init ) + bpy::class_<adios::ADIOSPy>("ADIOS", bpy::no_init ) .def("HelloMPI", &adios::ADIOSPy::HelloMPI ) .def("DefineVariableDouble", &adios::ADIOSPy::DefineVariablePy<double>, ReturnInternalReference() ) .def("DefineVariableFloat", &adios::ADIOSPy::DefineVariablePy<float>, ReturnInternalReference() ) .def("DeclareMethod", &adios::ADIOSPy::DeclareMethodPy, ReturnInternalReference() ) + .def("Open", &adios::ADIOSPy::OpenPy, open_overloads() ) ; - //classes - boost::python::class_<adios::VariablePy<double>>("VariableDouble", boost::python::no_init ) + bpy::class_<adios::VariablePy<double>>("VariableDouble", bpy::no_init ) .def("SetLocalDimensions", &adios::VariablePy<double>::SetLocalDimensions ) .def("GetLocalDimensions", &adios::VariablePy<double>::GetLocalDimensions ) ; - boost::python::class_<adios::MethodPy>("Method", boost::python::no_init ) - .def("SetParameters", boost::python::raw_function( &adios::MethodPy::SetParametersPy, 1 ) ) - .def("AddTransport", boost::python::raw_function( &adios::MethodPy::AddTransportPy, 1 ) ) + bpy::class_<adios::MethodPy>("Method", bpy::no_init ) + .def("SetParameters", bpy::raw_function( &adios::MethodPy::SetParametersPy, 1 ) ) + .def("AddTransport", bpy::raw_function( &adios::MethodPy::AddTransportPy, 1 ) ) .def("PrintAll", &adios::MethodPy::PrintAll ) ; + //Engines + bpy::class_<adios::EnginePy>("EnginePy", bpy::no_init ) + .def( "GetType", &adios::EnginePy::GetType ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<double>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<float>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def( "Close", &adios::EnginePy::Close ) + ; } diff --git a/bindings/python/test_hello.py b/bindings/python/test_hello.py index 80e805f57869b5f92d3cdb25e0eab3149417bd16..448753f80ba3a99f21cd908c92aba5350fb20dd0 100644 --- a/bindings/python/test_hello.py +++ b/bindings/python/test_hello.py @@ -4,39 +4,40 @@ from mpi4py import MPI from ADIOSPy import * +import numpy as np -adios = ADIOSPy( MPI.COMM_WORLD, True) -adios.HelloMPI( ) - +# Create ADIOS and verify MPI Comm is passed correctly +adios = ADIOSPy( MPI.COMM_WORLD, True ) #Pass communicator and debug flag is True rank = MPI.COMM_WORLD.Get_rank() -lDims = [rank+1, rank+2, rank+3] -Nx = 1 +size = MPI.COMM_WORLD.Get_size() + +# User data +myArray = np.array( [1,2,3,4], dtype=np.double ) -ioMyDoubles = adios.DefineVariableDouble( "ioMyDoubles", lDims, [], [] ) +if( rank % 2 == 1 ): # odd ranks only + oddRankArray = np.array([11,12,13,14],dtype=np.float) -dims = ioMyDoubles.GetLocalDimensions( ) -print "Old Dimensions" -for dim in dims: - print dim +# ADIOS Define Variables +ioMyDoubles = adios.DefineVariableDouble( "ioMyDoubles", [myArray.size], [], [] ) -ioMyDoubles.SetLocalDimensions( [20,20,20] ) +if( rank % 2 == 1 ): # odd ranks only + ioMyFloats = adios.DefineVariableDouble( "ioMyFloats", [oddRankArray.size], [], [] ) -dims = ioMyDoubles.GetLocalDimensions( ) -print "New Dimensions " -for dim in dims: - print dim - -method = adios.DeclareMethod("myMethod", "BPFileWriter") -method.SetParameters( max_buffer_size = '10000000' ) -method.AddTransport( 'File', have_metadata_file = 'yes', library = 'FStream' ) -method.AddTransport( "Mdtm", localIP='128.0.0.0.1', remoteIP='128.0.0.0.2', tolerances='1,2,3' ) -print -method.PrintAll( ) +#Setup method and print summary +ioSettings = adios.DeclareMethod("adiosSettings", "BPFileWriter") +ioSettings.SetParameters( max_buffer_size = '10000000' ) +ioSettings.AddTransport( 'File', have_metadata_file = 'yes', library = 'POSIX' ) # POSIX is default, just checking +#Start Engine +bpFileWriter = adios.Open( "file.bp", "w", ioSettings, None ) # Open files using N-to-N method, None means no new MPI communicator +bpFileWriter.Write( ioMyDoubles, myArray ) +if( rank % 2 == 1 ): + bpFileWriter.Write( ioMyFloats, oddRankArray ) -# bpWriter = adios.Open( ) -# ADIOS.SetEngineComm( bpWriter, comm ) -# bpWriter.Hello( ) +bpFileWriter.Close( ) -# challenge is to pass comm to C++ +if( rank == 0 ): + print "Done writing " + str( size ) + " bp files" + ioSettings.PrintAll( ) # just prints a summary of Method/Transport parameters + \ No newline at end of file diff --git a/include/core/Engine.h b/include/core/Engine.h index 985517704116a67717085598c10cc7bfc917fb27..3ea11ae7e50518cb29acd3c15b334edd009ebc61 100644 --- a/include/core/Engine.h +++ b/include/core/Engine.h @@ -129,6 +129,51 @@ public: Write( variableName, &val ); } + virtual void Write( Variable<char>& variable, const char* values ); + virtual void Write( Variable<unsigned char>& variable, const unsigned char* values ); + virtual void Write( Variable<short>& variable, const short* values ); + virtual void Write( Variable<unsigned short>& variable, const unsigned short* values ); + virtual void Write( Variable<int>& variable, const int* values ); + virtual void Write( Variable<unsigned int>& variable, const unsigned int* values ); + virtual void Write( Variable<long int>& variable, const long int* values ); + virtual void Write( Variable<unsigned long int>& variable, const unsigned long int* values ); + virtual void Write( Variable<long long int>& variable, const long long int* values ); + virtual void Write( Variable<unsigned long long int>& variable, const unsigned long long int* values ); + virtual void Write( Variable<float>& variable, const float* values ); + virtual void Write( Variable<double>& variable, const double* values ); + virtual void Write( Variable<long double>& variable, const long double* values ); + virtual void Write( Variable<std::complex<float>>& variable, const std::complex<float>* values ); + virtual void Write( Variable<std::complex<double>>& variable, const std::complex<double>* values ); + virtual void Write( Variable<std::complex<long double>>& variable, const std::complex<long double>* values ); + virtual void Write( VariableCompound& variable, const void* values ); + + + /** + * @brief Write functions can be overridden by derived classes. Base class behavior is to: + * 1) Write to Variable values (m_Values) using the pointer to default group *m_Group set with SetDefaultGroup function + * 2) Transform the data + * 3) Write to all capsules -> data and metadata + * @param variableName + * @param values coming from user app + */ + virtual void Write( const std::string variableName, const char* values ); + virtual void Write( const std::string variableName, const unsigned char* values ); + virtual void Write( const std::string variableName, const short* values ); + virtual void Write( const std::string variableName, const unsigned short* values ); + virtual void Write( const std::string variableName, const int* values ); + virtual void Write( const std::string variableName, const unsigned int* values ); + virtual void Write( const std::string variableName, const long int* values ); + virtual void Write( const std::string variableName, const unsigned long int* values ); + virtual void Write( const std::string variableName, const long long int* values ); + virtual void Write( const std::string variableName, const unsigned long long int* values ); + virtual void Write( const std::string variableName, const float* values ); + virtual void Write( const std::string variableName, const double* values ); + virtual void Write( const std::string variableName, const long double* values ); + virtual void Write( const std::string variableName, const std::complex<float>* values ); + virtual void Write( const std::string variableName, const std::complex<double>* values ); + virtual void Write( const std::string variableName, const std::complex<long double>* values ); + virtual void Write( const std::string variableName, const void* values ); + /** * Indicates that a new step is going to be written as new variables come in. @@ -164,7 +209,7 @@ public: virtual Variable<std::complex<long double>>* InquireVariableCLDouble( const std::string name, const bool readIn = true ); virtual VariableCompound* InquireVariableCompound( const std::string name, const bool readIn = true ); - virtual void Close( const int transportIndex = -1 ) = 0; ///< Closes a particular transport, or all if -1 + virtual void Close( const int transportIndex = -1 ); ///< Closes a particular transport, or all if -1 protected: @@ -183,53 +228,6 @@ protected: virtual void InitParameters( ); ///< Initialize parameters from Method, called from Initi in constructor virtual void InitTransports( ); ///< Initialize transports from Method, called from Init in constructor - - virtual void Write( Variable<char>& variable, const char* values ); - virtual void Write( Variable<unsigned char>& variable, const unsigned char* values ); - virtual void Write( Variable<short>& variable, const short* values ); - virtual void Write( Variable<unsigned short>& variable, const unsigned short* values ); - virtual void Write( Variable<int>& variable, const int* values ); - virtual void Write( Variable<unsigned int>& variable, const unsigned int* values ); - virtual void Write( Variable<long int>& variable, const long int* values ); - virtual void Write( Variable<unsigned long int>& variable, const unsigned long int* values ); - virtual void Write( Variable<long long int>& variable, const long long int* values ); - virtual void Write( Variable<unsigned long long int>& variable, const unsigned long long int* values ); - virtual void Write( Variable<float>& variable, const float* values ); - virtual void Write( Variable<double>& variable, const double* values ); - virtual void Write( Variable<long double>& variable, const long double* values ); - virtual void Write( Variable<std::complex<float>>& variable, const std::complex<float>* values ); - virtual void Write( Variable<std::complex<double>>& variable, const std::complex<double>* values ); - virtual void Write( Variable<std::complex<long double>>& variable, const std::complex<long double>* values ); - virtual void Write( VariableCompound& variable, const void* values ); - - - /** - * @brief Write functions can be overridden by derived classes. Base class behavior is to: - * 1) Write to Variable values (m_Values) using the pointer to default group *m_Group set with SetDefaultGroup function - * 2) Transform the data - * 3) Write to all capsules -> data and metadata - * @param variableName - * @param values coming from user app - */ - virtual void Write( const std::string variableName, const char* values ); - virtual void Write( const std::string variableName, const unsigned char* values ); - virtual void Write( const std::string variableName, const short* values ); - virtual void Write( const std::string variableName, const unsigned short* values ); - virtual void Write( const std::string variableName, const int* values ); - virtual void Write( const std::string variableName, const unsigned int* values ); - virtual void Write( const std::string variableName, const long int* values ); - virtual void Write( const std::string variableName, const unsigned long int* values ); - virtual void Write( const std::string variableName, const long long int* values ); - virtual void Write( const std::string variableName, const unsigned long long int* values ); - virtual void Write( const std::string variableName, const float* values ); - virtual void Write( const std::string variableName, const double* values ); - virtual void Write( const std::string variableName, const long double* values ); - virtual void Write( const std::string variableName, const std::complex<float>* values ); - virtual void Write( const std::string variableName, const std::complex<double>* values ); - virtual void Write( const std::string variableName, const std::complex<long double>* values ); - virtual void Write( const std::string variableName, const void* values ); - - /** * Used to verify parameters in m_Method containers * @param itParam iterator to a certain parameter diff --git a/include/engine/bp/BPFileWriter.h b/include/engine/bp/BPFileWriter.h index 89f9396d9593d4277003a12a5fb61db256662136..3b48ea1e53db072e63e93962ca7ca063a6e4f316 100644 --- a/include/engine/bp/BPFileWriter.h +++ b/include/engine/bp/BPFileWriter.h @@ -36,34 +36,6 @@ public: ~BPFileWriter( ); - - void Advance( ); - - /** - * Closes a single transport or all transports - * @param transportIndex, if -1 (default) closes all transports, otherwise it closes a transport in m_Transport[transportIndex]. In debug mode the latter is bounds-checked. - */ - void Close( const int transportIndex = -1 ); - -private: - - capsule::STLVector m_Buffer; ///< heap capsule using STL std::vector<char> - format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports - format::BP1MetadataSet m_MetadataSet; ///< metadata set accompanying the heap buffer data in bp format. Needed by m_BP1Writer - - bool m_IsFirstClose = true; ///< set to false after first Close is reached so metadata doesn't have to be accommodated for a subsequent Close - std::size_t m_MaxBufferSize; ///< maximum allowed memory to be allocated - float m_GrowthFactor = 1.5; ///< capsule memory growth factor, new_memory = m_GrowthFactor * current_memory - - bool m_TransportFlush = false; ///< true: transport flush happened, buffer must be reset - - void Init( ); - void InitParameters( ); - void InitTransports( ); - void InitProcessGroup( ); - - void WriteProcessGroupIndex( ); - void Write( Variable<char>& variable, const char* values ); void Write( Variable<unsigned char>& variable, const unsigned char* values ); void Write( Variable<short>& variable, const short* values ); @@ -101,6 +73,37 @@ private: void Write( const std::string variableName, const void* values ); + + void Advance( ); + + /** + * Closes a single transport or all transports + * @param transportIndex, if -1 (default) closes all transports, otherwise it closes a transport in m_Transport[transportIndex]. In debug mode the latter is bounds-checked. + */ + void Close( const int transportIndex = -1 ); + + +private: + + capsule::STLVector m_Buffer; ///< heap capsule using STL std::vector<char> + format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports + format::BP1MetadataSet m_MetadataSet; ///< metadata set accompanying the heap buffer data in bp format. Needed by m_BP1Writer + + bool m_IsFirstClose = true; ///< set to false after first Close is reached so metadata doesn't have to be accommodated for a subsequent Close + std::size_t m_MaxBufferSize; ///< maximum allowed memory to be allocated + float m_GrowthFactor = 1.5; ///< capsule memory growth factor, new_memory = m_GrowthFactor * current_memory + + bool m_TransportFlush = false; ///< true: transport flush happened, buffer must be reset + + void Init( ); + void InitParameters( ); + void InitTransports( ); + void InitProcessGroup( ); + + void WriteProcessGroupIndex( ); + + + /** * Common function for primitive (including std::complex) writes * @param group diff --git a/include/engine/dataman/DataManWriter.h b/include/engine/dataman/DataManWriter.h index 8f00973042b981cecec402ad92ab96ba8b4df6bb..66116bb422bfb911aa503f0d01b7f3e0e201c35e 100644 --- a/include/engine/dataman/DataManWriter.h +++ b/include/engine/dataman/DataManWriter.h @@ -46,22 +46,6 @@ public: void SetCallBack( std::function<void( const void*, std::string, std::string, std::string, Dims )> callback ); - void Close( const int transportIndex = -1 ); - -private: - - capsule::STLVector m_Buffer; ///< heap capsule, contains data and metadata buffers - format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports - - bool m_DoRealTime = false; - bool m_DoMonitor = false; - DataManager m_Man; - std::function<void( const void*, std::string, std::string, std::string, Dims )> m_CallBack; ///< call back function - - void Init( ); ///< calls InitCapsules and InitTransports based on Method, called from constructor - void InitCapsules( ); - void InitTransports( ); ///< from Transports - void Write( Variable<char>& variable, const char* values ); void Write( Variable<unsigned char>& variable, const unsigned char* values ); void Write( Variable<short>& variable, const short* values ); @@ -96,6 +80,22 @@ private: void Write( const std::string variableName, const std::complex<double>* values ); void Write( const std::string variableName, const std::complex<long double>* values ); + void Close( const int transportIndex = -1 ); + +private: + + capsule::STLVector m_Buffer; ///< heap capsule, contains data and metadata buffers + format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports + + bool m_DoRealTime = false; + bool m_DoMonitor = false; + DataManager m_Man; + std::function<void( const void*, std::string, std::string, std::string, Dims )> m_CallBack; ///< call back function + + void Init( ); ///< calls InitCapsules and InitTransports based on Method, called from constructor + void InitCapsules( ); + void InitTransports( ); ///< from Transports + /** * From transport Mdtm in m_Method * @param parameter must be an accepted parameter diff --git a/src/core/Engine.cpp b/src/core/Engine.cpp index 09d24d3ea4a66932520fe459ec7f0910dbaa9479..13bb8cd42c94282fe14215d1a7c50da65c88c45d 100644 --- a/src/core/Engine.cpp +++ b/src/core/Engine.cpp @@ -82,7 +82,9 @@ void Engine::Write( const std::string variableName, const std::complex<double>* void Engine::Write( const std::string variableName, const std::complex<long double>* values ){ } void Engine::Write( const std::string variableName, const void* values ){ } -void Engine::Advance( ){ } +void Engine::Advance(){ } + +void Engine::Close( const int transportIndex ){ } //READ Variable<void>* Engine::InquireVariable( const std::string name, const bool readIn ){ return nullptr; }