First working version of Python bindings using boost python and numpy

Added EnginePy as a thin manager wrapper to adios::Engine (member of
EnginePy)

Write accepts numpy arrays (ndarray) seamlessly

Write Outputs Verified with ADIOS 1.0 bpls

To do:
Advance

Makefile

@@ -69,4 +69,4 @@ nompi: $(HFiles) $(OBJNoMPI)
 	$(CC) $(CFLAGS) $(INC) -DADIOS_NOMPI -o $@ $<
 	
 clean:
-	rm ./bin/mpi/*.o ./lib/libadios.a ./bin/nompi/*.o ./lib/libadios_nompi.a	
+	rm ./bin/mpi/*.o ./bin/nompi/*.o ./lib/*.a ./lib/*.so	

bindings/python/Makefile

@@ -8,21 +8,21 @@ PYTHON_INC:= /usr/include/python$(PYTHON_VERSION)
 PYTHON_LIBLOC:= /usr/lib/python2.7/config-x86_64-linux-gnu/
  
 # location of the Boost Python include files and library 
-BOOST_INC:= /usr/include/boost
-BOOST_LIBLOC:= /lib/x86_64-linux-gnu/
+BOOST_INC:= /home/wgodoy/Apps/boost_1_63_0/
+BOOST_LIBLOC:= /home/wgodoy/Apps/boost_1_63_0/stage/lib/
 
 # location of ADIOS include and lib
 ADIOS_INC:= /home/wgodoy/workspace/ADIOSPP/include
 ADIOS_LIB:= /home/wgodoy/workspace/ADIOSPP/lib
 ADIOSPy_INC:=./include
 
-CFLAGS:=-c -Wall -fPIC -O0 -g -std=c++11
+CFLAGS:=-c -Wall -fPIC -O0 -g -std=c++11 -Wno-deprecated-declarations
 
 CPPFiles:=$(shell find ./src -type f -name "*.cpp")
 OBJS:=$(patsubst %.cpp, ./bin/%.o, $(notdir $(CPPFiles)) )
 
 INC:= -I$(PYTHON_INC) -I$(BOOST_INC) -I$(ADIOS_INC) -I$(ADIOSPy_INC)
-LIBS:= -L$(ADIOS_LIB) -ladios -L$(BOOST_LIBLOC) -lboost_python-py27 -L$(PYTHON_LIBLOC) -lpython$(PYTHON_VERSION)
+LIBS:= -L$(ADIOS_LIB) -ladios -L$(BOOST_LIBLOC) -lboost_python -lboost_numpy -L$(PYTHON_LIBLOC) -lpython$(PYTHON_VERSION)
 
  
 # compile mesh classes

bindings/python/include/ADIOSPy.h

@@ -9,14 +9,15 @@
 #define ADIOSPY_H_
 
 #include <string>
+#include <memory> //std::shared_ptr
 
-#include <boost/python.hpp>
+#include "boost/python.hpp"
 
 #include "ADIOS.h"
 #include "adiosPyFunctions.h" //ListToVector, VectorToList
 #include "VariablePy.h"
 #include "MethodPy.h"
-
+#include "EnginePy.h"
 
 namespace adios
 {
@@ -48,8 +49,8 @@ public:
 
     MethodPy& DeclareMethodPy( const std::string methodName, const std::string type = "" );
 
-
-
+    EnginePy OpenPy( const std::string name, const std::string accessMode,
+    		         const MethodPy&  method, boost::python::object py_comm = boost::python::object() );
 
 };
 

bindings/python/include/EnginePy.h

+/*
+ * EnginePy.h
+ *
+ *  Created on: Mar 15, 2017
+ *      Author: wgodoy
+ */
+
+#ifndef ENGINEPY_H_
+#define ENGINEPY_H_
+
+#include "boost/python.hpp"
+#include "boost/python/numpy.hpp"
+
+#include "core/Engine.h"
+#include "VariablePy.h"
+
+namespace adios
+{
+
+namespace np = boost::python::numpy;
+
+class EnginePy
+{
+
+public:
+
+	std::shared_ptr<Engine> m_Engine;
+
+	void WritePy( VariablePy<double>& variable, const np::ndarray& array );
+
+	void WritePy( VariablePy<float>& variable, const np::ndarray& array );
+
+	void Close( );
+
+	void GetType( ) const;
+
+};
+
+
+} //end namespace
+
+
+
+
+#endif /* BINDINGS_PYTHON_INCLUDE_ENGINEPY_H_ */

bindings/python/include/adiosPyFunctions.h

@@ -11,7 +11,8 @@
 #include <vector>
 #include <map>
 #include <string>
-#include <boost/python.hpp>
+
+#include "boost/python.hpp"
 
 
 namespace adios
@@ -27,8 +28,26 @@ Dims ListToVector( const boost::python::list& list );
 
 std::map<std::string, std::string> DictToMap( const boost::python::dict& dictionary );
 
+
+template< class T >
+T* PyObjectToPointer( const boost::python::object& object )
+{
+	return reinterpret_cast<T*>( object.ptr() );
+
+
+//	Py_buffer pyBuffer;
+//	if(PyObject_GetBuffer( object.ptr(), &pyBuffer, PyBUF_SIMPLE)!=-1)
+//	{
+//
+//	}
+//
+//	return nullptr;
 }
 
 
 
+} //end namespace
+
+
+
 #endif /* ADIOSPYFUNCTIONS_H_ */

bindings/python/src/ADIOSPy.cpp

@@ -7,8 +7,12 @@
 
 #include <iostream>
 
+#include <mpi4py/mpi4py.h>
+
 #include "ADIOSPy.h"
 
+#include "core/Engine.h"
+
 namespace adios
 {
 
@@ -35,6 +39,26 @@ MethodPy& ADIOSPy::DeclareMethodPy( const std::string methodName, const std::str
 }
 
 
+EnginePy ADIOSPy::OpenPy( const std::string name, const std::string accessMode,
+    		              const MethodPy& method, boost::python::object py_comm )
+{
+	EnginePy enginePy;
+	if( py_comm == boost::python::object() ) //None
+	{
+		enginePy.m_Engine = Open( name, accessMode, method );
+	}
+	else
+	{
+		if (import_mpi4py() < 0) throw std::logic_error( "ERROR: could not import mpi4py communicator in Open " + name + "\n" );
+		MPI_Comm* comm_p = PyMPIComm_Get( py_comm.ptr() );
+		if( comm_p == nullptr ) boost::python::throw_error_already_set();
+
+		enginePy.m_Engine = Open( name, accessMode, *comm_p, method );
+	}
+	//here downcast
+	return enginePy;
+}
+
 
 
 

bindings/python/src/EnginePy.cpp

+/*
+ * EnginePy.cpp
+ *
+ *  Created on: Mar 15, 2017
+ *      Author: wgodoy
+ */
+
+
+
+#include "EnginePy.h"
+
+#include "adiosPyFunctions.h"
+
+namespace adios
+{
+
+
+void EnginePy::WritePy( VariablePy<double>& variable, const np::ndarray& array )
+{
+    const double* values = reinterpret_cast<const double*>( array.get_data() );
+    m_Engine->Write( variable, values );
+}
+
+void EnginePy::WritePy( VariablePy<float>& variable, const np::ndarray& array )
+{
+	const float* values = reinterpret_cast<const float*>( array.get_data() );
+	m_Engine->Write( variable, values );
+}
+
+void EnginePy::GetType( ) const
+{
+	std::cout << "Engine type " << m_Engine->m_EngineType << "\n";
+}
+
+void EnginePy::Close( )
+{
+	m_Engine->Close( -1 );
+}
+
+
+
+} //end namespace

bindings/python/src/MethodPy.cpp

@@ -51,15 +51,16 @@ void MethodPy::PrintAll( ) const
 {
     std::cout << "Method parameters\n";
     for( const auto& param : m_Parameters )
-        std::cout << "Key: " << param.first << "\t Value: " << param.second << "\n";
+        std::cout << "Parameter: " << param.first << "\t Value: " << param.second << "\n";
 
     std::cout << "\n";
-    std::cout << "Method transports\n";
-    std::cout << "\n";
+    std::cout << "Transport Parameters\n";
+
     for( const auto& transportParameters : m_TransportParameters )
     {
-        for( const auto& param : transportParameters )
-            std::cout << "Key: " << param.first << "\t Value: " << param.second << "\n";
+        std::cout << "Transport:\n";
+    	for( const auto& param : transportParameters )
+            std::cout << "Parameter: " << param.first << "\t Value: " << param.second << "\n";
 
         std::cout << "\n";
     }

bindings/python/src/glue.cpp

@@ -6,24 +6,33 @@
  */
 
 #include <mpi4py/mpi4py.h>
-#include <boost/python.hpp>
-#include <boost/python/suite/indexing/vector_indexing_suite.hpp>
-#include <boost/python/raw_function.hpp>
+
+#include "boost/python.hpp"
+#include "boost/python/suite/indexing/vector_indexing_suite.hpp"
+#include "boost/python/raw_function.hpp"
+#include "boost/python/numpy.hpp"
+
 
 #include "ADIOSPy.h"
 #include "adiosPyFunctions.h"
 
 
-adios::ADIOSPy ADIOSPy( boost::python::object py_comm, const bool debug )
+namespace bpy = boost::python;
+namespace np = boost::python::numpy;
+
+
+adios::ADIOSPy ADIOSPy( bpy::object py_comm, const bool debug )
 {
     MPI_Comm* comm_p = PyMPIComm_Get( py_comm.ptr() );
-    if (comm_p == NULL) boost::python::throw_error_already_set();
+    if (comm_p == NULL) bpy::throw_error_already_set();
     return adios::ADIOSPy( *comm_p, debug );
 }
 
 
+using ReturnInternalReference = bpy::return_internal_reference<>;
+BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS( open_overloads, adios::ADIOSPy::OpenPy, 3, 4 )
 
-using ReturnInternalReference = boost::python::return_internal_reference<>;
+BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS( write_overload, adios::EnginePy::WritePy, 2, 2 )
 
 
 
@@ -31,31 +40,43 @@ BOOST_PYTHON_MODULE( ADIOSPy )
 {
     if (import_mpi4py() < 0) return; /* Python 2.X */
 
+    Py_Initialize();
+    np::initialize();
 
-    boost::python::class_< adios::Dims >("Dims")
+
+    bpy::class_< adios::Dims >("Dims")
         .def(boost::python::vector_indexing_suite< adios::Dims >() );
     //functions
-    boost::python::def("ADIOSPy", ADIOSPy );
+    bpy::def("ADIOSPy", ADIOSPy );
 
     //classes
-    boost::python::class_<adios::ADIOSPy>("ADIOS", boost::python::no_init )
+    bpy::class_<adios::ADIOSPy>("ADIOS", bpy::no_init )
         .def("HelloMPI", &adios::ADIOSPy::HelloMPI )
         .def("DefineVariableDouble", &adios::ADIOSPy::DefineVariablePy<double>, ReturnInternalReference() )
         .def("DefineVariableFloat", &adios::ADIOSPy::DefineVariablePy<float>, ReturnInternalReference() )
         .def("DeclareMethod", &adios::ADIOSPy::DeclareMethodPy, ReturnInternalReference() )
+		.def("Open", &adios::ADIOSPy::OpenPy, open_overloads() )
     ;
 
-    //classes
-    boost::python::class_<adios::VariablePy<double>>("VariableDouble", boost::python::no_init )
+    bpy::class_<adios::VariablePy<double>>("VariableDouble", bpy::no_init )
 		.def("SetLocalDimensions", &adios::VariablePy<double>::SetLocalDimensions )
 		.def("GetLocalDimensions", &adios::VariablePy<double>::GetLocalDimensions )
 	;
 
-    boost::python::class_<adios::MethodPy>("Method", boost::python::no_init )
-        .def("SetParameters", boost::python::raw_function( &adios::MethodPy::SetParametersPy, 1 )  )
-        .def("AddTransport", boost::python::raw_function( &adios::MethodPy::AddTransportPy, 1 ) )
+    bpy::class_<adios::MethodPy>("Method", bpy::no_init )
+        .def("SetParameters", bpy::raw_function( &adios::MethodPy::SetParametersPy, 1 )  )
+        .def("AddTransport", bpy::raw_function( &adios::MethodPy::AddTransportPy, 1 ) )
         .def("PrintAll", &adios::MethodPy::PrintAll )
     ;
 
+    //Engines
+    bpy::class_<adios::EnginePy>("EnginePy", bpy::no_init )
+    	.def( "GetType", &adios::EnginePy::GetType )
+		.def("Write", static_cast< void( adios::EnginePy::*)
+            (adios::VariablePy<double>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() )
+	    .def("Write", static_cast< void( adios::EnginePy::*)
+	        (adios::VariablePy<float>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() )
+		.def( "Close", &adios::EnginePy::Close )
+    ;
 
 }

bindings/python/test_hello.py

@@ -4,39 +4,40 @@
 
 from mpi4py import MPI
 from ADIOSPy import *
+import numpy as np
 
-adios = ADIOSPy( MPI.COMM_WORLD, True)
-adios.HelloMPI( )
-
+# Create ADIOS and verify MPI Comm is passed correctly
+adios = ADIOSPy( MPI.COMM_WORLD, True ) #Pass communicator and debug flag is True
 rank = MPI.COMM_WORLD.Get_rank()
-lDims = [rank+1, rank+2, rank+3]
-Nx = 1
+size = MPI.COMM_WORLD.Get_size()
+
+# User data
+myArray = np.array( [1,2,3,4], dtype=np.double )
 
-ioMyDoubles = adios.DefineVariableDouble( "ioMyDoubles", lDims, [], [] )
+if( rank % 2 == 1 ):  # odd ranks only
+    oddRankArray = np.array([11,12,13,14],dtype=np.float)
 
-dims = ioMyDoubles.GetLocalDimensions( )
-print "Old Dimensions" 
-for dim in dims:
-    print dim
+# ADIOS Define Variables    
+ioMyDoubles = adios.DefineVariableDouble( "ioMyDoubles", [myArray.size], [], [] )
 
-ioMyDoubles.SetLocalDimensions( [20,20,20] )
+if( rank % 2 == 1 ): # odd ranks only
+    ioMyFloats = adios.DefineVariableDouble( "ioMyFloats", [oddRankArray.size], [], [] )
 
-dims = ioMyDoubles.GetLocalDimensions( )
-print "New Dimensions " 
-for dim in dims:
-    print dim
-    
-method = adios.DeclareMethod("myMethod", "BPFileWriter")
-method.SetParameters( max_buffer_size = '10000000' )
-method.AddTransport( 'File', have_metadata_file = 'yes', library = 'FStream' )
-method.AddTransport( "Mdtm", localIP='128.0.0.0.1', remoteIP='128.0.0.0.2', tolerances='1,2,3' )
-print
-method.PrintAll( )
+#Setup method and print summary
+ioSettings = adios.DeclareMethod("adiosSettings", "BPFileWriter")
+ioSettings.SetParameters( max_buffer_size = '10000000' )
+ioSettings.AddTransport( 'File', have_metadata_file = 'yes', library = 'POSIX' )  # POSIX is default, just checking
 
+#Start Engine
+bpFileWriter = adios.Open( "file.bp", "w", ioSettings, None )  # Open files using N-to-N method, None means no new MPI communicator
+bpFileWriter.Write( ioMyDoubles, myArray )
 
+if( rank % 2 == 1 ): 
+    bpFileWriter.Write( ioMyFloats, oddRankArray )
 
-# bpWriter = adios.Open( )
-# ADIOS.SetEngineComm( bpWriter, comm ) 
-# bpWriter.Hello( )
+bpFileWriter.Close( ) 
 
-# challenge is to pass comm to C++
+if( rank == 0 ):
+    print "Done writing " + str( size ) + " bp files"
+    ioSettings.PrintAll( ) # just prints a summary of Method/Transport parameters
+    
\ No newline at end of file

include/core/Engine.h

@@ -129,6 +129,51 @@ public:
         Write( variableName, &val );
     }
 
+    virtual void Write( Variable<char>& variable,                      const char* values );
+    virtual void Write( Variable<unsigned char>& variable,             const unsigned char* values );
+    virtual void Write( Variable<short>& variable,                     const short* values );
+    virtual void Write( Variable<unsigned short>& variable,            const unsigned short* values );
+    virtual void Write( Variable<int>& variable,                       const int* values );
+    virtual void Write( Variable<unsigned int>& variable,              const unsigned int* values );
+    virtual void Write( Variable<long int>& variable,                  const long int* values );
+    virtual void Write( Variable<unsigned long int>& variable,         const unsigned long int* values );
+    virtual void Write( Variable<long long int>& variable,             const long long int* values );
+    virtual void Write( Variable<unsigned long long int>& variable,    const unsigned long long int* values );
+    virtual void Write( Variable<float>& variable,                     const float* values );
+    virtual void Write( Variable<double>& variable,                    const double* values );
+    virtual void Write( Variable<long double>& variable,               const long double* values );
+    virtual void Write( Variable<std::complex<float>>& variable,       const std::complex<float>* values );
+    virtual void Write( Variable<std::complex<double>>& variable,      const std::complex<double>* values );
+    virtual void Write( Variable<std::complex<long double>>& variable, const std::complex<long double>* values );
+    virtual void Write( VariableCompound& variable,                    const void* values );
+
+
+    /**
+     * @brief Write functions can be overridden by derived classes. Base class behavior is to:
+     * 1) Write to Variable values (m_Values) using the pointer to default group *m_Group set with SetDefaultGroup function
+     * 2) Transform the data
+     * 3) Write to all capsules -> data and metadata
+     * @param variableName
+     * @param values coming from user app
+     */
+    virtual void Write( const std::string variableName, const char* values );
+    virtual void Write( const std::string variableName, const unsigned char* values );
+    virtual void Write( const std::string variableName, const short* values );
+    virtual void Write( const std::string variableName, const unsigned short* values );
+    virtual void Write( const std::string variableName, const int* values );
+    virtual void Write( const std::string variableName, const unsigned int* values );
+    virtual void Write( const std::string variableName, const long int* values );
+    virtual void Write( const std::string variableName, const unsigned long int* values );
+    virtual void Write( const std::string variableName, const long long int* values );
+    virtual void Write( const std::string variableName, const unsigned long long int* values );
+    virtual void Write( const std::string variableName, const float* values );
+    virtual void Write( const std::string variableName, const double* values );
+    virtual void Write( const std::string variableName, const long double* values );
+    virtual void Write( const std::string variableName, const std::complex<float>* values );
+    virtual void Write( const std::string variableName, const std::complex<double>* values );
+    virtual void Write( const std::string variableName, const std::complex<long double>* values );
+    virtual void Write( const std::string variableName, const void* values );
+
 
     /**
      * Indicates that a new step is going to be written as new variables come in.
@@ -164,7 +209,7 @@ public:
     virtual Variable<std::complex<long double>>* InquireVariableCLDouble( const std::string name, const bool readIn = true );
     virtual VariableCompound* InquireVariableCompound( const std::string name, const bool readIn = true );
 
-    virtual void Close( const int transportIndex = -1  ) = 0; ///< Closes a particular transport, or all if -1
+    virtual void Close( const int transportIndex = -1 ); ///< Closes a particular transport, or all if -1
 
 
 protected:
@@ -183,53 +228,6 @@ protected:
     virtual void InitParameters( ); ///< Initialize parameters from Method, called from Initi in constructor
     virtual void InitTransports( ); ///< Initialize transports from Method, called from Init in constructor
 
-
-    virtual void Write( Variable<char>& variable,                      const char* values );
-    virtual void Write( Variable<unsigned char>& variable,             const unsigned char* values );
-    virtual void Write( Variable<short>& variable,                     const short* values );
-    virtual void Write( Variable<unsigned short>& variable,            const unsigned short* values );
-    virtual void Write( Variable<int>& variable,                       const int* values );
-    virtual void Write( Variable<unsigned int>& variable,              const unsigned int* values );
-    virtual void Write( Variable<long int>& variable,                  const long int* values );
-    virtual void Write( Variable<unsigned long int>& variable,         const unsigned long int* values );
-    virtual void Write( Variable<long long int>& variable,             const long long int* values );
-    virtual void Write( Variable<unsigned long long int>& variable,    const unsigned long long int* values );
-    virtual void Write( Variable<float>& variable,                     const float* values );
-    virtual void Write( Variable<double>& variable,                    const double* values );
-    virtual void Write( Variable<long double>& variable,               const long double* values );
-    virtual void Write( Variable<std::complex<float>>& variable,       const std::complex<float>* values );
-    virtual void Write( Variable<std::complex<double>>& variable,      const std::complex<double>* values );
-    virtual void Write( Variable<std::complex<long double>>& variable, const std::complex<long double>* values );
-    virtual void Write( VariableCompound& variable,                    const void* values );
-
-
-    /**
-     * @brief Write functions can be overridden by derived classes. Base class behavior is to:
-     * 1) Write to Variable values (m_Values) using the pointer to default group *m_Group set with SetDefaultGroup function
-     * 2) Transform the data
-     * 3) Write to all capsules -> data and metadata
-     * @param variableName
-     * @param values coming from user app
-     */
-    virtual void Write( const std::string variableName, const char* values );
-    virtual void Write( const std::string variableName, const unsigned char* values );
-    virtual void Write( const std::string variableName, const short* values );
-    virtual void Write( const std::string variableName, const unsigned short* values );
-    virtual void Write( const std::string variableName, const int* values );
-    virtual void Write( const std::string variableName, const unsigned int* values );
-    virtual void Write( const std::string variableName, const long int* values );
-    virtual void Write( const std::string variableName, const unsigned long int* values );
-    virtual void Write( const std::string variableName, const long long int* values );
-    virtual void Write( const std::string variableName, const unsigned long long int* values );
-    virtual void Write( const std::string variableName, const float* values );
-    virtual void Write( const std::string variableName, const double* values );
-    virtual void Write( const std::string variableName, const long double* values );
-    virtual void Write( const std::string variableName, const std::complex<float>* values );
-    virtual void Write( const std::string variableName, const std::complex<double>* values );
-    virtual void Write( const std::string variableName, const std::complex<long double>* values );
-    virtual void Write( const std::string variableName, const void* values );
-
-
     /**
      * Used to verify parameters in m_Method containers
      * @param itParam iterator to a certain parameter

include/engine/bp/BPFileWriter.h

@@ -36,34 +36,6 @@ public:
 
     ~BPFileWriter( );
 
-
-    void Advance( );
-
-    /**
-     * Closes a single transport or all transports
-     * @param transportIndex, if -1 (default) closes all transports, otherwise it closes a transport in m_Transport[transportIndex]. In debug mode the latter is bounds-checked.
-     */
-    void Close( const int transportIndex = -1 );
-
-private:
-
-    capsule::STLVector m_Buffer; ///< heap capsule using STL std::vector<char>
-    format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
-    format::BP1MetadataSet m_MetadataSet; ///< metadata set accompanying the heap buffer data in bp format. Needed by m_BP1Writer
-
-    bool m_IsFirstClose = true; ///< set to false after first Close is reached so metadata doesn't have to be accommodated for a subsequent Close
-    std::size_t m_MaxBufferSize; ///< maximum allowed memory to be allocated
-    float m_GrowthFactor = 1.5; ///< capsule memory growth factor, new_memory = m_GrowthFactor * current_memory
-
-    bool m_TransportFlush = false; ///< true: transport flush happened, buffer must be reset
-
-    void Init( );
-    void InitParameters( );
-    void InitTransports( );
-    void InitProcessGroup( );
-
-    void WriteProcessGroupIndex( );
-
     void Write( Variable<char>& variable, const char* values );
     void Write( Variable<unsigned char>& variable, const unsigned char* values );
     void Write( Variable<short>& variable, const short* values );
@@ -101,6 +73,37 @@ private:
     void Write( const std::string variableName, const void* values );
 
 
+
+    void Advance( );
+
+    /**
+     * Closes a single transport or all transports
+     * @param transportIndex, if -1 (default) closes all transports, otherwise it closes a transport in m_Transport[transportIndex]. In debug mode the latter is bounds-checked.
+     */
+    void Close( const int transportIndex = -1 );
+
+
+private:
+
+    capsule::STLVector m_Buffer; ///< heap capsule using STL std::vector<char>
+    format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
+    format::BP1MetadataSet m_MetadataSet; ///< metadata set accompanying the heap buffer data in bp format. Needed by m_BP1Writer
+
+    bool m_IsFirstClose = true; ///< set to false after first Close is reached so metadata doesn't have to be accommodated for a subsequent Close
+    std::size_t m_MaxBufferSize; ///< maximum allowed memory to be allocated
+    float m_GrowthFactor = 1.5; ///< capsule memory growth factor, new_memory = m_GrowthFactor * current_memory
+
+    bool m_TransportFlush = false; ///< true: transport flush happened, buffer must be reset
+
+    void Init( );
+    void InitParameters( );
+    void InitTransports( );
+    void InitProcessGroup( );
+
+    void WriteProcessGroupIndex( );
+
+
+
     /**
      * Common function for primitive (including std::complex) writes
      * @param group

include/engine/dataman/DataManWriter.h

@@ -46,22 +46,6 @@ public:
 
     void SetCallBack( std::function<void( const void*, std::string, std::string, std::string, Dims )> callback );
 
-    void Close( const int transportIndex = -1 );
-
-private:
-
-    capsule::STLVector m_Buffer; ///< heap capsule, contains data and metadata buffers
-    format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
-
-    bool m_DoRealTime = false;
-    bool m_DoMonitor = false;
-    DataManager m_Man;
-    std::function<void( const void*, std::string, std::string, std::string, Dims )> m_CallBack; ///< call back function
-
-    void Init( );  ///< calls InitCapsules and InitTransports based on Method, called from constructor
-    void InitCapsules( );
-    void InitTransports( ); ///< from Transports
-
     void Write( Variable<char>& variable, const char* values );
     void Write( Variable<unsigned char>& variable, const unsigned char* values );
     void Write( Variable<short>& variable, const short* values );
@@ -96,6 +80,22 @@ private:
     void Write( const std::string variableName, const std::complex<double>* values );
     void Write( const std::string variableName, const std::complex<long double>* values );
 
+    void Close( const int transportIndex = -1 );
+
+private:
+
+    capsule::STLVector m_Buffer; ///< heap capsule, contains data and metadata buffers
+    format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
+
+    bool m_DoRealTime = false;
+    bool m_DoMonitor = false;
+    DataManager m_Man;
+    std::function<void( const void*, std::string, std::string, std::string, Dims )> m_CallBack; ///< call back function
+
+    void Init( );  ///< calls InitCapsules and InitTransports based on Method, called from constructor
+    void InitCapsules( );
+    void InitTransports( ); ///< from Transports
+
     /**
      * From transport Mdtm in m_Method
      * @param parameter must be an accepted parameter

src/core/Engine.cpp

@@ -82,7 +82,9 @@ void Engine::Write( const std::string variableName, const std::complex<double>*
 void Engine::Write( const std::string variableName, const std::complex<long double>* values ){ }
 void Engine::Write( const std::string variableName, const void* values ){ }
 
-void Engine::Advance( ){ }
+void Engine::Advance(){ }
+
+void Engine::Close( const int transportIndex ){ }
 
 //READ
 Variable<void>* Engine::InquireVariable( const std::string name, const bool readIn ){ return nullptr; }