From 66e7bd4d0c13433e9ec910dd726e309369cee6c3 Mon Sep 17 00:00:00 2001 From: wfg <wfg@pc0098504.ornl.gov> Date: Thu, 16 Mar 2017 18:29:42 -0400 Subject: [PATCH] Trying to add PyBind11 support (unsuccessfully) Adding Variable<void> map container in ADIOS for unknown variable types (until Write). This is required in Python. --- bindings/python/Makefile | 40 +++-- bindings/python/README.md | 12 ++ bindings/python/include/ADIOSPy.h | 19 +- bindings/python/include/EnginePy.h | 38 +++- bindings/python/include/MethodPy.h | 32 +++- bindings/python/include/VariablePy.h | 14 +- bindings/python/include/adiosPyFunctions.h | 46 +++-- bindings/python/src/ADIOSPy.cpp | 8 +- bindings/python/src/EnginePy.cpp | 55 ++++-- bindings/python/src/MethodPy.cpp | 33 +++- bindings/python/src/adiosPyFunctions.cpp | 33 +++- bindings/python/src/glue.cpp | 82 --------- bindings/python/src/glueBoostPython.cpp | 198 +++++++++++++++++++++ bindings/python/src/gluePyBind11.cpp | 47 +++++ bindings/python/test_hello.py | 22 +-- include/ADIOS.h | 17 ++ include/core/Variable.h | 2 +- include/functions/adiosTemplates.h | 3 +- 18 files changed, 531 insertions(+), 170 deletions(-) create mode 100644 bindings/python/README.md delete mode 100644 bindings/python/src/glue.cpp create mode 100644 bindings/python/src/glueBoostPython.cpp create mode 100644 bindings/python/src/gluePyBind11.cpp diff --git a/bindings/python/Makefile b/bindings/python/Makefile index 8a00f5b0a..76ea1342a 100644 --- a/bindings/python/Makefile +++ b/bindings/python/Makefile @@ -2,27 +2,39 @@ # Created on: Mar 13, 2017 # Author: wfg +MPICC:=mpic++ + +CPPFiles:=$(shell find ./src -type f -name "*.cpp") +OBJS:=$(patsubst %.cpp, ./bin/%.o, $(notdir $(CPPFiles)) ) + # location of the Python header files PYTHON_VERSION:= 2.7 PYTHON_INC:= /usr/include/python$(PYTHON_VERSION) -PYTHON_LIBLOC:= /usr/lib/python2.7/config-x86_64-linux-gnu/ +PYTHON_LIBLOC:= /usr/lib/python$(PYTHON_VERSION)/config-x86_64-linux-gnu/ -# location of the Boost Python include files and library -BOOST_INC:= /home/wgodoy/Apps/boost_1_63_0/ -BOOST_LIBLOC:= /home/wgodoy/Apps/boost_1_63_0/stage/lib/ + +ifeq ($(HAVE_PYBIND11),yes) # USE PYBIND11 (headers only) + BIND_FLAG:= -DHAVE_PYBIND11 + BIND_INC:= /home/wfg/workspace/pybind11/include + BIND_LIBS:= + OBJS:=$(patsubst ./bin/glueBoostPython.o, ,$(OBJS) ) #remove BoostPython +else # USE BOOST PYTHON (default) + BIND_FLAG:= -DHAVE_BOOSTPYTHON + BIND_INC:= /opt/boost_1_63_0/ + BIND_LIBS:= -L/opt/boost_1_63_0/stage/lib/ -lboost_python -lboost_numpy + OBJS:=$(patsubst ./bin/gluePyBind11.o, ,$(OBJS) ) #remove PyBind11 +endif + # location of ADIOS include and lib -ADIOS_INC:= /home/wgodoy/workspace/ADIOSPP/include -ADIOS_LIB:= /home/wgodoy/workspace/ADIOSPP/lib +ADIOS_INC:= /home/wfg/workspace/ADIOSPP/include +ADIOS_LIB:= /home/wfg/workspace/ADIOSPP/lib ADIOSPy_INC:=./include -CFLAGS:=-c -Wall -fPIC -O0 -g -std=c++11 -Wno-deprecated-declarations - -CPPFiles:=$(shell find ./src -type f -name "*.cpp") -OBJS:=$(patsubst %.cpp, ./bin/%.o, $(notdir $(CPPFiles)) ) -INC:= -I$(PYTHON_INC) -I$(BOOST_INC) -I$(ADIOS_INC) -I$(ADIOSPy_INC) -LIBS:= -L$(ADIOS_LIB) -ladios -L$(BOOST_LIBLOC) -lboost_python -lboost_numpy -L$(PYTHON_LIBLOC) -lpython$(PYTHON_VERSION) +CFLAGS:=-c -Wall -fPIC -O0 -g -std=c++11 -Wno-deprecated-declarations $(BIND_FLAG) +INC:= -I$(PYTHON_INC) -I$(BIND_INC) -I$(ADIOS_INC) -I$(ADIOSPy_INC) +LIBS:= -L$(ADIOS_LIB) -ladios $(BIND_LIBS) -L$(PYTHON_LIBLOC) -lpython$(PYTHON_VERSION) # compile mesh classes @@ -31,11 +43,11 @@ TARGET = ADIOSPy all: $(TARGET).so $(TARGET).so: $(OBJS) - mpic++ -shared -o $(TARGET).so -Wl,--export-dynamic $(OBJS) $(LIBS) + $(MPICC) -shared -o $(TARGET).so -Wl,--export-dynamic $(OBJS) $(LIBS) ./bin/%.o: ./src/%.cpp @( mkdir -p ./bin ); - mpic++ $(INC) $(CFLAGS) -o $@ $< + $(MPICC) $(INC) $(CFLAGS) -o $@ $< clean: rm ./bin/*.o *.so \ No newline at end of file diff --git a/bindings/python/README.md b/bindings/python/README.md new file mode 100644 index 000000000..dcee25ef0 --- /dev/null +++ b/bindings/python/README.md @@ -0,0 +1,12 @@ +Python bindings for ADIOS 2.0 + +For Boost Python use Makefile.BoostPython +Requirements: +Boost.Python version 1.63 +Don't forget to add boost python libraries path in LD_LIBRARY_PATH + + + +For PyBind11 use Makefile.PyBind11 +Requirements: +PyBind11 version 2.0.1 diff --git a/bindings/python/include/ADIOSPy.h b/bindings/python/include/ADIOSPy.h index 278453bf1..c6753173f 100644 --- a/bindings/python/include/ADIOSPy.h +++ b/bindings/python/include/ADIOSPy.h @@ -11,7 +11,13 @@ #include <string> #include <memory> //std::shared_ptr -#include "boost/python.hpp" +#ifdef HAVE_BOOSTPYTHON + #include "boost/python.hpp" +#endif + +#ifdef HAVE_PYBIND11 + #include "pybind11/pybind11.h" +#endif #include "ADIOS.h" #include "adiosPyFunctions.h" //ListToVector, VectorToList @@ -22,7 +28,16 @@ namespace adios { +#ifdef HAVE_BOOSTPYTHON using pyList = boost::python::list; +using pyObject = boost::python::object; +#endif + +#ifdef HAVE_PYBIND11 +using pyList = pybind11::list; +using pyObject = pybind11::object; +#endif + class ADIOSPy : public ADIOS @@ -50,7 +65,7 @@ public: MethodPy& DeclareMethodPy( const std::string methodName, const std::string type = "" ); EnginePy OpenPy( const std::string name, const std::string accessMode, - const MethodPy& method, boost::python::object py_comm = boost::python::object() ); + const MethodPy& method, pyObject py_comm = pyObject() ); }; diff --git a/bindings/python/include/EnginePy.h b/bindings/python/include/EnginePy.h index cfe6019d6..489120253 100644 --- a/bindings/python/include/EnginePy.h +++ b/bindings/python/include/EnginePy.h @@ -8,16 +8,31 @@ #ifndef ENGINEPY_H_ #define ENGINEPY_H_ +#ifdef HAVE_BOOSTPYTHON #include "boost/python.hpp" #include "boost/python/numpy.hpp" +#endif + +#ifdef HAVE_PYBIND11 +#include "pybind11/pybind11.h" +#include "pybind11/numpy.h" +#endif #include "core/Engine.h" #include "VariablePy.h" +#include "adiosPyFunctions.h" namespace adios { -namespace np = boost::python::numpy; +#ifdef HAVE_BOOSTPYTHON +using pyArray = boost::python::numpy::ndarray; +#endif + +#ifdef HAVE_PYBIND11 +using pyArray = pybind11::array; +#endif + class EnginePy { @@ -26,9 +41,22 @@ public: std::shared_ptr<Engine> m_Engine; - void WritePy( VariablePy<double>& variable, const np::ndarray& array ); - - void WritePy( VariablePy<float>& variable, const np::ndarray& array ); + void WritePy( VariablePy<char>& variable, const pyArray& array ); + void WritePy( VariablePy<unsigned char>& variable, const pyArray& array ); + void WritePy( VariablePy<short>& variable, const pyArray& array ); + void WritePy( VariablePy<unsigned short>& variable, const pyArray& array ); + void WritePy( VariablePy<int>& variable, const pyArray& array ); + void WritePy( VariablePy<unsigned int>& variable, const pyArray& array ); + void WritePy( VariablePy<long int>& variable, const pyArray& array ); + void WritePy( VariablePy<unsigned long int>& variable, const pyArray& array ); + void WritePy( VariablePy<long long int>& variable, const pyArray& array ); + void WritePy( VariablePy<unsigned long long int>& variable, const pyArray& array ); + void WritePy( VariablePy<float>& variable, const pyArray& array ); + void WritePy( VariablePy<double>& variable, const pyArray& array ); + void WritePy( VariablePy<long double>& variable, const pyArray& array ); + void WritePy( VariablePy<std::complex<float>>& variable, const pyArray& array ); + void WritePy( VariablePy<std::complex<double>>& variable, const pyArray& array ); + void WritePy( VariablePy<std::complex<long double>>& variable, const pyArray& array ); void Close( ); @@ -42,4 +70,4 @@ public: -#endif /* BINDINGS_PYTHON_INCLUDE_ENGINEPY_H_ */ +#endif /* ENGINEPY_H_ */ diff --git a/bindings/python/include/MethodPy.h b/bindings/python/include/MethodPy.h index e2d8046e8..edd85d7f2 100644 --- a/bindings/python/include/MethodPy.h +++ b/bindings/python/include/MethodPy.h @@ -8,14 +8,33 @@ #ifndef METHODPY_H_ #define METHODPY_H_ -#include <boost/python.hpp> +#ifdef HAVE_BOOSTPYTHON + #include "boost/python.hpp" +#endif + +#ifdef HAVE_PYBIND11 + #include "pybind11/pybind11.h" + #include "pybind11/cast.h" +#endif + #include "core/Method.h" namespace adios { -using pyList = boost::python::list; +#ifdef HAVE_BOOSTPYTHON +using pyObject = boost::python::object; +using pyTuple = boost::python::tuple; +using pyDict = boost::python::dict; +#endif + +#ifdef HAVE_PYBIND11 +using pyObject = pybind11::object; +using pyTuple = pybind11::tuple; +using pyDict = pybind11::dict; +#endif + class MethodPy : public Method { @@ -31,20 +50,15 @@ public: * @param dictionary * @return */ - static boost::python::object SetParametersPy( boost::python::tuple args, boost::python::dict kwargs ); + static pyObject SetParametersPy( pyTuple args, pyDict kwargs ); - static boost::python::object AddTransportPy( boost::python::tuple args, boost::python::dict kwargs ); + static pyObject AddTransportPy( pyTuple args, pyDict kwargs ); void PrintAll( ) const; }; - - - - - } diff --git a/bindings/python/include/VariablePy.h b/bindings/python/include/VariablePy.h index 85798c862..2b96ba2cf 100644 --- a/bindings/python/include/VariablePy.h +++ b/bindings/python/include/VariablePy.h @@ -14,6 +14,15 @@ namespace adios { +#ifdef HAVE_BOOSTPYTHON +using pyList = boost::python::list; +#endif + +#ifdef HAVE_PYBIND11 +using pyList = pybind11::list; +#endif + + template< class T> class VariablePy : public Variable<T> { @@ -28,18 +37,17 @@ public: ~VariablePy( ) { } - void SetLocalDimensions( const boost::python::list list ) + void SetLocalDimensions( const pyList list ) { this->m_Dimensions = ListToVector( list ); } - void SetGlobalDimensionsAndOffsets( const boost::python::list globalDimensions, const boost::python::list globalOffsets ) + void SetGlobalDimensionsAndOffsets( const pyList globalDimensions, const pyList globalOffsets ) { this->m_GlobalDimensions = ListToVector( globalDimensions ); this->m_GlobalOffsets = ListToVector( globalOffsets ); } - Dims GetLocalDimensions( ) { return this->m_Dimensions; diff --git a/bindings/python/include/adiosPyFunctions.h b/bindings/python/include/adiosPyFunctions.h index ec21e8652..33a342cb1 100644 --- a/bindings/python/include/adiosPyFunctions.h +++ b/bindings/python/include/adiosPyFunctions.h @@ -12,40 +12,56 @@ #include <map> #include <string> -#include "boost/python.hpp" +#ifdef HAVE_BOOSTPYTHON + #include "boost/python.hpp" + #include "boost/python/numpy.hpp" +#endif + +#ifdef HAVE_PYBIND11 + #include "pybind11/pybind11.h" + #include "pybind11/numpy.h" +#endif namespace adios { using Dims = std::vector<std::size_t>; + +#ifdef HAVE_BOOSTPYTHON +using pyList = boost::python::list; +using pyDict = boost::python::dict; +using pyArray = boost::python::numpy::ndarray; +#endif + +#ifdef HAVE_PYBIND11 +using pyList = pybind11::list; +using pyDict = pybind11::dict; +using pyArray = pybind11::array; +#endif + /** * Transforms a boost python list to a Dims (std::vector<std::size_t>) object * @param list input boost python list from python program * @return Dims (std::vector<std::size_t>) object than can be passed to python */ -Dims ListToVector( const boost::python::list& list ); - -std::map<std::string, std::string> DictToMap( const boost::python::dict& dictionary ); +Dims ListToVector( const pyList& list ); +std::map<std::string, std::string> DictToMap( const pyDict& dictionary ); template< class T > -T* PyObjectToPointer( const boost::python::object& object ) +const T* PyArrayToPointer( const pyArray& array ) { - return reinterpret_cast<T*>( object.ptr() ); - + #ifdef HAVE_BOOSTPYTHON + return reinterpret_cast<const T*>( array.get_data() ); + #endif -// Py_buffer pyBuffer; -// if(PyObject_GetBuffer( object.ptr(), &pyBuffer, PyBUF_SIMPLE)!=-1) -// { -// -// } -// -// return nullptr; + #ifdef HAVE_PYBIND11 + return reinterpret_cast<const T*>( array.data() ); + #endif } - } //end namespace diff --git a/bindings/python/src/ADIOSPy.cpp b/bindings/python/src/ADIOSPy.cpp index 2f3bdb46b..b11d19cb8 100644 --- a/bindings/python/src/ADIOSPy.cpp +++ b/bindings/python/src/ADIOSPy.cpp @@ -40,10 +40,10 @@ MethodPy& ADIOSPy::DeclareMethodPy( const std::string methodName, const std::str EnginePy ADIOSPy::OpenPy( const std::string name, const std::string accessMode, - const MethodPy& method, boost::python::object py_comm ) + const MethodPy& method, pyObject py_comm ) { EnginePy enginePy; - if( py_comm == boost::python::object() ) //None + if( py_comm == pyObject() ) //None { enginePy.m_Engine = Open( name, accessMode, method ); } @@ -51,11 +51,11 @@ EnginePy ADIOSPy::OpenPy( const std::string name, const std::string accessMode, { if (import_mpi4py() < 0) throw std::logic_error( "ERROR: could not import mpi4py communicator in Open " + name + "\n" ); MPI_Comm* comm_p = PyMPIComm_Get( py_comm.ptr() ); - if( comm_p == nullptr ) boost::python::throw_error_already_set(); + if( comm_p == nullptr ) throw std::invalid_argument( "ERROR: MPI communicator is nullptr in Open " + name + "\n" ); enginePy.m_Engine = Open( name, accessMode, *comm_p, method ); } - //here downcast + return enginePy; } diff --git a/bindings/python/src/EnginePy.cpp b/bindings/python/src/EnginePy.cpp index b98b8265a..9ca125411 100644 --- a/bindings/python/src/EnginePy.cpp +++ b/bindings/python/src/EnginePy.cpp @@ -14,18 +14,53 @@ namespace adios { +void EnginePy::WritePy( VariablePy<char>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<char>( array ) ); } -void EnginePy::WritePy( VariablePy<double>& variable, const np::ndarray& array ) -{ - const double* values = reinterpret_cast<const double*>( array.get_data() ); - m_Engine->Write( variable, values ); -} +void EnginePy::WritePy( VariablePy<unsigned char>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<unsigned char>( array ) ); } -void EnginePy::WritePy( VariablePy<float>& variable, const np::ndarray& array ) -{ - const float* values = reinterpret_cast<const float*>( array.get_data() ); - m_Engine->Write( variable, values ); -} +void EnginePy::WritePy( VariablePy<short>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<short>( array ) ); } + +void EnginePy::WritePy( VariablePy<unsigned short>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<unsigned short>( array ) ); } + +void EnginePy::WritePy( VariablePy<int>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<int>( array ) ); } + +void EnginePy::WritePy( VariablePy<unsigned int>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<unsigned int>( array ) ); } + +void EnginePy::WritePy( VariablePy<long int>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<long int>( array ) ); } + +void EnginePy::WritePy( VariablePy<unsigned long int>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<unsigned long int>( array ) ); } + +void EnginePy::WritePy( VariablePy<long long int>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<long long int>( array ) ); } + +void EnginePy::WritePy( VariablePy<unsigned long long int>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<unsigned long long int>( array ) ); } + +void EnginePy::WritePy( VariablePy<float>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<float>( array ) ); } + +void EnginePy::WritePy( VariablePy<double>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<double>( array ) ); } + +void EnginePy::WritePy( VariablePy<long double>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<long double>( array ) ); } + +void EnginePy::WritePy( VariablePy<std::complex<float>>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<std::complex<float>>( array ) ); } + +void EnginePy::WritePy( VariablePy<std::complex<double>>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<std::complex<double>>( array ) ); } + +void EnginePy::WritePy( VariablePy<std::complex<long double>>& variable, const pyArray& array ) +{ m_Engine->Write( variable, PyArrayToPointer<std::complex<long double>>( array ) ); } void EnginePy::GetType( ) const { diff --git a/bindings/python/src/MethodPy.cpp b/bindings/python/src/MethodPy.cpp index 231c22e20..ffcd05290 100644 --- a/bindings/python/src/MethodPy.cpp +++ b/bindings/python/src/MethodPy.cpp @@ -12,6 +12,15 @@ namespace adios { +#ifdef HAVE_BOOSTPYTHON +namespace py = boost::python; +#endif + +#ifdef HAVE_PYBIND11 +namespace py = pybind11; +#endif + + MethodPy::MethodPy( const std::string type, const bool debugMode ): Method( type, debugMode ) { } @@ -21,24 +30,38 @@ MethodPy::~MethodPy( ) { } -boost::python::object MethodPy::SetParametersPy( boost::python::tuple args, boost::python::dict kwargs ) +pyObject MethodPy::SetParametersPy( pyTuple args, pyDict kwargs ) { - if( boost::python::len( args ) > 1 ) + if( py::len( args ) > 1 ) throw std::invalid_argument( "ERROR: syntax of Method SetParameters function is incorrect, only use dictionary\n" ); + #ifdef HAVE_BOOSTPYTHON MethodPy& self = boost::python::extract<MethodPy&>( args[0] ); + #endif + + #ifdef HAVE_PYBIND11 + MethodPy& self = (*this)( &args[0] ); + #endif + self.m_Parameters = DictToMap( kwargs ); return args[0]; } -boost::python::object MethodPy::AddTransportPy( boost::python::tuple args, boost::python::dict kwargs ) +pyObject MethodPy::AddTransportPy( pyTuple args, pyDict kwargs ) { - if( boost::python::len( args ) != 2 ) + if( py::len( args ) != 2 ) throw std::invalid_argument( "ERROR: syntax of Method AddTransport function is incorrect, only use one string for transport followed by a dictionary for parameters\n" ); + #ifdef HAVE_BOOSTPYTHON MethodPy& self = boost::python::extract<MethodPy&>( args[0] ); - std::string type = boost::python::extract<std::string>( args[1] ); + const std::string type = boost::python::extract<std::string>( args[1] ); + #endif + + #ifdef HAVE_PYBIND11 + MethodPy& self = (*this)( &args[0] ); + const std::string type = reinterpret_cast<std::string>( args[1] ); + #endif auto parameters = DictToMap( kwargs ); parameters.insert( std::make_pair( "transport", type ) ); diff --git a/bindings/python/src/adiosPyFunctions.cpp b/bindings/python/src/adiosPyFunctions.cpp index ed130c974..27601c5ff 100644 --- a/bindings/python/src/adiosPyFunctions.cpp +++ b/bindings/python/src/adiosPyFunctions.cpp @@ -12,29 +12,45 @@ namespace adios { -Dims ListToVector( const boost::python::list& list ) +#ifdef HAVE_BOOSTPYTHON +namespace py = boost::python; +using pyCastString = py::extract<std::string>; +using pyCastSize_t = py::extract<std::size_t>; + +#endif + +#ifdef HAVE_PYBIND11 +namespace py = pybind11; +using pyCastString = pybind11::cast<std::string>; +using pyCastSize_t = pybind11::cast<std::size_t>; +#endif + + + +Dims ListToVector( const pyList& list ) { - const boost::python::ssize_t length = boost::python::len( list ); + const unsigned int length = py::len( list ); Dims vec; vec.reserve( length ); for( unsigned int i=0; i<length;i++ ) - vec.push_back( boost::python::extract<std::size_t>( list[i]) ); + vec.push_back( pyCastSize_t( list[i]) ); return vec; } -std::map<std::string, std::string> DictToMap( const boost::python::dict& dictionary ) + +std::map<std::string, std::string> DictToMap( const pyDict& dictionary ) { - boost::python::list keys = dictionary.keys(); - unsigned int length = boost::python::len( keys ); + pyList keys = dictionary.keys(); + const unsigned int length = py::len( keys ); std::map<std::string, std::string> parameters; for( unsigned int k = 0; k < length; ++k ) { - const std::string key( boost::python::extract<std::string>( keys[k] ) ); - const std::string value( boost::python::extract<std::string>( dictionary[ keys[k] ] ) ); + const std::string key( pyCastString( keys[k] ) ); + const std::string value( pyCastString( dictionary[ keys[k] ] ) ); parameters.insert( std::make_pair( key, value ) ); } @@ -42,6 +58,5 @@ std::map<std::string, std::string> DictToMap( const boost::python::dict& diction } - } //end namespace diff --git a/bindings/python/src/glue.cpp b/bindings/python/src/glue.cpp deleted file mode 100644 index e716c1bcc..000000000 --- a/bindings/python/src/glue.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * glue.cpp - * - * Created on: Mar 13, 2017 - * Author: wfg - */ - -#include <mpi4py/mpi4py.h> - -#include "boost/python.hpp" -#include "boost/python/suite/indexing/vector_indexing_suite.hpp" -#include "boost/python/raw_function.hpp" -#include "boost/python/numpy.hpp" - - -#include "ADIOSPy.h" -#include "adiosPyFunctions.h" - - -namespace bpy = boost::python; -namespace np = boost::python::numpy; - - -adios::ADIOSPy ADIOSPy( bpy::object py_comm, const bool debug ) -{ - MPI_Comm* comm_p = PyMPIComm_Get( py_comm.ptr() ); - if (comm_p == NULL) bpy::throw_error_already_set(); - return adios::ADIOSPy( *comm_p, debug ); -} - - -using ReturnInternalReference = bpy::return_internal_reference<>; -BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS( open_overloads, adios::ADIOSPy::OpenPy, 3, 4 ) - -BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS( write_overload, adios::EnginePy::WritePy, 2, 2 ) - - - -BOOST_PYTHON_MODULE( ADIOSPy ) -{ - if (import_mpi4py() < 0) return; /* Python 2.X */ - - Py_Initialize(); - np::initialize(); - - - bpy::class_< adios::Dims >("Dims") - .def(boost::python::vector_indexing_suite< adios::Dims >() ); - //functions - bpy::def("ADIOSPy", ADIOSPy ); - - //classes - bpy::class_<adios::ADIOSPy>("ADIOS", bpy::no_init ) - .def("HelloMPI", &adios::ADIOSPy::HelloMPI ) - .def("DefineVariableDouble", &adios::ADIOSPy::DefineVariablePy<double>, ReturnInternalReference() ) - .def("DefineVariableFloat", &adios::ADIOSPy::DefineVariablePy<float>, ReturnInternalReference() ) - .def("DeclareMethod", &adios::ADIOSPy::DeclareMethodPy, ReturnInternalReference() ) - .def("Open", &adios::ADIOSPy::OpenPy, open_overloads() ) - ; - - bpy::class_<adios::VariablePy<double>>("VariableDouble", bpy::no_init ) - .def("SetLocalDimensions", &adios::VariablePy<double>::SetLocalDimensions ) - .def("GetLocalDimensions", &adios::VariablePy<double>::GetLocalDimensions ) - ; - - bpy::class_<adios::MethodPy>("Method", bpy::no_init ) - .def("SetParameters", bpy::raw_function( &adios::MethodPy::SetParametersPy, 1 ) ) - .def("AddTransport", bpy::raw_function( &adios::MethodPy::AddTransportPy, 1 ) ) - .def("PrintAll", &adios::MethodPy::PrintAll ) - ; - - //Engines - bpy::class_<adios::EnginePy>("EnginePy", bpy::no_init ) - .def( "GetType", &adios::EnginePy::GetType ) - .def("Write", static_cast< void( adios::EnginePy::*) - (adios::VariablePy<double>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) - .def("Write", static_cast< void( adios::EnginePy::*) - (adios::VariablePy<float>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) - .def( "Close", &adios::EnginePy::Close ) - ; - -} diff --git a/bindings/python/src/glueBoostPython.cpp b/bindings/python/src/glueBoostPython.cpp new file mode 100644 index 000000000..b0114261e --- /dev/null +++ b/bindings/python/src/glueBoostPython.cpp @@ -0,0 +1,198 @@ +/* + * glue.cpp + * + * Created on: Mar 13, 2017 + * Author: wfg + */ + +#include <mpi4py/mpi4py.h> + +#include "boost/python.hpp" +#include "boost/python/suite/indexing/vector_indexing_suite.hpp" +#include "boost/python/raw_function.hpp" +#include "boost/python/numpy.hpp" + + +#include "ADIOSPy.h" +#include "adiosPyFunctions.h" + + +namespace py = boost::python; +namespace np = boost::python::numpy; + + +adios::ADIOSPy ADIOSPy( py::object py_comm, const bool debug ) +{ + MPI_Comm* comm_p = PyMPIComm_Get( py_comm.ptr() ); + if (comm_p == NULL) py::throw_error_already_set(); + return adios::ADIOSPy( *comm_p, debug ); +} + + +using ReturnInternalReference = py::return_internal_reference<>; +BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS( open_overloads, adios::ADIOSPy::OpenPy, 3, 4 ) + +BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS( write_overload, adios::EnginePy::WritePy, 2, 2 ) + + + +BOOST_PYTHON_MODULE( ADIOSPy ) +{ + if (import_mpi4py() < 0) return; /* Python 2.X */ + + Py_Initialize(); + np::initialize(); + + py::class_< adios::Dims >("Dims") + .def(boost::python::vector_indexing_suite< adios::Dims >() ); + //functions + py::def("ADIOSPy", ADIOSPy ); + + //classes + py::class_<adios::ADIOSPy>("ADIOS", py::no_init ) + .def("HelloMPI", &adios::ADIOSPy::HelloMPI ) + .def("DefineVariableChar", &adios::ADIOSPy::DefineVariablePy<char>, ReturnInternalReference() ) + .def("DefineVariableUChar", &adios::ADIOSPy::DefineVariablePy<unsigned char>, ReturnInternalReference() ) + .def("DefineVariableShort", &adios::ADIOSPy::DefineVariablePy<short>, ReturnInternalReference() ) + .def("DefineVariableUShort", &adios::ADIOSPy::DefineVariablePy<unsigned short>, ReturnInternalReference() ) + .def("DefineVariableInt", &adios::ADIOSPy::DefineVariablePy<int>, ReturnInternalReference() ) + .def("DefineVariableUInt", &adios::ADIOSPy::DefineVariablePy<unsigned int>, ReturnInternalReference() ) + .def("DefineVariableLInt", &adios::ADIOSPy::DefineVariablePy<long int>, ReturnInternalReference() ) + .def("DefineVariableULInt", &adios::ADIOSPy::DefineVariablePy<unsigned long int>, ReturnInternalReference() ) + .def("DefineVariableLLInt", &adios::ADIOSPy::DefineVariablePy<long long int>, ReturnInternalReference() ) + .def("DefineVariableULLInt", &adios::ADIOSPy::DefineVariablePy<unsigned long long int>, ReturnInternalReference() ) + .def("DefineVariableFloat", &adios::ADIOSPy::DefineVariablePy<float>, ReturnInternalReference() ) + .def("DefineVariableDouble", &adios::ADIOSPy::DefineVariablePy<double>, ReturnInternalReference() ) + .def("DefineVariableLDouble", &adios::ADIOSPy::DefineVariablePy<long double>, ReturnInternalReference() ) + .def("DefineVariableCFloat", &adios::ADIOSPy::DefineVariablePy<std::complex<float>>, ReturnInternalReference() ) + .def("DefineVariableCDouble", &adios::ADIOSPy::DefineVariablePy<std::complex<double>>, ReturnInternalReference() ) + .def("DefineVariableCLDouble", &adios::ADIOSPy::DefineVariablePy<std::complex<long double>>, ReturnInternalReference() ) + .def("DeclareMethod", &adios::ADIOSPy::DeclareMethodPy, ReturnInternalReference() ) + .def("Open", &adios::ADIOSPy::OpenPy, open_overloads() ) + ; + + py::class_<adios::VariablePy<char>>("VariableChar", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<char>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<char>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<unsigned char>>("VariableUChar", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<unsigned char>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<unsigned char>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<short>>("VariableShort", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<short>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<short>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<unsigned short>>("VariableUShort", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<unsigned short>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<unsigned short>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<int>>("VariableInt", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<int>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<int>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<unsigned int>>("VariableUInt", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<unsigned int>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<unsigned int>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<long int>>("VariableLInt", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<long int>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<long int>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<unsigned long int>>("VariableULInt", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<unsigned long int>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<unsigned long int>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<long long int>>("VariableLLInt", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<long long int>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<long long int>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<unsigned long long int>>("VariableULLInt", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<unsigned long long int>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<unsigned long long int>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<float>>("VariableFloat", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<float>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<float>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<double>>("VariableDouble", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<double>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<double>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<long double>>("VariableLDouble", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<long double>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<long double>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<std::complex<float>>>("VariableCFloat", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<std::complex<float>>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<std::complex<float>>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<std::complex<double>>>("VariableCDouble", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<std::complex<double>>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<std::complex<double>>::GetLocalDimensions ) + ; + + py::class_<adios::VariablePy<std::complex<long double>>>("VariableCLDouble", py::no_init ) + .def("SetLocalDimensions", &adios::VariablePy<std::complex<long double>>::SetLocalDimensions ) + .def("GetLocalDimensions", &adios::VariablePy<std::complex<long double>>::GetLocalDimensions ) + ; + + py::class_<adios::MethodPy>("Method", py::no_init ) + .def("SetParameters", py::raw_function( &adios::MethodPy::SetParametersPy, 1 ) ) + .def("AddTransport", py::raw_function( &adios::MethodPy::AddTransportPy, 1 ) ) + .def("PrintAll", &adios::MethodPy::PrintAll ) + ; + + //Engine + py::class_<adios::EnginePy>("EnginePy", py::no_init ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<char>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<unsigned char>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<short>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<unsigned short>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<int>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<unsigned int>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<long int>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<unsigned long int>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<long long int>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<unsigned long long int>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<float>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<double>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<long double>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<std::complex<float>>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<std::complex<double>>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + .def("Write", static_cast< void( adios::EnginePy::*) + (adios::VariablePy<std::complex<long double>>&, const np::ndarray& )>( &adios::EnginePy::WritePy ), write_overload() ) + + .def( "Close", &adios::EnginePy::Close ) + ; +} + diff --git a/bindings/python/src/gluePyBind11.cpp b/bindings/python/src/gluePyBind11.cpp new file mode 100644 index 000000000..0e83fd352 --- /dev/null +++ b/bindings/python/src/gluePyBind11.cpp @@ -0,0 +1,47 @@ +/* + * gluePyBind11.cpp + * + * Created on: Mar 16, 2017 + * Author: wfg + */ + +#include <stdexcept> + +#include <mpi4py/mpi4py.h> + +#include "pybind11/pybind11.h" + +#include "ADIOSPy.h" + + +namespace py = pybind11; + + +adios::ADIOSPy ADIOSPy( py::object py_comm, const bool debug ) +{ + MPI_Comm* comm_p = PyMPIComm_Get( py_comm.ptr() ); + if( comm_p == NULL ) py::error_already_set(); + return adios::ADIOSPy( *comm_p, debug ); +} + + +PYBIND11_PLUGIN( ADIOSPy ) +{ + if (import_mpi4py() < 0) throw std::runtime_error("ERROR: mpi4py not loaded correctly\n"); /* Python 2.X */ + + py::module m( "ADIOSPy", "ADIOS Python bindings using pybind11" ); + + m.def("ADIOSPy", &ADIOSPy, "Function that creates an ADIOS object" ); + + py::class_<adios::ADIOSPy>( m, "ADIOS" ) + .def("HelloMPI", &adios::ADIOSPy::HelloMPI ) + .def("DefineVariableDouble", &adios::ADIOSPy::DefineVariablePy<double>, py::return_value_policy::reference_internal ) + .def("DefineVariableFloat", &adios::ADIOSPy::DefineVariablePy<float>, py::return_value_policy::reference_internal ) +// py::arg("localDimensionsPy") = py::list(), +// py::arg("globalDimensionsPy") = py::list(), py::arg("globalOffsetsPy") = py::list() ) + .def("DeclareMethod", &adios::ADIOSPy::DeclareMethodPy, py::return_value_policy::reference_internal ) + .def("Open", &adios::ADIOSPy::OpenPy ) + ; + + return m.ptr(); +} diff --git a/bindings/python/test_hello.py b/bindings/python/test_hello.py index 448753f80..d8cc94dcd 100644 --- a/bindings/python/test_hello.py +++ b/bindings/python/test_hello.py @@ -12,28 +12,30 @@ rank = MPI.COMM_WORLD.Get_rank() size = MPI.COMM_WORLD.Get_size() # User data -myArray = np.array( [1,2,3,4], dtype=np.double ) +myArray = np.array( [1,2,3,4] ) -if( rank % 2 == 1 ): # odd ranks only - oddRankArray = np.array([11,12,13,14],dtype=np.float) +# if( rank % 2 == 1 ): # odd ranks only +oddRankArray = np.array( [11.,12.,13.,14.]) # ADIOS Define Variables -ioMyDoubles = adios.DefineVariableDouble( "ioMyDoubles", [myArray.size], [], [] ) +# ioMyDoubles = adios.DefineVariableDouble( "ioMyDoubles", [myArray.size], [], [] ) -if( rank % 2 == 1 ): # odd ranks only - ioMyFloats = adios.DefineVariableDouble( "ioMyFloats", [oddRankArray.size], [], [] ) +# if( rank % 2 == 1 ): # odd ranks only +ioMyFloats = adios.DefineVariable( "ioMyFloats", [oddRankArray.size], [], [] ) +print oddRankArray.dtype +print myArray.dtype #Setup method and print summary ioSettings = adios.DeclareMethod("adiosSettings", "BPFileWriter") ioSettings.SetParameters( max_buffer_size = '10000000' ) -ioSettings.AddTransport( 'File', have_metadata_file = 'yes', library = 'POSIX' ) # POSIX is default, just checking +ioSettings.AddTransport( 'File', have_metadata_file = 'yes', library = 'POSIX', blah = 'sds' ) # POSIX is default, just checking #Start Engine bpFileWriter = adios.Open( "file.bp", "w", ioSettings, None ) # Open files using N-to-N method, None means no new MPI communicator -bpFileWriter.Write( ioMyDoubles, myArray ) +# bpFileWriter.Write( ioMyDoubles, myArray ) -if( rank % 2 == 1 ): - bpFileWriter.Write( ioMyFloats, oddRankArray ) +# if( rank % 2 == 1 ): +bpFileWriter.Write( ioMyFloats, oddRankArray ) bpFileWriter.Close( ) diff --git a/include/ADIOS.h b/include/ADIOS.h index 00c58e52a..1c8f0f65e 100644 --- a/include/ADIOS.h +++ b/include/ADIOS.h @@ -192,6 +192,8 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO protected: //no const to allow default empty and copy constructors + std::map<unsigned int, Variable<void> > m_Unknown; ///< C style void* Variable, type is unknown at DefineVariable, used in Python + std::map<unsigned int, Variable<char> > m_Char; std::map<unsigned int, Variable<unsigned char> > m_UChar; std::map<unsigned int, Variable<short> > m_Short; @@ -266,6 +268,17 @@ protected: //no const to allow default empty and copy constructors }; //template specializations of DefineVariable: +template<> inline +Variable<void>& ADIOS::DefineVariable( const std::string name, const Dims dimensions, + const Dims globalDimensions, const Dims globalOffsets ) +{ + CheckVariableInput( name, dimensions ); + const unsigned int size = m_Unknown.size(); + m_Unknown.emplace( size, Variable<void>( name, dimensions, globalDimensions, globalOffsets, m_DebugMode ) ); + m_Variables.emplace( name, std::make_pair( GetType<void>(), size ) ); + return m_Unknown.at( size ); +} + template<> inline Variable<char>& ADIOS::DefineVariable( const std::string name, const Dims dimensions, const Dims globalDimensions, const Dims globalOffsets ) @@ -458,6 +471,10 @@ Variable<std::complex<long double>>& ADIOS::DefineVariable( const std::string na //Get template specialization +template<> inline +Variable<void>& ADIOS::GetVariable( const std::string name ) +{ return m_Unknown.at( GetVariableIndex<void>(name) ); } + template<> inline Variable<char>& ADIOS::GetVariable( const std::string name ) { return m_Char.at( GetVariableIndex<char>(name) ); } diff --git a/include/core/Variable.h b/include/core/Variable.h index 95532571c..fd572fad1 100644 --- a/include/core/Variable.h +++ b/include/core/Variable.h @@ -40,7 +40,7 @@ class Variable : public VariableBase public: const T* m_AppValues = nullptr; ///< pointer to values passed from user in ADIOS Write, it might change in ADIOS Read - //std::vector<T> m_Values; ///< Vector variable returned to user, might be used for zero-copy? + std::vector< TransformData > m_Transforms; ///< associated transforms, sequence determines application order, e.g. first Transforms[0] then Transforms[1]. Pointer used as reference (no memory management). Variable<T>( const std::string name, const Dims dimensions, const Dims globalDimensions, const Dims globalOffsets, diff --git a/include/functions/adiosTemplates.h b/include/functions/adiosTemplates.h index 65998950a..22bad9f6f 100644 --- a/include/functions/adiosTemplates.h +++ b/include/functions/adiosTemplates.h @@ -25,7 +25,8 @@ namespace adios * Get the primitive type in a string from a template * @return if T is a char, returns string = "char" */ -template< class T> inline std::string GetType( ) noexcept { return "compound"; } +template<class T> inline std::string GetType( ) noexcept { return "compound"; } +template<> inline std::string GetType<void>() noexcept { return "unknown"; } template<> inline std::string GetType<char>() noexcept { return "char"; } template<> inline std::string GetType<unsigned char>() noexcept { return "unsigned char"; } template<> inline std::string GetType<short>() noexcept { return "short"; } -- GitLab