From b67c089ec83b3e7c21bc9e1f2b427fb599d88d9a Mon Sep 17 00:00:00 2001
From: wfg <wfg@pc0098504.ornl.gov>
Date: Fri, 17 Mar 2017 17:57:34 -0400
Subject: [PATCH] Changing the Python bindings to recognize any variable at
Write (not at DefineVariable)
---
bindings/python/include/ADIOSPy.h | 25 +++----
bindings/python/include/EnginePy.h | 50 ++++++++-----
bindings/python/include/VariablePy.h | 35 ++++-----
bindings/python/include/adiosPyFunctions.h | 24 ++++++
bindings/python/src/ADIOSPy.cpp | 23 +++++-
bindings/python/src/EnginePy.cpp | 87 ++++++++++++----------
bindings/python/src/VariablePy.cpp | 49 ++++++++++++
bindings/python/src/adiosPyFunctions.cpp | 14 ++++
bindings/python/test_hello.py | 2 +-
include/core/Engine.h | 3 +-
src/core/Engine.cpp | 3 +-
11 files changed, 212 insertions(+), 103 deletions(-)
create mode 100644 bindings/python/src/VariablePy.cpp
diff --git a/bindings/python/include/ADIOSPy.h b/bindings/python/include/ADIOSPy.h
index c6753173f..e2e7b6095 100644
--- a/bindings/python/include/ADIOSPy.h
+++ b/bindings/python/include/ADIOSPy.h
@@ -10,6 +10,7 @@
#include <string>
#include <memory> //std::shared_ptr
+#include <map>
#ifdef HAVE_BOOSTPYTHON
#include "boost/python.hpp"
@@ -23,7 +24,6 @@
#include "adiosPyFunctions.h" //ListToVector, VectorToList
#include "VariablePy.h"
#include "MethodPy.h"
-#include "EnginePy.h"
namespace adios
{
@@ -39,6 +39,8 @@ using pyObject = pybind11::object;
#endif
+class EnginePy;
+
class ADIOSPy : public ADIOS
{
@@ -50,22 +52,19 @@ public:
void HelloMPI( ); ///< says hello from rank/size for testing
+ VariablePy DefineVariablePy( const std::string name, const pyList localDimensionsPy = pyList(),
+ const pyList globalDimensionsPy = pyList(), const pyList globalOffsetsPy = pyList() );
- template<class T> inline
- VariablePy<T>& DefineVariablePy( const std::string name,
- const pyList localDimensionsPy = pyList(),
- const pyList globalDimensionsPy = pyList(),
- const pyList globalOffsetsPy = pyList() )
- {
- Variable<T>& var = DefineVariable<T>( name, ListToVector( localDimensionsPy ), ListToVector( globalDimensionsPy ), ListToVector( globalOffsetsPy ) );
- return *reinterpret_cast<VariablePy<T>*>( &var );
- }
+ MethodPy& DeclareMethodPy( const std::string methodName, const std::string type = "" );
+ EnginePy OpenPy( const std::string name, const std::string accessMode, const MethodPy& method, pyObject py_comm = pyObject() );
- MethodPy& DeclareMethodPy( const std::string methodName, const std::string type = "" );
+ void DefineVariableType( VariablePy& variablePy );
+
+
+private:
- EnginePy OpenPy( const std::string name, const std::string accessMode,
- const MethodPy& method, pyObject py_comm = pyObject() );
+ std::set<std::string> m_VariablesPyNames;
};
diff --git a/bindings/python/include/EnginePy.h b/bindings/python/include/EnginePy.h
index 489120253..3d0d033ed 100644
--- a/bindings/python/include/EnginePy.h
+++ b/bindings/python/include/EnginePy.h
@@ -18,6 +18,7 @@
#include "pybind11/numpy.h"
#endif
+#include "ADIOSPy.h"
#include "core/Engine.h"
#include "VariablePy.h"
#include "adiosPyFunctions.h"
@@ -26,11 +27,13 @@ namespace adios
{
#ifdef HAVE_BOOSTPYTHON
-using pyArray = boost::python::numpy::ndarray;
+//using pyArray = boost::python::numpy::ndarray;
+using dtype = boost::python::numpy::dtype;
#endif
#ifdef HAVE_PYBIND11
using pyArray = pybind11::array;
+using dtype = pybind11::dtype;
#endif
@@ -39,29 +42,38 @@ class EnginePy
public:
- std::shared_ptr<Engine> m_Engine;
-
- void WritePy( VariablePy<char>& variable, const pyArray& array );
- void WritePy( VariablePy<unsigned char>& variable, const pyArray& array );
- void WritePy( VariablePy<short>& variable, const pyArray& array );
- void WritePy( VariablePy<unsigned short>& variable, const pyArray& array );
- void WritePy( VariablePy<int>& variable, const pyArray& array );
- void WritePy( VariablePy<unsigned int>& variable, const pyArray& array );
- void WritePy( VariablePy<long int>& variable, const pyArray& array );
- void WritePy( VariablePy<unsigned long int>& variable, const pyArray& array );
- void WritePy( VariablePy<long long int>& variable, const pyArray& array );
- void WritePy( VariablePy<unsigned long long int>& variable, const pyArray& array );
- void WritePy( VariablePy<float>& variable, const pyArray& array );
- void WritePy( VariablePy<double>& variable, const pyArray& array );
- void WritePy( VariablePy<long double>& variable, const pyArray& array );
- void WritePy( VariablePy<std::complex<float>>& variable, const pyArray& array );
- void WritePy( VariablePy<std::complex<double>>& variable, const pyArray& array );
- void WritePy( VariablePy<std::complex<long double>>& variable, const pyArray& array );
+ EnginePy( ADIOSPy& adiosPy );
+
+ ~EnginePy( );
+
+ std::shared_ptr<Engine> m_Engine;
+
+ void WritePy( VariablePy& variable, const pyArray& array );
void Close( );
void GetType( ) const;
+private:
+
+ ADIOSPy& m_ADIOSPy;
+ bool m_IsVariableTypeDefined = false;
+
+ template< class T >
+ void DefineVariableInADIOS( VariablePy& variable )
+ {
+ auto& var = m_ADIOSPy.DefineVariable<T>( variable.m_Name, variable.m_LocalDimensions,
+ variable.m_GlobalDimensions, variable.m_GlobalOffsets );
+ variable.m_VariablePtr = &var;
+ variable.m_IsVariableDefined = true;
+ }
+
+ template< class T >
+ void WriteVariableInADIOS( VariablePy& variable, const pyArray& array )
+ {
+ m_Engine->Write( *reinterpret_cast<Variable<T>*>( variable.m_VariablePtr ), PyArrayToPointer<T>( array ) );
+ }
+
};
diff --git a/bindings/python/include/VariablePy.h b/bindings/python/include/VariablePy.h
index 2b96ba2cf..934b3b523 100644
--- a/bindings/python/include/VariablePy.h
+++ b/bindings/python/include/VariablePy.h
@@ -23,38 +23,29 @@ using pyList = pybind11::list;
#endif
-template< class T>
-class VariablePy : public Variable<T>
+class VariablePy
{
public:
- VariablePy<T>( const std::string name, const Dims dimensions, const Dims globalDimensions, const Dims globalOffsets,
- const bool debugMode ):
- Variable<T>( name, dimensions, globalDimensions, globalOffsets, debugMode )
- { }
+ VariablePy( const std::string name, const pyList localDimensionsPy, const pyList globalDimensionsPy, const pyList globalOffsetsPy );
- ~VariablePy( )
- { }
+ ~VariablePy( );
- void SetLocalDimensions( const pyList list )
- {
- this->m_Dimensions = ListToVector( list );
- }
+ void SetLocalDimensions( const pyList list );
- void SetGlobalDimensionsAndOffsets( const pyList globalDimensions, const pyList globalOffsets )
- {
- this->m_GlobalDimensions = ListToVector( globalDimensions );
- this->m_GlobalOffsets = ListToVector( globalOffsets );
- }
+ void SetGlobalDimensionsAndOffsets( const pyList globalDimensions, const pyList globalOffsets );
- Dims GetLocalDimensions( )
- {
- return this->m_Dimensions;
- }
+ Dims GetLocalDimensions( );
-};
+ void* m_VariablePtr = nullptr;
+ bool m_IsVariableDefined = false;
+ const std::string m_Name;
+ Dims m_LocalDimensions;
+ Dims m_GlobalDimensions;
+ Dims m_GlobalOffsets;
+};
diff --git a/bindings/python/include/adiosPyFunctions.h b/bindings/python/include/adiosPyFunctions.h
index 33a342cb1..a25834736 100644
--- a/bindings/python/include/adiosPyFunctions.h
+++ b/bindings/python/include/adiosPyFunctions.h
@@ -32,12 +32,14 @@ using Dims = std::vector<std::size_t>;
using pyList = boost::python::list;
using pyDict = boost::python::dict;
using pyArray = boost::python::numpy::ndarray;
+using dtype = boost::python::numpy::dtype;
#endif
#ifdef HAVE_PYBIND11
using pyList = pybind11::list;
using pyDict = pybind11::dict;
using pyArray = pybind11::array;
+using dtype = pybind11::dtype;
#endif
/**
@@ -62,6 +64,28 @@ const T* PyArrayToPointer( const pyArray& array )
}
+template< class T >
+dtype GetDType( )
+{
+ #ifdef HAVE_BOOSTPYTHON
+ return dtype::get_builtin<T>();
+ #endif
+
+ #ifdef HAVE_PYBIND11
+ return dtype::of<T>();
+ #endif
+}
+
+dtype DType( const pyArray& array );
+
+
+
+
+
+
+
+
+
} //end namespace
diff --git a/bindings/python/src/ADIOSPy.cpp b/bindings/python/src/ADIOSPy.cpp
index b11d19cb8..ab5ef1542 100644
--- a/bindings/python/src/ADIOSPy.cpp
+++ b/bindings/python/src/ADIOSPy.cpp
@@ -39,6 +39,26 @@ MethodPy& ADIOSPy::DeclareMethodPy( const std::string methodName, const std::str
}
+VariablePy ADIOSPy::DefineVariablePy( const std::string name, const pyList localDimensionsPy,
+ const pyList globalDimensionsPy, const pyList globalOffsetsPy )
+{
+ if( m_DebugMode == true )
+ {
+ if( m_VariablesPyNames.count( name ) == 1 )
+ throw std::invalid_argument( "ERROR: Variable " + name + " is already defined\n" );
+ }
+
+ m_VariablesPyNames.insert( name );
+ return VariablePy( name, localDimensionsPy, globalDimensionsPy, globalOffsetsPy );
+}
+
+void ADIOSPy::DefineVariableType( VariablePy& variablePy )
+{
+
+}
+
+
+
EnginePy ADIOSPy::OpenPy( const std::string name, const std::string accessMode,
const MethodPy& method, pyObject py_comm )
{
@@ -61,7 +81,4 @@ EnginePy ADIOSPy::OpenPy( const std::string name, const std::string accessMode,
-
} //end namespace
-
-
diff --git a/bindings/python/src/EnginePy.cpp b/bindings/python/src/EnginePy.cpp
index 9ca125411..c21454f76 100644
--- a/bindings/python/src/EnginePy.cpp
+++ b/bindings/python/src/EnginePy.cpp
@@ -14,53 +14,58 @@
namespace adios
{
-void EnginePy::WritePy( VariablePy<char>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<char>( array ) ); }
+EnginePy::EnginePy( ADIOSPy& adiosPy ):
+ m_ADIOSPy{ adiosPy }
+{ }
-void EnginePy::WritePy( VariablePy<unsigned char>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<unsigned char>( array ) ); }
-void EnginePy::WritePy( VariablePy<short>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<short>( array ) ); }
+EnginePy::~EnginePy( )
+{ }
-void EnginePy::WritePy( VariablePy<unsigned short>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<unsigned short>( array ) ); }
-void EnginePy::WritePy( VariablePy<int>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<int>( array ) ); }
-
-void EnginePy::WritePy( VariablePy<unsigned int>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<unsigned int>( array ) ); }
-
-void EnginePy::WritePy( VariablePy<long int>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<long int>( array ) ); }
-
-void EnginePy::WritePy( VariablePy<unsigned long int>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<unsigned long int>( array ) ); }
-
-void EnginePy::WritePy( VariablePy<long long int>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<long long int>( array ) ); }
-
-void EnginePy::WritePy( VariablePy<unsigned long long int>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<unsigned long long int>( array ) ); }
-
-void EnginePy::WritePy( VariablePy<float>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<float>( array ) ); }
-
-void EnginePy::WritePy( VariablePy<double>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<double>( array ) ); }
-
-void EnginePy::WritePy( VariablePy<long double>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<long double>( array ) ); }
-
-void EnginePy::WritePy( VariablePy<std::complex<float>>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<std::complex<float>>( array ) ); }
+void EnginePy::WritePy( VariablePy& variable, const pyArray& array )
+{
+ const dtype arrayDType = DType( array );
+
+ if( variable.m_IsVariableDefined == false ) //here define variable
+ {
+ if( arrayDType == GetDType<char>() ) DefineVariableInADIOS<char>( variable );
+ else if( arrayDType == GetDType<unsigned char>() ) DefineVariableInADIOS<unsigned char>( variable );
+ else if( arrayDType == GetDType<short>() ) DefineVariableInADIOS<short>( variable );
+ else if( arrayDType == GetDType<unsigned short>() ) DefineVariableInADIOS<unsigned short>( variable );
+ else if( arrayDType == GetDType<int>() ) DefineVariableInADIOS<int>( variable );
+ else if( arrayDType == GetDType<unsigned int>() ) DefineVariableInADIOS<unsigned int>( variable );
+ else if( arrayDType == GetDType<long int>() ) DefineVariableInADIOS<long int>( variable );
+ else if( arrayDType == GetDType<unsigned long int>() ) DefineVariableInADIOS<unsigned long int>( variable );
+ else if( arrayDType == GetDType<long long int>() ) DefineVariableInADIOS<long long int>( variable );
+ else if( arrayDType == GetDType<unsigned long long int>() ) DefineVariableInADIOS<unsigned long long int>( variable );
+ else if( arrayDType == GetDType<float>() ) DefineVariableInADIOS<float>( variable );
+ else if( arrayDType == GetDType<double>() ) DefineVariableInADIOS<double>( variable );
+ else if( arrayDType == GetDType<long double>() ) DefineVariableInADIOS<long double>( variable );
+ else if( arrayDType == GetDType<std::complex<float>>() ) DefineVariableInADIOS<std::complex<float>>( variable );
+ else if( arrayDType == GetDType<std::complex<double>>() ) DefineVariableInADIOS<std::complex<double>>( variable );
+ else if( arrayDType == GetDType<std::complex<long double>>() ) DefineVariableInADIOS<std::complex<long double>>( variable );
+ }
+
+ if( arrayDType == GetDType<char>() ) WriteVariableInADIOS<char>( variable, array );
+ else if( arrayDType == GetDType<unsigned char>() ) WriteVariableInADIOS<unsigned char>( variable, array );
+ else if( arrayDType == GetDType<short>() ) WriteVariableInADIOS<short>( variable, array );
+ else if( arrayDType == GetDType<unsigned short>() ) WriteVariableInADIOS<unsigned short>( variable, array );
+ else if( arrayDType == GetDType<int>() ) WriteVariableInADIOS<int>( variable, array );
+ else if( arrayDType == GetDType<unsigned int>() ) WriteVariableInADIOS<unsigned int>( variable, array );
+ else if( arrayDType == GetDType<long int>() ) WriteVariableInADIOS<long int>( variable, array );
+ else if( arrayDType == GetDType<unsigned long int>() ) WriteVariableInADIOS<unsigned long int>( variable, array );
+ else if( arrayDType == GetDType<long long int>() ) WriteVariableInADIOS<long long int>( variable, array );
+ else if( arrayDType == GetDType<unsigned long long int>() ) WriteVariableInADIOS<unsigned long long int>( variable, array );
+ else if( arrayDType == GetDType<float>() ) WriteVariableInADIOS<float>( variable, array );
+ else if( arrayDType == GetDType<double>() ) WriteVariableInADIOS<double>( variable, array );
+ else if( arrayDType == GetDType<long double>() ) WriteVariableInADIOS<long double>( variable, array );
+ else if( arrayDType == GetDType<std::complex<float>>() ) WriteVariableInADIOS<std::complex<float>>( variable, array );
+ else if( arrayDType == GetDType<std::complex<double>>() ) WriteVariableInADIOS<std::complex<double>>( variable, array );
+ else if( arrayDType == GetDType<std::complex<long double>>() ) WriteVariableInADIOS<std::complex<long double>>( variable, array );
+}
-void EnginePy::WritePy( VariablePy<std::complex<double>>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<std::complex<double>>( array ) ); }
-void EnginePy::WritePy( VariablePy<std::complex<long double>>& variable, const pyArray& array )
-{ m_Engine->Write( variable, PyArrayToPointer<std::complex<long double>>( array ) ); }
void EnginePy::GetType( ) const
{
diff --git a/bindings/python/src/VariablePy.cpp b/bindings/python/src/VariablePy.cpp
new file mode 100644
index 000000000..562bbb894
--- /dev/null
+++ b/bindings/python/src/VariablePy.cpp
@@ -0,0 +1,49 @@
+/*
+ * VariablePy.cpp
+ *
+ * Created on: Mar 17, 2017
+ * Author: wfg
+ */
+
+
+#include "VariablePy.h"
+
+
+namespace adios
+{
+
+VariablePy::VariablePy( const std::string name, const pyList localDimensionsPy,
+ const pyList globalDimensionsPy, const pyList globalOffsetsPy ):
+ m_Name{ name },
+ m_LocalDimensions{ ListToVector( localDimensionsPy ) },
+ m_GlobalDimensions{ ListToVector( globalDimensionsPy ) },
+ m_GlobalOffsets{ ListToVector( globalOffsetsPy ) }
+{ }
+
+
+VariablePy::~VariablePy( )
+{ }
+
+
+void VariablePy::SetLocalDimensions( const pyList list )
+{
+// this->m_Dimensions = ListToVector( list );
+
+}
+
+
+void VariablePy::SetGlobalDimensionsAndOffsets( const pyList globalDimensions, const pyList globalOffsets )
+{
+// this->m_GlobalDimensions = ListToVector( globalDimensions );
+// this->m_GlobalOffsets = ListToVector( globalOffsets );
+}
+
+
+Dims VariablePy::GetLocalDimensions( )
+{
+ return this->m_LocalDimensions;
+}
+
+
+
+} //end namespace
diff --git a/bindings/python/src/adiosPyFunctions.cpp b/bindings/python/src/adiosPyFunctions.cpp
index 27601c5ff..e3155c037 100644
--- a/bindings/python/src/adiosPyFunctions.cpp
+++ b/bindings/python/src/adiosPyFunctions.cpp
@@ -58,5 +58,19 @@ std::map<std::string, std::string> DictToMap( const pyDict& dictionary )
}
+dtype DType( const pyArray& array )
+{
+
+#ifdef HAVE_BOOSTPYTHON
+ return array.get_dtype();
+#endif
+
+#ifdef HAVE_PYBIND11
+ return array.dtype();
+#endif
+
+}
+
+
} //end namespace
diff --git a/bindings/python/test_hello.py b/bindings/python/test_hello.py
index d8cc94dcd..f167ab0db 100644
--- a/bindings/python/test_hello.py
+++ b/bindings/python/test_hello.py
@@ -12,7 +12,7 @@ rank = MPI.COMM_WORLD.Get_rank()
size = MPI.COMM_WORLD.Get_size()
# User data
-myArray = np.array( [1,2,3,4] )
+myArray = np.array( [1,2,3,4])
# if( rank % 2 == 1 ): # odd ranks only
oddRankArray = np.array( [11.,12.,13.,14.])
diff --git a/include/core/Engine.h b/include/core/Engine.h
index 3ea11ae7e..05839734c 100644
--- a/include/core/Engine.h
+++ b/include/core/Engine.h
@@ -172,7 +172,6 @@ public:
virtual void Write( const std::string variableName, const std::complex<float>* values );
virtual void Write( const std::string variableName, const std::complex<double>* values );
virtual void Write( const std::string variableName, const std::complex<long double>* values );
- virtual void Write( const std::string variableName, const void* values );
/**
@@ -209,7 +208,7 @@ public:
virtual Variable<std::complex<long double>>* InquireVariableCLDouble( const std::string name, const bool readIn = true );
virtual VariableCompound* InquireVariableCompound( const std::string name, const bool readIn = true );
- virtual void Close( const int transportIndex = -1 ); ///< Closes a particular transport, or all if -1
+ virtual void Close( const int transportIndex = -1 ) = 0; ///< Closes a particular transport, or all if -1
protected:
diff --git a/src/core/Engine.cpp b/src/core/Engine.cpp
index 13bb8cd42..76445c1d5 100644
--- a/src/core/Engine.cpp
+++ b/src/core/Engine.cpp
@@ -26,7 +26,7 @@ Engine::Engine( ADIOS& adios, const std::string engineType, const std::string na
m_ADIOS{ adios },
m_DebugMode{ debugMode },
m_Cores{ cores },
- m_EndMessage{ endMessage }
+ m_EndMessage( endMessage )
{
if( m_DebugMode == true )
{
@@ -80,7 +80,6 @@ void Engine::Write( const std::string variableName, const long double* values ){
void Engine::Write( const std::string variableName, const std::complex<float>* values ){ }
void Engine::Write( const std::string variableName, const std::complex<double>* values ){ }
void Engine::Write( const std::string variableName, const std::complex<long double>* values ){ }
-void Engine::Write( const std::string variableName, const void* values ){ }
void Engine::Advance(){ }
--
GitLab