From c715a4b9571988f73c95ea7b1c3db36b5f63af77 Mon Sep 17 00:00:00 2001
From: William F Godoy <williamfgc@yahoo.com>
Date: Thu, 8 Jun 2017 17:56:05 -0400
Subject: [PATCH] Bug in Python bindings, changed IO SetParameters and
AddTransport
Python bindings running but not printing values with bpls. Min and Max
captured correctly.
IO SetParameters and AddTransport will accept vector and map
Variable.cpp and Variable.tcc for complete separation of core components
in the public include
---
bindings/python/Makefile | 43 ++++++
bindings/python/helloBPWriter.py | 26 ++++
bindings/python/source/ADIOSPy.cpp | 61 ++-------
bindings/python/source/ADIOSPy.h | 11 +-
bindings/python/source/EnginePy.cpp | 122 ++++++------------
bindings/python/source/EnginePy.h | 59 ++++-----
bindings/python/source/EnginePy.inl | 43 ++++++
bindings/python/source/IOPy.cpp | 48 ++++++-
bindings/python/source/IOPy.h | 26 ++--
bindings/python/source/VariablePy.cpp | 40 +++---
bindings/python/source/VariablePy.h | 34 +++--
bindings/python/source/adiosPyFunctions.cpp | 70 ++--------
bindings/python/source/adiosPyFunctions.h | 50 ++-----
bindings/python/source/adiosPyTypes.h | 7 +-
bindings/python/source/gluePyBind11.cpp | 79 +++++++-----
bindings/python/test_hello.py | 52 --------
.../multistep/reader_allsteps.cpp | 2 +-
.../multistep/writer_multistep.cpp | 4 +-
.../heatTransfer/read/heatRead_adios2.cpp | 4 +-
examples/heatTransfer/write/IO_ph5_adios2.cpp | 2 +-
.../hello/adios1Writer/helloADIOS1Writer.cpp | 2 +-
.../datamanReader/helloDataManReader.cpp | 7 +-
.../helloDataManReader_nompi.cpp | 6 +-
.../datamanWriter/helloDataManWriter.cpp | 4 +-
.../helloDataManWriter_nompi.cpp | 4 +-
source/adios2/CMakeLists.txt | 1 +
source/adios2/core/Engine.cpp | 2 +
source/adios2/core/Engine.h | 1 +
source/adios2/core/IO.cpp | 62 +++++----
source/adios2/core/IO.h | 35 +++--
source/adios2/core/IO.inl | 25 +---
source/adios2/core/Variable.cpp | 17 +++
source/adios2/core/Variable.h | 5 +-
source/adios2/core/Variable.inl | 40 ------
source/adios2/core/Variable.tcc | 43 ++++++
source/adios2/core/VariableBase.cpp | 20 ++-
source/adios2/core/VariableBase.h | 26 ++--
source/adios2/core/VariableCompound.tcc | 2 +-
.../adios2/engine/dataman/DataManWriter.tcc | 1 +
source/adios2/helper/adiosString.cpp | 10 +-
40 files changed, 552 insertions(+), 544 deletions(-)
create mode 100644 bindings/python/Makefile
create mode 100644 bindings/python/helloBPWriter.py
create mode 100644 bindings/python/source/EnginePy.inl
delete mode 100644 bindings/python/test_hello.py
create mode 100644 source/adios2/core/Variable.cpp
delete mode 100644 source/adios2/core/Variable.inl
create mode 100644 source/adios2/core/Variable.tcc
diff --git a/bindings/python/Makefile b/bindings/python/Makefile
new file mode 100644
index 000000000..d7f98bb07
--- /dev/null
+++ b/bindings/python/Makefile
@@ -0,0 +1,43 @@
+#Makefile
+# Created on: Jun 8, 2017
+# Author: William F Godoy
+
+
+MPICC:=mpic++
+
+CPPFiles:=$(shell find ./source -type f -name "*.cpp")
+OBJS:=$(patsubst %.cpp, ./bin/%.o, $(notdir $(CPPFiles)) )
+
+# Python
+PYTHON_VERSION:= 2.7
+PYTHON_INC:= /usr/include/python$(PYTHON_VERSION)
+PYTHON_LIBLOC:= /usr/lib/python$(PYTHON_VERSION)/config-x86_64-linux-gnu/
+
+
+# ADIOS2
+ADIOS2_INC:= /usr/local/include
+ADIOS2_LIB:= /usr/local/lib
+
+# PyBind11
+PyBind11_INC:= /home/wfg/workspace/ADIOS2/thirdparty/pybind11/include
+
+
+CFLAGS:=-c -Wall -fPIC -O0 -g -std=c++11 -Wno-deprecated-declarations
+INC:= -I$(PYTHON_INC) -I$(ADIOS2_INC) -I$(PyBind11_INC)
+LIBS:= -L$(ADIOS2_LIB) -ladios2 -L$(PYTHON_LIBLOC) -lpython$(PYTHON_VERSION)
+
+
+# compile mesh classes
+TARGET = adios2py
+
+all: $(TARGET).so
+
+$(TARGET).so: $(OBJS)
+ $(MPICC) -shared -o $(TARGET).so -Wl,--export-dynamic $(OBJS) $(LIBS)
+
+./bin/%.o: ./source/%.cpp
+ @( mkdir -p ./bin );
+ $(MPICC) $(INC) $(CFLAGS) -o $@ $<
+
+clean:
+ rm ./bin/*.o *.so
\ No newline at end of file
diff --git a/bindings/python/helloBPWriter.py b/bindings/python/helloBPWriter.py
new file mode 100644
index 000000000..f5f90c5ca
--- /dev/null
+++ b/bindings/python/helloBPWriter.py
@@ -0,0 +1,26 @@
+#
+# Distributed under the OSI-approved Apache License, Version 2.0. See
+# accompanying file Copyright.txt for details.
+#
+# test_hello.py
+# Created on: Feb 2, 2017
+# Author: wfg
+
+from mpi4py import MPI
+from adios2py import *
+import numpy as np
+
+
+# User data
+myArray = np.array([1, 2, 3, 4])
+
+print "Read " + str(Read)
+
+adios = ADIOS(MPI.COMM_WORLD, True)
+
+bpIO = adios.DeclareIO("BPN2N")
+ioArray = bpIO.DefineVariable("bpArray", [myArray.size], [0], [myArray.size], True)
+
+bpFileWriter = bpIO.Open("myArray.bp", Write)
+bpFileWriter.Write(ioArray, myArray)
+bpFileWriter.Close()
diff --git a/bindings/python/source/ADIOSPy.cpp b/bindings/python/source/ADIOSPy.cpp
index f820b568d..c613882bb 100644
--- a/bindings/python/source/ADIOSPy.cpp
+++ b/bindings/python/source/ADIOSPy.cpp
@@ -10,68 +10,23 @@
#include "ADIOSPy.h"
-#include "bindings/python/source/adiosPyTypes.h"
+#include "adiosPyTypes.h"
+
+#include <iostream>
namespace adios
{
ADIOSPy::ADIOSPy(MPI_Comm mpiComm, const bool debug)
-: m_DebugMode(debug), ADIOS(mpiComm, debug)
+: m_DebugMode(debug), m_ADIOS(mpiComm, debug)
{
}
-IOPy &ADIOSPy::DeclareIO(const std::string name)
+IOPy ADIOSPy::DeclareIO(const std::string name)
{
+ std::cout << "Declaring IO " << name << "\n";
+
return IOPy(m_ADIOS.DeclareIO(name), m_DebugMode);
}
-// VariablePy ADIOSPy::DefineVariablePy(const std::string name,
-// const pyList localDimensionsPy,
-// const pyList globalDimensionsPy,
-// const pyList globalOffsetsPy)
-//{
-// if (m_DebugMode == true)
-// {
-// if (m_VariablesPyNames.count(name) == 1)
-// throw std::invalid_argument("ERROR: Variable " + name +
-// " is already defined\n");
-// }
-//
-// m_VariablesPyNames.insert(name);
-// return VariablePy(name, localDimensionsPy, globalDimensionsPy,
-// globalOffsetsPy);
-//}
-//
-// void ADIOSPy::DefineVariableType(VariablePy &variablePy) {}
-//
-// EnginePy ADIOSPy::OpenPy(const std::string name, const std::string
-// accessMode,
-// const MethodPy &method, pyObject py_comm)
-//{
-// EnginePy enginePy(*this);
-//
-// bool isEmpty = IsEmpty(py_comm);
-//
-// if (isEmpty == true) // None
-// {
-// enginePy.m_Engine = Open(name, accessMode, method);
-// }
-// else
-// {
-// if (import_mpi4py() < 0)
-// throw std::logic_error(
-// "ERROR: could not import mpi4py communicator in Open " + name
-// +
-// "\n");
-// MPI_Comm *comm_p = PyMPIComm_Get(py_comm.ptr());
-// if (comm_p == nullptr)
-// throw std::invalid_argument(
-// "ERROR: MPI communicator is nullptr in Open " + name + "\n");
-//
-// enginePy.m_Engine = Open(name, accessMode, *comm_p, method);
-// }
-//
-// return enginePy;
-//}
-
-} // end namespace
+} // end namespace adios
diff --git a/bindings/python/source/ADIOSPy.h b/bindings/python/source/ADIOSPy.h
index 2de1bfdac..f390697d5 100644
--- a/bindings/python/source/ADIOSPy.h
+++ b/bindings/python/source/ADIOSPy.h
@@ -8,16 +8,14 @@
* Author: William F Godoy godoywf@ornl.gov
*/
-#ifndef BINDINGS_PYTHON_SOURCE_ADIOSPY_H_
-#define BINDINGS_PYTHON_SOURCE_ADIOSPY_H_
+#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPY_H_
+#define ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPY_H_
/// \cond EXCLUDE_FROM_DOXYGEN
#include <string>
/// \endcond
#include "IOPy.h"
-#include "adios2/ADIOSMPICommOnly.h"
-#include "adios2/core/ADIOS.h"
namespace adios
{
@@ -29,10 +27,7 @@ public:
ADIOSPy(MPI_Comm mpiComm, const bool debug = false);
~ADIOSPy() = default;
- /** testing mpi4py should go away*/
- void HelloMPI();
-
- IOPy &DeclareIO(const std::string methodName);
+ IOPy DeclareIO(const std::string name);
private:
const bool m_DebugMode;
diff --git a/bindings/python/source/EnginePy.cpp b/bindings/python/source/EnginePy.cpp
index 91310f2a7..b60efe971 100644
--- a/bindings/python/source/EnginePy.cpp
+++ b/bindings/python/source/EnginePy.cpp
@@ -8,99 +8,61 @@
* Author: wgodoy
*/
-#include <string>
+#include "EnginePy.h"
-#include "adios2/EnginePy.h"
+#include "adios2/ADIOSMacros.h"
-#include "adios2/adiosPyFunctions.h"
+#include "adiosPyFunctions.h"
+
+#include <iostream>
namespace adios
{
-EnginePy::EnginePy(ADIOSPy &adiosPy) : m_ADIOSPy{adiosPy} {}
-
-EnginePy::~EnginePy() {}
-
-void EnginePy::WritePy(VariablePy &variable, const pyArray &array)
+EnginePy::EnginePy(IO &io, const std::string &name, const OpenMode openMode,
+ MPI_Comm mpiComm)
+: m_IO(io), m_Engine(m_IO.Open(name, openMode, mpiComm)),
+ m_DebugMode(m_IO.m_DebugMode)
{
+}
- if (variable.m_IsVariableDefined == false) // here define variable
+void EnginePy::Write(VariablePy &variable, const pyArray &array)
+{
+ if (variable.m_IsDefined)
{
- if (IsType<char>(array))
- DefineVariableInADIOS<char>(variable);
- else if (IsType<unsigned char>(array))
- DefineVariableInADIOS<unsigned char>(variable);
- else if (IsType<short>(array))
- DefineVariableInADIOS<short>(variable);
- else if (IsType<unsigned short>(array))
- DefineVariableInADIOS<unsigned short>(variable);
- else if (IsType<int>(array))
- DefineVariableInADIOS<int>(variable);
- else if (IsType<unsigned int>(array))
- DefineVariableInADIOS<unsigned int>(variable);
- else if (IsType<long int>(array))
- DefineVariableInADIOS<long int>(variable);
- else if (IsType<unsigned long int>(array))
- DefineVariableInADIOS<unsigned long int>(variable);
- else if (IsType<long long int>(array))
- DefineVariableInADIOS<long long int>(variable);
- else if (IsType<unsigned long long int>(array))
- DefineVariableInADIOS<unsigned long long int>(variable);
- else if (IsType<float>(array))
- DefineVariableInADIOS<float>(variable);
- else if (IsType<double>(array))
- DefineVariableInADIOS<double>(variable);
- else if (IsType<long double>(array))
- DefineVariableInADIOS<long double>(variable);
- else if (IsType<std::complex<float>>(array))
- DefineVariableInADIOS<std::complex<float>>(variable);
- else if (IsType<std::complex<double>>(array))
- DefineVariableInADIOS<std::complex<double>>(variable);
- else if (IsType<std::complex<long double>>(array))
- DefineVariableInADIOS<std::complex<long double>>(variable);
+ // do nothing, not supporting compound types in Python, yet
+ }
+#define declare_type(T) \
+ else if (pybind11::isinstance<pybind11::array_t<T>>(array)) \
+ { \
+ DefineVariableInIO<T>(variable); \
}
+ ADIOS2_FOREACH_TYPE_1ARG(declare_type)
+#undef declare_type
- if (IsType<char>(array))
- WriteVariableInADIOS<char>(variable, array);
- else if (IsType<unsigned char>(array))
- WriteVariableInADIOS<unsigned char>(variable, array);
- else if (IsType<short>(array))
- WriteVariableInADIOS<short>(variable, array);
- else if (IsType<unsigned short>(array))
- WriteVariableInADIOS<unsigned short>(variable, array);
- else if (IsType<int>(array))
- WriteVariableInADIOS<int>(variable, array);
- else if (IsType<unsigned int>(array))
- WriteVariableInADIOS<unsigned int>(variable, array);
- else if (IsType<long int>(array))
- WriteVariableInADIOS<long int>(variable, array);
- else if (IsType<unsigned long int>(array))
- WriteVariableInADIOS<unsigned long int>(variable, array);
- else if (IsType<long long int>(array))
- WriteVariableInADIOS<long long int>(variable, array);
- else if (IsType<unsigned long long int>(array))
- WriteVariableInADIOS<unsigned long long int>(variable, array);
- else if (IsType<float>(array))
- WriteVariableInADIOS<float>(variable, array);
- else if (IsType<double>(array))
- WriteVariableInADIOS<double>(variable, array);
- else if (IsType<long double>(array))
- WriteVariableInADIOS<long double>(variable, array);
- else if (IsType<std::complex<float>>(array))
- WriteVariableInADIOS<std::complex<float>>(variable, array);
- else if (IsType<std::complex<double>>(array))
- WriteVariableInADIOS<std::complex<double>>(variable, array);
- else if (IsType<std::complex<long double>>(array))
- WriteVariableInADIOS<std::complex<long double>>(variable, array);
+ if (!variable.m_IsDefined)
+ {
+ if (m_DebugMode)
+ {
+ throw std::runtime_error("ERROR: variable " + variable.m_Name +
+ " couldn't not be created in IO " +
+ m_IO.m_Name + " , in call to Write\n");
+ }
+ }
+#define declare_type(T) \
+ else if (pybind11::isinstance<pybind11::array_t<T>>(array)) \
+ { \
+ WriteInIO<T>(variable, array); \
+ }
+ ADIOS2_FOREACH_TYPE_1ARG(declare_type)
+#undef declare_type
}
-void EnginePy::Advance() { m_Engine->Advance(); }
-
-void EnginePy::Close() { m_Engine->Close(-1); }
-
-void EnginePy::GetEngineType() const
+void EnginePy::Advance(const float timeoutSeconds)
{
- std::cout << "Engine type " << m_Engine->m_EngineType << "\n";
+ m_Engine->Advance(timeoutSeconds);
}
-} // end namespace
+void EnginePy::Close() { m_Engine->Close(); }
+
+} // end namespace adios
diff --git a/bindings/python/source/EnginePy.h b/bindings/python/source/EnginePy.h
index 70ad66d44..19f6e1dea 100644
--- a/bindings/python/source/EnginePy.h
+++ b/bindings/python/source/EnginePy.h
@@ -5,61 +5,54 @@
* EnginePy.h
*
* Created on: Mar 15, 2017
- * Author: wgodoy
+ * Author: William F Godoy godoywf@ornl.gov
*/
-#ifndef BINDINGS_PYTHON_SOURCE_ENGINEPY_H_
-#define BINDINGS_PYTHON_SOURCE_ENGINEPY_H_
+#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_H_
+#define ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_H_
+
+/// \cond EXCLUDE_FROM_DOXYGEN
+#include <memory> //std::shared_ptr
+#include <string>
+/// \endcond
+
+#include <adios2.h>
-#include "ADIOSPy.h"
#include "VariablePy.h"
-#include "adios2/core/Engine.h"
-#include "adiosPyFunctions.h"
+#include "adiosPyTypes.h" //pyArray
namespace adios
{
+
class EnginePy
{
public:
- EnginePy();
+ EnginePy(IO &io, const std::string &name, const OpenMode openMode,
+ MPI_Comm mpiComm);
- ~EnginePy();
+ ~EnginePy() = default;
- Engine *m_Engine;
+ void Write(VariablePy &variable, const pyArray &array);
- void WritePy(VariablePy &variable, const pyArray &array);
-
- void Advance();
+ void Advance(const float timeoutSeconds = 0.);
void Close();
private:
+ IO &m_IO;
+ std::shared_ptr<Engine> m_Engine;
+ const bool m_DebugMode;
+
template <class T>
- void DefineVariableInADIOS(VariablePy &variable);
+ void DefineVariableInIO(VariablePy &variable);
template <class T>
- void WriteVariableInADIOS(VariablePy &variable, const pyArray &array);
-
- // template <class T>
- // void DefineVariableInADIOS(VariablePy &variable)
- // {
- // auto &var = m_ADIOSPy.DefineVariable<T>(
- // variable.m_Name, variable.m_LocalDimensions,
- // variable.m_GlobalDimensions, variable.m_GlobalOffsets);
- // variable.m_VariablePtr = &var;
- // variable.m_IsVariableDefined = true;
- // }
- //
- // template <class T>
- // void WriteVariableInADIOS(VariablePy &variable, const pyArray &array)
- // {
- // m_Engine->Write(
- // *reinterpret_cast<Variable<T> *>(variable.m_VariablePtr),
- // PyArrayToPointer<T>(array));
- // }
+ void WriteInIO(VariablePy &variable, const pyArray &array);
};
-} // end namespace
+} // end namespace adios
+
+#include "EnginePy.inl"
#endif /* BINDINGS_PYTHON_SOURCE_ENGINEPY_H_ */
diff --git a/bindings/python/source/EnginePy.inl b/bindings/python/source/EnginePy.inl
new file mode 100644
index 000000000..33b7cae0c
--- /dev/null
+++ b/bindings/python/source/EnginePy.inl
@@ -0,0 +1,43 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * EnginePy.tcc
+ *
+ * Created on: Jun 8, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_INL_
+#define ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_INL_
+#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ENGINEPY_H_
+#error "Inline file should only be included from it's header, never on it's own"
+#endif
+
+#include "adiosPyFunctions.h"
+
+namespace adios
+{
+
+template <class T>
+void EnginePy::DefineVariableInIO(VariablePy &variable)
+{
+ auto &var = m_IO.DefineVariable<T>(
+ variable.m_Name, PyListToDims(variable.m_Shape),
+ PyListToDims(variable.m_Start), PyListToDims(variable.m_Count),
+ variable.m_IsConstantDims);
+
+ variable.m_VariableBase = &var;
+ variable.m_IsDefined = true;
+}
+
+template <class T>
+void EnginePy::WriteInIO(VariablePy &variable, const pyArray &array)
+{
+ m_Engine->Write(*dynamic_cast<Variable<T> *>(variable.m_VariableBase),
+ reinterpret_cast<const T *>(array.data()));
+}
+
+} // end namespace adios
+
+#endif /* BINDINGS_PYTHON_SOURCE_ENGINEPY_TCC_ */
diff --git a/bindings/python/source/IOPy.cpp b/bindings/python/source/IOPy.cpp
index 646c52f57..692b2b018 100644
--- a/bindings/python/source/IOPy.cpp
+++ b/bindings/python/source/IOPy.cpp
@@ -10,20 +10,54 @@
#include "IOPy.h"
+#include <mpi4py/mpi4py.h>
+
namespace adios
{
-IOPy::IOPy(IO &io, const bool debugMode) : m_IO(io), m_DebugMode(debugMode) {}
+IOPy::IOPy(IO &io, const bool debugMode) : m_IO(io), m_DebugMode(debugMode)
+{
+ m_IO.m_HostLanguage = "Python";
+}
-void IOPy::SetParameters(const pybind11::kwargs kwargs)
-{ // transform to vector and call m_IO SetParameters
+void IOPy::SetParameters(const pyKwargs &kwargs)
+{
+ m_IO.SetParameters(KwargsToParams(kwargs));
}
-unsigned int IOPy::AddTransport(const std::string type,
- const pybind11::kwargs kwargs)
+unsigned int IOPy::AddTransport(const std::string type, const pyKwargs &kwargs)
{
- // transform to vector and call AddTransport
- return -1;
+ return m_IO.AddTransport(type, KwargsToParams(kwargs));
}
+VariablePy IOPy::DefineVariable(const std::string &name, const pyList shape,
+ const pyList start, const pyList count,
+ const bool isConstantDims)
+{
+ return VariablePy(name, shape, start, count, isConstantDims, m_DebugMode);
+}
+
+EnginePy IOPy::Open(const std::string &name, const int openMode)
+{
+ // MPI_Comm *mpiCommPtr = PyMPIComm_Get(pyMPIComm.ptr());
+ //
+ // if (m_DebugMode)
+ // {
+ // if (mpiCommPtr == NULL)
+ // {
+ // throw std::runtime_error("ERROR: mpi4py communicator for
+ // engine " +
+ // name + " is null, in call to IO
+ // Open\n");
+ // }
+ // }
+ return EnginePy(m_IO, name, static_cast<adios::OpenMode>(openMode),
+ m_IO.m_MPIComm);
+}
+
+// EnginePy IOPy::Open(const std::string &name, const OpenMode openMode)
+//{
+// return EnginePy(m_IO, name, openMode, m_IO.m_MPIComm);
+//}
+
} // end namespace
diff --git a/bindings/python/source/IOPy.h b/bindings/python/source/IOPy.h
index 5bc5bf10e..6bacc3a41 100644
--- a/bindings/python/source/IOPy.h
+++ b/bindings/python/source/IOPy.h
@@ -8,10 +8,14 @@
* Author: William F Godoy godoywf@ornl.gov
*/
-#ifndef BINDINGS_PYTHON_SOURCE_IOPY_H_
-#define BINDINGS_PYTHON_SOURCE_IOPY_H_
+#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_IOPY_H_
+#define ADIOS2_BINDINGS_PYTHON_SOURCE_IOPY_H_
-#include "adios2/core/IO.h"
+/// \cond EXCLUDE_FROM_DOXYGEN
+#include <string>
+/// \endcond
+
+#include "EnginePy.h"
#include "adiosPyTypes.h"
namespace adios
@@ -21,23 +25,23 @@ class IOPy
{
public:
+ IO &m_IO;
+ const bool m_DebugMode;
+
IOPy(IO &io, const bool debugMode);
~IOPy() = default;
- void SetParameters(const pyKwargs kwargs);
- unsigned int AddTransport(const std::string type, const pyKwargs kwargs);
+ void SetParameters(const pyKwargs &kwargs);
+ unsigned int AddTransport(const std::string type, const pyKwargs &kwargs);
VariablePy DefineVariable(const std::string &name, const pyList shape,
const pyList start, const pyList count,
- const bool isConstantShape = false);
+ const bool isConstantDims);
- EnginePy Open(const std::string name, const std::string openMode,
- pyObject py_comm = pyObject());
+ EnginePy Open(const std::string &name, const int openMode);
-private:
- IO &m_IO;
- const bool m_DebugMode;
+ // EnginePy Open(const std::string &name, const OpenMode openMode);
};
} // end namespace adios
diff --git a/bindings/python/source/VariablePy.cpp b/bindings/python/source/VariablePy.cpp
index 8480f70f6..76813c6cd 100644
--- a/bindings/python/source/VariablePy.cpp
+++ b/bindings/python/source/VariablePy.cpp
@@ -13,29 +13,35 @@
namespace adios
{
-VariablePy::VariablePy(const std::string name, const pyList localDimensionsPy,
- const pyList globalDimensionsPy,
- const pyList globalOffsetsPy)
-: m_Name{name}, m_LocalDimensions{ListToVector(localDimensionsPy)},
- m_GlobalDimensions{ListToVector(globalDimensionsPy)},
- m_GlobalOffsets{ListToVector(globalOffsetsPy)}
+VariablePy::VariablePy(const std::string &name, const pyList shape,
+ const pyList start, const pyList count,
+ const bool isConstantDims, const bool debugMode)
+: m_Name(name), m_Shape(shape), m_Start(count), m_Count(count),
+ m_IsConstantDims(isConstantDims), m_DebugMode(debugMode)
{
}
-VariablePy::~VariablePy() {}
-
-void VariablePy::SetLocalDimensions(const pyList list)
+void VariablePy::SetDimensions(const pyList shape, const pyList start,
+ const pyList count)
{
- // this->m_Dimensions = ListToVector( list );
+ if (m_DebugMode)
+ {
+ if (m_IsConstantDims)
+ {
+ throw std::invalid_argument(
+ "ERROR: variable " + m_Name +
+ " dimensions are constant, in call from SetDimensions\n");
+ }
+ }
+
+ m_Shape = shape;
+ m_Start = start;
+ m_Count = count;
}
-void VariablePy::SetGlobalDimensionsAndOffsets(const pyList globalDimensions,
- const pyList globalOffsets)
+std::string VariablePy::GetType() const noexcept
{
- // this->m_GlobalDimensions = ListToVector( globalDimensions );
- // this->m_GlobalOffsets = ListToVector( globalOffsets );
+ return m_VariableBase->m_Type;
}
-Dims VariablePy::GetLocalDimensions() { return this->m_LocalDimensions; }
-
-} // end namespace
+} // end namespace adios
diff --git a/bindings/python/source/VariablePy.h b/bindings/python/source/VariablePy.h
index 401ac13d4..980fcc46f 100644
--- a/bindings/python/source/VariablePy.h
+++ b/bindings/python/source/VariablePy.h
@@ -8,10 +8,11 @@
* Author: William F Godoy godoywf@ornl.gov
*/
-#ifndef BINDINGS_PYTHON_SOURCE_VARIABLEPY_H_
-#define BINDINGS_PYTHON_SOURCE_VARIABLEPY_H_
+#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_VARIABLEPY_H_
+#define ADIOS2_BINDINGS_PYTHON_SOURCE_VARIABLEPY_H_
+
+#include <adios2.h>
-#include "adios2/core/VariableBase.h"
#include "adiosPyFunctions.h"
namespace adios
@@ -21,23 +22,28 @@ class VariablePy
{
public:
- VariablePy(const std::string name, const pyList shape, const pyList start,
- const pyList count, const bool isConstateShape);
+ const std::string m_Name;
+ pyList m_Shape;
+ pyList m_Start;
+ pyList m_Count;
+ const bool m_IsConstantDims;
+
+ VariableBase *m_VariableBase = nullptr;
+ bool m_IsDefined = false;
- ~VariablePy();
+ VariablePy(const std::string &name, const pyList shape, const pyList start,
+ const pyList count, const bool isConstantDims,
+ const bool debugMode);
+
+ ~VariablePy() = default;
void SetDimensions(const pyList shape, const pyList start,
const pyList count);
- Dims GetLocalDimensions();
-
- VariableBase *m_VariableBase = nullptr;
- bool m_IsVariableDefined = false;
+ std::string GetType() const noexcept;
- const std::string m_Name;
- Dims m_LocalDimensions;
- Dims m_GlobalDimensions;
- Dims m_GlobalOffsets;
+private:
+ const bool m_DebugMode;
};
} // end namespace adios
diff --git a/bindings/python/source/adiosPyFunctions.cpp b/bindings/python/source/adiosPyFunctions.cpp
index f9396e05b..806fd3181 100644
--- a/bindings/python/source/adiosPyFunctions.cpp
+++ b/bindings/python/source/adiosPyFunctions.cpp
@@ -9,79 +9,35 @@
*/
#include <iostream>
-#include "adios2/adiosPyFunctions.h"
+#include "adiosPyFunctions.h"
namespace adios
{
-#ifdef HAVE_BOOSTPYTHON
-namespace py = boost::python;
-#endif
-
-#ifdef HAVE_PYBIND11
-namespace py = pybind11;
-#endif
-
-Dims ListToVector(const pyList &list)
-{
- const unsigned int length = py::len(list);
- Dims vec;
- vec.reserve(length);
-
- for (unsigned int i = 0; i < length; i++)
- vec.push_back(PyCast<std::size_t>(list[i]));
-
- return vec;
-}
-
-#ifdef HAVE_BOOSTPYTHON
-std::map<std::string, std::string> DictToMap(const pyDict &dictionary)
+Dims PyListToDims(const pyList list)
{
- std::map<std::string, std::string> parameters;
+ const unsigned int length = pybind11::len(list);
+ Dims dimensions;
+ dimensions.reserve(length);
- pyList keys = dictionary.keys();
- const unsigned int length = py::len(keys);
-
- for (unsigned int k = 0; k < length; ++k)
+ for (unsigned int i = 0; i < length; ++i)
{
- const std::string key(PyCast<std::string>(keys[k]));
- const std::string value(PyCast<std::string>(dictionary[keys[k]]));
- parameters.insert(std::make_pair(key, value));
+ dimensions.push_back(pybind11::cast<size_t>(list[i]));
}
- return parameters;
+ return dimensions;
}
-#endif
-#ifdef HAVE_PYBIND11
-std::map<std::string, std::string> KwargsToMap(const pybind11::kwargs &kwargs)
+Params KwargsToParams(const pyKwargs &kwargs)
{
- std::map<std::string, std::string> parameters;
+ Params parameters;
for (const auto &pair : kwargs)
{
- const std::string key(PyCast<std::string>(pair.first));
- const std::string value(PyCast<std::string>(pair.second));
- parameters.insert(std::make_pair(key, value));
+ parameters.emplace(pybind11::cast<std::string>(pair.first),
+ pybind11::cast<std::string>(pair.second));
}
return parameters;
}
-#endif
-
-bool IsEmpty(pyObject object)
-{
- bool isEmpty = false;
-
-#ifdef HAVE_BOOSTPYTHON
- if (object == boost::python::object())
- isEmpty = true;
-#endif
-
-#ifdef HAVE_PYBIND11
- if (object == pybind11::none())
- isEmpty = true;
-#endif
- return isEmpty;
-}
-} // end namespace
+} // end namespace adios
diff --git a/bindings/python/source/adiosPyFunctions.h b/bindings/python/source/adiosPyFunctions.h
index fbb3acec7..ed97bbaf4 100644
--- a/bindings/python/source/adiosPyFunctions.h
+++ b/bindings/python/source/adiosPyFunctions.h
@@ -8,55 +8,33 @@
* Author: William F Godoy godoywf@ornl.gov
*/
-#ifndef BINDINGS_PYTHON_SOURCE_ADIOSPYFUNCTIONS_H_
-#define BINDINGS_PYTHON_SOURCE_ADIOSPYFUNCTIONS_H_
+#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYFUNCTIONS_H_
+#define ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYFUNCTIONS_H_
#include <map>
#include <string>
#include <vector>
#include "adios2/ADIOSTypes.h"
-#include "bindings/python/source/adiosPyTypes.h"
+#include "adiosPyTypes.h"
namespace adios
{
/**
- * Transforms a boost python list to a Dims (std::vector<std::size_t>) object
- * @param list input boost python list from python program
- * @return Dims (std::vector<std::size_t>) object than can be passed to python
+ * Python list to vector of dimensions (Dims)
+ * @param list python list of numbers
+ * @return adios::Dims
*/
-Dims ListToVector(const pyList &list);
+Dims PyListToDims(const pyList list);
-std::map<std::string, std::string> KwargsToMap(const pyKwargs &dictionary);
-
-template <class T>
-const T *PyArrayToPointer(const pyArray &array)
-{
-
- return reinterpret_cast<const T *>(array.data());
-}
-
-template <class T>
-bool IsType(const pyArray &array)
-{
- bool isType = false;
-
- if (pybind11::isinstance<pybind11::array_t<T>>(array))
- {
- isType = true;
- }
-
- return isType;
-}
-
-template <class T, class U>
-T PyCast(U object)
-{
- return pybind11::cast<T>(object);
-}
-
-bool IsEmpty(pyObject object);
+/**
+ * Python dictionary kwargs to adios::Params (std::map<std::string,
+ * std::string>)
+ * @param kwargs dictionary
+ * @return adios::Params
+ */
+Params KwargsToParams(const pyKwargs &kwargs);
} // end namespace adios
diff --git a/bindings/python/source/adiosPyTypes.h b/bindings/python/source/adiosPyTypes.h
index cd81c85a6..656649e56 100644
--- a/bindings/python/source/adiosPyTypes.h
+++ b/bindings/python/source/adiosPyTypes.h
@@ -8,15 +8,16 @@
* Author: William F Godoy godoywf@ornl.gov
*/
-#ifndef BINDINGS_PYTHON_SOURCE_ADIOSPYTYPES_H_
-#define BINDINGS_PYTHON_SOURCE_ADIOSPYTYPES_H_
+#ifndef ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYTYPES_H_
+#define ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPYTYPES_H_
+#include <adios2.h>
#include <pybind11/numpy.h>
#include <pybind11/pytypes.h>
namespace adios
{
-// python
+// pytypes
using pyObject = pybind11::object;
using pyTuple = pybind11::tuple;
using pyDict = pybind11::dict;
diff --git a/bindings/python/source/gluePyBind11.cpp b/bindings/python/source/gluePyBind11.cpp
index efff5d3e7..aaf903399 100644
--- a/bindings/python/source/gluePyBind11.cpp
+++ b/bindings/python/source/gluePyBind11.cpp
@@ -10,31 +10,33 @@
#include <stdexcept>
+#include <adios2.h>
#include <mpi4py/mpi4py.h>
-#include <pybind11.h>
+#include <pybind11/pybind11.h>
-#include "bindings/python/source/ADIOSPy.h"
-#include "bindings/python/source/EnginePy.h"
-#include "bindings/python/source/IOPy.h"
+#include "ADIOSPy.h"
+#include "EnginePy.h"
+#include "IOPy.h"
+#include "VariablePy.h"
+#include "adiosPyTypes.h"
-namespace py = pybind11;
-
-adios::ADIOSPy ADIOSPy(py::object py_comm, const bool debugMode)
+adios::ADIOSPy ADIOSPyInit(adios::pyObject py_comm, const bool debugMode)
{
- MPI_Comm *comm_p = PyMPIComm_Get(py_comm.ptr());
+ MPI_Comm *mpiCommPtr = PyMPIComm_Get(py_comm.ptr());
if (debugMode)
{
- if (comm_p == NULL)
+ if (mpiCommPtr == NULL)
{
- py::error_already_set(
- "ERROR: mpi4py communicator in ADIOS in null\n");
+ throw std::runtime_error(
+ "ERROR: mpi4py communicator is null, in call "
+ "to ADIOS constructor\n");
}
}
- return adios::ADIOSPy(*comm_p, debugMode);
+ return adios::ADIOSPy(*mpiCommPtr, debugMode);
}
-PYBIND11_PLUGIN(ADIOSPy)
+PYBIND11_PLUGIN(adios2py)
{
if (import_mpi4py() < 0)
{
@@ -42,31 +44,42 @@ PYBIND11_PLUGIN(ADIOSPy)
"ERROR: mpi4py not loaded correctly\n"); /* Python 2.X */
}
- py::module m("ADIOSPy", "ADIOS Python bindings using pybind11");
-
- m.def("ADIOSPy", &ADIOSPy, "Function that creates an ADIOS object");
+ pybind11::module m("adios2py", "ADIOS2 Python bindings using pybind11");
+ m.attr("DebugON") = true;
+ m.attr("DebugOFF") = false;
+ m.attr("Write") = static_cast<int>(adios::OpenMode::Write);
+ m.attr("Read") = static_cast<int>(adios::OpenMode::Read);
+ m.attr("Append") = static_cast<int>(adios::OpenMode::Append);
+ m.attr("ReadWrite") = static_cast<int>(adios::OpenMode::ReadWrite);
+ m.def("ADIOS", &ADIOSPyInit, "Function that creates an ADIOS object");
- py::class_<adios::ADIOSPy>(m, "ADIOS")
- .def("HelloMPI", &adios::ADIOSPy::HelloMPI)
- //.def("DefineVariable", &adios::ADIOSPy::DefineVariablePy)
- .def("DeclareIO", &adios::ADIOSPy::DeclareIOPy,
- py::return_value_policy::reference_internal)
- //.def("Open", &adios::ADIOSPy::OpenPy);
+ pybind11::class_<adios::ADIOSPy>(m, "ADIOSPy")
+ .def("DeclareIO", &adios::ADIOSPy::DeclareIO);
- py::class_<adios::VariablePy>(m, "Variable")
- .def("SetLocalDimensions", &adios::VariablePy::SetLocalDimensions)
- .def("GetLocalDimensions", &adios::VariablePy::GetLocalDimensions);
+ pybind11::class_<adios::IOPy>(m, "IOPy")
+ .def("SetParameters", &adios::IOPy::SetParameters)
+ .def("AddTransport", &adios::IOPy::AddTransport)
+ .def("DefineVariable", &adios::IOPy::DefineVariable)
+ .def("Open", &adios::IOPy::Open);
- py::class_<adios::IOPy>(m, "IO")
- .def("SetParameters", &adios::IOPy::SetParametersPy)
- .def("AddTransport", &adios::IOPy::AddTransportPy)
- .def("PrintAll", &adios::MethodPy::PrintAll);
+ pybind11::class_<adios::VariablePy>(m, "VariablePy")
+ .def("SetDimensions", &adios::VariablePy::SetDimensions);
- // Engine
- py::class_<adios::EnginePy>(m, "Engine")
- .def("Write", &adios::EnginePy::WritePy)
- .def("Advance", &adios::EnginePy::WritePy)
+ pybind11::class_<adios::EnginePy>(m, "EnginePy")
+ .def("Write", &adios::EnginePy::Write)
+ .def("Advance", &adios::EnginePy::Advance)
.def("Close", &adios::EnginePy::Close);
+ // Trying overloaded function
+ // (adios::EnginePy (adios::IOPy::*)(
+ // const std::string &, const adios::OpenMode,
+ // adios::pyObject)) &
+ // &adios::IOPy::Open)
+ // .def("Open", (adios::EnginePy (adios::IOPy::*)(const std::string
+ // &,
+ // const
+ // adios::OpenMode)) &
+ // &adios::IOPy::Open);
+
return m.ptr();
}
diff --git a/bindings/python/test_hello.py b/bindings/python/test_hello.py
deleted file mode 100644
index b66f26289..000000000
--- a/bindings/python/test_hello.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Distributed under the OSI-approved Apache License, Version 2.0. See
-# accompanying file Copyright.txt for details.
-#
-# test_hello.py
-# Created on: Feb 2, 2017
-# Author: wfg
-
-from mpi4py import MPI
-from ADIOSPy import *
-import numpy as np
-
-# Create ADIOS and verify MPI Comm is passed correctly
-# Pass communicator and debug flag is True
-adios = ADIOSPy(MPI.COMM_WORLD, True)
-rank = MPI.COMM_WORLD.Get_rank()
-size = MPI.COMM_WORLD.Get_size()
-
-# User data
-myArray = np.array([1, 2, 3, 4])
-
-if(rank % 2 == 1): # odd ranks only
- oddRankArray = np.array([11., 12., 13., 14.])
-
-
-# ADIOS Define Variables
-ioArray = adios.DefineVariable("ioArray", [myArray.size], [], [])
-
-if(rank % 2 == 1): # odd ranks only
- ioOddRankArray = adios.DefineVariable(
- "ioMyFloats", [oddRankArray.size], [], [])
-
-
-# Setup method and print summary
-ioSettings = adios.DeclareMethod("adiosSettings")
-ioSettings.SetParameters(profile_units='mus')
-# POSIX is default, just checking
-ioSettings.AddTransport('File', have_metadata_file='no', profile_units='mus')
-
-# Start Engine
-# Open files using N-to-N method, None means no new MPI communicator
-bpFileWriter = adios.Open("file.bp", "w", ioSettings, None)
-bpFileWriter.Write(ioArray, myArray)
-
-if(rank % 2 == 1):
- bpFileWriter.Write(ioOddRankArray, oddRankArray)
-
-bpFileWriter.Close()
-
-if(rank == 0):
- print "Done writing " + str(size) + " bp files"
- ioSettings.PrintAll()
diff --git a/examples/experimental/multistep/reader_allsteps.cpp b/examples/experimental/multistep/reader_allsteps.cpp
index 7b50a98ba..972537f0a 100644
--- a/examples/experimental/multistep/reader_allsteps.cpp
+++ b/examples/experimental/multistep/reader_allsteps.cpp
@@ -115,7 +115,7 @@ int main(int argc, char *argv[])
bpReaderSettings.SetEngine(
"ADIOS1Reader"); // BP is the default engine
// see only one step at a time
- bpReaderSettings.SetParameters("OpenAsFile");
+ bpReaderSettings.SetParameters({"OpenAsFile=yes"});
}
// Create engine smart pointer due to polymorphism,
diff --git a/examples/experimental/multistep/writer_multistep.cpp b/examples/experimental/multistep/writer_multistep.cpp
index dc2c47064..dc80c2e42 100644
--- a/examples/experimental/multistep/writer_multistep.cpp
+++ b/examples/experimental/multistep/writer_multistep.cpp
@@ -74,11 +74,11 @@ int main(int argc, char *argv[])
bpWriterSettings.AddTransport("file"
#ifdef ADIOS2_HAVE_MPI
,
- "library=MPI-IO"
+ { "library=MPI-IO" }
#endif
);
// Passing parameters to the engine
- bpWriterSettings.SetParameters("have_metadata_file=yes");
+ bpWriterSettings.SetParameters({"have_metadata_file=yes"});
// number of aggregators
// bpWriterSettings.SetParameters("Aggregation", (nproc + 1) / 2);
}
diff --git a/examples/heatTransfer/read/heatRead_adios2.cpp b/examples/heatTransfer/read/heatRead_adios2.cpp
index 93bc3faf9..e9920eba1 100644
--- a/examples/heatTransfer/read/heatRead_adios2.cpp
+++ b/examples/heatTransfer/read/heatRead_adios2.cpp
@@ -55,11 +55,11 @@ int main(int argc, char *argv[])
// if not defined by user, we can change the default settings
// BPFileWriter is the default engine
bpReaderIO.SetEngine("ADIOS1Reader");
- bpReaderIO.SetParameters("num_threads=2");
+ bpReaderIO.SetParameters({"num_threads=2"});
// ISO-POSIX file is the default transport
// Passing parameters to the transport
- bpReaderIO.AddTransport("File", "verbose=4");
+ bpReaderIO.AddTransport("File", {"verbose=4"});
}
auto bpReader =
diff --git a/examples/heatTransfer/write/IO_ph5_adios2.cpp b/examples/heatTransfer/write/IO_ph5_adios2.cpp
index 01fca5621..e75a71315 100644
--- a/examples/heatTransfer/write/IO_ph5_adios2.cpp
+++ b/examples/heatTransfer/write/IO_ph5_adios2.cpp
@@ -39,7 +39,7 @@ IO::IO(const Settings &s, MPI_Comm comm)
const std::string aggregatorsParam("Aggregators=" +
std::to_string((s.nproc + 1) / 2));
- h5io.SetParameters("have_metadata_file=yes", aggregatorsParam);
+ h5io.SetParameters({"have_metadata_file=yes", aggregatorsParam});
}
// ad->DefineScalar<unsigned int>("gndx", true);
diff --git a/examples/hello/adios1Writer/helloADIOS1Writer.cpp b/examples/hello/adios1Writer/helloADIOS1Writer.cpp
index 1041f895c..e6bcb4f87 100644
--- a/examples/hello/adios1Writer/helloADIOS1Writer.cpp
+++ b/examples/hello/adios1Writer/helloADIOS1Writer.cpp
@@ -42,7 +42,7 @@ int main(int argc, char *argv[])
* Parameters, Transports, and Execution: Engines */
adios::IO &adios1IO = adios.DeclareIO("ADIOS1IO");
adios1IO.SetEngine("ADIOS1Writer");
- adios1IO.AddTransport("file", "library=MPI");
+ adios1IO.AddTransport("file", {"library=MPI"});
/** global array : name, { shape (total) }, { start (local) }, { count
* (local) }, all are constant dimensions */
diff --git a/examples/hello/datamanReader/helloDataManReader.cpp b/examples/hello/datamanReader/helloDataManReader.cpp
index 7dc984af3..408c9e7c6 100644
--- a/examples/hello/datamanReader/helloDataManReader.cpp
+++ b/examples/hello/datamanReader/helloDataManReader.cpp
@@ -8,8 +8,10 @@
* Author: Jason Wang
*/
+#include <chrono>
#include <iostream>
#include <numeric>
+#include <thread>
#include <vector>
#include <adios2.h>
@@ -43,8 +45,9 @@ int main(int argc, char *argv[])
adios::IO &dataManIO = adios.DeclareIO("WAN");
dataManIO.SetEngine("DataManReader");
- dataManIO.SetParameters("real_time=yes", "method_type=stream",
- "method=dump");
+ dataManIO.SetParameters(
+ {"real_time=yes", "method_type=stream", "method=dump"});
+
auto dataManReader =
dataManIO.Open("myDoubles.bp", adios::OpenMode::Read);
diff --git a/examples/hello/datamanReader/helloDataManReader_nompi.cpp b/examples/hello/datamanReader/helloDataManReader_nompi.cpp
index 965e47c36..1e525fc24 100644
--- a/examples/hello/datamanReader/helloDataManReader_nompi.cpp
+++ b/examples/hello/datamanReader/helloDataManReader_nompi.cpp
@@ -8,8 +8,10 @@
* Author: Jason Wang
*/
+#include <chrono>
#include <iostream>
#include <numeric>
+#include <thread>
#include <vector>
#include <adios2.h>
@@ -38,8 +40,8 @@ int main(int argc, char *argv[])
adios::IO &dataManIO = adios.DeclareIO("WAN");
dataManIO.SetEngine("DataManReader");
- dataManIO.SetParameters("real_time=yes", "method_type=stream",
- "method=dump");
+ dataManIO.SetParameters(
+ {"real_time=yes", "method_type=stream", "method=dump"});
auto dataManReader =
dataManIO.Open("myDoubles.bp", adios::OpenMode::Read);
diff --git a/examples/hello/datamanWriter/helloDataManWriter.cpp b/examples/hello/datamanWriter/helloDataManWriter.cpp
index dcdf4bff7..1c5071379 100644
--- a/examples/hello/datamanWriter/helloDataManWriter.cpp
+++ b/examples/hello/datamanWriter/helloDataManWriter.cpp
@@ -31,8 +31,8 @@ int main(int argc, char *argv[])
adios::ADIOS adios(MPI_COMM_WORLD, adios::DebugON);
adios::IO &dataManIO = adios.DeclareIO("WANIO");
dataManIO.SetEngine("DataManWriter");
- dataManIO.SetParameters("peer-to-peer=yes", "real_time=yes",
- "compress=no", "method=dump");
+ dataManIO.SetParameters({"peer-to-peer=yes", "real_time=yes",
+ "compress=no", "method=dump"});
// Define variable and local size
auto bpFloats =
diff --git a/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp b/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp
index cbc08fb60..fa9fb7f79 100644
--- a/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp
+++ b/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp
@@ -24,8 +24,8 @@ int main(int argc, char *argv[])
adios::ADIOS adios(adios::DebugON);
adios::IO &dataManIO = adios.DeclareIO("WANIO");
dataManIO.SetEngine("DataManWriter");
- dataManIO.SetParameters("peer-to-peer=yes", "real_time=yes",
- "compress=no", "method=dump");
+ dataManIO.SetParameters({"peer-to-peer=yes", "real_time=yes",
+ "compress=no", "method=dump"});
// Define variable and local size
auto bpFloats =
diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt
index 30c544588..084fe733e 100644
--- a/source/adios2/CMakeLists.txt
+++ b/source/adios2/CMakeLists.txt
@@ -11,6 +11,7 @@ add_library(adios2
core/SelectionBoundingBox.cpp
core/SelectionPoints.cpp
core/Transform.cpp
+ core/Variable.cpp core/Variable.tcc
core/VariableBase.cpp
core/VariableCompound.cpp core/VariableCompound.tcc
diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp
index 8595d6db4..dbf25da83 100644
--- a/source/adios2/core/Engine.cpp
+++ b/source/adios2/core/Engine.cpp
@@ -16,6 +16,8 @@
#include <set>
/// \endcond
+#include "adios2/helper/adiosFunctions.h" //GetType<T>
+
namespace adios
{
diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h
index f18e561df..5cc189a64 100644
--- a/source/adios2/core/Engine.h
+++ b/source/adios2/core/Engine.h
@@ -18,6 +18,7 @@
#include <functional> //std::function
#include <map>
#include <memory> //std::shared_ptr
+#include <set>
#include <string>
#include <vector>
/// \endcond
diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp
index a9753acc3..a91d1cc89 100644
--- a/source/adios2/core/IO.cpp
+++ b/source/adios2/core/IO.cpp
@@ -8,11 +8,12 @@
* Author: William F Godoy godoywf@ornl.gov
*/
+#include "adios2/helper/adiosFunctions.h" //BuildParametersMap
+
#include "IO.h"
#include "IO.tcc"
#include "adios2/ADIOSMPI.h"
-#include "adios2/helper/adiosFunctions.h" //BuildParametersMap
#include "adios2/engine/bp/BPFileWriter.h"
@@ -44,6 +45,26 @@ IO::IO(const std::string name, MPI_Comm mpiComm, const bool inConfigFile,
void IO::SetEngine(const std::string engineType) { m_EngineType = engineType; }
void IO::SetIOMode(const IOMode ioMode) { m_IOMode = ioMode; };
+void IO::SetParameters(const std::vector<std::string> ¶metersVector)
+{
+ m_Parameters = BuildParametersMap(parametersVector, m_DebugMode);
+}
+
+void IO::SetParameters(const Params ¶meters) { m_Parameters = parameters; }
+
+unsigned int IO::AddTransport(const std::string type,
+ const std::vector<std::string> ¶metersVector)
+{
+ Params parametersMap(BuildParametersMap(parametersVector, m_DebugMode));
+ return AddTransportCommon(type, parametersMap);
+}
+
+unsigned int IO::AddTransport(const std::string type, const Params ¶meters)
+{
+ Params parametersMap(parameters);
+ return AddTransportCommon(type, parametersMap);
+}
+
VariableCompound &IO::GetVariableCompound(const std::string &name)
{
return m_Compound.at(GetVariableIndex(name));
@@ -219,34 +240,15 @@ std::shared_ptr<Engine> IO::Open(const std::string &name,
}
// PRIVATE Functions
-unsigned int
-IO::AddTransportParameters(const std::string type,
- const std::vector<std::string> ¶meters)
+unsigned int IO::AddTransportCommon(const std::string type, Params ¶meters)
{
if (m_DebugMode)
{
- if (type.empty() || type.find("=") != type.npos)
- {
- throw std::invalid_argument(
- "ERROR: first argument in AddTransport must "
- "be a single word for transport\n");
- }
- }
-
- Params mapParameters = BuildParametersMap(parameters, m_DebugMode);
-
- if (m_DebugMode)
- {
- if (mapParameters.count("transport") == 1)
- {
- std::invalid_argument("ERROR: transport can't be redefined with "
- "\"transport=type\", "
- "type must be the first argument\n");
- }
+ CheckTransportType(type);
}
- mapParameters["transport"] = type;
- m_TransportsParameters.push_back(std::move(mapParameters));
+ parameters["transport"] = type;
+ m_TransportsParameters.push_back(parameters);
return static_cast<unsigned int>(m_TransportsParameters.size() - 1);
}
@@ -276,6 +278,18 @@ bool IO::VariableExists(const std::string &name) const
return exists;
}
+void IO::CheckTransportType(const std::string type) const
+{
+ if (type.empty() || type.find("=") != type.npos)
+ {
+ throw std::invalid_argument(
+ "ERROR: wrong first argument " + type +
+ ", must "
+ "be a single word for a supported transport type, in "
+ "call to IO AddTransport \n");
+ }
+}
+
// Explicitly instantiate the necessary public template implementations
#define define_template_instantiation(T) \
template Variable<T> &IO::DefineVariable<T>( \
diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h
index 87df704d8..ebb848167 100644
--- a/source/adios2/core/IO.h
+++ b/source/adios2/core/IO.h
@@ -13,6 +13,7 @@
/// \cond EXCLUDE_FROM_DOXYGEN
#include <map>
+#include <set>
#include <string>
#include <utility> //std::pair
#include <vector>
@@ -48,7 +49,7 @@ public:
const bool m_DebugMode = false;
/** from ADIOS class passed to Engine created with Open */
- const std::string m_HostLanguage = "C++";
+ std::string m_HostLanguage = "C++";
/** From SetParameter, parameters for a particular engine from m_Type */
Params m_Parameters;
@@ -79,12 +80,18 @@ public:
void SetIOMode(const IOMode mode);
/**
- * Sets parameters for the method in "parameter=value" format
- * @param args list of parameters with format "parameter1=value1", ...,
- * "parameterN=valueN"
+ * Sets IO parameters in "parameter=value" format
+ * @param paramsVector each vector entry with format
+ * "parameter1=value1", ..., "parameterN=valueN"
+ */
+ void SetParameters(const std::vector<std::string> ¶metersVector);
+
+ /**
+ * Version that passes a map to fill out parameters
+ * initializer list = { "param1", "value1" }, {"param2", "value2"},
+ * @param params adios::Params std::map<std::string, std::string>
*/
- template <class... Args>
- void SetParameters(Args... args);
+ void SetParameters(const Params ¶meters = Params());
/**
* Adds a transport and its parameters for the method
@@ -92,8 +99,11 @@ public:
* @param args list of parameters for a transport with format
* "parameter1=value1", ..., "parameterN=valueN"
*/
- template <class... Args>
- unsigned int AddTransport(const std::string type, Args... args);
+ unsigned int AddTransport(const std::string type,
+ const std::vector<std::string> ¶msVector);
+
+ unsigned int AddTransport(const std::string type,
+ const Params ¶ms = Params());
/**
* Define a Variable of primitive data type for I/O.
@@ -124,6 +134,7 @@ public:
* change over time
* @return reference to Variable object
*/
+
template <class T>
VariableCompound &
DefineVariableCompound(const std::string &name, const Dims shape = Dims{},
@@ -242,14 +253,12 @@ private:
std::set<std::string> m_EngineNames;
/**
- * Called from AddTransport to transform parameter to a map
+ * Called from AddTransport overloads
* @param type
* @param parameters
* @return transport index
*/
- unsigned int
- AddTransportParameters(const std::string type,
- const std::vector<std::string> ¶meters);
+ unsigned int AddTransportCommon(const std::string type, Params ¶meters);
/** Gets the internal reference to a variable map for type T
* This function is specialized in IO.tcc */
@@ -265,6 +274,8 @@ private:
* @return true: variable name exists, false: variable name doesn't exist
*/
bool VariableExists(const std::string &name) const;
+
+ void CheckTransportType(const std::string type) const;
};
// Explicit declaration of the public template methods
diff --git a/source/adios2/core/IO.inl b/source/adios2/core/IO.inl
index d7a850544..4b9efbfb3 100644
--- a/source/adios2/core/IO.inl
+++ b/source/adios2/core/IO.inl
@@ -14,26 +14,11 @@
#error "Inline file should only be included from it's header, never on it's own"
#endif
-#include "adios2/ADIOSMacros.h"
-#include "adios2/helper/adiosFunctions.h" //BuildParametersMap
+#include "IO.h"
namespace adios
{
-template <class... Args>
-void IO::SetParameters(Args... args)
-{
- std::vector<std::string> parameters = {args...};
- m_Parameters = BuildParametersMap(parameters, m_DebugMode);
-}
-
-template <class... Args>
-unsigned int IO::AddTransport(const std::string type, Args... args)
-{
- std::vector<std::string> parameters = {args...};
- return AddTransportParameters(type, parameters);
-}
-
template <class T>
VariableCompound &IO::DefineVariableCompound(const std::string &name,
const Dims shape, const Dims start,
@@ -44,16 +29,16 @@ VariableCompound &IO::DefineVariableCompound(const std::string &name,
{
if (VariableExists(name))
{
- std::invalid_argument("ERROR: variable " + name +
- " exists in IO object " + m_Name +
- ", in call to DefineVariable\n");
+ throw std::invalid_argument("ERROR: variable " + name +
+ " exists in IO object " + m_Name +
+ ", in call to DefineVariable\n");
}
}
const unsigned int size = m_Compound.size();
auto itVariableCompound = m_Compound.emplace(
size, VariableCompound(name, sizeof(T), shape, start, count,
constantShape, m_DebugMode));
- m_Variables.emplace(name, std::make_pair(GetType<T>(), size));
+ m_Variables.emplace(name, std::make_pair("compound", size));
return itVariableCompound.first->second;
}
diff --git a/source/adios2/core/Variable.cpp b/source/adios2/core/Variable.cpp
new file mode 100644
index 000000000..424844882
--- /dev/null
+++ b/source/adios2/core/Variable.cpp
@@ -0,0 +1,17 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * Variable.cpp needed for template separation using Variable.tcc
+ *
+ * Created on: Jun 8, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#include "Variable.h"
+#include "Variable.tcc"
+
+namespace adios
+{
+
+} // end namespace adios
diff --git a/source/adios2/core/Variable.h b/source/adios2/core/Variable.h
index 9b859f3fa..2cc4cad36 100644
--- a/source/adios2/core/Variable.h
+++ b/source/adios2/core/Variable.h
@@ -18,10 +18,9 @@
#include <vector>
/// \endcond
-#include "VariableBase.h"
-#include "adios2/ADIOSConfig.h"
#include "adios2/ADIOSMacros.h"
#include "adios2/core/Transform.h"
+#include "adios2/core/VariableBase.h"
namespace adios
{
@@ -62,6 +61,4 @@ public:
} // end namespace adios
-#include "Variable.inl"
-
#endif /* ADIOS2_CORE_VARIABLE_H_ */
diff --git a/source/adios2/core/Variable.inl b/source/adios2/core/Variable.inl
deleted file mode 100644
index 4abb6e3c4..000000000
--- a/source/adios2/core/Variable.inl
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Distributed under the OSI-approved Apache License, Version 2.0. See
- * accompanying file Copyright.txt for details.
- *
- * Variable.inl
- *
- * Created on: May 1, 2017
- * Author: William F Godoy godoywf@ornl.gov
- */
-
-#ifndef ADIOS2_CORE_VARIABLE_INL_
-#define ADIOS2_CORE_VARIABLE_INL_
-#ifndef ADIOS2_CORE_VARIABLE_H_
-#error "Inline file should only be included from it's header, never on it's own"
-#endif
-
-#include "Variable.h"
-
-#include "adios2/helper/adiosFunctions.h" //GetType
-
-namespace adios
-{
-
-template <class T>
-Variable<T>::Variable(const std::string &name, const Dims shape,
- const Dims start, const Dims count,
- const bool constantShape, const bool debugMode)
-: VariableBase(name, GetType<T>(), sizeof(T), shape, start, count,
- constantShape, debugMode)
-{
-}
-
-template <class T>
-void Variable<T>::ApplyTransforms()
-{
-}
-
-} // end namespace adios
-
-#endif /* ADIOS2_CORE_VARIABLE_INL_ */
diff --git a/source/adios2/core/Variable.tcc b/source/adios2/core/Variable.tcc
new file mode 100644
index 000000000..4da41c61e
--- /dev/null
+++ b/source/adios2/core/Variable.tcc
@@ -0,0 +1,43 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * Variable.tcc
+ *
+ * Created on: May 1, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#ifndef ADIOS2_CORE_VARIABLE_TCC_
+#define ADIOS2_CORE_VARIABLE_TCC_
+
+#include "Variable.h"
+
+#include "adios2/ADIOSMacros.h"
+#include "adios2/helper/adiosFunctions.h" //GetType
+
+namespace adios
+{
+
+#define declare_type(T) \
+ \
+ template <> \
+ Variable<T>::Variable(const std::string &name, const Dims shape, \
+ const Dims start, const Dims count, \
+ const bool constantShape, const bool debugMode) \
+ : VariableBase(name, GetType<T>(), sizeof(T), shape, start, count, \
+ constantShape, debugMode) \
+ { \
+ } \
+ \
+ template <> \
+ void Variable<T>::ApplyTransforms() \
+ { \
+ }
+
+ADIOS2_FOREACH_TYPE_1ARG(declare_type)
+#undef declare_type
+
+} // end namespace adios
+
+#endif /* ADIOS2_CORE_VARIABLE_TCC_ */
diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp
index db50a21db..4d7a30a22 100644
--- a/source/adios2/core/VariableBase.cpp
+++ b/source/adios2/core/VariableBase.cpp
@@ -15,15 +15,17 @@
#include <stdexcept> //std::invalid_argument
/// \endcond
+#include "adios2/helper/adiosFunctions.h" //GetTotalSize
+
namespace adios
{
VariableBase::VariableBase(const std::string &name, const std::string type,
const size_t elementSize, const Dims shape,
const Dims start, const Dims count,
- const bool constantShape, const bool debugMode)
+ const bool constantDims, const bool debugMode)
: m_Name(name), m_Type(type), m_ElementSize(elementSize), m_Shape(shape),
- m_Start(start), m_Count(count), m_ConstantShape(constantShape),
+ m_Start(start), m_Count(count), m_ConstantDims(constantDims),
m_DebugMode(debugMode)
{
InitShapeType();
@@ -50,7 +52,7 @@ void VariableBase::SetSelection(const Dims start, const Dims count)
m_Name + ", in call to SetSelection\n");
}
- if (m_ConstantShape)
+ if (m_ConstantDims)
{
throw std::invalid_argument(
"ERROR: selection is not valid for constant shape variable " +
@@ -123,6 +125,16 @@ void VariableBase::SetStepSelection(const unsigned int startStep,
m_ReadNSteps = countStep;
}
+void VariableBase::AddTransform(
+ Transform &transform, const std::vector<std::string> ¶metersVector)
+{
+}
+
+void VariableBase::AddTransform(Transform &transform,
+ const Params ¶metersVector)
+{
+}
+
// transforms related functions
void VariableBase::ClearTransforms() { m_TransformsInfo.clear(); }
@@ -133,7 +145,7 @@ void VariableBase::InitShapeType()
{
if (m_DebugMode)
{
- if (m_ConstantShape)
+ if (m_ConstantDims)
{
throw std::invalid_argument(
"ERROR: isConstantShape (true) argument is invalid "
diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h
index 7c17a1687..d6a69264d 100644
--- a/source/adios2/core/VariableBase.h
+++ b/source/adios2/core/VariableBase.h
@@ -22,7 +22,6 @@
#include "adios2/ADIOSTypes.h"
#include "adios2/core/SelectionBoundingBox.h"
#include "adios2/core/Transform.h"
-#include "adios2/helper/adiosFunctions.h"
namespace adios
{
@@ -41,10 +40,10 @@ public:
* VariableCompound -> from constructor sizeof(struct) */
const size_t m_ElementSize;
- ShapeID m_ShapeID; ///< see shape types in ADIOSTypes.h
- bool m_SingleValue = false; ///< true: single value, false: array
- const bool m_ConstantShape = false; ///< true: fix m_Shape, m_Start, m_Count
- Dims m_Shape; ///< total dimensions across MPI
+ ShapeID m_ShapeID; ///< see shape types in ADIOSTypes.h
+ bool m_SingleValue = false; ///< true: single value, false: array
+ const bool m_ConstantDims = false; ///< true: fix m_Shape, m_Start, m_Count
+ Dims m_Shape; ///< total dimensions across MPI
Dims m_Start; ///< starting point (offsets) in global shape
Dims m_Count; ///< dimensions from m_Start in global shape
@@ -106,14 +105,11 @@ public:
void SetStepSelection(const unsigned int startStep,
const unsigned int countStep);
- template <class... Args>
- void AddTransform(Transform &transform, Args... args)
- {
- std::vector<std::string> parameters = {args...};
- m_TransformsInfo.emplace_back(
- transform,
- BuildParametersMap(parameters, m_DebugMode)); // need to check
- }
+ void AddTransform(Transform &transform,
+ const std::vector<std::string> ¶metersVector);
+
+ void AddTransform(Transform &transform,
+ const Params ¶metersVector = Params());
/** Apply current sequence of transforms defined by AddTransform */
virtual void ApplyTransforms() = 0;
@@ -136,9 +132,9 @@ private:
/** reference to object derived from Transform class */
Transform &Object;
/** parameters from AddTransform */
- std::map<std::string, std::string> Parameters;
+ Params Parameters;
/** resulting sizes from transformation */
- std::vector<std::size_t> Sizes;
+ Dims Sizes;
};
/**
diff --git a/source/adios2/core/VariableCompound.tcc b/source/adios2/core/VariableCompound.tcc
index 8293bce14..1c382e24b 100644
--- a/source/adios2/core/VariableCompound.tcc
+++ b/source/adios2/core/VariableCompound.tcc
@@ -13,7 +13,7 @@
#include "VariableCompound.h"
-#include "adios2/ADIOSConfig.h"
+#include "adios2/helper/adiosFunctions.h" //GetType
namespace adios
{
diff --git a/source/adios2/engine/dataman/DataManWriter.tcc b/source/adios2/engine/dataman/DataManWriter.tcc
index 9c3023e45..113b0771d 100644
--- a/source/adios2/engine/dataman/DataManWriter.tcc
+++ b/source/adios2/engine/dataman/DataManWriter.tcc
@@ -14,6 +14,7 @@
#include "DataManWriter.h"
#include "adios2/ADIOSMPI.h"
+#include "adios2/helper/adiosFunctions.h" //GetType<T>
namespace adios
{
diff --git a/source/adios2/helper/adiosString.cpp b/source/adios2/helper/adiosString.cpp
index 69d34efd7..90b6399b1 100644
--- a/source/adios2/helper/adiosString.cpp
+++ b/source/adios2/helper/adiosString.cpp
@@ -49,15 +49,15 @@ Params BuildParametersMap(const std::vector<std::string> ¶meters,
if (equalPosition == parameter.npos)
{
throw std::invalid_argument(
- "ERROR: wrong format for parameter " + parameter +
- ", format must be field=value \n");
+ "ERROR: wrong format for IO parameter " + parameter +
+ ", format must be key=value for each entry \n");
}
if (equalPosition == parameter.size() - 1)
{
- throw std::invalid_argument("ERROR: empty value in parameter " +
- parameter +
- ", format must be field=value \n");
+ throw std::invalid_argument(
+ "ERROR: empty value in IO parameter " + parameter +
+ ", format must be key=value \n");
}
}
--
GitLab