Skip to content
Snippets Groups Projects
Commit 2dcb03b1 authored by William F Godoy's avatar William F Godoy
Browse files

Working example for Write and Read with Python 

Now passing numpy array in DefineVariable
autopep8 syntax errors
parent 5c64052a
No related branches found
No related tags found
1 merge request!337Working example for Write and Read with Python
Hide whitespace changes
Inline Side-by-side
Showing
with 117 additions and 122 deletions
bindings/python/EnginePy.cpp
+ 4
− 52
View file @ 2dcb03b1
......@@ -18,10 +18,8 @@ namespace adios2
{
EnginePy::EnginePy(IO &io, const std::string &name, const Mode openMode,
MPI_Comm mpiComm,
std::map<std::string, VariableBase> &variablesPlaceholder)
: m_Engine(io.Open(name, openMode, mpiComm)),
m_VariablesPlaceholder(variablesPlaceholder), m_DebugMode(io.m_DebugMode)
MPI_Comm mpiComm)
: m_Engine(io.Open(name, openMode, mpiComm)), m_DebugMode(io.m_DebugMode)
{
}
......@@ -29,8 +27,6 @@ void EnginePy::BeginStep() { m_Engine.BeginStep(); }
void EnginePy::PutSync(VariableBase *variable, const pybind11::array &array)
{
DefineInIO(variable, array);
if (variable->m_Type == "compound")
{
// not supported
......@@ -57,15 +53,12 @@ void EnginePy::PutSync(VariableBase *variable, const pybind11::array &array)
void EnginePy::PutSync(VariableBase *variable, const std::string &string)
{
DefineInIO(variable, string);
m_Engine.PutSync(*dynamic_cast<adios2::Variable<std::string> *>(variable),
string);
}
void EnginePy::PutDeferred(VariableBase *variable, const pybind11::array &array)
{
DefineInIO(variable, array);
if (variable->m_Type == "compound")
{
// not supported
......@@ -92,7 +85,6 @@ void EnginePy::PutDeferred(VariableBase *variable, const pybind11::array &array)
void EnginePy::PutDeferred(VariableBase *variable, const std::string &string)
{
DefineInIO(variable, string);
m_Engine.PutDeferred(
*dynamic_cast<adios2::Variable<std::string> *>(variable), string);
}
......@@ -196,51 +188,11 @@ void EnginePy::PerformGets() { m_Engine.PerformGets(); }
void EnginePy::EndStep() { m_Engine.EndStep(); }
void EnginePy::WriteStep() { m_Engine.WriteStep(); }
void EnginePy::Close(const int transportIndex)
{
m_Engine.Close(transportIndex);
}
// PRIVATE
void EnginePy::DefineInIO(VariableBase *variable, const pybind11::array &array)
{
if (variable->m_Type.empty()) // Define in IO
{
auto &io = m_Engine.GetIO();
if (array.is(pybind11::array()))
{
if (m_DebugMode)
{
throw std::invalid_argument(
"ERROR: passing an empty numpy array for variable " +
variable->m_Name + ", in call to Put/Get");
}
}
#define declare_type(T) \
else if (pybind11::isinstance< \
pybind11::array_t<T, pybind11::array::c_style>>(array)) \
{ \
variable = &io.DefineVariable<T>(variable->m_Name, variable->m_Shape, \
variable->m_Start, variable->m_Count, \
variable->m_ConstantDims); \
m_VariablesPlaceholder.erase(variable->m_Name); \
}
ADIOS2_FOREACH_NUMPY_TYPE_1ARG(declare_type)
#undef declare_type
}
}
void EnginePy::DefineInIO(VariableBase *variable, const std::string &string)
{
if (variable->m_Type.empty()) // Define in IO
{
auto &io = m_Engine.GetIO();
variable = &io.DefineVariable<std::string>(
variable->m_Name, variable->m_Shape, variable->m_Start,
variable->m_Count, variable->m_ConstantDims);
m_VariablesPlaceholder.erase(variable->m_Name);
}
}
} // end namespace adios2
bindings/python/EnginePy.h
+ 3
− 6
View file @ 2dcb03b1
......@@ -27,8 +27,7 @@ class EnginePy
public:
EnginePy(IO &io, const std::string &name, const Mode openMode,
MPI_Comm mpiComm,
std::map<std::string, VariableBase> &variablesPlaceholder);
MPI_Comm mpiComm);
~EnginePy() = default;
......@@ -52,15 +51,13 @@ public:
void EndStep();
void WriteStep();
void Close(const int transportIndex = -1);
private:
Engine &m_Engine;
std::map<std::string, VariableBase> &m_VariablesPlaceholder;
const bool m_DebugMode;
void DefineInIO(VariableBase *variable, const pybind11::array &array);
void DefineInIO(VariableBase *variable, const std::string &string);
};
} // end namespace adios2
......
bindings/python/IOPy.cpp
+ 5
− 37
View file @ 2dcb03b1
......@@ -42,30 +42,15 @@ unsigned int IOPy::AddTransport(const std::string type,
return m_IO.AddTransport(type, parameters);
}
VariableBase &IOPy::DefineVariable(const std::string &name, const Dims &shape,
VariableBase *IOPy::DefineVariable(const std::string &name, const Dims &shape,
const Dims &start, const Dims &count,
const bool isConstantDims,
pybind11::array &array)
{
if (m_DebugMode)
{
if (m_VariablesPlaceholder.count(name) == 1)
{
throw std::invalid_argument("ERROR: variable " + name +
" already exists, in "
"call to DefineVariable\n");
}
}
VariableBase *variable = nullptr;
if (array.is(pybind11::array()))
if (false)
{
// put in placeholder
auto itVariableEmplace = m_VariablesPlaceholder.emplace(
name, VariableBase(name, "unknown", 0, shape, start, count,
isConstantDims, m_DebugMode));
variable = &itVariableEmplace.first->second;
}
#define declare_type(T) \
else if (pybind11::isinstance< \
......@@ -89,33 +74,16 @@ VariableBase &IOPy::DefineVariable(const std::string &name, const Dims &shape,
}
}
return *variable;
return variable;
}
VariableBase *IOPy::InquireVariable(const std::string &name) noexcept
{
// first check in placeholder
if (!m_VariablesPlaceholder.empty())
{
auto itVariablePlaceholder = m_VariablesPlaceholder.find(name);
if (itVariablePlaceholder != m_VariablesPlaceholder.end())
{
return &itVariablePlaceholder->second;
}
}
const std::string type(m_IO.InquireVariableType(name));
if (type.empty())
{
return nullptr;
}
adios2::VariableBase *variable = nullptr;
if (type == "compound")
if (type == "unknown")
{
// not supported
}
#define declare_template_instantiation(T) \
else if (type == adios2::GetType<T>()) \
......@@ -131,7 +99,7 @@ VariableBase *IOPy::InquireVariable(const std::string &name) noexcept
EnginePy IOPy::Open(const std::string &name, const int openMode)
{
return EnginePy(m_IO, name, static_cast<adios2::Mode>(openMode),
m_IO.m_MPIComm, m_VariablesPlaceholder);
m_IO.m_MPIComm);
}
} // end namespace adios2
bindings/python/IOPy.h
+ 1
− 6
View file @ 2dcb03b1
......@@ -39,7 +39,7 @@ public:
const Params &GetParameters() const noexcept;
VariableBase &DefineVariable(const std::string &name, const Dims &shape,
VariableBase *DefineVariable(const std::string &name, const Dims &shape,
const Dims &start, const Dims &count,
const bool isConstantDims,
pybind11::array &array);
......@@ -50,11 +50,6 @@ public:
private:
const bool m_DebugMode;
/**
* Placeholder map needed as Variables are not created in ADIOS at
* DefineVariable, but until Put when type is known from numpy
*/
std::map<std::string, VariableBase> m_VariablesPlaceholder;
};
} // end namespace adios2
......
bindings/python/gluePyBind11.cpp
+ 2
− 0
View file @ 2dcb03b1
......@@ -135,6 +135,7 @@ PYBIND11_MODULE(adios2, m)
pybind11::class_<adios2::VariableBase>(m, "Variable")
.def("SetSelection", &adios2::VariableBase::SetSelection)
.def("SetStepSelection", &adios2::VariableBase::SetStepSelection)
.def("SelectionSize", &adios2::VariableBase::SelectionSize);
pybind11::class_<adios2::IOPy>(m, "IOPy")
......@@ -188,6 +189,7 @@ PYBIND11_MODULE(adios2, m)
adios2::EnginePy::GetDeferred)
.def("PerformGets", &adios2::EnginePy::PerformGets)
.def("EndStep", &adios2::EnginePy::EndStep)
.def("WriteStep", &adios2::EnginePy::WriteStep)
.def("Close", &adios2::EnginePy::Close,
pybind11::arg("transportIndex") = -1);
}
examples/hello/bpReader/helloBPReaderHeatMap2D.py 0 → 100644
+ 77
− 0
View file @ 2dcb03b1
#
# Distributed under the OSI-approved Apache License, Version 2.0. See
# accompanying file Copyright.txt for details.
#
# helloBPReaderHeatMap2D.py
#
#
# Created on: Dec 5th, 2017
# Author: William F Godoy godoywf@ornl.gov
#
from mpi4py import MPI
import numpy
import adios2
# MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
# User data
Nx = 10
Ny = 10
count = [Nx, Ny]
start = [rank * Nx, 0]
shape = [size * Nx, Ny]
temperatures = numpy.zeros(Nx * Ny, dtype=numpy.int)
for i in range(0, Nx):
iGlobal = start[0] + i
for j in range(0, Ny):
value = iGlobal * shape[1] + j
temperatures[i * Nx + j] = value
print(str(i) + "," + str(j) + " " + str(value))
# ADIOS portion
adios = adios2.ADIOS(comm)
ioWrite = adios.DeclareIO("ioWriter")
varTemperature = ioWrite.DefineVariable("temperature2D", shape,
start, count, adios2.ConstantDims,
temperatures)
obpStream = ioWrite.Open('HeatMap2D_py.bp', adios2.Mode.Write)
obpStream.PutSync(varTemperature, temperatures)
obpStream.Close()
if rank == 0:
ioRead = adios.DeclareIO("ioReader")
ibpStream = ioRead.Open('HeatMap2D_py.bp', adios2.Mode.Read)
var_inTemperature = ioRead.InquireVariable("temperature2D")
if var_inTemperature is not None:
var_inTemperature.SetSelection([[2, 2], [4, 4]])
inSize = var_inTemperature.SelectionSize()
print('Incoming size ' + str(inSize))
inTemperatures = numpy.zeros(inSize, dtype=numpy.int)
ibpStream.GetSync(var_inTemperature, inTemperatures)
print('Incoming temperature map')
for i in range(0, inTemperatures.size):
print(str(inTemperatures[i]) + ' ')
if (i + 1) % 4 == 0:
print()
ibpStream.Close()
examples/hello/bpReader/helloBPReaderHeatMap3D.f90
+ 5
− 1
View file @ 2dcb03b1
......@@ -88,7 +88,7 @@ program helloBPReaderHeatMap3D
do k=1,sel_count(3)
do j=1,sel_count(2)
do i=1,sel_count(1)
write(6,'(I4) ', advance="no") sel_temperatures(i,j,k)
write(6,'(I5) ', advance="no") sel_temperatures(i,j,k)
end do
write(*,*)
end do
......@@ -99,6 +99,10 @@ program helloBPReaderHeatMap3D
end if
call adios2_close( bpReader, ierr )
end if
call MPI_Finalize(ierr)
end program helloBPReaderHeatMap3D
testing/adios2/bindings/python/TestBPWriteTypes.py
+ 10
− 10
View file @ 2dcb03b1
......@@ -25,28 +25,28 @@ bpIO = adios.DeclareIO("NPTypes")
# ADIOS Variable name, shape, start, offset, constant dims
# All local variables
varI8 = bpIO.DefineVariable(
"varI8", [], [], [data.I8.size], adios2.ConstantDims)
"varI8", [], [], [data.I8.size], adios2.ConstantDims, data.I8)
varI16 = bpIO.DefineVariable(
"varI16", [], [], [data.I16.size], adios2.ConstantDims)
"varI16", [], [], [data.I16.size], adios2.ConstantDims, data.I16)
varI32 = bpIO.DefineVariable(
"varI32", [], [], [data.I32.size], adios2.ConstantDims)
"varI32", [], [], [data.I32.size], adios2.ConstantDims, data.I32)
varI64 = bpIO.DefineVariable(
"varI64", [], [], [data.I64.size], adios2.ConstantDims)
"varI64", [], [], [data.I64.size], adios2.ConstantDims, data.I64)
varU8 = bpIO.DefineVariable(
"varUI8", [], [], [data.U8.size], adios2.ConstantDims)
"varUI8", [], [], [data.U8.size], adios2.ConstantDims, data.U8)
varU16 = bpIO.DefineVariable(
"varUI16", [], [], [data.U16.size], adios2.ConstantDims)
"varUI16", [], [], [data.U16.size], adios2.ConstantDims, data.U16)
varU32 = bpIO.DefineVariable(
"varUI32", [], [], [data.U32.size], adios2.ConstantDims)
"varUI32", [], [], [data.U32.size], adios2.ConstantDims, data.U32)
varU64 = bpIO.DefineVariable(
"varUI64", [], [], [data.U64.size], adios2.ConstantDims)
"varUI64", [], [], [data.U64.size], adios2.ConstantDims, data.U64)
varR32 = bpIO.DefineVariable(
"varR32", [], [], [data.R32.size], adios2.ConstantDims)
"varR32", [], [], [data.R32.size], adios2.ConstantDims, data.R32)
varR64 = bpIO.DefineVariable(
"varR64", [], [], [data.R64.size], adios2.ConstantDims)
"varR64", [], [], [data.R64.size], adios2.ConstantDims, data.R64)
# ADIOS Engine
......
testing/adios2/bindings/python/TestBPWriteTypes_nompi.py
+ 10
− 10
View file @ 2dcb03b1
......@@ -23,28 +23,28 @@ bpIO = adios.DeclareIO("NPTypes")
# ADIOS Variable name, shape, start, offset, constant dims
# All local variables
varI8 = bpIO.DefineVariable(
"varI8", [], [], [data.I8.size], adios2.ConstantDims)
"varI8", [], [], [data.I8.size], adios2.ConstantDims, data.I8)
varI16 = bpIO.DefineVariable(
"varI16", [], [], [data.I16.size], adios2.ConstantDims)
"varI16", [], [], [data.I16.size], adios2.ConstantDims, data.I16)
varI32 = bpIO.DefineVariable(
"varI32", [], [], [data.I32.size], adios2.ConstantDims)
"varI32", [], [], [data.I32.size], adios2.ConstantDims, data.I32)
varI64 = bpIO.DefineVariable(
"varI64", [], [], [data.I64.size], adios2.ConstantDims)
"varI64", [], [], [data.I64.size], adios2.ConstantDims, data.I64)
varU8 = bpIO.DefineVariable(
"varUI8", [], [], [data.U8.size], adios2.ConstantDims)
"varUI8", [], [], [data.U8.size], adios2.ConstantDims, data.U8)
varU16 = bpIO.DefineVariable(
"varUI16", [], [], [data.U16.size], adios2.ConstantDims)
"varUI16", [], [], [data.U16.size], adios2.ConstantDims, data.U16)
varU32 = bpIO.DefineVariable(
"varUI32", [], [], [data.U32.size], adios2.ConstantDims)
"varUI32", [], [], [data.U32.size], adios2.ConstantDims, data.U32)
varU64 = bpIO.DefineVariable(
"varUI64", [], [], [data.U64.size], adios2.ConstantDims)
"varUI64", [], [], [data.U64.size], adios2.ConstantDims, data.U64)
varR32 = bpIO.DefineVariable(
"varR32", [], [], [data.R32.size], adios2.ConstantDims)
"varR32", [], [], [data.R32.size], adios2.ConstantDims, data.R32)
varR64 = bpIO.DefineVariable(
"varR64", [], [], [data.R64.size], adios2.ConstantDims)
"varR64", [], [], [data.R64.size], adios2.ConstantDims, data.R64)
# ADIOS Engine
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment