Skip to content

GitLab

Unverified Commit 3e2361f7 authored by William F Godoy's avatar William F Godoy Committed by GitHub
Browse files

Merge pull request #1463 from williamfgc/highlevel_local 

Reading local variables in C++ Python high-level APIs
parents cfe490f1 dac5aefc
Expand all Hide whitespace changes
Inline Side-by-side
bindings/CXX11/cxx11/fstream/ADIOS2fstream.cpp
View file @ 3e2361f7
......@@ -120,7 +120,10 @@ bool getstep(adios2::fstream &stream, adios2::fstep &step)
return step.m_Stream->GetStep();
}
size_t fstream::currentstep() const noexcept { return m_Stream->CurrentStep(); }
size_t fstream::current_step() const noexcept
{
return m_Stream->CurrentStep();
}
adios2::Mode fstream::ToMode(const openmode mode) const noexcept
{
......@@ -177,35 +180,38 @@ ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(declare_template_instantiation)
const vParams &, const bool); \
\
template void fstream::write<T>(const std::string &, const T &, \
const bool); \
\
template std::vector<T> fstream::read<T>(const std::string &); \
const bool, const bool); \
\
template std::vector<T> fstream::read<T>(const std::string &, \
const size_t, const size_t); \
const size_t); \
\
template std::vector<T> fstream::read<T>(const std::string &, \
const Dims &, const Dims &); \
template std::vector<T> fstream::read<T>( \
const std::string &, const size_t, const size_t, const size_t); \
\
template std::vector<T> fstream::read<T>(const std::string &, \
const Dims &, const Dims &, \
const size_t, const size_t); \
template std::vector<T> fstream::read<T>( \
const std::string &, const Dims &, const Dims &, const size_t); \
\
template void fstream::read<T>(const std::string &, T *); \
template std::vector<T> fstream::read<T>( \
const std::string &, const Dims &, const Dims &, const size_t, \
const size_t, const size_t); \
\
template void fstream::read(const std::string &, T *, const size_t, \
const size_t); \
template void fstream::read<T>(const std::string &, T *, const size_t); \
\
template void fstream::read<T>(const std::string &name, T &); \
template void fstream::read(const std::string &, T *, const size_t, \
const size_t, const size_t); \
\
template void fstream::read<T>(const std::string &name, T &, \
const size_t); \
\
template void fstream::read<T>(const std::string &name, T &, const size_t, \
const size_t); \
\
template void fstream::read<T>(const std::string &, T *, const Dims &, \
const Dims &); \
const Dims &, const size_t); \
\
template void fstream::read<T>(const std::string &, T *, const Dims &, \
const Dims &, const size_t, const size_t);
const Dims &, const size_t, const size_t, \
const size_t);
ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation)
#undef declare_template_instantiation
......
bindings/CXX11/cxx11/fstream/ADIOS2fstream.h
View file @ 3e2361f7
......@@ -293,6 +293,8 @@ public:
* Write a self-describing single-value variable
* @param name variable name
* @param value variable data value (can be r-value)
* @param isLocalValue true: local value (returned as GlobalArray), false:
* global value (returned as global value)
* @param endStep similar to std::endStep, end current step and flush
* (default). Use adios2::endStep for true.
* @exception std::invalid_argument (user input error) or
......@@ -300,28 +302,33 @@ public:
*/
template <class T>
void write(const std::string &name, const T &value,
const bool endStep = false);
const bool isLocalValue = false, const bool endStep = false);
/**
* Reads into a pre-allocated pointer a selection piece in dimension. When
* Reads into a pre-allocated pointer. When
* used with adios2::getstep reads current step
* @param name variable name
* @param data pre-allocated pointer to hold read data, if variable is
* not found (name and type don't match) it becomes nullptr
* @param data pre-allocated pointer to hold read data
* @param blockID required for local variables, specify current block to be
* selected
* @exception throws exception if variable name, dimensions or step not
* found
*/
template <class T>
void read(const std::string &name, T *data);
void read(const std::string &name, T *data, const size_t blockID = 0);
/**
* Reads a value. When used with adios2::getstep reads current step value
* @param name variable name
* @param value output value, if variable is not found (name and type don't
* match) the returned value address becomes nullptr
* @param blockID required for local variables, specify current block to
* be selected
* @exception throws exception if variable name, dimensions or step not
* found
*/
template <class T>
void read(const std::string &name, T &value);
void read(const std::string &name, T &value, const size_t blockID = 0);
/**
* Read accessing steps in random access mode. Not be used with
......@@ -333,12 +340,14 @@ public:
* appearance, not absolute step in stream)
* @param stepsCount variable number of steps form step_start, don't have to
* be contiguous, necessarily
* @param blockID required for local variables, specify current block to
* be selected
* @exception throws exception if variable name, dimensions or step not
* found
*/
template <class T>
void read(const std::string &name, T *data, const size_t stepsStart,
const size_t stepsCount = 1);
const size_t stepsCount = 1, const size_t blockID = 0);
/**
* Reads into a single value for a single step. Not be used with
......@@ -348,11 +357,14 @@ public:
* if variable is not found (name, type and step don't match) the returned
* value address becomes nullptr
* @param step selected single step
* @param blockID required for local variables, specify current block to
* be selected
* @exception throws exception if variable name, dimensions or step not
* found
*/
template <class T>
void read(const std::string &name, T &value, const size_t step);
void read(const std::string &name, T &value, const size_t step,
const size_t blockID = 0);
/**
* Reads into a pre-allocated pointer a selection piece in dimension. When
......@@ -362,12 +374,14 @@ public:
* not found (name and type don't match) it becomes nullptr
* @param start variable local offset selection
* @param count variable local dimension selection from start
* @param blockID required for local variables, specify current block to
* be selected
* @exception throws exception if variable name, dimensions or step not
* found
*/
template <class T>
void read(const std::string &name, T *data, const adios2::Dims &start,
const adios2::Dims &count);
const adios2::Dims &count, const size_t blockID = 0);
/**
* Reads into a pre-allocated pointer a selection piece in dimensions and
......@@ -382,24 +396,28 @@ public:
* appearance, not absolute step in stream)
* @param stepsCount variable number of steps form step_start, don't have to
* be necessarily contiguous
* @param blockID required for local variables, specify current block to
* be selected
* @exception throws exception if variable name, dimensions or step not
* found
*/
template <class T>
void read(const std::string &name, T *data, const adios2::Dims &start,
const adios2::Dims &count, const size_t stepsStart,
const size_t stepsCount);
const size_t stepsCount, const size_t blockID = 0);
/**
* Reads entire variable for current step (streaming mode: step by step)
* @param name variable name
* @param blockID required for local variables, specify current block to
* be selected
* @return data of variable name for current step. Single data will have
* a size=1 vector
* @exception throws exception if variable name, dimensions or step not
* found
*/
template <class T>
std::vector<T> read(const std::string &name);
std::vector<T> read(const std::string &name, const size_t blockID = 0);
/**
* Returns a vector with full variable dimensions for the current step
......@@ -410,6 +428,8 @@ public:
* appearance, not absolute step in stream)
* @param stepsCount variable number of steps form step_start, don't have to
* be contiguous, necessarily
* @param blockID required for local variables, specify current block to
* be selected
* @return data of variable name for current step, empty if exception is
* thrown
* @exception throws exception if variable name, dimensions or step not
......@@ -417,7 +437,7 @@ public:
*/
template <class T>
std::vector<T> read(const std::string &name, const size_t stepsStart,
const size_t stepsCount = 1);
const size_t stepsCount = 1, const size_t blockID = 0);
/**
* Reads a selection piece in dimension for current step (streaming mode:
......@@ -425,6 +445,8 @@ public:
* @param name variable name
* @param start variable local offset selection
* @param count variable local dimension selection from start
* @param blockID required for local variables, specify current block to
* be selected
* @return data of variable name for current step, empty if exception is
* thrown
* @exception throws exception if variable name, dimensions or step not
......@@ -432,7 +454,7 @@ public:
*/
template <class T>
std::vector<T> read(const std::string &name, const Dims &start,
const Dims &count);
const Dims &count, const size_t blockID = 0);
/**
* Reads a selection piece in dimension and a selection piece in steps
......@@ -445,6 +467,8 @@ public:
* appearance, not absolute step in stream)
* @param stepsCount variable number of steps form step_start, don't have to
* be contiguous, necessarily
* @param blockID required for local variables, specify current block to
* be selected
* @return variable data, empty if exception is thrown
* @exception throws exception if variable name, dimensions or step not
* found
......@@ -452,7 +476,7 @@ public:
template <class T>
std::vector<T> read(const std::string &name, const Dims &start,
const Dims &count, const size_t stepsStart,
const size_t stepsCount);
const size_t stepsCount, const size_t blockID = 0);
/**
* Reads an attribute returning a vector
......@@ -495,7 +519,7 @@ public:
* Return current step when getstep is called in a loop, read mode only
* @return current step
*/
size_t currentstep() const noexcept;
size_t current_step() const noexcept;
size_t steps() const;
......@@ -535,39 +559,42 @@ ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(declare_template_instantiation)
const Dims &, const vParams &, const bool); \
\
extern template void fstream::write<T>(const std::string &, const T &, \
const bool); \
const bool, const bool); \
\
extern template std::vector<T> fstream::read<T>(const std::string &); \
extern template std::vector<T> fstream::read<T>(const std::string &, \
const size_t); \
\
extern template std::vector<T> fstream::read<T>( \
const std::string &, const size_t, const size_t); \
const std::string &, const size_t, const size_t, const size_t); \
\
extern template std::vector<T> fstream::read<T>( \
const std::string &, const Dims &, const Dims &); \
const std::string &, const Dims &, const Dims &, const size_t); \
\
extern template std::vector<T> fstream::read<T>( \
const std::string &, const Dims &, const Dims &, const size_t, \
const size_t); \
const size_t, const size_t); \
\
extern template void fstream::read<T>(const std::string &, T *); \
extern template void fstream::read<T>(const std::string &, T *, \
const size_t); \
\
extern template void fstream::read(const std::string &, T *, const size_t, \
const size_t); \
const size_t, const size_t); \
\
extern template void fstream::read<T>(const std::string &name, T &); \
extern template void fstream::read<T>(const std::string &name, T &, \
const size_t); \
\
extern template void fstream::read<T>(const std::string &name, T &, \
const size_t); \
\
extern template void fstream::read<T>(const std::string &, T *, \
const Dims &, const Dims &); \
extern template void fstream::read<T>( \
const std::string &, T *, const Dims &, const Dims &, const size_t); \
\
extern template void fstream::read<T>(const std::string &, T *, \
const size_t, const size_t); \
extern template void fstream::read<T>( \
const std::string &, T *, const size_t, const size_t, const size_t); \
\
extern template void fstream::read<T>(const std::string &, T *, \
const Dims &, const Dims &, \
const size_t, const size_t);
extern template void fstream::read<T>( \
const std::string &, T *, const Dims &, const Dims &, const size_t, \
const size_t, const size_t);
ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation)
#undef declare_template_instantiation
......
bindings/CXX11/cxx11/fstream/ADIOS2fstream.tcc
View file @ 3e2361f7
......@@ -60,103 +60,102 @@ void fstream::write(const std::string &name, const T *data,
}
template <class T>
void fstream::write(const std::string &name, const T &value, const bool endStep)
void fstream::write(const std::string &name, const T &value,
const bool isLocalValue, const bool endStep)
{
using IOType = typename TypeInfo<T>::IOType;
m_Stream->Write(name, reinterpret_cast<const IOType &>(value), endStep);
m_Stream->Write(name, reinterpret_cast<const IOType &>(value), isLocalValue,
endStep);
}
template <class T>
void fstream::read(const std::string &name, T *data)
void fstream::read(const std::string &name, T *data, const size_t blockID)
{
using IOType = typename TypeInfo<T>::IOType;
m_Stream->Read(name, reinterpret_cast<IOType *>(data));
m_Stream->Read(name, reinterpret_cast<IOType *>(data), blockID);
}
template <class T>
void fstream::read(const std::string &name, T *data, const size_t stepStart,
const size_t stepCount)
const size_t stepCount, const size_t blockID)
{
using IOType = typename TypeInfo<T>::IOType;
m_Stream->Read(name, reinterpret_cast<IOType *>(data),
Box<size_t>{stepStart, stepCount});
Box<size_t>{stepStart, stepCount}, blockID);
}
template <class T>
void fstream::read(const std::string &name, T &value)
void fstream::read(const std::string &name, T &value, const size_t blockID)
{
using IOType = typename TypeInfo<T>::IOType;
m_Stream->Read(name, &reinterpret_cast<IOType &>(value));
m_Stream->Read(name, &reinterpret_cast<IOType &>(value), blockID);
}
template <class T>
void fstream::read(const std::string &name, T *data, const Dims &selectionStart,
const Dims &selectionCount)
const Dims &selectionCount, const size_t blockID)
{
using IOType = typename TypeInfo<T>::IOType;
m_Stream->Read(name, reinterpret_cast<IOType *>(data),
Box<Dims>(selectionStart, selectionCount));
Box<Dims>(selectionStart, selectionCount), blockID);
}
template <class T>
void fstream::read(const std::string &name, T *data, const Dims &selectionStart,
const Dims &selectionCount, const size_t stepSelectionStart,
const size_t stepSelectionCount)
void fstream::read(const std::string &name, T *data, const Dims &start,
const Dims &count, const size_t stepStart,
const size_t stepCount, const size_t blockID)
{
using IOType = typename TypeInfo<T>::IOType;
m_Stream->Read(name, reinterpret_cast<IOType *>(data),
Box<Dims>(selectionStart, selectionCount),
Box<size_t>(stepSelectionStart, stepSelectionCount));
Box<Dims>(start, count), Box<size_t>(stepStart, stepCount),
blockID);
}
template <class T>
void fstream::read(const std::string &name, T &value,
const size_t stepSelectionStart)
void fstream::read(const std::string &name, T &value, const size_t stepStart,
const size_t blockID)
{
using IOType = typename TypeInfo<T>::IOType;
m_Stream->Read(name, &reinterpret_cast<IOType &>(value),
Box<size_t>(stepSelectionStart, 1));
Box<size_t>(stepStart, 1), blockID);
}
template <class T>
std::vector<T> fstream::read(const std::string &name)
std::vector<T> fstream::read(const std::string &name, const size_t blockID)
{
using IOType = typename TypeInfo<T>::IOType;
auto vec = m_Stream->Read<IOType>(name);
auto vec = m_Stream->Read<IOType>(name, blockID);
return reinterpret_cast<std::vector<T> &>(vec);
}
template <class T>
std::vector<T> fstream::read(const std::string &name, const size_t stepsStart,
const size_t stepsCount)
const size_t stepsCount, const size_t blockID)
{
using IOType = typename TypeInfo<T>::IOType;
auto vec =
m_Stream->Read<IOType>(name, Box<size_t>(stepsStart, stepsCount));
auto vec = m_Stream->Read<IOType>(name, Box<size_t>(stepsStart, stepsCount),
blockID);
return reinterpret_cast<std::vector<T> &>(vec);
}
template <class T>
std::vector<T> fstream::read(const std::string &name,
const Dims &selectionStart,
const Dims &selectionCount)
std::vector<T> fstream::read(const std::string &name, const Dims &start,
const Dims &count, const size_t blockID)
{
using IOType = typename TypeInfo<T>::IOType;
auto vec =
m_Stream->Read<IOType>(name, Box<Dims>(selectionStart, selectionCount));
auto vec = m_Stream->Read<IOType>(name, Box<Dims>(start, count), blockID);
return reinterpret_cast<std::vector<T> &>(vec);
}
template <class T>
std::vector<T>
fstream::read(const std::string &name, const Dims &selectionStart,
const Dims &selectionCount, const size_t stepSelectionStart,
const size_t stepSelectionCount)
std::vector<T> fstream::read(const std::string &name, const Dims &start,
const Dims &count, const size_t stepStart,
const size_t stepCount, const size_t blockID)
{
using IOType = typename TypeInfo<T>::IOType;
auto vec = m_Stream->Read<IOType>(
name, Box<Dims>(selectionStart, selectionCount),
Box<size_t>(stepSelectionStart, stepSelectionCount));
auto vec =
m_Stream->Read<IOType>(name, Box<Dims>(start, count),
Box<size_t>(stepStart, stepCount), blockID);
return reinterpret_cast<std::vector<T> &>(vec);
}
......
bindings/Python/CMakeLists.txt
View file @ 3e2361f7
......@@ -5,7 +5,7 @@ pybind11_add_module(adios2py MODULE
py11Attribute.cpp
py11Engine.cpp
py11Operator.cpp
py11File.cpp
py11File.cpp py11File.tcc
py11glue.cpp
)
target_link_libraries(adios2py PRIVATE adios2)
......
bindings/Python/py11File.cpp
View file @ 3e2361f7
......@@ -9,6 +9,7 @@
*/
#include "py11File.h"
#include "py11File.tcc"
#include <algorithm>
#include <iostream>
......@@ -173,15 +174,22 @@ void File::Write(const std::string &name, const pybind11::array &array,
}
void File::Write(const std::string &name, const pybind11::array &array,
const bool endStep)
const bool isLocalValue, const bool endStep)
{
Write(name, array, {}, {}, {}, endStep);
if (isLocalValue)
{
Write(name, array, {adios2::LocalValueDim}, {}, {}, endStep);
}
else
{
Write(name, array, {}, {}, {}, endStep);
}
}
void File::Write(const std::string &name, const std::string &stringValue,
const bool endStep)
const bool isLocalValue, const bool endStep)
{
m_Stream->Write(name, stringValue, endStep);
m_Stream->Write(name, stringValue, isLocalValue, endStep);
}
bool File::GetStep() const
......@@ -189,25 +197,29 @@ bool File::GetStep() const
return const_cast<File *>(this)->m_Stream->GetStep();
}
std::vector<std::string> File::ReadString(const std::string &name)
std::vector<std::string> File::ReadString(const std::string &name,
const size_t blockID)
{
return m_Stream->Read<std::string>(name);
return m_Stream->Read<std::string>(name, blockID);
}
std::vector<std::string> File::ReadString(const std::string &name,
const size_t stepStart,
const size_t stepCount)
const size_t stepCount,
const size_t blockID)
{
return m_Stream->Read<std::string>(name, Box<size_t>(stepStart, stepCount));
return m_Stream->Read<std::string>(name, Box<size_t>(stepStart, stepCount),
blockID);
}
pybind11::array File::Read(const std::string &name)
pybind11::array File::Read(const std::string &name, const size_t blockID)
{
const std::string type = m_Stream->m_IO->InquireVariableType(name);
if (type == helper::GetType<std::string>())
{
const std::string value = m_Stream->Read<std::string>(name).front();
const std::string value =
m_Stream->Read<std::string>(name, blockID).front();
pybind11::array pyArray(pybind11::dtype::of<char>(),
Dims{value.size()});
char *pyPtr =
......@@ -220,20 +232,7 @@ pybind11::array File::Read(const std::string &name)
{ \
core::Variable<T> &variable = \
*m_Stream->m_IO->InquireVariable<T>(name); \
Dims pyCount; \
if (variable.m_SingleValue) \
{ \
pyCount = {1}; \
pybind11::array pyArray(pybind11::dtype::of<T>(), pyCount); \
m_Stream->Read<T>(name, reinterpret_cast<T *>( \
const_cast<void *>(pyArray.data()))); \
return pyArray; \
} \
else \
{ \
const Dims zerosStart(variable.m_Shape.size(), 0); \
return Read(name, zerosStart, variable.m_Shape); \
} \
return DoRead(variable, blockID); \
}
ADIOS2_FOREACH_NUMPY_TYPE_1ARG(declare_type)
#undef declare_type
......@@ -246,8 +245,8 @@ pybind11::array File::Read(const std::string &name)
return pybind11::array();
}
pybind11::array File::Read(const std::string &name, const Dims &selectionStart,
const Dims &selectionCount)
pybind11::array File::Read(const std::string &name, const Dims &start,
const Dims &count, const size_t blockID)
{
const std::string type = m_Stream->m_IO->InquireVariableType(name);
......@@ -257,10 +256,10 @@ pybind11::array File::Read(const std::string &name, const Dims &selectionStart,
#define declare_type(T) \
else if (type == helper::GetType<T>()) \
{ \
pybind11::array pyArray(pybind11::dtype::of<T>(), selectionCount); \
pybind11::array pyArray(pybind11::dtype::of<T>(), count); \
m_Stream->Read<T>( \
name, reinterpret_cast<T *>(const_cast<void *>(pyArray.data())), \
Box<Dims>(selectionStart, selectionCount)); \
Box<Dims>(start, count), blockID); \
return pyArray; \
}
ADIOS2_FOREACH_NUMPY_TYPE_1ARG(declare_type)
......@@ -271,17 +270,16 @@ pybind11::array File::Read(const std::string &name, const Dims &selectionStart,
", type can't be mapped to a numpy type, in call to read\n");
}
pybind11::array File::Read(const std::string &name, const Dims &selectionStart,
const Dims &selectionCount,
const size_t stepSelectionStart,
const size_t stepSelectionCount)
pybind11::array File::Read(const std::string &name, const Dims &start,
const Dims &count, const size_t stepStart,
const size_t stepCount, const size_t blockID)
{
// shape of the returned numpy array
Dims shapePy(selectionCount.size() + 1);
shapePy[0] = stepSelectionCount;
Dims shapePy(count.size() + 1);
shapePy[0] = stepCount;
for (auto i = 1; i < shapePy.size(); ++i)
{
shapePy[i] = selectionCount[i - 1];
shapePy[i] = count[i - 1];
}
const std::string type = m_Stream->m_IO->InquireVariableType(name);
......@@ -295,8 +293,8 @@ pybind11::array File::Read(const std::string &name, const Dims &selectionStart,
pybind11::array pyArray(pybind11::dtype::of<T>(), shapePy); \
m_Stream->Read<T>( \
name, reinterpret_cast<T *>(const_cast<void *>(pyArray.data())), \
Box<Dims>(selectionStart, selectionCount), \
Box<size_t>(stepSelectionStart, stepSelectionCount)); \
Box<Dims>(start, count), Box<size_t>(stepStart, stepCount), \
blockID); \
return pyArray; \
}
ADIOS2_FOREACH_NUMPY_TYPE_1ARG(declare_type)
......
bindings/Python/py11File.h
View file @ 3e2361f7
......@@ -78,28 +78,29 @@ public: