diff --git a/CMakeLists.txt b/CMakeLists.txt
index 65c892099bf1f43c6fad87f03bba814321af9ecb..e31f749bf9acf63d09836b41d882dec7323219db 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -55,31 +55,40 @@ set(CMAKE_CXX_STANDARD_REQUIRED True)
include(CMakeDependentOption)
-# Setup shared library / -fPIC stuff
+# Setup shared library defaults. If explicitly specified somehow, then default
+# to that. Otherwise base the default on whether or not shared libs are even
+# supported.
get_property(SHARED_LIBS_SUPPORTED GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS)
-if(DEFINED BUILD_SHARED_LIBS)
- set(ADIOS2_BUILD_SHARED_LIBS_DEFAULT ${BUILD_SHARED_LIBS})
-elseif(DEFINED ADIOS2_BUILD_SHARED_LIBS)
+if(DEFINED ADIOS2_BUILD_SHARED_LIBS)
set(ADIOS2_BUILD_SHARED_LIBS_DEFAULT ${ADIOS2_BUILD_SHARED_LIBS})
+elseif(DEFINED BUILD_SHARED_LIBS)
+ set(ADIOS2_BUILD_SHARED_LIBS_DEFAULT ${BUILD_SHARED_LIBS})
else()
set(ADIOS2_BUILD_SHARED_LIBS_DEFAULT ${SHARED_LIBS_SUPPORTED})
endif()
-unset(BUILD_SHARED_LIBS)
-option(ADIOS2_BUILD_SHARED_LIBS "Build shared libraries (so/dylib/dll)." ${ADIOS2_BUILD_SHARED_LIBS})
+cmake_dependent_option(ADIOS2_BUILD_SHARED_LIBS
+ "Build shared libraries (so/dylib/dll)." ${ADIOS2_BUILD_SHARED_LIBS_DEFAULT}
+ "SHARED_LIBS_SUPPORTED" OFF
+)
set(BUILD_SHARED_LIBS ${ADIOS2_BUILD_SHARED_LIBS})
-if(NOT SHARED_LIBS_SUPPORTED)
- if(BUILD_SHARED_LIBS)
- message(WARNING "A shared library build was requested but is not supported on this platform / compiler. Unexpected build results will likely occur")
- endif()
- set(BUILD_SHARED_LIBS OFF)
+mark_as_advanced(BUILD_SHARED_LIBS)
+
+# Setup PIC defaults. If explicitly specified somehow, then default
+# to that. Otherwise base the default on whether or not shared libs are even
+# supported.
+if(DEFINED ADIOS2_ENABLE_PIC)
+ set(ADIOS2_ENABLE_PIC_DEFAULT ${ADIOS2_ENABLE_PIC})
+elseif(DEFINED CMAKE_POSITION_INDEPENDENT_CODE)
+ set(ADIOS2_ENABLE_PIC_DEFAULT ${CMAKE_POSITION_INDEPENDENT_CODE})
+else()
+ set(ADIOS2_ENABLE_PIC_DEFAULT ${SHARED_LIBS_SUPPORTED})
endif()
-
cmake_dependent_option(ADIOS2_ENABLE_PIC
- "Build with Position Independent Code" ON
- "SHARED_LIBS_SUPPORTED" OFF)
-if(ADIOS2_ENABLE_PIC)
- set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-endif()
+ "Build with Position Independent Code" ${ADIOS2_ENABLE_PIC_DEFAULT}
+ "SHARED_LIBS_SUPPORTED" OFF
+)
+set(CMAKE_POSITION_INDEPENDENT_CODE ${ADIOS2_ENABLE_PIC})
+mark_as_advanced(CMAKE_POSITION_INDEPENDENT_CODE)
adios_option(BZip2 "Enable support for BZip2 transforms" AUTO)
adios_option(ZFP "Enable support for ZFP transforms" AUTO)
@@ -162,10 +171,13 @@ message("")
message(" Installation prefix: ${CMAKE_INSTALL_PREFIX}")
message(" Features:")
if(BUILD_SHARED_LIBS)
- message(" Library Type: shared")
+ set(msg_lib_type "shared")
+elseif(CMAKE_POSITION_INDEPENDENT_CODE)
+ set(msg_lib_type "static (with PIC)")
else()
- message(" Library Type: static")
+ set(msg_lib_type "static (without PIC)")
endif()
+message(" Library Type: ${msg_lib_type}")
message(" Build Type: ${CMAKE_BUILD_TYPE}")
message(" Testing: ${BUILD_TESTING}")
message(" Build Options:")
diff --git a/bindings/python/ADIOSPy.cpp b/bindings/python/ADIOSPy.cpp
index 96c8033a80176655e44130dc7dd5c956671b6163..7f196f40a1d4806fd28bcad89c3b1fa7041b7a4f 100644
--- a/bindings/python/ADIOSPy.cpp
+++ b/bindings/python/ADIOSPy.cpp
@@ -19,7 +19,8 @@ namespace adios2
ADIOSPy::ADIOSPy(const std::string configFile, MPI_Comm mpiComm,
const bool debugMode)
-: m_DebugMode(debugMode), m_ADIOS(configFile, mpiComm, debugMode)
+: m_DebugMode(debugMode),
+ m_ADIOS(std::make_shared<adios2::ADIOS>(configFile, mpiComm, debugMode))
{
}
@@ -29,7 +30,7 @@ ADIOSPy::ADIOSPy(MPI_Comm mpiComm, const bool debugMode)
}
ADIOSPy::ADIOSPy(const std::string configFile, const bool debugMode)
-: ADIOSPy("", MPI_COMM_SELF, debugMode)
+: ADIOSPy(configFile, MPI_COMM_SELF, debugMode)
{
}
@@ -39,7 +40,7 @@ ADIOSPy::ADIOSPy(const bool debugMode) : ADIOSPy("", MPI_COMM_SELF, debugMode)
IOPy ADIOSPy::DeclareIO(const std::string name)
{
- return IOPy(m_ADIOS.DeclareIO(name), m_DebugMode);
+ return IOPy(m_ADIOS->DeclareIO(name), m_DebugMode);
}
} // end namespace adios
diff --git a/bindings/python/ADIOSPy.h b/bindings/python/ADIOSPy.h
index 64cd3fbbe0c2ef6b1f1e6d6a27ecf7e92f66a7d4..d3c9f5e85ddf7fbad16e24f326a46baba5e50640 100644
--- a/bindings/python/ADIOSPy.h
+++ b/bindings/python/ADIOSPy.h
@@ -12,6 +12,7 @@
#define ADIOS2_BINDINGS_PYTHON_SOURCE_ADIOSPY_H_
/// \cond EXCLUDE_FROM_DOXYGEN
+#include <memory> //std::shared_ptr
#include <string>
/// \endcond
@@ -32,13 +33,14 @@ public:
ADIOSPy(MPI_Comm mpiComm, const bool debugMode);
ADIOSPy(const std::string configFile, const bool debugMode);
ADIOSPy(const bool debugMode);
+
~ADIOSPy() = default;
IOPy DeclareIO(const std::string name);
private:
const bool m_DebugMode;
- adios2::ADIOS m_ADIOS;
+ std::shared_ptr<adios2::ADIOS> m_ADIOS;
};
} // end namespace adios
diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt
index 8403e7306894667f7b63a93f844a4562d938d0d3..90d030a55245ad5ee4e0d691a093d6e160567fa7 100644
--- a/bindings/python/CMakeLists.txt
+++ b/bindings/python/CMakeLists.txt
@@ -1,7 +1,3 @@
-if(NOT BUILD_SHARED_LIBS)
- message(ERROR "Python bindings are only supported for shared libraries")
-endif()
-
set(Python_ADDITIONAL_VERSIONS 3 2.7)
find_package(PythonInterp REQUIRED)
find_package(PythonLibsNew REQUIRED)
diff --git a/bindings/python/gluePyBind11.cpp b/bindings/python/gluePyBind11.cpp
index 86eab2cb14aad75939642f69d9147b7e377bb8b5..85f5ec219461a07de5c4a89ce690b5f426f100dd 100644
--- a/bindings/python/gluePyBind11.cpp
+++ b/bindings/python/gluePyBind11.cpp
@@ -68,7 +68,7 @@ adios2::ADIOSPy ADIOSPyInit(adios2::pyObject &object, const bool debugMode)
adios2::ADIOSPy ADIOSPyInitConfig(const std::string configFile,
const bool debugMode)
{
- return adios2::ADIOSPy(debugMode);
+ return adios2::ADIOSPy(configFile, debugMode);
}
adios2::ADIOSPy ADIOSPyInit(const bool debugMode)
diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake
index a484a5957f7c862b724cd102639b6339e7460c98..5022c676d3c31fe1dbaa82907f0ec9b72417a614 100644
--- a/cmake/DetectOptions.cmake
+++ b/cmake/DetectOptions.cmake
@@ -94,7 +94,7 @@ list(INSERT CMAKE_MODULE_PATH 0
"${ADIOS2_SOURCE_DIR}/thirdparty/pybind11/pybind11/tools"
)
if(ADIOS2_USE_Python STREQUAL AUTO)
- if(BUILD_SHARED_LIBS)
+ if(SHARED_LIBS_SUPPORTED AND ADIOS2_ENABLE_PIC)
set(Python_ADDITIONAL_VERSIONS 3 2.7)
find_package(PythonInterp)
find_package(PythonLibsNew)
diff --git a/examples/basics/joinedArray/joinedArray_write.cpp b/examples/basics/joinedArray/joinedArray_write.cpp
index 73aa4fc6b251665d8c1f4f205a127c58755dc77f..d22438dda7dfe985b869d9abc8c9911efc186ad0 100644
--- a/examples/basics/joinedArray/joinedArray_write.cpp
+++ b/examples/basics/joinedArray/joinedArray_write.cpp
@@ -70,6 +70,8 @@ int main(int argc, char *argv[])
// Get io settings from the config file or
// create one with default settings here
adios2::IO &io = adios.DeclareIO("Output");
+ // io.SetEngine("ADIOS1Writer");
+ // io.AddTransport("File", {{"library", "MPI"}});
/*
* Define joinable local array: type, name, global and local size
diff --git a/examples/heatTransfer/read/PrintData.cpp b/examples/heatTransfer/read/PrintData.cpp
index ef84e6a40a69e1ad1c37514c69a0843093055e06..edbc6688fba93569b4d3eb9b74190735d9b21be7 100644
--- a/examples/heatTransfer/read/PrintData.cpp
+++ b/examples/heatTransfer/read/PrintData.cpp
@@ -15,6 +15,44 @@
#include "PrintData.h"
+void printData(double *xy, size_t *size, size_t *offset, int rank, int steps)
+{
+ std::ofstream myfile;
+ std::string filename = "data." + std::to_string(rank);
+ myfile.open(filename);
+ double *data = xy;
+ uint64_t nelems = size[0] * size[1];
+ for (int step = 0; step < steps; step++)
+ {
+ myfile << "rank=" << rank << " size=" << size[0] << "x" << size[1]
+ << " offsets=" << offset[0] << ":" << offset[1]
+ << " step=" << step << std::endl;
+
+ myfile << " time row columns " << offset[1] << "..."
+ << offset[1] + size[1] - 1 << std::endl;
+ myfile << " ";
+ for (int j = 0; j < size[1]; j++)
+ {
+ myfile << std::setw(9) << offset[1] + j;
+ }
+ myfile << std::endl;
+ myfile << "------------------------------------------------------------"
+ "--\n";
+ for (int i = 0; i < size[0]; i++)
+ {
+ myfile << std::setw(5) << step << std::setw(5) << offset[0] + i;
+ for (int j = 0; j < size[1]; j++)
+ {
+ myfile << std::setw(9) << std::setprecision(2)
+ << data[i * size[1] + j];
+ }
+ myfile << std::endl;
+ }
+ data += nelems;
+ }
+ myfile.close();
+}
+
void printData(double *xy, uint64_t *size, uint64_t *offset, int rank,
int steps)
{
diff --git a/examples/heatTransfer/read/PrintData.h b/examples/heatTransfer/read/PrintData.h
index 308ace34007f88c15498464b1555b5cca3ed1f6b..13e1e527ce1a27036d0f65c540be48a459b3b3eb 100644
--- a/examples/heatTransfer/read/PrintData.h
+++ b/examples/heatTransfer/read/PrintData.h
@@ -13,6 +13,7 @@
#include <cstdint>
+void printData(double *xy, size_t *size, size_t *offset, int rank, int steps);
void printData(double *xy, uint64_t *size, uint64_t *offset, int rank,
int steps);
diff --git a/examples/heatTransfer/read/heatRead_adios2.cpp b/examples/heatTransfer/read/heatRead_adios2.cpp
index dd8ef0cf0e7038a99eae8e3d7a90c71ced2d9f15..35ec668372a3c950002659d1487cac773a4515c7 100644
--- a/examples/heatTransfer/read/heatRead_adios2.cpp
+++ b/examples/heatTransfer/read/heatRead_adios2.cpp
@@ -93,8 +93,8 @@ int main(int argc, char *argv[])
}
// 1D decomposition of the columns, which is inefficient for reading!
- std::vector<uint64_t> readsize({gndx, gndy / nproc});
- std::vector<uint64_t> offset({0LL, rank * readsize[1]});
+ adios2::Dims readsize({gndx, gndy / nproc});
+ adios2::Dims offset({0LL, rank * readsize[1]});
if (rank == nproc - 1)
{
// last process should read all the rest of columns
diff --git a/examples/hello/CMakeLists.txt b/examples/hello/CMakeLists.txt
index 43b9e6a4bc0f6d23aef3571dc119cb7a67579a1e..2850407da804fce34afa2c23bdfce603865e4795 100644
--- a/examples/hello/CMakeLists.txt
+++ b/examples/hello/CMakeLists.txt
@@ -19,3 +19,12 @@ endif()
if(ADIOS2_HAVE_HDF5)
add_subdirectory(hdf5Writer)
endif()
+
+if(ADIOS2_HAVE_BZip2)
+ add_subdirectory(bpBZip2Wrapper)
+endif()
+
+if(ADIOS2_HAVE_ZFP)
+ add_subdirectory(bpZfpWrapper)
+endif()
+
diff --git a/examples/hello/bpBZip2Wrapper/CMakeLists.txt b/examples/hello/bpBZip2Wrapper/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3803a998a7103673cdee602633c71a03a049da24
--- /dev/null
+++ b/examples/hello/bpBZip2Wrapper/CMakeLists.txt
@@ -0,0 +1,18 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0. See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+if(ADIOS2_HAVE_MPI)
+ find_package(MPI COMPONENTS C REQUIRED)
+
+ add_executable(hello_bpBZip2Wrapper helloBPBZip2Wrapper.cpp)
+ target_include_directories(hello_bpBZip2Wrapper PRIVATE ${MPI_C_INCLUDE_PATH})
+ target_link_libraries(hello_bpBZip2Wrapper ${MPI_C_LIBRARIES})
+
+else()
+ add_executable(hello_bpBZip2Wrapper helloBPBZip2Wrapper_nompi.cpp)
+
+endif()
+
+target_link_libraries(hello_bpBZip2Wrapper adios2)
diff --git a/examples/hello/bpBZip2Wrapper/helloBPBZip2Wrapper.cpp b/examples/hello/bpBZip2Wrapper/helloBPBZip2Wrapper.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bb425051ccd39c7bfdbb56f1925f1d014476be42
--- /dev/null
+++ b/examples/hello/bpBZip2Wrapper/helloBPBZip2Wrapper.cpp
@@ -0,0 +1,120 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * helloBPBZip2Wrapper.cpp: Simple self-descriptive example of how to write a
+ * variable to a BP File that lives in several MPI processes and is compressed
+ * with BZip2 http://www.bzip.org/
+ *
+ * Created on: Jul 26, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+#include <mpi.h>
+
+#include <ios> //std::ios_base::failure
+#include <iostream> //std::cout
+#include <numeric> //std::iota
+#include <stdexcept> //std::invalid_argument std::exception
+#include <vector>
+
+#include <adios2.h>
+
+int main(int argc, char *argv[])
+{
+ MPI_Init(&argc, &argv);
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ /** Application variable uints from 0 to 1000 */
+ std::vector<unsigned int> myUInts(1000);
+ std::iota(myUInts.begin(), myUInts.end(), 0.f);
+ const std::size_t Nx = myUInts.size();
+ const std::size_t inputBytes = Nx * sizeof(unsigned int);
+
+ try
+ {
+ /** ADIOS class factory of IO class objects, DebugON is recommended */
+ adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON);
+
+ // Get a Transform of type BZip2
+ adios2::Transform &adiosBZip2 = adios.GetTransform("bzip2");
+
+ /*** IO class object: settings and factory of Settings: Variables,
+ * Parameters, Transports, and Execution: Engines */
+ adios2::IO &bpIO = adios.DeclareIO("BPFile_N2N_BZip2");
+
+ /** global array : name, { shape (total) }, { start (local) }, { count
+ * (local) }, all are constant dimensions */
+ adios2::Variable<unsigned int> &bpUInts =
+ bpIO.DefineVariable<unsigned int>("bpUInts", {size * Nx},
+ {rank * Nx}, {Nx},
+ adios2::ConstantDims);
+
+ // 1st way: adding transform metadata to variable to Engine can decide:
+ // &adiosBZip2 gets mapped to bpUInts.TransformInfo[bzip2ID].Operator
+ const unsigned int bzip2ID =
+ bpUInts.AddTransform(adiosBZip2, {{"BlockSize100K", "9"}});
+
+ // 2nd way: treat Transforms as wrappers to underlying library.
+ // you can redefine parameters
+ const std::size_t estimatedSize =
+ adiosBZip2.BufferMaxSize(Nx * bpUInts.m_ElementSize);
+ std::vector<char> compressedBuffer(estimatedSize);
+ size_t compressedSize = adiosBZip2.Compress(
+ myUInts.data(), bpUInts.m_Count, bpUInts.m_ElementSize,
+ bpUInts.m_Type, compressedBuffer.data(), {{"BlockSize100K", "1"}});
+
+ compressedBuffer.resize(compressedSize);
+
+ std::cout << "Rank " << rank << "\n";
+ std::cout << "Compression summary:\n";
+ std::cout << "Input data size: " << inputBytes << " bytes\n";
+ std::cout << "BZip2 estimated output size: " << estimatedSize
+ << " bytes\n";
+ std::cout << "BZip2 final output size: " << compressedSize
+ << " bytes\n\n";
+
+ // Allocate original data size
+ std::vector<unsigned int> decompressedBuffer(Nx);
+ size_t decompressedSize = adiosBZip2.Decompress(
+ compressedBuffer.data(), compressedSize, decompressedBuffer.data(),
+ decompressedBuffer.size() * sizeof(unsigned int));
+
+ std::cout << "Decompression summary:\n";
+ std::cout << "Decompressed size: " << decompressedSize << " bytes\n";
+ std::cout << "Data:\n";
+
+ for (const auto number : decompressedBuffer)
+ {
+ if (number % 25 == 0)
+ {
+ std::cout << "\n";
+ }
+ std::cout << number << " ";
+ }
+ std::cout << "\n";
+ }
+ catch (std::invalid_argument &e)
+ {
+ std::cout << "Invalid argument exception, STOPPING PROGRAM from rank "
+ << rank << "\n";
+ std::cout << e.what() << "\n";
+ }
+ catch (std::ios_base::failure &e)
+ {
+ std::cout
+ << "IO System base failure exception, STOPPING PROGRAM from rank "
+ << rank << "\n";
+ std::cout << e.what() << "\n";
+ }
+ catch (std::exception &e)
+ {
+ std::cout << "Exception, STOPPING PROGRAM from rank " << rank << "\n";
+ std::cout << e.what() << "\n";
+ }
+
+ MPI_Finalize();
+
+ return 0;
+}
diff --git a/examples/hello/bpBZip2Wrapper/helloBPBZip2Wrapper_nompi.cpp b/examples/hello/bpBZip2Wrapper/helloBPBZip2Wrapper_nompi.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fe7e2b7b027a031f5b63c3d61f94b2788763c6d9
--- /dev/null
+++ b/examples/hello/bpBZip2Wrapper/helloBPBZip2Wrapper_nompi.cpp
@@ -0,0 +1,103 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * helloBPBZip2Writer_nompi.cpp sequential non-mpi version of helloBPBZip2Writer
+ * using BZip2 http://www.bzip.org/
+ *
+ * Created on: Jul 26, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#include <ios> //std::ios_base::failure
+#include <iostream> //std::cout
+#include <numeric> //std::iota
+#include <stdexcept> //std::invalid_argument std::exception
+#include <vector>
+
+#include <adios2.h>
+
+int main(int argc, char *argv[])
+{
+ /** Application variable uints from 0 to 1000 */
+ std::vector<unsigned int> myUInts(1000);
+ std::iota(myUInts.begin(), myUInts.end(), 0.f);
+ const std::size_t Nx = myUInts.size();
+ const std::size_t inputBytes = Nx * sizeof(unsigned int);
+
+ try
+ {
+ /** ADIOS class factory of IO class objects, DebugON is recommended */
+ adios2::ADIOS adios(adios2::DebugON);
+
+ // Get a Transform of type BZip2
+ adios2::Transform &adiosBZip2 = adios.GetTransform("bzip2");
+
+ /*** IO class object: settings and factory of Settings: Variables,
+ * Parameters, Transports, and Execution: Engines */
+ adios2::IO &bpIO = adios.DeclareIO("BPFile_N2N_BZip2");
+
+ /** global array : name, { shape (total) }, { start (local) }, { count
+ * (local) }, all are constant dimensions */
+ adios2::Variable<unsigned int> &bpUInts =
+ bpIO.DefineVariable<unsigned int>("bpUInts", {}, {}, {Nx},
+ adios2::ConstantDims);
+
+ // 1st way: adding transform metadata to variable to Engine can decide:
+ bpUInts.AddTransform(adiosBZip2, {{"BlockSize100K", "10"}});
+
+ // 2nd way: treat Transforms as wrappers to underlying library:
+ const std::size_t estimatedSize =
+ adiosBZip2.BufferMaxSize(Nx * bpUInts.m_ElementSize);
+ std::vector<char> compressedBuffer(estimatedSize);
+ size_t compressedSize = adiosBZip2.Compress(
+ myUInts.data(), bpUInts.m_Count, bpUInts.m_ElementSize,
+ bpUInts.m_Type, compressedBuffer.data(), {{"BlockSize100K", "9"}});
+
+ compressedBuffer.resize(compressedSize);
+
+ std::cout << "Compression summary:\n";
+ std::cout << "Input data size: " << inputBytes << " bytes\n";
+ std::cout << "BZip2 estimated output size: " << estimatedSize
+ << " bytes\n";
+ std::cout << "BZip2 final output size: " << compressedSize
+ << " bytes\n\n";
+
+ // Allocate original data size
+ std::vector<unsigned int> decompressedBuffer(Nx);
+ size_t decompressedSize = adiosBZip2.Decompress(
+ compressedBuffer.data(), compressedSize, decompressedBuffer.data(),
+ decompressedBuffer.size() * sizeof(unsigned int));
+
+ std::cout << "Decompression summary:\n";
+ std::cout << "Decompressed size: " << decompressedSize << " bytes\n";
+ std::cout << "Data:\n";
+
+ for (const auto number : decompressedBuffer)
+ {
+ if (number % 25 == 0)
+ {
+ std::cout << "\n";
+ }
+ std::cout << number << " ";
+ }
+ std::cout << "\n";
+ }
+ catch (std::invalid_argument &e)
+ {
+ std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ catch (std::ios_base::failure &e)
+ {
+ std::cout << "IO System base failure exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ catch (std::exception &e)
+ {
+ std::cout << "Exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+
+ return 0;
+}
diff --git a/examples/hello/bpWriter/helloBPWriter_nompi.cpp b/examples/hello/bpWriter/helloBPWriter_nompi.cpp
index 7217e12b6ac48e8dafa6062593bb5d4f41c3ca8d..0a08af072a9e949079e48b2172e46c266368a1c1 100644
--- a/examples/hello/bpWriter/helloBPWriter_nompi.cpp
+++ b/examples/hello/bpWriter/helloBPWriter_nompi.cpp
@@ -29,7 +29,6 @@ int main(int argc, char *argv[])
/*** IO class object: settings and factory of Settings: Variables,
* Parameters, Transports, and Execution: Engines */
adios2::IO &bpIO = adios.DeclareIO("BPFile_N2N");
- bpIO.AddTransport("File", {{"Library", "stdio"}});
/** global array: name, { shape (total dimensions) }, { start (local) },
* { count (local) }, all are constant dimensions */
diff --git a/examples/hello/bpZfpWrapper/CMakeLists.txt b/examples/hello/bpZfpWrapper/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c5cb52b74f20dc3bdced1ea0e63a5a5373a33329
--- /dev/null
+++ b/examples/hello/bpZfpWrapper/CMakeLists.txt
@@ -0,0 +1,18 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0. See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+if(ADIOS2_HAVE_MPI)
+ find_package(MPI COMPONENTS C REQUIRED)
+
+ add_executable(hello_bpZfpWrapper helloBPZfpWrapper.cpp)
+ target_include_directories(hello_bpZfpWrapper PRIVATE ${MPI_C_INCLUDE_PATH})
+ target_link_libraries(hello_bpZfpWrapper ${MPI_C_LIBRARIES})
+
+else()
+ add_executable(hello_bpZfpWrapper helloBPZfpWrapper_nompi.cpp)
+
+endif()
+
+target_link_libraries(hello_bpZfpWrapper adios2)
diff --git a/examples/hello/bpZfpWrapper/helloBPZfpWrapper.cpp b/examples/hello/bpZfpWrapper/helloBPZfpWrapper.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a9227e20c7c266d94f61e64eb71bbac290e48476
--- /dev/null
+++ b/examples/hello/bpZfpWrapper/helloBPZfpWrapper.cpp
@@ -0,0 +1,122 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * helloBPZfpWrapper.cpp: Simple self-descriptive example of how to write a
+ * variable to a BP File that lives in several MPI processes and is compressed
+ * with Zfp https://computation.llnl.gov/projects/floating-point-compression
+ *
+ * Created on: Jul 26, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+#include <mpi.h>
+
+#include <cstdint> //std::int32_t
+#include <ios> //std::ios_base::failure
+#include <iostream> //std::cout
+#include <numeric> //std::iota
+#include <stdexcept> //std::invalid_argument std::exception
+#include <vector>
+
+#include <adios2.h>
+
+int main(int argc, char *argv[])
+{
+ MPI_Init(&argc, &argv);
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ /** Application variable uints from 0 to 100 */
+ std::vector<float> myFloats(100);
+ std::iota(myFloats.begin(), myFloats.end(), 0.f);
+ const std::size_t Nx = myFloats.size();
+ const std::size_t inputBytes = Nx * sizeof(float);
+
+ try
+ {
+ /** ADIOS class factory of IO class objects, DebugON is recommended */
+ adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON);
+
+ // Get a Transform of type BZip2
+ adios2::Transform &adiosZfp = adios.GetTransform("zfp");
+
+ /*** IO class object: settings and factory of Settings: Variables,
+ * Parameters, Transports, and Execution: Engines */
+ adios2::IO &bpIO = adios.DeclareIO("BPFile_N2N_Zfp");
+
+ /** global array : name, { shape (total) }, { start (local) }, { count
+ * (local) }, all are constant dimensions */
+ adios2::Variable<float> &bpFloats = bpIO.DefineVariable<float>(
+ "bpFloats", {size * Nx}, {rank * Nx}, {Nx}, adios2::ConstantDims);
+
+ // 1st way: adding transform metadata to variable to Engine can decide:
+ const unsigned int zfpID =
+ bpFloats.AddTransform(adiosZfp, {{"Rate", "8"}});
+
+ // 2nd way: treat Transforms as wrappers to underlying library:
+ const std::size_t estimatedSize =
+ adiosZfp.BufferMaxSize(myFloats.data(), bpFloats.m_Count,
+ bpFloats.m_TransformsInfo[zfpID].Parameters);
+
+ // Compress
+ std::vector<char> compressedBuffer(estimatedSize);
+
+ size_t compressedSize = adiosZfp.Compress(
+ myFloats.data(), bpFloats.m_Count, bpFloats.m_ElementSize,
+ bpFloats.m_Type, compressedBuffer.data(),
+ bpFloats.m_TransformsInfo[zfpID].Parameters);
+
+ compressedBuffer.resize(compressedSize);
+
+ std::cout << "Compression summary:\n";
+ std::cout << "Input data size: " << inputBytes << " bytes\n";
+ std::cout << "Zfp estimated output size: " << estimatedSize
+ << " bytes\n";
+ std::cout << "Zfp final output size: " << compressedSize
+ << " bytes\n\n";
+
+ // Allocate original data size
+ std::vector<float> decompressedBuffer(Nx);
+ size_t decompressedSize = adiosZfp.Decompress(
+ compressedBuffer.data(), compressedBuffer.size(),
+ decompressedBuffer.data(), bpFloats.m_Count, bpFloats.m_Type,
+ bpFloats.m_TransformsInfo[zfpID].Parameters);
+
+ std::cout << "Decompression summary:\n";
+ std::cout << "Decompressed size: " << decompressedSize << " bytes\n ";
+ std::cout << "Data:\n";
+
+ for (const auto number : decompressedBuffer)
+ {
+ if (static_cast<int>(number) % 25 == 0 && number != 0)
+ {
+ std::cout << "\n";
+ }
+ std::cout << number << " ";
+ }
+ std::cout << "\n";
+ }
+ catch (std::invalid_argument &e)
+ {
+ std::cout << "Invalid argument exception, STOPPING PROGRAM from rank "
+ << rank << "\n";
+ std::cout << e.what() << "\n";
+ }
+ catch (std::ios_base::failure &e)
+ {
+ std::cout
+ << "IO System base failure exception, STOPPING PROGRAM from rank "
+ << rank << "\n";
+ std::cout << e.what() << "\n";
+ }
+ catch (std::exception &e)
+ {
+ std::cout << "Exception, STOPPING PROGRAM from rank " << rank << "\n";
+ std::cout << e.what() << "\n";
+ }
+
+ MPI_Finalize();
+
+ return 0;
+}
diff --git a/examples/hello/bpZfpWrapper/helloBPZfpWrapper_nompi.cpp b/examples/hello/bpZfpWrapper/helloBPZfpWrapper_nompi.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e84feee7b8c75401da3ee360ac22b29a291ce70f
--- /dev/null
+++ b/examples/hello/bpZfpWrapper/helloBPZfpWrapper_nompi.cpp
@@ -0,0 +1,114 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * helloBPZfpWriter_nompi.cpp sequential non-mpi version of helloBPZfpWrapper
+ * using Zfp https://computation.llnl.gov/projects/floating-point-compression
+ *
+ * Created on: Jul 26, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#include <cstdint> //std::int32_t
+#include <ios> //std::ios_base::failure
+#include <iostream> //std::cout
+#include <numeric> //std::iota
+#include <stdexcept> //std::invalid_argument std::exception
+#include <vector>
+
+#include <adios2.h>
+
+int main(int argc, char *argv[])
+{
+ /** Application variable uints from 0 to 100 */
+ std::vector<float> myFloats(100);
+ std::iota(myFloats.begin(), myFloats.end(), 0.f);
+ const std::size_t Nx = myFloats.size();
+ const std::size_t inputBytes = Nx * sizeof(float);
+
+ try
+ {
+ /** ADIOS class factory of IO class objects, DebugON is recommended */
+ adios2::ADIOS adios(adios2::DebugON);
+
+ // Get a Transform of type BZip2
+ adios2::Transform &adiosZfp = adios.GetTransform("zfp");
+
+ /*** IO class object: settings and factory of Settings: Variables,
+ * Parameters, Transports, and Execution: Engines */
+ adios2::IO &bpIO = adios.DeclareIO("BPVariable_Zfp");
+
+ /** global array : name, { shape (total) }, { start (local) }, { count
+ * (local) }, all are constant dimensions */
+ adios2::Variable<float> &bpFloats = bpIO.DefineVariable<float>(
+ "bpUInts", {}, {}, {Nx}, adios2::ConstantDims);
+
+ // Adding transform metadata to variable:
+ // Options:
+ // Rate double
+ // Tolerance
+ // Precision
+ const unsigned int zfpID =
+ bpFloats.AddTransform(adiosZfp, {{"Rate", "8"}});
+
+ // Treat Transforms as wrappers to underlying library (zfp):
+ const std::size_t estimatedSize =
+ adiosZfp.BufferMaxSize(myFloats.data(), bpFloats.m_Count,
+ bpFloats.m_TransformsInfo[zfpID].Parameters);
+
+ // Compress
+ std::vector<char> compressedBuffer(estimatedSize);
+
+ size_t compressedSize = adiosZfp.Compress(
+ myFloats.data(), bpFloats.m_Count, bpFloats.m_ElementSize,
+ bpFloats.m_Type, compressedBuffer.data(),
+ bpFloats.m_TransformsInfo[zfpID].Parameters);
+
+ compressedBuffer.resize(compressedSize);
+
+ std::cout << "Compression summary:\n";
+ std::cout << "Input data size: " << inputBytes << " bytes\n";
+ std::cout << "Zfp estimated output size: " << estimatedSize
+ << " bytes\n";
+ std::cout << "Zfp final output size: " << compressedSize
+ << " bytes\n\n";
+
+ // Decompress, allocate original data size
+ std::vector<float> decompressedBuffer(Nx);
+ size_t decompressedSize = adiosZfp.Decompress(
+ compressedBuffer.data(), compressedBuffer.size(),
+ decompressedBuffer.data(), bpFloats.m_Count, bpFloats.m_Type,
+ bpFloats.m_TransformsInfo[zfpID].Parameters);
+
+ std::cout << "Decompression summary:\n";
+ std::cout << "Decompressed size: " << decompressedSize << " bytes\n ";
+ std::cout << "Data:\n";
+
+ for (const auto number : decompressedBuffer)
+ {
+ if (static_cast<int>(number) % 25 == 0 && number != 0)
+ {
+ std::cout << "\n";
+ }
+ std::cout << number << " ";
+ }
+ std::cout << "\n";
+ }
+ catch (std::invalid_argument &e)
+ {
+ std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ catch (std::ios_base::failure &e)
+ {
+ std::cout << "IO System base failure exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ catch (std::exception &e)
+ {
+ std::cout << "Exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+
+ return 0;
+}
diff --git a/source/adios2/ADIOSConfig.h.in b/source/adios2/ADIOSConfig.h.in
index 7bfbd537dec1992ed8b55ba1f4fdfe954d64218b..44b9128c1f50b81c4d0458fe163833c1138aecdb 100644
--- a/source/adios2/ADIOSConfig.h.in
+++ b/source/adios2/ADIOSConfig.h.in
@@ -27,10 +27,10 @@
/* CMake Option: ADIOS_USE_MPI=OFF */
#cmakedefine ADIOS2_HAVE_MPI
-/* CMake Option: ADIOS_USE_ZFP=OFF */
+/* CMake Option: ADIOS_USE_ZFP=ON */
#cmakedefine ADIOS2_HAVE_ZFP
-/* CMake Option: ADIOS_USE_BZip2=OFF */
+/* CMake Option: ADIOS_USE_BZip2=ON */
#cmakedefine ADIOS2_HAVE_BZIP2
/* CMake Option: ADIOS_USE_ADIOS1=ON */
diff --git a/source/adios2/ADIOSMacros.h b/source/adios2/ADIOSMacros.h
index 8f1471a4a7067ea274f5620ba439320856679df0..f3043732b588cbde469d3b5f9e06adcc85afa03b 100644
--- a/source/adios2/ADIOSMacros.h
+++ b/source/adios2/ADIOSMacros.h
@@ -56,4 +56,10 @@
MACRO(float) \
MACRO(double)
+#define ADIOS2_FOREACH_ZFP_TYPE_1ARG(MACRO) \
+ MACRO(int32_t) \
+ MACRO(int64_t) \
+ MACRO(float) \
+ MACRO(double)
+
#endif /* ADIOS2_ADIOSMACROS_H */
diff --git a/source/adios2/ADIOSTypes.h b/source/adios2/ADIOSTypes.h
index 352d62ef2c0ade4981f629e0c65c8252639de082..bb806e56960f57681c2edd4ff500a94fc67103d5 100644
--- a/source/adios2/ADIOSTypes.h
+++ b/source/adios2/ADIOSTypes.h
@@ -144,7 +144,7 @@ enum class SelectionType
// adios defaults
#ifdef _WIN32
-const std::string DefaultFileLibrary("stdio");
+const std::string DefaultFileLibrary("fstream");
#else
const std::string DefaultFileLibrary("POSIX");
#endif
@@ -160,7 +160,7 @@ constexpr size_t DefaultMaxBufferSize(std::numeric_limits<size_t>::max() - 1);
/** default buffer growth factor. Needs to be studied
* for optimizing applications*/
-constexpr float DefaultBufferGrowthFactor(1.05);
+constexpr float DefaultBufferGrowthFactor(1.05f);
/** default size for writing/reading files using POSIX/fstream/stdio write
* 2Gb - 100Kb (tolerance)*/
diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt
index da981d4b4e66e8c6e7f946f00e88c47ea076fbf9..f0891b33b70a60ff70f7a3d075c4ba6404c76f28 100644
--- a/source/adios2/CMakeLists.txt
+++ b/source/adios2/CMakeLists.txt
@@ -10,13 +10,14 @@ add_library(adios2
core/Selection.cpp
core/SelectionBoundingBox.cpp
core/SelectionPoints.cpp
- core/Transform.cpp
+ core/Transform.cpp core/Transform.tcc
core/Variable.cpp core/Variable.tcc
core/VariableBase.cpp
core/VariableCompound.cpp core/VariableCompound.tcc
#helper
helper/adiosMath.cpp
+ helper/adiosMPIFunctions.cpp
helper/adiosString.cpp
helper/adiosSystem.cpp
helper/adiosType.cpp
@@ -79,15 +80,14 @@ endif()
if(ADIOS2_HAVE_BZip2)
find_package(BZip2 REQUIRED)
- target_sources(adios2 PRIVATE transform/compression/BZip2.cpp)
+ target_sources(adios2 PRIVATE transform/compress/CompressBZip2.cpp)
target_link_libraries(adios2 PRIVATE BZip2::BZip2)
endif()
if(ADIOS2_HAVE_ZFP)
find_package(ZFP REQUIRED)
- message("ADIOS ZFP support not yet implemented")
-# target_sources(adios2 PRIVATE transform/compression/ZFP.cpp)
-# target_link_libraries(adios2 PRIVATE zfp::zfp)
+ target_sources(adios2 PRIVATE transform/compress/CompressZfp.cpp)
+ target_link_libraries(adios2 PRIVATE zfp::zfp)
endif()
if(ADIOS2_HAVE_MPI)
diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp
index af7020254e84fd396c9e4c8be71c984deef9b8f0..d4853c603d064962abc543a8caa94dfc36a01643 100644
--- a/source/adios2/core/ADIOS.cpp
+++ b/source/adios2/core/ADIOS.cpp
@@ -21,6 +21,15 @@
#include "adios2/ADIOSMPI.h"
#include "adios2/helper/adiosFunctions.h"
+// transforms
+#ifdef ADIOS2_HAVE_BZIP2
+#include "adios2/transform/compress/CompressBZip2.h"
+#endif
+
+#ifdef ADIOS2_HAVE_ZFP
+#include "adios2/transform/compress/CompressZfp.h"
+#endif
+
namespace adios2
{
@@ -86,11 +95,59 @@ IO &ADIOS::GetIO(const std::string name)
{
throw std::invalid_argument(
"ERROR: Unable to find previously defined IO object with name \"" +
- name + "\" in call to GetIO.");
+ name + "\", in call to GetIO.");
}
return itIO->second;
}
+Transform &ADIOS::GetTransform(const std::string transform)
+{
+ auto itTransform = m_Transforms.find(transform);
+
+ if (itTransform != m_Transforms.end())
+ {
+ return *itTransform->second.get();
+ }
+
+ if (transform == "bzip2" || transform == "BZip2")
+ {
+#ifdef ADIOS2_HAVE_BZIP2
+ auto itPair = m_Transforms.emplace(
+ "bzip2",
+ std::make_shared<adios2::transform::CompressBZip2>(m_DebugMode));
+ return *itPair.first->second;
+#else
+ throw std::invalid_argument(
+ "ERROR: this version of ADIOS2 didn't compile with "
+ "bzip2 library, in call to GetTransport\n");
+#endif
+ }
+ else if (transform == "zfp" || transform == "Zfp")
+ {
+#ifdef ADIOS2_HAVE_ZFP
+ auto itPair = m_Transforms.emplace(
+ "zfp",
+ std::make_shared<adios2::transform::CompressZfp>(m_DebugMode));
+ return *itPair.first->second;
+#else
+ throw std::invalid_argument(
+ "ERROR: this version of ADIOS2 didn't compile with "
+ "zfp library, in call to GetTransport\n");
+#endif
+ }
+ else
+ {
+ if (m_DebugMode)
+ {
+ throw std::invalid_argument(
+ "ERROR: transform " + transform +
+ " not supported by ADIOS2, in call to GetTransport\n");
+ }
+ }
+
+ return *itTransform->second.get();
+}
+
// PRIVATE FUNCTIONS
void ADIOS::CheckMPI() const
{
diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h
index c9e9937076f33f3d15f06c0d4cc02f304f38a2ef..eba4e72430fa5b313253fb1fc8793c0f0adeef29 100644
--- a/source/adios2/core/ADIOS.h
+++ b/source/adios2/core/ADIOS.h
@@ -33,7 +33,9 @@ class ADIOS
public:
/** Passed from parallel constructor, MPI_Comm is a pointer itself. */
MPI_Comm m_MPIComm;
- std::string m_HostLanguage = "C++"; ///< changed by language bindings
+
+ /** Changed by language bindings */
+ std::string m_HostLanguage = "C++";
/**
* @brief Constructor for MPI applications WITH a XML config file
@@ -43,7 +45,7 @@ public:
* @param debugMode true: extra exception checks (recommended)
*/
ADIOS(const std::string configFile, MPI_Comm mpiComm,
- const bool debugMode = false);
+ const bool debugMode = true);
/**
* @brief Constructor for non-MPI applications WITH a XML config file
@@ -51,20 +53,27 @@ public:
* future?)
* @param debugMode true: extra exception checks (recommended)
*/
- ADIOS(const std::string configFile, const bool debugMode = false);
+ ADIOS(const std::string configFile, const bool debugMode = true);
/**
* @brief Constructor for MPI apps WITHOUT a XML config file
* @param mpiComm MPI communicator from application
* @param debugMode true: extra exception checks (recommended)
*/
- ADIOS(MPI_Comm mpiComm, const bool debugMode = false);
+ ADIOS(MPI_Comm mpiComm, const bool debugMode = true);
/**
* @brief ADIOS no-MPI default empty constructor
* @param debugMode true: extra exception checks (recommended)
*/
- ADIOS(const bool debugMode = false);
+ ADIOS(const bool debugMode = true);
+
+ /**
+ * Delete copy constructor explicitly. Objects shouldn't be allowed to be
+ * redefined. Use smart pointers if this is absolutely necessary.
+ * @param adios reference to another adios object
+ */
+ ADIOS(const ADIOS &adios) = delete;
~ADIOS() = default;
@@ -81,19 +90,23 @@ public:
IO &DeclareIO(const std::string ioName);
/**
- * Retrieve an already defined IO object
+ * Retrieve an already defined IO object. Make sure IO was previously
+ * created with DeclareIO. Otherwise, throws an exception in debug mode.
+ * @return reference to existing IO object inside ADIOS
*/
IO &GetIO(const std::string name);
-protected: // no const member to allow default empty and copy constructors
+ Transform &GetTransform(const std::string transform);
+
+private:
/** XML File to be read containing configuration information */
- std::string m_ConfigFile;
+ const std::string m_ConfigFile;
/** if true will do more checks, exceptions, warnings, expect slower code */
- bool m_DebugMode = false;
+ const bool m_DebugMode = true;
/** transforms associated with ADIOS run */
- std::vector<std::shared_ptr<Transform>> m_Transforms;
+ std::map<std::string, std::shared_ptr<Transform>> m_Transforms;
/**
* @brief List of IO class objects defined from either ADIOS
diff --git a/source/adios2/core/Engine.inl b/source/adios2/core/Engine.inl
index 1dde7f3c449f7dbb862113200883e84193ec4324..dccf770fd98215718fb83fdf6c0ba771827d1b8f 100644
--- a/source/adios2/core/Engine.inl
+++ b/source/adios2/core/Engine.inl
@@ -29,7 +29,7 @@ void Engine::Write(Variable<T> &variable, const T *values)
{
if (m_DebugMode)
{
- variable.CheckDims("in call to Write");
+ variable.CheckDimsBeforeWrite("Write(" + variable.m_Name + ")");
}
DoWrite(variable, values);
diff --git a/source/adios2/core/Transform.cpp b/source/adios2/core/Transform.cpp
index dc4079aed7aa725493c67f6eab8cda3c1f4617de..02973f2460c664fc9f214a9b6692728ed4c91229 100644
--- a/source/adios2/core/Transform.cpp
+++ b/source/adios2/core/Transform.cpp
@@ -9,20 +9,90 @@
*/
#include "Transform.h"
+#include "Transform.tcc"
namespace adios2
{
-Transform::Transform(const std::string method) : m_Method(method) {}
+Transform::Transform(const std::string library, const bool debugMode)
+: m_Library(library), m_DebugMode(debugMode)
+{
+}
+
+size_t Transform::BufferMaxSize(const size_t sizeIn) const
+{
+ if (m_DebugMode)
+ {
+ throw std::invalid_argument(
+ "ERROR: signature (const size_t) not supported "
+ "by derived class implemented with " +
+ m_Library + ", in call to BufferMaxSize\n");
+ }
+ return 0;
+}
+
+size_t Transform::Compress(const void * /*dataIn*/, const Dims & /*dimensions*/,
+ const size_t /*elementSize*/,
+ const std::string /*type*/, void * /*bufferOut*/,
+ const Params & /*params*/) const
+{
+ if (m_DebugMode)
+ {
+ throw std::invalid_argument("ERROR: signature (const void*, const "
+ "Dims, const size_t, const std::string, "
+ "void*, const Params&) not supported "
+ "by derived class implemented with " +
+ m_Library + ", in call to Compress\n");
+ }
+ return 0;
+}
-void Transform::Compress(const std::vector<char> & /*bufferIn*/,
- std::vector<char> & /*bufferOut*/)
+size_t Transform::Decompress(const void *bufferIn, const size_t sizeIn,
+ void *dataOut, const size_t sizeOut) const
{
+ if (m_DebugMode)
+ {
+ throw std::invalid_argument(
+ "ERROR: signature (const void*, const size_t, void) not supported "
+ "by derived class implemented with " +
+ m_Library + ", in call to Decompress\n");
+ }
+
+ return 0;
}
-void Transform::Decompress(const std::vector<char> & /*bufferIn*/,
- std::vector<char> & /*bufferOut*/)
+size_t Transform::Decompress(const void * /*bufferIn*/, const size_t /*sizeIn*/,
+ void * /*dataOut*/, const Dims & /*dimensions*/,
+ const std::string /*type*/,
+ const Params & /*parameters*/) const
{
+ if (m_DebugMode)
+ {
+ throw std::invalid_argument("ERROR: signature (const void*, const "
+ "size_t, void*, const Dims&, const "
+ "std::string ) not supported "
+ "by derived class implemented with " +
+ m_Library + ", in call to Decompress\n");
+ }
+
+ return 0;
+}
+
+// PROTECTED
+size_t Transform::DoBufferMaxSize(const void *dataIn, const Dims &dimensions,
+ const std::string type,
+ const Params ¶meters) const
+{
+ if (m_DebugMode)
+ {
+ throw std::invalid_argument(
+ "ERROR: signature (const void*, const Dims& "
+ "std::string ) not supported "
+ "by derived class implemented with " +
+ m_Library + ", in call to BufferMaxSize\n");
+ }
+
+ return 0;
}
-} // end namespace adios
+} // end namespace adios2
diff --git a/source/adios2/core/Transform.h b/source/adios2/core/Transform.h
index 32c857293ef9d7ee29ecb02abda652c81d9d261b..19d617432da299dfd62ac0e2f279946c1b8e31ff 100644
--- a/source/adios2/core/Transform.h
+++ b/source/adios2/core/Transform.h
@@ -16,33 +16,96 @@
#include <vector>
/// \endcond
-#include "adios2/ADIOSConfig.h"
+#include "adios2/ADIOSTypes.h"
namespace adios2
{
-/** Base class that defines data variable transformations */
+/** Base class that defines data variable transformations implemented under
+ * adios2/transform */
class Transform
{
public:
- /** From derived class to identify the transform */
- const std::string m_Method;
+ /** From derived class */
+ const std::string m_Library;
/**
- * Initialize parent method
- * @param method zlib, bzip2, szip
+ * Unique base class constructor
+ * @param method bzip2, zfp
+ * @param debugMode true: extra exceptions checks
*/
- Transform(const std::string method);
+ Transform(const std::string library, const bool debugMode);
virtual ~Transform() = default;
- virtual void Compress(const std::vector<char> &bufferIn,
- std::vector<char> &bufferOut);
+ /**
+ * Returns a conservative buffer size to hold input data for classes
+ * @param sizeIn size of input data to be compressed in bytes
+ * @return recommended allocation for output buffer
+ */
+ virtual size_t BufferMaxSize(const size_t sizeIn) const;
+
+ /**
+ * Used by Zfp
+ * Returns a conservative buffer size to hold input data for classes
+ * @param dataIn
+ * @param dimensions
+ * @return recommended allocation for output buffer in bytes
+ */
+ template <class T>
+ size_t BufferMaxSize(const T *dataIn, const Dims &dimensions,
+ const Params ¶ms) const;
+
+ /**
+ * BZip2 and Zfp common call
+ * @param dataIn
+ * @param dimensions
+ * @param elementSize
+ * @param type
+ * @param bufferOut
+ * @param parameters
+ * @return size of compressed buffer
+ */
+ virtual size_t Compress(const void *dataIn, const Dims &dimensions,
+ const size_t elementSize, const std::string type,
+ void *bufferOut,
+ const Params ¶meters = Params()) const;
- virtual void Decompress(const std::vector<char> &bufferIn,
- std::vector<char> &bufferOut);
+ virtual size_t Decompress(const void *bufferIn, const size_t sizeIn,
+ void *dataOut, const size_t sizeOut) const;
+
+ /**
+ * Zfp signature
+ * @param bufferIn
+ * @param sizeIn
+ * @param dataOut
+ * @param dimensions
+ * @param type
+ * @return
+ */
+ virtual size_t Decompress(const void *bufferIn, const size_t sizeIn,
+ void *dataOut, const Dims &dimensions,
+ const std::string type,
+ const Params ¶meters) const;
+
+protected:
+ /** true: extra exception checks, false: skip exception checks */
+ const bool m_DebugMode = false;
+
+ /**
+ * Used by CompressZfp
+ * Returns a conservative buffer size to hold input data for classes
+ * @param dataIn
+ * @param dimensions
+ * @param type
+ * @return conservative buffer size for allocation
+ */
+ virtual size_t DoBufferMaxSize(const void *dataIn, const Dims &dimensions,
+ const std::string type,
+ const Params ¶meters) const;
};
-} // end namespace adios
+} // end namespace adios2
+
#endif /* ADIOS2_CORE_TRANSFORM_H_ */
diff --git a/source/adios2/core/Transform.tcc b/source/adios2/core/Transform.tcc
new file mode 100644
index 0000000000000000000000000000000000000000..3fac195fda0603970a939ab6f817c9c553c5079f
--- /dev/null
+++ b/source/adios2/core/Transform.tcc
@@ -0,0 +1,35 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * Transform.tcc : specialization of template functions
+ *
+ * Created on: Jul 25, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#ifndef ADIOS2_CORE_TRANSFORM_TCC_
+#define ADIOS2_CORE_TRANSFORM_TCC_
+
+#include "Transform.h"
+
+#include "adios2/ADIOSMacros.h"
+#include "adios2/helper/adiosFunctions.h"
+
+namespace adios2
+{
+
+#define declare_type(T) \
+ template <> \
+ size_t Transform::BufferMaxSize<T>( \
+ const T *dataIn, const Dims &dimensions, const Params ¶meters) \
+ const \
+ { \
+ return DoBufferMaxSize(dataIn, dimensions, GetType<T>(), parameters); \
+ }
+ADIOS2_FOREACH_ZFP_TYPE_1ARG(declare_type)
+#undef declare_type
+
+} // end namespace adios2
+
+#endif // ADIOS2_CORE_TRANSFORM_TCC_
diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp
index 3a13c739ad5ea212d8f00815c85d6170f01e95ed..d98f572d15db6b399d008e6d5c4364cbbd29f91c 100644
--- a/source/adios2/core/VariableBase.cpp
+++ b/source/adios2/core/VariableBase.cpp
@@ -60,8 +60,7 @@ void VariableBase::SetSelection(const Dims start, const Dims count)
}
if (m_ShapeID == ShapeID::GlobalArray &&
- (m_Shape.size() != m_Count.size() ||
- m_Shape.size() != m_Start.size()))
+ (m_Shape.size() != count.size() || m_Shape.size() != start.size()))
{
throw std::invalid_argument("ERROR: count and start must be the "
"same size as shape for variable " +
@@ -125,105 +124,176 @@ void VariableBase::SetStepSelection(const unsigned int startStep,
m_ReadNSteps = countStep;
}
-void VariableBase::AddTransform(
- Transform &transform, const std::vector<std::string> ¶metersVector)
+// transforms related functions
+unsigned int VariableBase::AddTransform(Transform &transform,
+ const Params ¶meters) noexcept
{
+ m_TransformsInfo.push_back(TransformInfo{transform, parameters});
+ return static_cast<unsigned int>(m_TransformsInfo.size() - 1);
}
-void VariableBase::AddTransform(Transform &transform,
- const Params ¶metersVector)
+void VariableBase::ResetTransformParameters(const unsigned int transformIndex,
+ const Params ¶meters)
{
+ if (m_DebugMode)
+ {
+ if (transformIndex < m_TransformsInfo.size())
+ {
+ m_TransformsInfo[transformIndex].Parameters = parameters;
+ }
+ }
+ else
+ {
+ m_TransformsInfo[transformIndex].Parameters = parameters;
+ }
}
-// transforms related functions
-void VariableBase::ClearTransforms() { m_TransformsInfo.clear(); }
+void VariableBase::ClearTransforms() noexcept { m_TransformsInfo.clear(); }
// PRIVATE
void VariableBase::InitShapeType()
{
- if (!m_Shape.empty() && m_Start.empty() && m_Count.empty())
+ if (!m_Shape.empty())
{
- if (m_DebugMode)
+ if (std::count(m_Shape.begin(), m_Shape.end(), JoinedDim) == 1)
{
- if (m_ConstantDims)
+ if (!m_Start.empty() &&
+ std::count(m_Start.begin(), m_Start.end(), 0) != m_Start.size())
{
- throw std::invalid_argument(
- "ERROR: isConstantShape (true) argument is invalid "
- "with empty start and count "
- "arguments\n");
+ throw std::invalid_argument("ERROR: The Start array must be "
+ "empty or full-zero when defining "
+ "a Joined Array in call to "
+ "DefineVariable " +
+ m_Name + "\n");
}
+ m_ShapeID = ShapeID::JoinedArray;
}
-
- m_ShapeID = ShapeID::GlobalArray;
- }
- else if (!m_Shape.empty() && m_Shape.size() == m_Start.size() &&
- m_Shape.size() == m_Count.size())
- {
- if (m_DebugMode)
+ else if (m_Start.empty() && m_Count.empty())
{
- auto lf_LargerThanError = [&](const unsigned int i,
- const std::string dims1,
- const std::string dims2) {
-
- const std::string iString(std::to_string(i));
- throw std::invalid_argument(
- "ERROR: " + dims1 + "[" + iString + "] > " + dims2 + "[" +
- iString + "], in DefineVariable " + m_Name + "\n");
- };
-
- for (unsigned int i = 0; i < m_Shape.size(); ++i)
+ if (m_Shape.size() == 1 && m_Shape.front() == LocalValueDim)
+ {
+ m_ShapeID = ShapeID::LocalValue;
+ m_SingleValue = true;
+ }
+ else
{
- if (m_Count[i] > m_Shape[i])
+ if (m_DebugMode)
{
- lf_LargerThanError(i, "count", "shape");
+ if (m_ConstantDims)
+ {
+ throw std::invalid_argument(
+ "ERROR: isConstantShape (true) argument is invalid "
+ "with empty start and count "
+ "arguments in call to "
+ "DefineVariable " +
+ m_Name + "\n");
+ }
}
- if (m_Start[i] > m_Shape[i])
+
+ m_ShapeID = ShapeID::GlobalArray;
+ }
+ }
+ else if (m_Shape.size() == m_Start.size() &&
+ m_Shape.size() == m_Count.size())
+ {
+ if (m_DebugMode)
+ {
+ auto lf_LargerThanError = [&](const unsigned int i,
+ const std::string dims1,
+ const std::string dims2) {
+
+ const std::string iString(std::to_string(i));
+ throw std::invalid_argument(
+ "ERROR: " + dims1 + "[" + iString + "] > " + dims2 +
+ "[" + iString + "], in DefineVariable " + m_Name +
+ "\n");
+ };
+
+ for (unsigned int i = 0; i < m_Shape.size(); ++i)
{
- lf_LargerThanError(i, "start", "shape");
+ if (m_Count[i] > m_Shape[i])
+ {
+ lf_LargerThanError(i, "count", "shape");
+ }
+ if (m_Start[i] > m_Shape[i])
+ {
+ lf_LargerThanError(i, "start", "shape");
+ }
}
}
+ m_ShapeID = ShapeID::GlobalArray;
+ }
+ else
+ {
+ throw std::invalid_argument("ERROR: the "
+ "combination of shape, start and count "
+ "arguments is inconsistent, in call to "
+ "DefineVariable " +
+ m_Name + "\n");
}
-
- m_ShapeID = ShapeID::GlobalArray;
- }
- else if (m_Shape.empty() && m_Start.empty() && m_Count.empty())
- {
- m_ShapeID = ShapeID::GlobalValue;
- m_SingleValue = true;
- }
- else if (m_Shape.empty() && m_Start.empty() && !m_Count.empty())
- {
- m_ShapeID = ShapeID::LocalArray;
- }
- else if (m_Shape.size() == 1 && m_Shape.front() == LocalValueDim)
- {
- m_ShapeID = ShapeID::LocalValue;
- m_SingleValue = true;
}
- else if (!m_Shape.empty() &&
- std::count(m_Shape.begin(), m_Shape.end(), JoinedDim) == 1)
+ else //(m_Shape.empty())
{
- m_ShapeID = ShapeID::JoinedArray;
+ if (m_Start.empty())
+ {
+ if (m_Count.empty())
+ {
+ m_ShapeID = ShapeID::GlobalValue;
+ m_SingleValue = true;
+ }
+ else if (m_Start.empty() && !m_Count.empty())
+ {
+ m_ShapeID = ShapeID::LocalArray;
+ }
+ }
+ else
+ {
+ throw std::invalid_argument(
+ "ERROR: if the "
+ "shape is empty, start must be empty as well, in call to "
+ "DefineVariable " +
+ m_Name + "\n");
+ }
}
- else if (!m_Shape.empty() &&
- std::count(m_Shape.begin(), m_Shape.end(), JoinedDim) > 1)
+
+ /* Extra checks for invalid settings */
+ if (m_DebugMode)
+ CheckDimsCommon("DefineVariable(" + m_Name + ")");
+}
+
+void VariableBase::CheckDimsCommon(const std::string hint) const
+{
+ if (m_ShapeID != ShapeID::LocalValue)
{
- throw std::invalid_argument("ERROR: variable can't have more than one "
- "JoinedDim in shape argument, in call to "
- "DefineVariable " +
- m_Name + "\n");
+ if ((!m_Shape.empty() &&
+ std::count(m_Shape.begin(), m_Shape.end(), LocalValueDim) > 0) ||
+ (!m_Start.empty() &&
+ std::count(m_Start.begin(), m_Start.end(), LocalValueDim) > 0) ||
+ (!m_Count.empty() &&
+ std::count(m_Count.begin(), m_Count.end(), LocalValueDim) > 0))
+ {
+ throw std::invalid_argument("ERROR: LocalValueDim is only "
+ "allowed in a {LocalValueDim} "
+ "shape in call to " +
+ hint + "\n");
+ }
}
- else
+
+ if ((!m_Shape.empty() &&
+ std::count(m_Shape.begin(), m_Shape.end(), JoinedDim) > 1) ||
+ (!m_Start.empty() &&
+ std::count(m_Start.begin(), m_Start.end(), JoinedDim) > 0) ||
+ (!m_Count.empty() &&
+ std::count(m_Count.begin(), m_Count.end(), JoinedDim) > 0))
{
- throw std::invalid_argument("ERROR: the "
- "combination of shape, start and count "
- "arguments is inconsistent, in call to "
- "DefineVariable " +
- m_Name + "\n");
+ throw std::invalid_argument("ERROR: JoinedDim is only allowed once in "
+ "Shape and cannot appear in Start/Count in "
+ "call to " +
+ hint + "\n");
}
}
-void VariableBase::CheckDims(const std::string hint) const
+void VariableBase::CheckDimsBeforeWrite(const std::string hint) const
{
if (m_ShapeID == ShapeID::GlobalArray)
{
@@ -232,10 +302,12 @@ void VariableBase::CheckDims(const std::string hint) const
throw std::invalid_argument(
"ERROR: GlobalArray variable " + m_Name +
" start and count dimensions must be defined by either "
- "DefineVariable or a Selection " +
+ "DefineVariable or a Selection in call to " +
hint + "\n");
}
}
+
+ CheckDimsCommon(hint);
// TODO need to think more exceptions here
}
diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h
index 424a93892bd4142e1db38622909afc7c99a1f5c6..61fc093dfd72f5860565edc4eb5da69d5ec72611 100644
--- a/source/adios2/core/VariableBase.h
+++ b/source/adios2/core/VariableBase.h
@@ -105,44 +105,51 @@ public:
void SetStepSelection(const unsigned int startStep,
const unsigned int countStep);
- void AddTransform(Transform &transform,
- const std::vector<std::string> ¶metersVector);
+ /**
+ * Pushed a new transform to a sequence of transports
+ * @param transform reference to an object derived from the Transform class
+ * @param parameters transform specific parameters
+ * @return transformID handler
+ */
+ unsigned int AddTransform(Transform &transform,
+ const Params ¶meters = Params()) noexcept;
- void AddTransform(Transform &transform,
- const Params ¶metersVector = Params());
-
- /** Apply current sequence of transforms defined by AddTransform */
- virtual void ApplyTransforms() = 0;
+ void ResetTransformParameters(const unsigned int transformIndex,
+ const Params ¶meters = Params());
/** Clears out the transform sequence defined by AddTransform */
- void ClearTransforms();
+ void ClearTransforms() noexcept;
- /** Self-check dims according to type, called from Engine before Write
- * @param hint extra debugging info for the exception */
- void CheckDims(const std::string hint) const;
-
-private:
- const bool m_DebugMode = false;
-
- void InitShapeType();
+ /** Apply current sequence of transforms defined by AddTransform */
+ virtual void ApplyTransforms() = 0;
/** Transforms metadata info */
struct TransformInfo
{
/** reference to object derived from Transform class */
- Transform &Object;
+ Transform &Operator;
/** parameters from AddTransform */
Params Parameters;
/** resulting sizes from transformation */
Dims Sizes;
};
- /**
- * Sequence determines application order, e.g.
- * first Transforms[0] then Transforms[1]. Pointer used as
- * reference (no memory management).
- */
+ /** Registered transforms */
std::vector<TransformInfo> m_TransformsInfo;
+
+ /** Self-check dims according to type, called right after DefineVariable and
+ * SetSelection.
+ * @param hint extra debugging info for the exception */
+ void CheckDimsCommon(const std::string hint) const;
+
+ /** Self-check dims according to type, called from Engine before Write
+ * @param hint extra debugging info for the exception */
+ void CheckDimsBeforeWrite(const std::string hint) const;
+
+private:
+ const bool m_DebugMode = false;
+
+ void InitShapeType();
};
} // end namespace
diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp
index 54dde23fc04fa1dc657926d4152570caba84a6d2..a4d661fd3ace86496c321cf1ef1977bacc9b14ee 100644
--- a/source/adios2/engine/dataman/DataManReader.cpp
+++ b/source/adios2/engine/dataman/DataManReader.cpp
@@ -17,20 +17,19 @@ namespace adios2
DataManReader::DataManReader(IO &io, const std::string &name,
const OpenMode openMode, MPI_Comm mpiComm)
-: Engine("DataManReader", io, name, openMode, mpiComm),
- m_Man(mpiComm, true)
+: Engine("DataManReader", io, name, openMode, mpiComm), m_Man(mpiComm, true)
{
m_EndMessage = " in call to IO Open DataManReader " + m_Name + "\n";
Init();
}
-void DataManReader::SetCallBack(
+void DataManReader::SetCallback(
std::function<void(const void *, std::string, std::string, std::string,
Dims)>
callback)
{
m_CallBack = callback;
-// m_Man.reg_callback(callback);
+ m_Man.SetCallback(callback);
}
void DataManReader::Close(const int transportIndex) {}
@@ -81,6 +80,7 @@ void DataManReader::Init()
}) == s.end();
};
+ /*
json jmsg;
for (auto &i : m_IO.m_Parameters)
{
@@ -94,7 +94,19 @@ void DataManReader::Init()
}
}
jmsg["stream_mode"] = "receiver";
-// m_Man.add_stream(jmsg);
+ */
+
+ int n_Transports = 1;
+ std::vector<Params> para(n_Transports);
+
+ for (unsigned int i = 0; i < para.size(); i++)
+ {
+ para[i]["type"] = "wan";
+ para[i]["transport"] = "zmq";
+ para[i]["name"] = "stream";
+ para[i]["ipaddress"] = "127.0.0.1";
+ }
+ m_Man.OpenWANTransports("zmq", adios2::OpenMode::Read, para, true);
std::string method_type;
int num_channels = 0;
diff --git a/source/adios2/engine/dataman/DataManReader.h b/source/adios2/engine/dataman/DataManReader.h
index 1e8ef433fb9fdbbcf3f85a7b99877056df6acb71..c1c2c42d1403213e1530e1a14fac856f2d983a66 100644
--- a/source/adios2/engine/dataman/DataManReader.h
+++ b/source/adios2/engine/dataman/DataManReader.h
@@ -13,9 +13,9 @@
#include <iostream> //std::cout << Needs to go
-#include "adios2/toolkit/transportman/DataMan/DataMan.h"
#include "adios2/ADIOSConfig.h"
#include "adios2/core/Engine.h"
+#include "adios2/toolkit/transportman/dataman/DataMan.h"
namespace adios2
{
@@ -45,7 +45,7 @@ public:
* @param callback function (get) provided by the user to be applied in
* DataMan
*/
- void SetCallBack(std::function<void(const void *, std::string, std::string,
+ void SetCallback(std::function<void(const void *, std::string, std::string,
std::string, Dims)>
callback);
diff --git a/source/adios2/engine/dataman/DataManWriter.cpp b/source/adios2/engine/dataman/DataManWriter.cpp
index 6e7357a44d92d48270dd398e0fca5b723be414c3..95132cc67110cfcbba2791ce6e4bbc4b1b9df5b6 100644
--- a/source/adios2/engine/dataman/DataManWriter.cpp
+++ b/source/adios2/engine/dataman/DataManWriter.cpp
@@ -20,8 +20,7 @@ namespace adios2
DataManWriter::DataManWriter(IO &io, const std::string &name,
const OpenMode openMode, MPI_Comm mpiComm)
-: Engine("DataManWriter", io, name, openMode, mpiComm),
- m_Man(mpiComm, true)
+: Engine("DataManWriter", io, name, openMode, mpiComm), m_Man(mpiComm, true)
{
m_EndMessage = ", in call to Open DataManWriter\n";
Init();
@@ -33,15 +32,17 @@ void DataManWriter::SetCallBack(
callback)
{
m_CallBack = callback;
-// m_Man.reg_callback(callback);
+ // m_Man.reg_callback(callback);
}
-void DataManWriter::Advance(const float timeoutSeconds) {
-// m_Man.flush();
+void DataManWriter::Advance(const float timeoutSeconds)
+{
+ // m_Man.flush();
}
-void DataManWriter::Close(const int transportIndex) {
-// m_Man.flush();
+void DataManWriter::Close(const int transportIndex)
+{
+ // m_Man.flush();
}
// PRIVATE functions below
@@ -102,39 +103,32 @@ void DataManWriter::Init()
}) == s.end();
};
-// json jmsg;
-// for (const auto &i : m_IO.m_Parameters)
-// {
-// if (lf_IsNumber(i.second))
-// {
-// jmsg[i.first] = std::stoi(i.second);
-// }
-// else
-// {
-// jmsg[i.first] = i.second;
-// }
-// }
-// jmsg["stream_mode"] = "sender";
-// m_Man.add_stream(jmsg);
+ // json jmsg;
+ // for (const auto &i : m_IO.m_Parameters)
+ // {
+ // if (lf_IsNumber(i.second))
+ // {
+ // jmsg[i.first] = std::stoi(i.second);
+ // }
+ // else
+ // {
+ // jmsg[i.first] = i.second;
+ // }
+ // }
+ // jmsg["stream_mode"] = "sender";
+ // m_Man.add_stream(jmsg);
int n_Transports = 1;
std::vector<Params> para(n_Transports);
- std::cout << para.size() << std::endl;
-
- for(unsigned int i=0; i<para.size(); i++){
+ for (unsigned int i = 0; i < para.size(); i++)
+ {
para[i]["type"] = "wan";
para[i]["transport"] = "zmq";
para[i]["name"] = "stream";
para[i]["ipaddress"] = "127.0.0.1";
}
- for(auto &i :para){
- for(auto &j :i){
- std::cout << j.first << " " << j.second << std::endl;
- }
- }
-
m_Man.OpenWANTransports("zmq", adios2::OpenMode::Write, para, true);
std::string method_type;
diff --git a/source/adios2/engine/dataman/DataManWriter.h b/source/adios2/engine/dataman/DataManWriter.h
index f087a7b74f52f5dc61449f9a988071b5176cd757..d1a840ab60a03bdf857c5b7033875f2329fedf78 100644
--- a/source/adios2/engine/dataman/DataManWriter.h
+++ b/source/adios2/engine/dataman/DataManWriter.h
@@ -14,9 +14,9 @@
#include <iostream> //std::cout must be removed, only used for hello example
#include <unistd.h> //sleep must be removed
-#include "adios2/toolkit/transportman/DataMan/DataMan.h"
#include "adios2/ADIOSConfig.h"
#include "adios2/core/Engine.h"
+#include "adios2/toolkit/transportman/dataman/DataMan.h"
namespace adios2
{
@@ -25,8 +25,6 @@ class DataManWriter : public Engine
{
public:
- using json = nlohmann::json;
-
DataManWriter(IO &io, const std::string &name, const OpenMode openMode,
MPI_Comm mpiComm);
diff --git a/source/adios2/engine/dataman/DataManWriter.tcc b/source/adios2/engine/dataman/DataManWriter.tcc
index e86cf3a53a537763f9937de4ad03f6161685acbe..05e1b816b64ef91e11bef9a6f0430febac6aa3f6 100644
--- a/source/adios2/engine/dataman/DataManWriter.tcc
+++ b/source/adios2/engine/dataman/DataManWriter.tcc
@@ -40,7 +40,8 @@ void DataManWriter::DoWriteCommon(Variable<T> &variable, const T *values)
variable.m_Start.assign(variable.m_Count.size(), 0);
}
- json jmsg;
+ std::cout << "DoWriteCommon begin" << std::endl;
+ nlohmann::json jmsg;
jmsg["doid"] = m_Name;
jmsg["var"] = variable.m_Name;
jmsg["dtype"] = GetType<T>();
@@ -48,7 +49,12 @@ void DataManWriter::DoWriteCommon(Variable<T> &variable, const T *values)
jmsg["varshape"] = variable.m_Shape;
jmsg["offset"] = variable.m_Start;
jmsg["timestep"] = 0;
-// m_Man.put(values, jmsg);
+ jmsg["bytes"] =
+ std::accumulate(variable.m_Shape.begin(), variable.m_Shape.end(),
+ sizeof(T), std::multiplies<size_t>());
+
+ std::cout << "DoWriteCommon end" << std::endl;
+ m_Man.WriteWAN(values, jmsg);
if (m_DoMonitor)
{
diff --git a/source/adios2/helper/adiosFunctions.h b/source/adios2/helper/adiosFunctions.h
index af3e98f18f93c75e745c2903ce5f7ab4e98c662f..e088875c7a4ec65bdbbed370a710b71dc834c71f 100644
--- a/source/adios2/helper/adiosFunctions.h
+++ b/source/adios2/helper/adiosFunctions.h
@@ -12,9 +12,10 @@
#ifndef ADIOS2_HELPER_ADIOSFUNCTIONS_H_
#define ADIOS2_HELPER_ADIOSFUNCTIONS_H_
-#include "adios2/helper/adiosMath.h" //math functions (cmath, algorithm)
-#include "adios2/helper/adiosMemory.h" //memcpy, std::copy, insert, resize
-#include "adios2/helper/adiosString.h" //std::string manipulation
+#include "adios2/helper/adiosMPIFunctions.h" //MPI functions (Bcast, send/recv)
+#include "adios2/helper/adiosMath.h" //math functions (cmath, algorithm)
+#include "adios2/helper/adiosMemory.h" //memcpy, std::copy, insert, resize
+#include "adios2/helper/adiosString.h" //std::string manipulation
#include "adios2/helper/adiosSystem.h" //OS functionality, POSIX, filesystem
#include "adios2/helper/adiosType.h" //Type casting, conversion, checks, etc.
#include "adios2/helper/adiosXML.h" //XML parsing
diff --git a/source/adios2/helper/adiosMPIFunctions.cpp b/source/adios2/helper/adiosMPIFunctions.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bd550b00ef5c2e1fb0f4baaa9f83a3a8da49cbd3
--- /dev/null
+++ b/source/adios2/helper/adiosMPIFunctions.cpp
@@ -0,0 +1,46 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * adiosMPIFunctions.cpp
+ *
+ * Created on: Jul 20, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+#include "adiosMPIFunctions.h"
+
+#include "adios2/ADIOSMPI.h"
+#include "adios2/ADIOSTypes.h"
+
+namespace adios2
+{
+
+std::string BroadcastString(const std::string &input, MPI_Comm mpiComm,
+ const int rankSource)
+{
+ int rank;
+ MPI_Comm_rank(mpiComm, &rank);
+ size_t length = 0;
+ std::string output;
+
+ if (rank == rankSource)
+ {
+ length = input.size();
+ MPI_Bcast(&length, 1, ADIOS2_MPI_SIZE_T, rankSource, mpiComm);
+
+ MPI_Bcast(const_cast<char *>(input.data()), length, MPI_CHAR,
+ rankSource, mpiComm);
+
+ return input;
+ }
+ else
+ {
+ MPI_Bcast(&length, 1, ADIOS2_MPI_SIZE_T, rankSource, mpiComm);
+ output.resize(length);
+ MPI_Bcast(const_cast<char *>(output.data()), length, MPI_CHAR,
+ rankSource, mpiComm);
+ }
+ return output;
+}
+
+} // end namespace adios2
diff --git a/source/adios2/helper/adiosMPIFunctions.h b/source/adios2/helper/adiosMPIFunctions.h
new file mode 100644
index 0000000000000000000000000000000000000000..a79345d6af4b735a57f718887c6d21f38b8878bb
--- /dev/null
+++ b/source/adios2/helper/adiosMPIFunctions.h
@@ -0,0 +1,36 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * adiosMPIFunctions.h : collection of MPI functions used across adios2
+ *
+ * Created on: Jul 20, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#ifndef ADIOS2_HELPER_ADIOMPIFUNCTIONS_H_
+#define ADIOS2_HELPER_ADIOMPIFUNCTIONS_H_
+
+/// \cond EXCLUDE_FROM_DOXYGEN
+#include <string>
+/// \endcond
+
+#include "adios2/ADIOSMPICommOnly.h"
+
+namespace adios2
+{
+
+/**
+ * rankSource: owns a string and broadcast to all other ranks
+ * Others: receive std::vector<char> and copy to a string
+ * @param mpiComm MPI communicator defining all ranks and size domain
+ * @param input string input from rankSource
+ * @param rankSource rank that broadcast the string, (default = 0)
+ * @return input contents for each rank
+ */
+std::string BroadcastString(const std::string &input, MPI_Comm mpiComm,
+ const int rankSource = 0);
+
+} // end namespace adios2
+
+#endif /* ADIOS2_HELPER_ADIOMPIFUNCTIONS_H_ */
diff --git a/source/adios2/helper/adiosMemory.h b/source/adios2/helper/adiosMemory.h
index 20ab7739562cd8cb7650ae0f136020f35f1ffe72..ad5dd0116d123da9a885c99ca940efa1e825b969 100644
--- a/source/adios2/helper/adiosMemory.h
+++ b/source/adios2/helper/adiosMemory.h
@@ -83,6 +83,14 @@ void MemcpyToBufferThreads(std::vector<char> &buffer, size_t &position,
const T *source, size_t size,
const unsigned int threads = 1);
+/**
+ * Cast an element to uint64 and insert to a buffer
+ * @param buffer data destination
+ * @param element to be added to buffer
+ */
+template <class T>
+void InsertU64(std::vector<char> &buffer, const T element) noexcept;
+
} // end namespace adios
#include "adiosMemory.inl"
diff --git a/source/adios2/helper/adiosMemory.inl b/source/adios2/helper/adiosMemory.inl
index ee50425e2b1083eb2e076d8579b631de49b406eb..0eaa2662a765b1e8e5523c624d5ce853b3194a10 100644
--- a/source/adios2/helper/adiosMemory.inl
+++ b/source/adios2/helper/adiosMemory.inl
@@ -25,7 +25,7 @@ namespace adios2
template <class T>
void InsertToBuffer(std::vector<char> &buffer, const T *source,
- const std::size_t elements) noexcept
+ const size_t elements) noexcept
{
const char *src = reinterpret_cast<const char *>(source);
buffer.insert(buffer.end(), src, src + elements * sizeof(T));
@@ -159,6 +159,13 @@ void MemcpyToBufferThreads(std::vector<char> &buffer, size_t &position,
}
}
+template <class T>
+void InsertU64(std::vector<char> &buffer, const T element) noexcept
+{
+ const uint64_t element64 = static_cast<const uint64_t>(element);
+ InsertToBuffer(buffer, &element64);
+}
+
} // end namespace
#endif /* ADIOS2_HELPER_ADIOSMEMORY_INL_ */
diff --git a/source/adios2/helper/adiosString.cpp b/source/adios2/helper/adiosString.cpp
index 21e4bb138c253a69b78788955df28a04ef7166b1..f294951ab6021ba1e1bd7cda44859bbea5617caa 100644
--- a/source/adios2/helper/adiosString.cpp
+++ b/source/adios2/helper/adiosString.cpp
@@ -130,25 +130,103 @@ void SetParameterValue(const std::string key, const Params ¶meters,
}
std::string GetParameter(const std::string key, const Params ¶ms,
- const bool isMandatory,
- const bool debugMode, const std::string hint){
- std::string value;
- auto itParameter = params.find(key);
- if (itParameter == params.end())
+ const bool isMandatory, const bool debugMode,
+ const std::string hint)
+{
+ std::string value;
+ auto itParameter = params.find(key);
+ if (itParameter == params.end())
+ {
+ if (debugMode && isMandatory)
{
- if (debugMode && isMandatory)
- {
- throw std::invalid_argument(
- "ERROR: mandatory parameter " + key +
- " not found, " + hint );
- }
+ throw std::invalid_argument("ERROR: mandatory parameter " + key +
+ " not found, " + hint);
+ }
+ }
+ else
+ {
+ value = itParameter->second;
+ }
+ return value;
+}
+
+void SetParameterValueInt(const std::string key, const Params ¶meters,
+ int &value, const bool debugMode,
+ const std::string hint)
+{
+ auto itKey = parameters.find(key);
+
+ if (itKey == parameters.end())
+ {
+ return;
+ }
+
+ if (debugMode)
+ {
+ try
+ {
+ value = std::stoi(itKey->second);
+ }
+ catch (...)
+ {
+ std::throw_with_nested(std::invalid_argument(
+ "ERROR: could not cast " + itKey->second +
+ " to int from key parameter: " + itKey->first + ", " + hint));
+ }
+ }
+ else
+ {
+ value = std::stoi(itKey->second);
+ }
+}
+
+double StringToDouble(const std::string value, const bool debugMode,
+ const std::string hint)
+{
+ double valueDouble = -1.;
+
+ if (debugMode)
+ {
+ try
+ {
+ valueDouble = std::stod(value);
}
- else
+ catch (...)
{
- value = itParameter->second;
+ std::throw_with_nested(std::invalid_argument(
+ "ERROR: could not cast " + value + " to double, " + hint));
}
- return value;
}
+ else
+ {
+ valueDouble = std::stod(value);
+ }
+ return valueDouble;
+}
+unsigned int StringToUInt(const std::string value, const bool debugMode,
+ const std::string hint)
+{
+ unsigned int valueUInt = 0;
+
+ if (debugMode)
+ {
+ try
+ {
+ valueUInt = static_cast<unsigned int>(std::stoul(value));
+ }
+ catch (...)
+ {
+ std::throw_with_nested(
+ std::invalid_argument("ERROR: could not cast " + value +
+ " to unsigned int, " + hint));
+ }
+ }
+ else
+ {
+ valueUInt = static_cast<unsigned int>(std::stoul(value));
+ }
+ return valueUInt;
+}
} // end namespace adios
diff --git a/source/adios2/helper/adiosString.h b/source/adios2/helper/adiosString.h
index 45952bb6cd789c8b6aab7277787f7f83b87540d4..e323797e8a18ebaa7ae3bcec78875288a1e10476 100644
--- a/source/adios2/helper/adiosString.h
+++ b/source/adios2/helper/adiosString.h
@@ -69,10 +69,41 @@ void SetParameterValue(const std::string key, const Params ¶meters,
std::string &value) noexcept;
std::string GetParameter(const std::string key, const adios2::Params ¶ms,
- const bool isMandatory,
- const bool debugMode, const std::string hint);
-}
+ const bool isMandatory, const bool debugMode,
+ const std::string hint);
+/**
+ * Sets int value if found in parameters for input key
+ * @param key input
+ * @param parameters map with key: field, value: value
+ * @param value to be modified if key is found in parameters
+ * @param debugMode check for string conversion
+ * @param hint passed for extra debugging info if exception is thrown
+ */
+void SetParameterValueInt(const std::string key, const Params ¶meters,
+ int &value, const bool debugMode,
+ const std::string hint);
+/**
+ * function that cast a string to a double verifying validity of the cast with
+ * exceptions in debugMode
+ * @param value string to be casted
+ * @param debugMode check for string conversion
+ * @param hint passed for extra debugging info if exception is thrown
+ * @return value as a double
+ */
+double StringToDouble(const std::string value, const bool debugMode,
+ const std::string hint);
+/**
+ * function that cast a string to unsigned int verifying validity of the cast
+ * with exceptions in debugMode
+ * @param value string to be casted
+ * @param debugMode check for string conversion
+ * @param hint passed for extra debugging info if exception is thrown
+ * @return value as unsigned int
+ */
+unsigned int StringToUInt(const std::string value, const bool debugMode,
+ const std::string hint);
+}
#endif /* ADIOS2_HELPER_ADIOSSTRING_H_ */
diff --git a/source/adios2/helper/adiosSystem.cpp b/source/adios2/helper/adiosSystem.cpp
index 8e0b967c5e0d07b8620c29dce7beadaf8693a2f6..233f2d096cffde6bb1b482c75bcdcbbc5a36ee72 100644
--- a/source/adios2/helper/adiosSystem.cpp
+++ b/source/adios2/helper/adiosSystem.cpp
@@ -9,7 +9,7 @@
*/
#include "adiosSystem.h"
-#include <ctime> //std::ctime
+#include <ctime>
#include <chrono> //system_clock, now
@@ -19,6 +19,11 @@
#include "adios2/ADIOSTypes.h"
#include "adios2/helper/adiosString.h"
+// remove ctime warning on Windows
+#ifdef _WIN32
+#pragma warning(disable : 4996)
+#endif
+
namespace adios2
{
@@ -38,44 +43,7 @@ std::string LocalTimeDate() noexcept
std::time_t now =
std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
- return std::string(std::ctime(&now));
-}
-
-std::string BroadcastString(const std::string &input, MPI_Comm mpiComm)
-{
- std::string receivedInput;
- size_t characterCount = 0;
-
- int rank;
- MPI_Comm_rank(mpiComm, &rank);
-
- if (rank == 0) // sender
- {
- characterCount = input.size();
-
- // broadcast size for allocation
- MPI_Bcast(&characterCount, 1, ADIOS2_MPI_SIZE_T, 0, mpiComm);
-
- // broadcast contents
- MPI_Bcast(const_cast<char *>(input.c_str()),
- static_cast<int>(characterCount), MPI_CHAR, 0, mpiComm);
-
- return input;
- }
- else // receivers
- {
- // receive size
- MPI_Bcast(&characterCount, 1, ADIOS2_MPI_SIZE_T, 0, mpiComm);
-
- // allocate receiver
- std::vector<char> stringReceiver(characterCount);
- MPI_Bcast(stringReceiver.data(), static_cast<int>(characterCount),
- MPI_CHAR, 0, mpiComm);
-
- receivedInput.assign(stringReceiver.begin(), stringReceiver.end());
- }
-
- return receivedInput;
+ return std::string(ctime(&now));
}
} // end namespace adios
diff --git a/source/adios2/helper/adiosSystem.h b/source/adios2/helper/adiosSystem.h
index 844e73e47fbb4fbaed5772f28abb9a2569d1634f..650cba40987401a9452304de0a7e9d7b574307fd 100644
--- a/source/adios2/helper/adiosSystem.h
+++ b/source/adios2/helper/adiosSystem.h
@@ -17,8 +17,6 @@
#include <vector>
/// \endcond
-#include "adios2/ADIOSMPICommOnly.h"
-
namespace adios2
{
@@ -43,15 +41,6 @@ bool IsLittleEndian() noexcept;
*/
std::string LocalTimeDate() noexcept;
-/**
- * Rank 0: opens file, dumps contents string and broadcast.
- * Others: receive std::vector<char> and copy to a string
- * @param fileName
- * @param mpiComm
- * @return fileContents as a single string
- */
-std::string BroadcastString(const std::string &input, MPI_Comm mpiComm);
-
} // end namespace adios
#endif /* ADIOS2_HELPER_ADIOSSYSTEM_H_ */
diff --git a/source/adios2/helper/adiosXML.cpp b/source/adios2/helper/adiosXML.cpp
index 0045c4de336d3ec8b71fb5c85dbd1b5729ad7ec5..25bae1343c05e09b5776f9ca7af4ecf07f3ae6ae 100644
--- a/source/adios2/helper/adiosXML.cpp
+++ b/source/adios2/helper/adiosXML.cpp
@@ -6,6 +6,7 @@
*
* Created on: May 17, 2017
* Author: William F Godoy godoywf@ornl.gov
+ * Chuck Atkins chuck.atkins@kitware.com
*/
#include "adiosXML.h"
@@ -16,6 +17,7 @@
#include "adios2/ADIOSMPI.h"
#include "adios2/ADIOSTypes.h"
+#include "adios2/helper/adiosMPIFunctions.h"
#include "adios2/helper/adiosString.h"
#include <pugixml.hpp>
@@ -23,148 +25,173 @@
namespace adios2
{
-Params InitParametersXML(pugi::xml_node node, bool debugMode)
+Params InitParametersXML(const pugi::xml_node &node, const bool debugMode)
{
Params params;
- for (pugi::xml_node paramNode : node.children("parameter"))
+ for (const pugi::xml_node paramNode : node.children("parameter"))
{
- pugi::xml_attribute attrKey = paramNode.attribute("key");
- if (!attrKey)
+ const pugi::xml_attribute key = paramNode.attribute("key");
+ if (debugMode)
{
- if (debugMode)
+ if (!key)
{
throw std::invalid_argument("ERROR: XML: No \"key\" attribute "
- "found on <parameter> element.");
+ "found on <parameter> element, in "
+ "call to ADIOS constructor\n");
}
- continue;
}
- pugi::xml_attribute attrValue = paramNode.attribute("value");
- if (!attrValue)
+ const pugi::xml_attribute value = paramNode.attribute("value");
+
+ if (debugMode)
{
- if (debugMode)
+ if (!value)
{
+
throw std::invalid_argument("ERROR: XML: No \"value\" "
"attribute found on <parameter> "
- "element.");
+ "element, for key " +
+ std::string(key.value()) +
+ ", in call to ADIOS constructor\n");
+ continue;
}
- continue;
}
- params.emplace(attrKey.value(), attrValue.value());
+ params.emplace(key.value(), value.value());
}
return params;
}
-void InitIOXML(const pugi::xml_node ioNode, const MPI_Comm mpiComm,
+void InitIOXML(const pugi::xml_node &ioNode, MPI_Comm mpiComm,
const bool debugMode,
- std::vector<std::shared_ptr<Transform>> &transforms,
+ std::map<std::string, std::shared_ptr<Transform>> &transforms,
std::map<std::string, IO> &ios)
{
// Extract <io name=""> attribute
- pugi::xml_attribute nameAttr = ioNode.attribute("name");
+ const pugi::xml_attribute nameAttr = ioNode.attribute("name");
if (!nameAttr)
{
if (debugMode)
{
- throw std::invalid_argument(
- "ERROR: XML: No \"name\" attribute found on <io> element.");
+ throw std::invalid_argument("ERROR: XML: No \"name\" attribute "
+ "found on <io> element, in call to "
+ "ADIOS constructor.\n");
}
return;
}
- std::string ioName = nameAttr.value();
+ const std::string ioName = nameAttr.value();
// Build the IO object
auto ioIt = ios.emplace(ioName, IO(ioName, mpiComm, true, debugMode));
IO &io = ioIt.first->second;
// Extract <engine> element
- pugi::xml_node engineNode = ioNode.child("engine");
- if (!engineNode)
+ if (debugMode)
{
- throw std::invalid_argument(
- "ERROR: XML: No <engine> element found in <io> element.");
+ unsigned int count = 0;
+
+ for (const pugi::xml_node engineNode : ioNode.children("engine"))
+ {
+ ++count;
+ if (count == 2)
+ {
+ throw std::invalid_argument(
+ "ERROR: XML only one <engine> element "
+ "can exist inside an <io> element from io " +
+ ioName + ", in call to ADIOS constructor\n");
+ }
+ }
}
- pugi::xml_attribute engineTypeAttr = engineNode.attribute("type");
- if (!engineTypeAttr)
+
+ const pugi::xml_node engineNode = ioNode.child("engine");
+ if (engineNode)
{
- throw std::invalid_argument(
- "ERROR: XML: No \"type\" attribute found on <engine> element.");
+ const pugi::xml_attribute engineTypeAttr = engineNode.attribute("type");
+
+ if (debugMode)
+ {
+ if (!engineTypeAttr)
+ {
+ throw std::invalid_argument(
+ "ERROR: XML: No \"type\" attribute "
+ "found on <engine> element, in call to "
+ "ADIOS constructor");
+ }
+ }
+
+ io.SetEngine(engineTypeAttr.value());
}
- io.SetEngine(engineTypeAttr.value());
// Process <engine> parameters
io.SetParameters(InitParametersXML(engineNode, debugMode));
// Extract and process <transport> elements
- for (pugi::xml_node transportNode : ioNode.children("transport"))
+ for (const pugi::xml_node transportNode : ioNode.children("transport"))
{
- pugi::xml_attribute typeAttr = transportNode.attribute("type");
- if (!typeAttr)
+ const pugi::xml_attribute typeXMLAttribute =
+ transportNode.attribute("type");
+
+ if (debugMode)
{
- if (debugMode)
+ if (!typeXMLAttribute)
{
+
throw std::invalid_argument("ERROR: XML: No \"type\" attribute "
- "found on <transport> element.");
+ "found on <transport> element, in "
+ "call to ADIOS constructor\n");
}
- continue;
}
- io.AddTransport(typeAttr.value(),
+
+ io.AddTransport(typeXMLAttribute.value(),
InitParametersXML(transportNode, debugMode));
}
}
-void InitXML(const std::string configXML, const MPI_Comm mpiComm,
+void InitXML(const std::string configXML, MPI_Comm mpiComm,
const bool debugMode,
- std::vector<std::shared_ptr<Transform>> &transforms,
+ std::map<std::string, std::shared_ptr<Transform>> &transforms,
std::map<std::string, IO> &ios)
{
int mpiRank;
MPI_Comm_rank(mpiComm, &mpiRank);
std::string fileContents;
- unsigned long long len;
// Read the file on rank 0 and broadcast it to everybody else
if (mpiRank == 0)
{
fileContents = FileToString(configXML);
- len = static_cast<unsigned long long>(fileContents.size());
- }
- MPI_Bcast(&len, 1, MPI_UNSIGNED_LONG, 0, mpiComm);
- if (mpiRank != 0)
- {
- fileContents.resize(len);
}
- MPI_Bcast(const_cast<char *>(fileContents.data()), len, MPI_CHAR, 0,
- mpiComm);
+
+ fileContents = BroadcastString(fileContents, mpiComm);
pugi::xml_document doc;
auto parse_result = doc.load_buffer_inplace(
const_cast<char *>(fileContents.data()), fileContents.size());
- if (!parse_result)
+
+ if (debugMode)
{
- if (debugMode)
+ if (!parse_result)
{
throw std::invalid_argument(
- std::string("ERROR: XML: Parse error: ") +
- parse_result.description());
+ "ERROR: XML: parse error in file " + configXML +
+ " description: " + std::string(parse_result.description()) +
+ ", in call to ADIOS constructor\n");
}
- return;
}
- pugi::xml_node configNode = doc.child("adios-config");
- if (!configNode)
+ const pugi::xml_node configNode = doc.child("adios-config");
+
+ if (debugMode)
{
- if (debugMode)
+ if (!configNode)
{
throw std::invalid_argument(
- "ERROR: XML: No <adios-config> element found");
+ "ERROR: XML: No <adios-config> element found in file " +
+ configXML + ", in call to ADIOS constructor\n");
}
- return;
}
- ios.clear();
- for (pugi::xml_node ioNode : configNode.children("io"))
+ for (const pugi::xml_node ioNode : configNode.children("io"))
{
InitIOXML(ioNode, mpiComm, debugMode, transforms, ios);
}
diff --git a/source/adios2/helper/adiosXML.h b/source/adios2/helper/adiosXML.h
index 055c14605f43259a71783c4727c015ea4a34d476..0746eb47708041c3b19aef93a1aa3ee3c11b98fd 100644
--- a/source/adios2/helper/adiosXML.h
+++ b/source/adios2/helper/adiosXML.h
@@ -16,7 +16,6 @@
#include <memory> //std::shared_ptr
#include <string>
#include <utility> //std::pair
-#include <vector>
/// \endcond
#include "adios2/core/IO.h"
@@ -34,9 +33,9 @@ namespace adios2
* @param transforms
* @param ios
*/
-void InitXML(const std::string configXML, const MPI_Comm mpiComm,
+void InitXML(const std::string configXML, MPI_Comm mpiComm,
const bool debugMode,
- std::vector<std::shared_ptr<Transform>> &transforms,
+ std::map<std::string, std::shared_ptr<Transform>> &transforms,
std::map<std::string, IO> &ios);
}
diff --git a/source/adios2/mpidummy.cpp b/source/adios2/mpidummy.cpp
index 75f76646a12d8c80e149ca11e2cf5be46d94c7d2..c97938e95d78063c73195ac0cacd2275335f516c 100644
--- a/source/adios2/mpidummy.cpp
+++ b/source/adios2/mpidummy.cpp
@@ -26,7 +26,6 @@
#define open64 open
#endif
*/
-
#include <cinttypes>
#include <cstdio>
#include <cstring>
@@ -34,6 +33,12 @@
#include <chrono>
#include <string>
+// remove warnings on Windows
+#ifdef _WIN32
+#pragma warning(disable : 4996) // fopen
+#pragma warning(disable : 4477) // strcpy, sprintf
+#endif
+
namespace adios2
{
@@ -296,7 +301,7 @@ int MPI_File_open(MPI_Comm /*comm*/, const char *filename, int amode,
}
mode += "b";
- *fh = fopen(filename, mode.c_str());
+ *fh = std::fopen(filename, mode.c_str());
if (!*fh)
{
std::snprintf(mpierrmsg, MPI_MAX_ERROR_STRING, "File not found: %s",
@@ -310,10 +315,10 @@ int MPI_File_close(MPI_File *fh) { return fclose(*fh); }
int MPI_File_get_size(MPI_File fh, MPI_Offset *size)
{
- long curpos = ftell(fh);
+ long curpos = std::ftell(fh);
fseek(fh, 0, SEEK_END); // go to end, returned is the size in bytes
- long endpos = ftell(fh);
- fseek(fh, curpos, SEEK_SET); // go back where we were
+ long endpos = std::ftell(fh);
+ std::fseek(fh, curpos, SEEK_SET); // go back where we were
*size = static_cast<MPI_Offset>(endpos);
// printf("MPI_File_get_size: fh=%d, size=%lld\n", fh, *size);
return MPI_SUCCESS;
@@ -325,13 +330,14 @@ int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
// FIXME: int count can read only 2GB (*datatype size) array at max
size_t bytes_to_read = static_cast<size_t>(count) * datatype;
size_t bytes_read;
- bytes_read = fread(buf, 1, bytes_to_read, fh);
+ bytes_read = std::fread(buf, 1, bytes_to_read, fh);
if (bytes_read != bytes_to_read)
{
std::snprintf(mpierrmsg, MPI_MAX_ERROR_STRING,
- "could not read %" PRId64 " bytes. read only: %" PRId64
+ "could not read %llu bytes. read only: %llu"
"\n",
- bytes_to_read, bytes_read);
+ (unsigned long long)bytes_to_read,
+ (unsigned long long)bytes_read);
return -2;
}
*status = bytes_read;
@@ -342,7 +348,7 @@ int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence)
{
- return fseek(fh, offset, whence) == MPI_SUCCESS;
+ return std::fseek(fh, offset, whence) == MPI_SUCCESS;
}
int MPI_Get_count(const MPI_Status *status, MPI_Datatype, int *count)
diff --git a/source/adios2/mpidummy.h b/source/adios2/mpidummy.h
index f4607ea204c313de54f37a1a8b0a2f484fc50ade..5457f0dd7788c55653787571271a4294878d9079 100644
--- a/source/adios2/mpidummy.h
+++ b/source/adios2/mpidummy.h
@@ -14,9 +14,6 @@
#include <cstdint>
#include <cstdio>
-/// \cond EXCLUDE_FROM_DOXYGEN
-/// \endcond
-
namespace adios2
{
diff --git a/source/adios2/toolkit/capsule/Capsule.h b/source/adios2/toolkit/capsule/Capsule.h
index 09ba7fd4c3daf7b992decd735c9a68a37442d142..0534e75ec69912a4b012befb0249048c9fad2a12 100644
--- a/source/adios2/toolkit/capsule/Capsule.h
+++ b/source/adios2/toolkit/capsule/Capsule.h
@@ -61,8 +61,8 @@ public:
size_t GetAvailableDataSize() const;
- virtual void ResizeData(size_t size); ///< resize data buffer
- virtual void ResizeMetadata(size_t size); ///< resize metadata buffer
+ virtual void ResizeData(const size_t size); ///< resize data buffer
+ virtual void ResizeMetadata(const size_t size); ///< resize metadata buffer
protected:
const bool m_DebugMode = false; ///< true: extra exception checks
diff --git a/source/adios2/toolkit/capsule/heap/STLVector.cpp b/source/adios2/toolkit/capsule/heap/STLVector.cpp
index 57daca380aa539c9f7b765617bee9fb71132717e..5221f00faa3245c66a14d3dd44708d34a40b1f81 100644
--- a/source/adios2/toolkit/capsule/heap/STLVector.cpp
+++ b/source/adios2/toolkit/capsule/heap/STLVector.cpp
@@ -41,11 +41,12 @@ void STLVector::ResizeData(const size_t size)
{
m_Data.resize(size);
}
- catch (std::bad_alloc &e)
+ catch (...)
{
- throw std::runtime_error("ERROR: bad_alloc detected when resizing "
- "data buffer with size " +
- std::to_string(size) + "\n");
+ std::throw_with_nested(
+ std::runtime_error("ERROR: possible overflow when resizing "
+ "data buffer with size " +
+ std::to_string(size) + "\n"));
}
}
else
@@ -62,11 +63,12 @@ void STLVector::ResizeMetadata(const size_t size)
{
m_Metadata.resize(size);
}
- catch (std::bad_alloc &e)
+ catch (...)
{
- throw std::runtime_error("ERROR: bad_alloc detected when resizing "
- "metadata buffer with size " +
- std::to_string(size) + "\n");
+ std::throw_with_nested(
+ std::runtime_error("ERROR: possible overflow when resizing "
+ "metadata buffer with size " +
+ std::to_string(size) + "\n"));
}
}
else
diff --git a/source/adios2/toolkit/capsule/shmem/ShmSystemV.cpp b/source/adios2/toolkit/capsule/shmem/ShmSystemV.cpp
index 62486da1d39580ba1e211928cf9a1bbb5ca9cf60..21ba4efcd636c739cfa8a3517d03f4d7db9e14c5 100644
--- a/source/adios2/toolkit/capsule/shmem/ShmSystemV.cpp
+++ b/source/adios2/toolkit/capsule/shmem/ShmSystemV.cpp
@@ -2,10 +2,10 @@
* Distributed under the OSI-approved Apache License, Version 2.0. See
* accompanying file Copyright.txt for details.
*
- * ShmSystemV.cpp
+ * ShmSystemV.cpp : implementation of ShmSystemV class
*
* Created on: Dec 22, 2016
- * Author: wfg
+ * Author: William F Godoy godoywf@ornl.gov
*/
#include "ShmSystemV.h"
diff --git a/source/adios2/toolkit/capsule/shmem/ShmSystemV.h b/source/adios2/toolkit/capsule/shmem/ShmSystemV.h
index df997af7bf10a2a6cd46939c3fc544ffe15e89e6..b1bf0f48ea2c28a77c9571137c7e66d803b7eb09 100644
--- a/source/adios2/toolkit/capsule/shmem/ShmSystemV.h
+++ b/source/adios2/toolkit/capsule/shmem/ShmSystemV.h
@@ -1,6 +1,12 @@
/*
* Distributed under the OSI-approved Apache License, Version 2.0. See
* accompanying file Copyright.txt for details.
+ *
+ * ShmSystemV.h : ShmSystem class as a thin wrapper to a shared memory capsule
+ * using POSIX SystemV
+ *
+ * Created on: Dec 22, 2016
+ * Author: William F Godoy godoywf@ornl.gov
*/
#ifndef ADIOS2_TOOLKIT_CAPSULE_SHMEM_SHMSYSTEMV_H_
diff --git a/source/adios2/toolkit/format/bp1/BP1Base.cpp b/source/adios2/toolkit/format/bp1/BP1Base.cpp
index 387366b43fffc86c6736aa54d8799d3255dfd288..ecc990f38bef44f499eb417f0e9e609845ae5a6c 100644
--- a/source/adios2/toolkit/format/bp1/BP1Base.cpp
+++ b/source/adios2/toolkit/format/bp1/BP1Base.cpp
@@ -170,6 +170,8 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
if (m_DebugMode)
{
bool success = true;
+ std::string description;
+
try
{
m_GrowthFactor = std::stof(value);
@@ -177,6 +179,7 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
catch (std::exception &e)
{
success = false;
+ description = std::string(e.what());
}
if (!success || m_GrowthFactor <= 1.f)
@@ -184,7 +187,8 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
throw std::invalid_argument(
"ERROR: BufferGrowthFactor value "
"can't be less or equal than 1 (default = 1.5), or couldn't "
- "convert number, in call to Open\n");
+ "convert number,\n additional description:" +
+ description + "\n, in call to Open\n");
}
}
else
@@ -195,15 +199,14 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
void BP1Base::InitParameterInitBufferSize(const std::string value)
{
- const std::string errorMessage(
- "ERROR: wrong value for InitialBufferSize, it must be larger than "
- "16Kb (minimum default), in call to Open\n");
-
if (m_DebugMode)
{
if (value.size() < 2)
{
- throw std::invalid_argument(errorMessage);
+ throw std::invalid_argument(
+ "ERROR: wrong value for InitialBufferSize, it must be larger "
+ "than "
+ "16Kb (minimum default), in call to Open\n");
}
}
@@ -215,6 +218,8 @@ void BP1Base::InitParameterInitBufferSize(const std::string value)
if (m_DebugMode)
{
bool success = true;
+ std::string description;
+
try
{
bufferSize = static_cast<size_t>(std::stoul(number) * factor);
@@ -222,11 +227,16 @@ void BP1Base::InitParameterInitBufferSize(const std::string value)
catch (std::exception &e)
{
success = false;
+ description = std::string(e.what());
}
if (!success || bufferSize < DefaultInitialBufferSize) // 16384b
{
- throw std::invalid_argument(errorMessage);
+ throw std::invalid_argument(
+ "ERROR: wrong value for InitialBufferSize, it must be larger "
+ "than "
+ "16Kb (minimum default), additional description: " +
+ description + " in call to Open\n");
}
}
else
@@ -239,17 +249,15 @@ void BP1Base::InitParameterInitBufferSize(const std::string value)
void BP1Base::InitParameterMaxBufferSize(const std::string value)
{
- const std::string errorMessage(
- "ERROR: couldn't convert value of max_buffer_size IO "
- "SetParameter, valid syntax: MaxBufferSize=10Gb, "
- "MaxBufferSize=1000Mb, MaxBufferSize=16Kb (minimum default), "
- " in call to Open");
-
if (m_DebugMode)
{
if (value.size() < 2)
{
- throw std::invalid_argument(errorMessage);
+ throw std::invalid_argument(
+ "ERROR: couldn't convert value of max_buffer_size IO "
+ "SetParameter, valid syntax: MaxBufferSize=10Gb, "
+ "MaxBufferSize=1000Mb, MaxBufferSize=16Kb (minimum default), "
+ " in call to Open");
}
}
@@ -260,6 +268,8 @@ void BP1Base::InitParameterMaxBufferSize(const std::string value)
if (m_DebugMode)
{
bool success = true;
+ std::string description;
+
try
{
m_MaxBufferSize = static_cast<size_t>(std::stoul(number) * factor);
@@ -267,11 +277,17 @@ void BP1Base::InitParameterMaxBufferSize(const std::string value)
catch (std::exception &e)
{
success = false;
+ description = std::string(e.what());
}
if (!success || m_MaxBufferSize < 16 * 1024) // 16384b
{
- throw std::invalid_argument(errorMessage);
+ throw std::invalid_argument(
+ "ERROR: couldn't convert value of max_buffer_size IO "
+ "SetParameter, valid syntax: MaxBufferSize=10Gb, "
+ "MaxBufferSize=1000Mb, MaxBufferSize=16Kb (minimum default), "
+ "\nadditional description: " +
+ description + " in call to Open");
}
}
else
@@ -287,6 +303,7 @@ void BP1Base::InitParameterThreads(const std::string value)
if (m_DebugMode)
{
bool success = true;
+ std::string description;
try
{
@@ -295,13 +312,15 @@ void BP1Base::InitParameterThreads(const std::string value)
catch (std::exception &e)
{
success = false;
+ description = std::string(e.what());
}
if (!success || threads < 1)
{
throw std::invalid_argument(
"ERROR: value in Threads=value in IO SetParameters must be "
- "an integer >= 1 (default), in call to Open\n");
+ "an integer >= 1 (default) \nadditional description: " +
+ description + "\n, in call to Open\n");
}
}
else
@@ -319,6 +338,7 @@ void BP1Base::InitParameterVerbose(const std::string value)
if (m_DebugMode)
{
bool success = true;
+ std::string description;
try
{
@@ -327,13 +347,15 @@ void BP1Base::InitParameterVerbose(const std::string value)
catch (std::exception &e)
{
success = false;
+ description = std::string(e.what());
}
if (!success || verbosity < 0 || verbosity > 5)
{
throw std::invalid_argument(
"ERROR: value in Verbose=value in IO SetParameters must be "
- "an integer in the range [0,5], in call to Open\n");
+ "an integer in the range [0,5], \nadditional description: " +
+ description + "\n, in call to Open\n");
}
}
else
diff --git a/source/adios2/toolkit/format/bp1/BP1Writer.cpp b/source/adios2/toolkit/format/bp1/BP1Writer.cpp
index c7b9210b3e63d6ab65eb7d2590da3cc3439ac624..7bc9f34b18e379806764b3c36808272b03d67e76 100644
--- a/source/adios2/toolkit/format/bp1/BP1Writer.cpp
+++ b/source/adios2/toolkit/format/bp1/BP1Writer.cpp
@@ -42,7 +42,7 @@ void BP1Writer::WriteProcessGroupIndex(
dataPosition += 8; // skip pg length (8)
const std::size_t metadataPGLengthPosition = metadataBuffer.size();
- metadataBuffer.insert(metadataBuffer.end(), 2, 0); // skip pg length (2)
+ metadataBuffer.insert(metadataBuffer.end(), 2, '\0'); // skip pg length (2)
// write name to metadata
const std::string name(std::to_string(m_BP1Aggregator.m_RankMPI));
@@ -56,7 +56,8 @@ void BP1Writer::WriteProcessGroupIndex(
WriteNameRecord(name, dataBuffer, dataPosition);
// processID in metadata,
- const uint32_t processID = static_cast<uint32_t>(m_BP1Aggregator.m_RankMPI);
+ const uint32_t processID =
+ static_cast<const uint32_t>(m_BP1Aggregator.m_RankMPI);
InsertToBuffer(metadataBuffer, &processID);
// skip coordination var in data ....what is coordination var?
dataPosition += 4;
@@ -71,22 +72,23 @@ void BP1Writer::WriteProcessGroupIndex(
CopyToBuffer(dataBuffer, dataPosition, &m_MetadataSet.TimeStep);
// offset to pg in data in metadata which is the current absolute position
- InsertToBuffer(metadataBuffer, reinterpret_cast<uint64_t *>(
- &m_HeapBuffer.m_DataAbsolutePosition));
+ InsertU64(metadataBuffer, m_HeapBuffer.m_DataAbsolutePosition);
// Back to writing metadata pg index length (length of group)
- const uint16_t metadataPGIndexLength =
- metadataBuffer.size() - metadataPGLengthPosition - 2;
+ const uint16_t metadataPGIndexLength = static_cast<const uint16_t>(
+ metadataBuffer.size() - metadataPGLengthPosition - 2);
+
size_t backPosition = metadataPGLengthPosition;
CopyToBuffer(metadataBuffer, backPosition, &metadataPGIndexLength);
// DONE With metadataBuffer
// here write method in data
const std::vector<uint8_t> methodIDs = GetTransportIDs(transportsTypes);
- const uint8_t methodsCount = methodIDs.size();
+ const uint8_t methodsCount = static_cast<const uint8_t>(methodIDs.size());
CopyToBuffer(dataBuffer, dataPosition, &methodsCount); // count
// methodID (1) + method params length(2), no parameters for now
- const uint16_t methodsLength = methodIDs.size() * 3;
+ const uint16_t methodsLength =
+ static_cast<const uint16_t>(methodsCount * 3);
CopyToBuffer(dataBuffer, dataPosition, &methodsLength); // length
@@ -253,23 +255,19 @@ void BP1Writer::WriteDimensionsRecord(const Dims localDimensions,
{
if (offsets.empty())
{
- for (const auto &localDimension : localDimensions)
+ for (const auto localDimension : localDimensions)
{
- InsertToBuffer(buffer,
- reinterpret_cast<const uint64_t *>(&localDimension));
- buffer.insert(buffer.end(), 2 * sizeof(uint64_t), 0);
+ InsertU64(buffer, localDimension);
+ buffer.insert(buffer.end(), 2 * sizeof(uint64_t), '\0');
}
}
else
{
for (unsigned int d = 0; d < localDimensions.size(); ++d)
{
- InsertToBuffer(buffer, reinterpret_cast<const uint64_t *>(
- &localDimensions[d]));
- InsertToBuffer(buffer, reinterpret_cast<const uint64_t *>(
- &globalDimensions[d]));
- InsertToBuffer(buffer,
- reinterpret_cast<const uint64_t *>(&offsets[d]));
+ InsertU64(buffer, localDimensions[d]);
+ InsertU64(buffer, globalDimensions[d]);
+ InsertU64(buffer, offsets[d]);
}
}
}
@@ -290,8 +288,9 @@ void BP1Writer::WriteDimensionsRecord(const Dims localDimensions,
CopyToBuffer(buffer, position, &no);
}
- CopyToBuffer(buffer, position,
- reinterpret_cast<const uint64_t *>(&dimension));
+ const uint64_t dimension64 = static_cast<const uint64_t>(dimension);
+
+ CopyToBuffer(buffer, position, &dimension64);
};
// BODY Starts here
@@ -326,7 +325,7 @@ void BP1Writer::WriteDimensionsRecord(const Dims localDimensions,
void BP1Writer::WriteNameRecord(const std::string name,
std::vector<char> &buffer) noexcept
{
- const uint16_t length = name.length();
+ const uint16_t length = static_cast<const uint16_t>(name.length());
InsertToBuffer(buffer, &length);
InsertToBuffer(buffer, name.c_str(), length);
}
@@ -335,7 +334,7 @@ void BP1Writer::WriteNameRecord(const std::string name,
std::vector<char> &buffer,
size_t &position) noexcept
{
- const uint16_t length = name.length();
+ const uint16_t length = static_cast<const uint16_t>(name.length());
CopyToBuffer(buffer, position, &length);
CopyToBuffer(buffer, position, name.c_str(), length);
}
@@ -434,23 +433,22 @@ void BP1Writer::FlattenMetadata() noexcept
lf_IndexCountLength(m_MetadataSet.AttributesIndices, attributesCount,
attributesLength);
- const size_t footerSize = (pgLength + 16) + (varsLength + 12) +
- (attributesLength + 12) +
- m_MetadataSet.MiniFooterSize;
+ const size_t footerSize = static_cast<const size_t>(
+ (pgLength + 16) + (varsLength + 12) + (attributesLength + 12) +
+ m_MetadataSet.MiniFooterSize);
auto &buffer = m_HeapBuffer.m_Data;
auto &position = m_HeapBuffer.m_DataPosition;
// reserve data to fit metadata,
- // must replace with growth buffer strategy
+ // must replace with growth buffer strategy?
m_HeapBuffer.ResizeData(position + footerSize);
- // buffer.resize(position + footerSize);
// write pg index
CopyToBuffer(buffer, position, &pgCount);
CopyToBuffer(buffer, position, &pgLength);
CopyToBuffer(buffer, position, m_MetadataSet.PGIndex.Buffer.data(),
- pgLength);
+ static_cast<const size_t>(pgLength));
// Vars indices
lf_FlattenIndices(varsCount, varsLength, m_MetadataSet.VarsIndices, buffer,
@@ -460,9 +458,12 @@ void BP1Writer::FlattenMetadata() noexcept
m_MetadataSet.AttributesIndices, buffer, position);
// getting absolute offsets, minifooter is 28 bytes for now
- const uint64_t offsetPGIndex = m_HeapBuffer.m_DataAbsolutePosition;
- const uint64_t offsetVarsIndex = offsetPGIndex + (pgLength + 16);
- const uint64_t offsetAttributeIndex = offsetVarsIndex + (varsLength + 12);
+ const uint64_t offsetPGIndex =
+ static_cast<const uint64_t>(m_HeapBuffer.m_DataAbsolutePosition);
+ const uint64_t offsetVarsIndex =
+ static_cast<const uint64_t>(offsetPGIndex + (pgLength + 16));
+ const uint64_t offsetAttributeIndex =
+ static_cast<const uint64_t>(offsetVarsIndex + (varsLength + 12));
CopyToBuffer(buffer, position, &offsetPGIndex);
CopyToBuffer(buffer, position, &offsetVarsIndex);
diff --git a/source/adios2/toolkit/format/bp1/BP1Writer.h b/source/adios2/toolkit/format/bp1/BP1Writer.h
index 44d306c3f2500cf61a216f58be108026ab0f58df..7f65b749e2730dece386db1e273a4f77cf7208cf 100644
--- a/source/adios2/toolkit/format/bp1/BP1Writer.h
+++ b/source/adios2/toolkit/format/bp1/BP1Writer.h
@@ -72,7 +72,7 @@ public:
void Advance();
/** Flattens data buffer and close current process group, doesn't
- * advance time index */
+ * advance time index */
void Flush();
/**
diff --git a/source/adios2/toolkit/format/bp1/BP1Writer.tcc b/source/adios2/toolkit/format/bp1/BP1Writer.tcc
index 733f43195b2e8f85d864e0d78896e1bc28400c11..c1aa88ca6643d8e98fe48c3c4145b137b7ec7ff4 100644
--- a/source/adios2/toolkit/format/bp1/BP1Writer.tcc
+++ b/source/adios2/toolkit/format/bp1/BP1Writer.tcc
@@ -37,7 +37,7 @@ void BP1Writer::WriteVariableMetadata(const Variable<T> &variable) noexcept
stats.MemberID = variableIndex.MemberID;
// write metadata header in data and extract offsets
- stats.Offset = m_HeapBuffer.m_DataAbsolutePosition;
+ stats.Offset = static_cast<uint64_t>(m_HeapBuffer.m_DataAbsolutePosition);
WriteVariableMetadataInData(variable, stats);
stats.PayloadOffset = m_HeapBuffer.m_DataAbsolutePosition;
@@ -109,7 +109,8 @@ void BP1Writer::WriteVariableMetadataInData(
constexpr char no = 'n'; // isDimension
CopyToBuffer(buffer, position, &no);
- const uint8_t dimensions = variable.m_Count.size();
+ const uint8_t dimensions =
+ static_cast<const uint8_t>(variable.m_Count.size());
CopyToBuffer(buffer, position, &dimensions); // count
// 27 is from 9 bytes for each: var y/n + local, var y/n + global dimension,
@@ -145,11 +146,11 @@ void BP1Writer::WriteVariableMetadataInIndex(
if (isNew) // write variable header (might be shared with
// attributes index)
{
- buffer.insert(buffer.end(), 4, 0); // skip var length (4)
+ buffer.insert(buffer.end(), 4, '\0'); // skip var length (4)
InsertToBuffer(buffer, &stats.MemberID);
- buffer.insert(buffer.end(), 2, 0); // skip group name
+ buffer.insert(buffer.end(), 2, '\0'); // skip group name
WriteNameRecord(variable.m_Name, buffer);
- buffer.insert(buffer.end(), 2, 0); // skip path
+ buffer.insert(buffer.end(), 2, '\0'); // skip path
const std::uint8_t dataType = GetDataType<T>();
InsertToBuffer(buffer, &dataType);
@@ -256,15 +257,17 @@ void BP1Writer::WriteVariableCharacteristics(
// going back at the end
const size_t characteristicsCountPosition = buffer.size();
// skip characteristics count(1) + length (4)
- buffer.insert(buffer.end(), 5, 0);
+ buffer.insert(buffer.end(), 5, '\0');
uint8_t characteristicsCounter = 0;
// DIMENSIONS
uint8_t characteristicID = characteristic_dimensions;
InsertToBuffer(buffer, &characteristicID);
- const uint8_t dimensions = variable.m_Count.size();
+ const uint8_t dimensions =
+ static_cast<const uint8_t>(variable.m_Count.size());
InsertToBuffer(buffer, &dimensions); // count
- const uint16_t dimensionsLength = 24 * dimensions;
+ const uint16_t dimensionsLength =
+ static_cast<const uint16_t>(24 * dimensions);
InsertToBuffer(buffer, &dimensionsLength); // length
WriteDimensionsRecord(variable.m_Count, variable.m_Shape, variable.m_Start,
buffer);
@@ -276,9 +279,10 @@ void BP1Writer::WriteVariableCharacteristics(
WriteCharacteristicRecord(characteristic_time_index, characteristicsCounter,
stats.TimeIndex, buffer);
+ const uint32_t rankU32 =
+ static_cast<const uint32_t>(m_BP1Aggregator.m_RankMPI);
WriteCharacteristicRecord(characteristic_file_index, characteristicsCounter,
- static_cast<uint32_t>(m_BP1Aggregator.m_RankMPI),
- buffer);
+ rankU32, buffer);
WriteCharacteristicRecord(characteristic_offset, characteristicsCounter,
stats.Offset, buffer);
@@ -293,8 +297,8 @@ void BP1Writer::WriteVariableCharacteristics(
CopyToBuffer(buffer, backPosition, &characteristicsCounter); // count (1)
// remove its own length (4) + characteristic counter (1)
- const uint32_t characteristicsLength =
- buffer.size() - characteristicsCountPosition - 4 - 1;
+ const uint32_t characteristicsLength = static_cast<const uint32_t>(
+ buffer.size() - characteristicsCountPosition - 4 - 1);
CopyToBuffer(buffer, backPosition, &characteristicsLength); // length
}
@@ -315,9 +319,11 @@ void BP1Writer::WriteVariableCharacteristics(
uint8_t characteristicID = characteristic_dimensions;
CopyToBuffer(buffer, position, &characteristicID);
- const uint8_t dimensions = variable.m_Count.size();
+ const uint8_t dimensions =
+ static_cast<const uint8_t>(variable.m_Count.size());
CopyToBuffer(buffer, position, &dimensions); // count
- const uint16_t dimensionsLength = 24 * dimensions;
+ const uint16_t dimensionsLength =
+ static_cast<const uint16_t>(24 * dimensions);
CopyToBuffer(buffer, position, &dimensionsLength); // length
WriteDimensionsRecord(variable.m_Count, variable.m_Shape, variable.m_Start,
buffer, position, true); // isCharacteristic = true
@@ -333,8 +339,8 @@ void BP1Writer::WriteVariableCharacteristics(
CopyToBuffer(buffer, backPosition, &characteristicsCounter);
// remove its own length (4) + characteristic counter (1)
- const uint32_t characteristicsLength =
- position - characteristicsCountPosition - 4 - 1;
+ const uint32_t characteristicsLength = static_cast<const uint32_t>(
+ position - characteristicsCountPosition - 4 - 1);
CopyToBuffer(buffer, backPosition, &characteristicsLength);
}
diff --git a/source/adios2/toolkit/transport/file/FilePointer.cpp b/source/adios2/toolkit/transport/file/FilePointer.cpp
index 82b764bd58548119564bf60f8fda0416d2967a46..e11c4f9fb9dc17a6118f4cec5d8fc6f933061eb8 100644
--- a/source/adios2/toolkit/transport/file/FilePointer.cpp
+++ b/source/adios2/toolkit/transport/file/FilePointer.cpp
@@ -14,6 +14,11 @@
#include <ios> //std::ios_base::failure
/// \endcond
+// removes fopen warning on Windows
+#ifdef _WIN32
+#pragma warning(disable : 4996) // fopen
+#endif
+
namespace adios2
{
namespace transport
@@ -28,7 +33,7 @@ FilePointer::~FilePointer()
{
if (m_IsOpen)
{
- fclose(m_File);
+ std::fclose(m_File);
}
}
@@ -48,19 +53,19 @@ void FilePointer::Open(const std::string &name, const OpenMode openMode)
if (m_OpenMode == OpenMode::Write)
{
- m_File = std::fopen(name.c_str(), "w");
+ m_File = std::fopen(name.c_str(), "wb");
}
else if (m_OpenMode == OpenMode::Append)
{
// need to change when implemented
- m_File = std::fopen(name.c_str(), "a");
+ m_File = std::fopen(name.c_str(), "ab");
}
else if (m_OpenMode == OpenMode::Read)
{
- m_File = std::fopen(name.c_str(), "r");
+ m_File = std::fopen(name.c_str(), "rb");
}
- if (std::ferror(m_File))
+ if (ferror(m_File))
{
throw std::ios_base::failure("ERROR: couldn't open file " + name +
", "
diff --git a/source/adios2/toolkit/transport/file/FilePointer.h b/source/adios2/toolkit/transport/file/FilePointer.h
index 7c14b48066d1291790411adac6ffef5194bb6cb1..6d1521a3d5d99fff26f00806871194f9461fa5a8 100644
--- a/source/adios2/toolkit/transport/file/FilePointer.h
+++ b/source/adios2/toolkit/transport/file/FilePointer.h
@@ -45,7 +45,7 @@ public:
private:
/** C File pointer */
- std::FILE *m_File = nullptr; // NULL or nullptr?
+ FILE *m_File = nullptr; // NULL or nullptr?
};
} // end namespace transport
diff --git a/source/adios2/toolkit/transport/file/FileStream.cpp b/source/adios2/toolkit/transport/file/FileStream.cpp
index 50c05e0e22443b37be4721c0e60eb5568bdae081..3609394539420b2a3c3333bc31506f45b05b76a7 100644
--- a/source/adios2/toolkit/transport/file/FileStream.cpp
+++ b/source/adios2/toolkit/transport/file/FileStream.cpp
@@ -40,16 +40,17 @@ void FileStream::Open(const std::string &name, const OpenMode openMode)
if (m_OpenMode == OpenMode::Write)
{
- m_FileStream.open(name, std::fstream::out);
+ m_FileStream.open(name, std::fstream::out | std::fstream::binary);
}
else if (m_OpenMode == OpenMode::Append)
{
// to be changed to rw?
- m_FileStream.open(name, std::fstream::out | std::fstream::app);
+ m_FileStream.open(name, std::fstream::out | std::fstream::app |
+ std::fstream::binary);
}
else if (m_OpenMode == OpenMode::Read)
{
- m_FileStream.open(name, std::fstream::in);
+ m_FileStream.open(name, std::fstream::in | std::fstream::binary);
}
if (!m_FileStream)
diff --git a/source/adios2/toolkit/transport/wan/WANZmq.cpp b/source/adios2/toolkit/transport/wan/WANZmq.cpp
index 6627a6372c42ddebaab36572825787cabbfa29e2..7760f4bf7f00ab3e8f95c4bf3eea6294b1f8b286 100644
--- a/source/adios2/toolkit/transport/wan/WANZmq.cpp
+++ b/source/adios2/toolkit/transport/wan/WANZmq.cpp
@@ -10,6 +10,7 @@
#include "WANZmq.h"
+#include <iostream>
#include <zmq.h>
namespace adios2
@@ -22,7 +23,7 @@ WANZmq::WANZmq(const std::string ipAddress, const std::string port,
: Transport("wan", "zmq", mpiComm, debugMode), m_IPAddress(ipAddress),
m_Port(port)
{
-
+ m_Context = zmq_ctx_new();
if (m_DebugMode)
{
// TODO verify port is unsigned int
@@ -35,6 +36,10 @@ WANZmq::~WANZmq()
{
zmq_close(m_Socket);
}
+ if (m_Context)
+ {
+ zmq_ctx_destroy(m_Context);
+ }
}
void WANZmq::Open(const std::string &name, const OpenMode openMode)
@@ -51,7 +56,8 @@ void WANZmq::Open(const std::string &name, const OpenMode openMode)
m_Socket = zmq_socket(m_Context, ZMQ_REQ);
const std::string fullIP("tcp://" + m_IPAddress + ":" + m_Port);
- zmq_connect(m_Socket, fullIP.c_str());
+ std::cout << "full IP = " << fullIP << std::endl;
+ int err = zmq_connect(m_Socket, fullIP.c_str());
if (m_Profiler.IsActive)
{
@@ -78,6 +84,7 @@ void WANZmq::Open(const std::string &name, const OpenMode openMode)
m_Socket = zmq_socket(m_Context, ZMQ_REP);
const std::string fullIP("tcp://" + m_IPAddress + ":" + m_Port);
+ std::cout << "full IP = " << fullIP << std::endl;
zmq_bind(m_Socket, fullIP.c_str());
if (m_Profiler.IsActive)
@@ -108,10 +115,13 @@ void WANZmq::Write(const char *buffer, size_t size)
m_Profiler.Timers.at("write").Resume();
}
+ /*
+
int status = zmq_send(m_Socket, buffer, size, 0);
char ret[10];
zmq_recv(m_Socket, ret, 10, 0);
+
if (m_Profiler.IsActive)
{
m_Profiler.Timers.at("write").Pause();
@@ -128,6 +138,7 @@ void WANZmq::Write(const char *buffer, size_t size)
", in call to WANZmq write\n");
}
}
+ */
}
void WANZmq::Flush() {}
diff --git a/source/adios2/toolkit/transport/wan/WANZmq.h b/source/adios2/toolkit/transport/wan/WANZmq.h
index 7249f963cc31d0fd464c4c9d92c594c2ee600408..c24b3e70b02841f2e0d67710eec0875ea2dffd63 100644
--- a/source/adios2/toolkit/transport/wan/WANZmq.h
+++ b/source/adios2/toolkit/transport/wan/WANZmq.h
@@ -50,10 +50,10 @@ private:
const std::string m_IPAddress;
std::string m_Port;
- /** TODO: find out if is provided externally */
+ /** context handler created by zmq, thread safe */
void *m_Context = NULL;
- /** handler created by zmq */
+ /** socket handler created by zmq */
void *m_Socket = NULL;
};
diff --git a/source/adios2/toolkit/transportman/dataman/DataMan.cpp b/source/adios2/toolkit/transportman/dataman/DataMan.cpp
index 651337954c14bce8b25990f74ddf7a446f6b5837..95abf19ee8c735b42293c6084383dcb5a4005fd1 100644
--- a/source/adios2/toolkit/transportman/dataman/DataMan.cpp
+++ b/source/adios2/toolkit/transportman/dataman/DataMan.cpp
@@ -8,7 +8,7 @@
* Author: Jason Wang wangr1@ornl.gov
*/
-#include "adios2/toolkit/transportman/DataMan/DataMan.h"
+#include "adios2/toolkit/transportman/dataman/DataMan.h"
#include "adios2/helper/adiosString.h"
#ifdef ADIOS2_HAVE_ZEROMQ
@@ -32,16 +32,16 @@ void DataMan::OpenWANTransports(const std::string &name,
for (const auto ¶meters : parametersVector)
{
- std::shared_ptr<Transport> wanTransport;
-
- for(auto &i :parameters){
+ std::shared_ptr<Transport> wanTransport, controlTransport;
+ // to be removed
+ for (auto &i : parameters)
+ {
std::cout << i.first << " " << i.second << std::endl;
-
}
const std::string type(
- GetParameter("type", parameters, true, m_DebugMode, ""));
+ GetParameter("type", parameters, true, m_DebugMode, ""));
const std::string trans(
GetParameter("transport", parameters, true, m_DebugMode, ""));
@@ -49,14 +49,16 @@ void DataMan::OpenWANTransports(const std::string &name,
const std::string ipAddress(
GetParameter("ipaddress", parameters, true, m_DebugMode, ""));
- std::string port(
+ std::string port_control(
GetParameter("port", parameters, false, m_DebugMode, ""));
- if (port.empty())
+ if (port_control.empty())
{
- port = m_DefaultPort;
+ port_control = std::to_string(m_DefaultPort);
}
+ const std::string port_data(std::to_string(stoi(port_control) + 1));
+
std::string messageName(
GetParameter("name", parameters, false, m_DebugMode, ""));
@@ -71,7 +73,9 @@ void DataMan::OpenWANTransports(const std::string &name,
{
#ifdef ADIOS2_HAVE_ZEROMQ
wanTransport = std::make_shared<transport::WANZmq>(
- ipAddress, port, m_MPIComm, m_DebugMode);
+ ipAddress, port_data, m_MPIComm, m_DebugMode);
+ controlTransport = std::make_shared<transport::WANZmq>(
+ ipAddress, port_control, m_MPIComm, m_DebugMode);
#else
throw std::invalid_argument(
"ERROR: this version of ADIOS2 didn't compile with "
@@ -89,8 +93,46 @@ void DataMan::OpenWANTransports(const std::string &name,
}
}
}
+
wanTransport->Open(messageName, openMode);
m_Transports.push_back(std::move(wanTransport));
+ controlTransport->Open(messageName, openMode);
+ m_ControlTransports.push_back(std::move(controlTransport));
+ }
+}
+
+void DataMan::WriteWAN(const void *buffer, nlohmann::json jmsg)
+{
+ m_ControlTransports[m_CurrentTransport]->Write(jmsg.dump().c_str(),
+ jmsg.dump().size());
+ m_Transports[m_CurrentTransport]->Write(static_cast<const char *>(buffer),
+ jmsg["bytes"].get<size_t>());
+}
+
+void DataMan::SetCallback(std::function<void(const void *, std::string,
+ std::string, std::string, Dims)>
+ callback)
+{
+ m_CallBack = callback;
+}
+
+void DataMan::ReadThread(std::shared_ptr<Transport> trans,
+ std::shared_ptr<Transport> ctl_trans)
+{
+ while (m_Listening)
+ {
+ // Wait for Read API to be implemented
+ /*
+ if (ctl_trans->Read() >= 0)
+ {
+ std::string smsg;
+ nlohmann::json jmsg = json::parse(smsg);
+ }
+ else
+ {
+ usleep(1);
+ }
+ */
}
}
diff --git a/source/adios2/toolkit/transportman/dataman/DataMan.h b/source/adios2/toolkit/transportman/dataman/DataMan.h
index 0e7dacd258b5431a5994a1ce1b357dea6ce03303..8466068fa98c96ac2b1fc97a88970ff505efa53e 100644
--- a/source/adios2/toolkit/transportman/dataman/DataMan.h
+++ b/source/adios2/toolkit/transportman/dataman/DataMan.h
@@ -13,6 +13,7 @@
#include "adios2/toolkit/transportman/TransportMan.h"
#include <json.hpp>
+#include <thread>
namespace adios2
{
@@ -31,11 +32,29 @@ public:
const std::vector<Params> ¶metersVector,
const bool profile);
+ void WriteWAN(const void *buffer, nlohmann::json jmsg);
+
+ void SetCallback(std::function<void(const void *, std::string, std::string,
+ std::string, Dims)>
+ callback);
+
private:
+ void ReadThread(std::shared_ptr<Transport> trans,
+ std::shared_ptr<Transport> ctl_trans);
+
+ std::vector<std::shared_ptr<Transport>> m_ControlTransports;
+ std::vector<std::thread> m_ControlThreads;
+ size_t m_CurrentTransport = 0;
+ bool m_Listening = false;
+
+ std::function<void(const void *, std::string, std::string, std::string,
+ Dims)>
+ m_CallBack;
+
nlohmann::json m_JMessage;
/** Pick the appropriate default */
- const std::string m_DefaultPort = "22";
+ const int m_DefaultPort = 12306;
};
} // end namespace transportman
diff --git a/source/adios2/transform/compress/CompressBZip2.cpp b/source/adios2/transform/compress/CompressBZip2.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..59c378422c42f25edf77e0dd4d99f30da3c180b0
--- /dev/null
+++ b/source/adios2/transform/compress/CompressBZip2.cpp
@@ -0,0 +1,170 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * CompressBZip2.cpp
+ *
+ * Created on: Jul 24, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#include "CompressBZip2.h"
+
+/// \cond EXCLUDE_FROM_DOXYGEN
+#include <cmath> //std::ceil
+#include <ios> //std::ios_base::failure
+#include <stdexcept> //std::invalid_argument
+/// \endcond
+
+#include <bzlib.h>
+
+#include "adios2/helper/adiosFunctions.h"
+
+namespace adios2
+{
+namespace transform
+{
+
+CompressBZip2::CompressBZip2(const bool debugMode)
+: Transform("bzip2", debugMode)
+{
+}
+
+size_t CompressBZip2::BufferMaxSize(const size_t sizeIn) const
+{
+ return static_cast<size_t>(std::ceil(1.1 * sizeIn) + 600);
+}
+
+size_t CompressBZip2::Compress(const void *dataIn, const Dims &dimensions,
+ const size_t elementSize, const std::string type,
+ void *bufferOut, const Params ¶meters) const
+{
+ // defaults
+ int blockSize100k = 1;
+ int verbosity = 0;
+ int workFactor = 0;
+
+ if (!parameters.empty())
+ {
+ const std::string hint(" in call to CompressBZip2 Compress " + type +
+ "\n");
+ SetParameterValueInt("BlockSize100K", parameters, blockSize100k,
+ m_DebugMode, hint);
+ SetParameterValueInt("Verbosity", parameters, verbosity, m_DebugMode,
+ hint);
+ SetParameterValueInt("WorkFactor", parameters, workFactor, m_DebugMode,
+ hint);
+ if (m_DebugMode == true)
+ {
+
+ if (blockSize100k < 1 || blockSize100k > 9)
+ {
+ throw std::invalid_argument(
+ "ERROR: BlockSize100K must be an "
+ "integer between 1 (less "
+ "compression, less memory) and 9 "
+ "(more compression, more memory) inclusive, " +
+ hint);
+ }
+ }
+ }
+
+ const size_t sizeIn =
+ static_cast<const size_t>(GetTotalSize(dimensions) * elementSize);
+ // Build inputs to BZip2 compression function
+ char *dest = const_cast<char *>(reinterpret_cast<const char *>(bufferOut));
+ unsigned int destLen = static_cast<unsigned int>(BufferMaxSize(sizeIn));
+
+ char *source = const_cast<char *>(reinterpret_cast<const char *>(dataIn));
+ unsigned int sourceLen = static_cast<unsigned int>(sizeIn);
+
+ int status = BZ2_bzBuffToBuffCompress(dest, &destLen, source, sourceLen,
+ blockSize100k, verbosity, workFactor);
+
+ if (m_DebugMode)
+ {
+ CheckStatus(status, "in call to CompressBZip2 Compress\n");
+ }
+
+ return static_cast<size_t>(destLen);
+}
+
+size_t CompressBZip2::Decompress(const void *bufferIn, const size_t sizeIn,
+ void *dataOut, const size_t sizeOut) const
+{
+ // TODO: leave defaults at zero?
+ int small = 0;
+ int verbosity = 0;
+
+ char *dest = reinterpret_cast<char *>(dataOut);
+ unsigned int destLen = static_cast<unsigned int>(sizeOut);
+
+ char *source = const_cast<char *>(reinterpret_cast<const char *>(bufferIn));
+ unsigned int sourceLen = static_cast<unsigned int>(sizeIn);
+
+ int status = BZ2_bzBuffToBuffDecompress(dest, &destLen, source, sourceLen,
+ small, verbosity);
+
+ if (m_DebugMode)
+ {
+ CheckStatus(status, "in call to CompressBZip2 Decompress\n");
+ }
+
+ return static_cast<size_t>(destLen);
+}
+
+void CompressBZip2::CheckStatus(const int status, const std::string hint) const
+{
+ switch (status)
+ {
+
+ case (BZ_CONFIG_ERROR):
+ throw std::invalid_argument(
+ "ERROR: BZ_CONFIG_ERROR, bzip2 library is not configured "
+ "correctly" +
+ hint);
+ break;
+
+ case (BZ_PARAM_ERROR):
+ throw std::invalid_argument(
+ "ERROR: BZ_PARAM_ERROR bufferOut stream might be null" + hint);
+ break;
+
+ case (BZ_MEM_ERROR):
+ throw std::ios_base::failure(
+ "ERROR: BZ_MEM_ERROR bzip2 detected insufficient memory " + hint);
+ break;
+
+ case (BZ_OUTBUFF_FULL):
+ throw std::ios_base::failure("ERROR: BZ_OUTBUFF_FULL bzip2 detected "
+ "size of compressed data is larger than "
+ "destination length " +
+ hint);
+ break;
+
+ // decompression
+ case (BZ_DATA_ERROR):
+ throw std::invalid_argument("ERROR: BZ_DATA_ERROR, bzip2 library "
+ "detected integrity errors in compressed "
+ "data " +
+ hint);
+ break;
+
+ case (BZ_DATA_ERROR_MAGIC):
+ throw std::invalid_argument("ERROR: BZ_DATA_ERROR_MAGIC, bzip2 library "
+ "detected wrong magic numbers in "
+ "compressed data " +
+ hint);
+ break;
+
+ case (BZ_UNEXPECTED_EOF):
+ throw std::invalid_argument("ERROR: BZ_UNEXPECTED_EOF, bzip2 library "
+ "detected unexpected end of "
+ "compressed data " +
+ hint);
+ break;
+ }
+}
+
+} // end namespace transform
+} // end namespace adios2
diff --git a/source/adios2/transform/compress/CompressBZip2.h b/source/adios2/transform/compress/CompressBZip2.h
new file mode 100644
index 0000000000000000000000000000000000000000..81cd28759a041a0feca1408c80a2e1ab784cb194
--- /dev/null
+++ b/source/adios2/transform/compress/CompressBZip2.h
@@ -0,0 +1,74 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * CompressBZip2.h : wrapper to BZip2 compression library
+ *
+ * Created on: Jul 24, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#ifndef ADIOS2_TRANSFORM_COMPRESSION_COMPRESSBZIP2_H_
+#define ADIOS2_TRANSFORM_COMPRESSION_COMPRESSBZIP2_H_
+
+#include "adios2/core/Transform.h"
+
+namespace adios2
+{
+namespace transform
+{
+
+class CompressBZip2 : public Transform
+{
+
+public:
+ /**
+ * Unique constructor
+ * @param debugMode
+ */
+ CompressBZip2(const bool debugMode);
+
+ ~CompressBZip2() = default;
+
+ size_t BufferMaxSize(const size_t sizeIn) const final;
+
+ /**
+ * Compression signature for legacy libraries that use void*
+ * @param dataIn
+ * @param dimensions
+ * @param type
+ * @param bufferOut
+ * @param parameters
+ * @return size of compressed buffer in bytes
+ */
+ size_t Compress(const void *dataIn, const Dims &dimensions,
+ const size_t elementSize, const std::string type,
+ void *bufferOut,
+ const Params ¶meters = Params()) const final;
+
+ /**
+ * Decompression signature for legacy libraries that use void*
+ * @param bufferIn
+ * @param sizeIn
+ * @param dataOut
+ * @param dimensions
+ * @param type
+ * @return size of decompressed buffer in bytes
+ */
+ size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
+ const size_t sizeOut) const final;
+
+private:
+ /**
+ * In debug mode, check status from BZip compression and decompression
+ * functions
+ * @param status returned by BZip2 library
+ * @param hint extra exception information
+ */
+ void CheckStatus(const int status, const std::string hint) const;
+};
+
+} // end namespace transform
+} // end namespace adios2
+
+#endif /* ADIOS2_TRANSFORM_COMPRESSION_COMPRESSBZIP2_H_ */
diff --git a/source/adios2/transform/compress/CompressZfp.cpp b/source/adios2/transform/compress/CompressZfp.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b53f18c4f82e55281fb69e1cf210b3f5df8b6428
--- /dev/null
+++ b/source/adios2/transform/compress/CompressZfp.cpp
@@ -0,0 +1,278 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * CompressZfp.cpp
+ *
+ * Created on: Jul 25, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#include "CompressZfp.h"
+
+#include "adios2/helper/adiosFunctions.h"
+
+namespace adios2
+{
+namespace transform
+{
+
+CompressZfp::CompressZfp(const bool debugMode) : Transform("zfp", debugMode) {}
+
+size_t CompressZfp::DoBufferMaxSize(const void *dataIn, const Dims &dimensions,
+ const std::string type,
+ const Params ¶meters) const
+{
+ zfp_field *field = GetZFPField(dataIn, dimensions, type);
+ zfp_stream *stream = GetZFPStream(dimensions, type, parameters);
+ const size_t maxSize = zfp_stream_maximum_size(stream, field);
+ zfp_field_free(field);
+ zfp_stream_close(stream);
+ return maxSize;
+}
+
+size_t CompressZfp::Compress(const void *dataIn, const Dims &dimensions,
+ const size_t elementSize, const std::string type,
+ void *bufferOut, const Params ¶meters) const
+{
+
+ zfp_field *field = GetZFPField(dataIn, dimensions, type);
+ zfp_stream *stream = GetZFPStream(dimensions, type, parameters);
+ size_t maxSize = zfp_stream_maximum_size(stream, field);
+ // associate bitstream
+ bitstream *bitstream = stream_open(bufferOut, maxSize);
+ zfp_stream_set_bit_stream(stream, bitstream);
+ zfp_stream_rewind(stream);
+
+ size_t sizeOut = zfp_compress(stream, field);
+
+ if (m_DebugMode == true)
+ {
+ if (sizeOut == 0)
+ {
+ throw std::invalid_argument("ERROR: zfp failed, compressed buffer "
+ "size is 0, in call to Compress");
+ }
+ }
+
+ zfp_field_free(field);
+ zfp_stream_close(stream);
+ return sizeOut;
+}
+
+size_t CompressZfp::Decompress(const void *bufferIn, const size_t sizeIn,
+ void *dataOut, const Dims &dimensions,
+ const std::string type,
+ const Params ¶meters) const
+{
+ auto lf_GetTypeSize = [](const zfp_type zfpType) -> size_t {
+
+ size_t size = 0;
+ if (zfpType == zfp_type_int32 || zfpType == zfp_type_float)
+ {
+ size = 4;
+ }
+ else if (zfpType == zfp_type_int64 || zfpType == zfp_type_double)
+ {
+ size = 8;
+ }
+ return size;
+ };
+
+ zfp_field *field = GetZFPField(dataOut, dimensions, type);
+ zfp_stream *stream = GetZFPStream(dimensions, type, parameters);
+
+ // associate bitstream
+ bitstream *bitstream = stream_open(const_cast<void *>(bufferIn), sizeIn);
+ zfp_stream_set_bit_stream(stream, bitstream);
+ zfp_stream_rewind(stream);
+
+ int status = zfp_decompress(stream, field);
+
+ if (m_DebugMode)
+ {
+ if (!status)
+ {
+ throw std::invalid_argument(
+ "ERROR: zfp failed with status " + std::to_string(status) +
+ ", in call to CompressZfp Decompress\n");
+ }
+ }
+
+ zfp_field_free(field);
+ zfp_stream_close(stream);
+ stream_close(bitstream);
+
+ const size_t typeSizeBytes = lf_GetTypeSize(GetZfpType(type));
+ const size_t dataSizeBytes = GetTotalSize(dimensions) * typeSizeBytes;
+
+ return dataSizeBytes;
+}
+
+// PRIVATE
+zfp_type CompressZfp::GetZfpType(const std::string type) const
+{
+ zfp_type zfpType = zfp_type_none;
+
+ if (type == GetType<double>())
+ {
+ zfpType = zfp_type_double;
+ }
+ else if (type == GetType<float>())
+ {
+ zfpType = zfp_type_float;
+ }
+ else if (type == GetType<int64_t>())
+ {
+ zfpType = zfp_type_int64;
+ }
+ else if (type == GetType<int32_t>())
+ {
+ zfpType = zfp_type_int32;
+ }
+ else
+ {
+ if (m_DebugMode)
+ {
+
+ throw std::invalid_argument(
+ "ERROR: type " + type +
+ " not supported by zfp, only "
+ "signed int32_t, signed int64_t, float, and "
+ "double types are acceptable, from class "
+ "CompressZfp Transform\n");
+ }
+ }
+
+ return zfpType;
+}
+
+zfp_field *CompressZfp::GetZFPField(const void *data, const Dims &dimensions,
+ const std::string type) const
+{
+ auto lf_CheckField = [](const zfp_field *field,
+ const std::string zfpFieldFunction,
+ const std::string type) {
+
+ if (field == nullptr || field == NULL)
+ {
+ throw std::invalid_argument(
+ "ERROR: " + zfpFieldFunction + " failed for data of type " +
+ type + ", data pointer might be corrupted, from "
+ "class CompressZfp Transform\n");
+ }
+ };
+
+ zfp_type zfpType = GetZfpType(type);
+ zfp_field *field = nullptr;
+
+ if (dimensions.size() == 1)
+ {
+ field = zfp_field_1d(const_cast<void *>(data), zfpType, dimensions[0]);
+ if (m_DebugMode)
+ {
+ lf_CheckField(field, "zfp_field_1d", type);
+ }
+ }
+ else if (dimensions.size() == 2)
+ {
+ field = zfp_field_2d(const_cast<void *>(data), zfpType, dimensions[0],
+ dimensions[1]);
+ if (m_DebugMode)
+ {
+ lf_CheckField(field, "zfp_field_2d", type);
+ }
+ }
+ else if (dimensions.size() == 3)
+ {
+ field = zfp_field_3d(const_cast<void *>(data), zfpType, dimensions[0],
+ dimensions[1], dimensions[2]);
+ if (m_DebugMode)
+ {
+ lf_CheckField(field, "zfp_field_3d", type);
+ }
+ }
+ else
+ {
+ if (m_DebugMode)
+ {
+ throw std::invalid_argument(
+ "ERROR: zfp_field* failed for data of type " + type +
+ ", only 1D, 2D and 3D dimensions are supported, from "
+ "class CompressZfp Transform\n");
+ }
+ }
+
+ return field;
+}
+
+zfp_stream *CompressZfp::GetZFPStream(const Dims &dimensions,
+ const std::string type,
+ const Params ¶meters) const
+{
+ auto lf_HasKey = [](Params::const_iterator itKey,
+ const Params ¶meters) -> bool {
+
+ bool hasKey = false;
+ if (itKey != parameters.end())
+ {
+ hasKey = true;
+ }
+ return hasKey;
+ };
+
+ zfp_stream *stream = zfp_stream_open(NULL);
+
+ auto itTolerance = parameters.find("Tolerance");
+ const bool hasTolerance = lf_HasKey(itTolerance, parameters);
+
+ auto itRate = parameters.find("Rate");
+ const bool hasRate = lf_HasKey(itRate, parameters);
+
+ auto itPrecision = parameters.find("Precision");
+ const bool hasPrecision = lf_HasKey(itPrecision, parameters);
+
+ if (m_DebugMode)
+ {
+ if ((hasTolerance && hasRate) || (hasTolerance && hasPrecision) ||
+ (hasRate && hasPrecision) ||
+ !(hasTolerance || hasRate || hasPrecision))
+ {
+ throw std::invalid_argument("ERROR: zfp parameters Tolerance, "
+ "Rate, Precision are mutually "
+ "exclusive, only one of them is "
+ "mandatory, from "
+ "class CompressZfp Transform\n");
+ }
+ }
+
+ if (hasTolerance)
+ {
+ const double tolerance = StringToDouble(
+ itTolerance->second, m_DebugMode,
+ "setting Tolerance in call to CompressZfp class Transform\n");
+
+ zfp_stream_set_accuracy(stream, tolerance);
+ }
+ else if (hasRate)
+ {
+ const double rate = StringToDouble(
+ itRate->second, m_DebugMode,
+ "setting Rate in call to CompressZfp class Transform\n");
+ // TODO support last argument write random access?
+ zfp_stream_set_rate(stream, rate, GetZfpType(type),
+ static_cast<unsigned int>(dimensions.size()), 0);
+ }
+ else if (hasPrecision)
+ {
+ const unsigned int precision = StringToUInt(
+ itPrecision->second, m_DebugMode,
+ "setting Precision in call to CompressZfp class Transform\n");
+ zfp_stream_set_precision(stream, precision);
+ }
+
+ return stream;
+}
+
+} // end namespace transform
+} // end namespace adios2
diff --git a/source/adios2/transform/compress/CompressZfp.h b/source/adios2/transform/compress/CompressZfp.h
new file mode 100644
index 0000000000000000000000000000000000000000..251d14b455a86b8d7dd86b8bf4cc756e912cd312
--- /dev/null
+++ b/source/adios2/transform/compress/CompressZfp.h
@@ -0,0 +1,101 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * CompressZfp.h : wrapper to Zfp compression library
+ *
+ * Created on: Jul 25, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#ifndef ADIOS2_TRANSFORM_COMPRESS_COMPRESSZFP_H_
+#define ADIOS2_TRANSFORM_COMPRESS_COMPRESSZFP_H_
+
+#include <zfp.h>
+
+#include "adios2/core/Transform.h"
+
+namespace adios2
+{
+
+namespace transform
+{
+
+class CompressZfp : public Transform
+{
+
+public:
+ /**
+ * Unique constructor
+ * @param debugMode
+ */
+ CompressZfp(const bool debugMode);
+
+ ~CompressZfp() = default;
+
+ /**
+ * Wrapper around zfp compression
+ * @param dataIn
+ * @param dimensions
+ * @param type
+ * @param bufferOut
+ * @param parameters
+ * @return size of compressed buffer in bytes
+ */
+ size_t Compress(const void *dataIn, const Dims &dimensions,
+ const size_t elementSize, const std::string type,
+ void *bufferOut, const Params ¶meters) const final;
+
+ /**
+ * Wrapper around zfp decompression
+ * @param bufferIn
+ * @param sizeIn
+ * @param dataOut
+ * @param dimensions
+ * @param type
+ * @return size of decompressed data in dataOut
+ */
+ size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
+ const Dims &dimensions, const std::string type,
+ const Params ¶meters) const final;
+
+private:
+ /**
+ * Returns Zfp supported zfp_type based on adios string type
+ * @param type adios type as string, see GetType<T> in
+ * helper/adiosType.inl
+ * @return zfp_type
+ */
+ zfp_type GetZfpType(const std::string type) const;
+
+ /**
+ * Constructor Zfp zfp_field based on input information around the data
+ * pointer
+ * @param data
+ * @param shape
+ * @param type
+ * @return zfp_field*
+ */
+ zfp_field *GetZFPField(const void *data, const Dims &shape,
+ const std::string type) const;
+
+ zfp_stream *GetZFPStream(const Dims &dimensions, const std::string type,
+ const Params ¶meters) const;
+
+ size_t DoBufferMaxSize(const void *dataIn, const Dims &dimensions,
+ const std::string type,
+ const Params ¶meters) const final;
+
+ /**
+ * In debug mode, check status from BZip compression and decompression
+ * functions
+ * @param status returned by BZip2 library
+ * @param hint extra exception information
+ */
+ void CheckStatus(const int status, const std::string hint) const;
+};
+
+} // end namespace transform
+} // end namespace adios2
+
+#endif /* ADIOS2_TRANSFORM_COMPRESS_COMPRESSZFP_H_ */
diff --git a/source/adios2/transform/compression/BZip2.cpp b/source/adios2/transform/compression/BZip2.cpp
deleted file mode 100644
index 89a61737a93408a1d44660cd32bda558be6443cb..0000000000000000000000000000000000000000
--- a/source/adios2/transform/compression/BZip2.cpp
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Distributed under the OSI-approved Apache License, Version 2.0. See
- * accompanying file Copyright.txt for details.
- *
- * BZIP2.cpp
- *
- * Created on: Oct 19, 2016
- * Author: William F Godoy godoywf@ornl.gov
- */
-
-#include "BZip2.h"
-
-namespace adios2
-{
-namespace transform
-{
-
-BZip2::BZip2() : Transform("bzip2") {}
-
-void BZip2::Compress(const std::vector<char> & /*bufferIn*/,
- std::vector<char> & /*bufferOut*/)
-{
-}
-
-void BZip2::Decompress(const std::vector<char> & /*bufferIn*/,
- std::vector<char> & /*bufferOut*/)
-{
-}
-
-} // end namespace transform
-} // end namespace adios
diff --git a/source/adios2/transform/compression/BZip2.h b/source/adios2/transform/compression/BZip2.h
deleted file mode 100644
index 1da04ed6d1656e954f5f99f966eca63300a407bc..0000000000000000000000000000000000000000
--- a/source/adios2/transform/compression/BZip2.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Distributed under the OSI-approved Apache License, Version 2.0. See
- * accompanying file Copyright.txt for details.
- *
- * BZip2.h not yet implemented
- *
- * Created on: Oct 17, 2016
- * Author: William F Godoy godoywf@ornl.gov
- */
-
-#ifndef ADIOS2_TRANSFORM_COMPRESSION_BZIP2_H_
-#define ADIOS2_TRANSFORM_COMPRESSION_BZIP2_H_
-
-#include "adios2/ADIOSConfig.h"
-#include "adios2/core/Transform.h"
-
-namespace adios2
-{
-namespace transform
-{
-
-class BZip2 : public Transform
-{
-
-public:
- /**
- * Initialize parent method
- * @param compressionLevel
- * @param variable
- */
- BZip2();
-
- virtual ~BZip2() = default;
-
- void Compress(const std::vector<char> &bufferIn,
- std::vector<char> &bufferOut);
-
- void Decompress(const std::vector<char> &bufferIn,
- std::vector<char> &bufferOut);
-};
-
-} // end namespace transform
-} // end namespace adios
-
-#endif /* ADIOS2_TRANSFORM_BZIP2_H_ */
diff --git a/testing/adios2/CMakeLists.txt b/testing/adios2/CMakeLists.txt
index 544e4099f8a6d3d309e8d97dbbf80c3deff1c91c..2115e42d9a8d60498548be59448f610ec898ff22 100644
--- a/testing/adios2/CMakeLists.txt
+++ b/testing/adios2/CMakeLists.txt
@@ -7,3 +7,4 @@ add_subdirectory(interface)
add_subdirectory(engine)
add_subdirectory(bindings)
add_subdirectory(xml)
+add_subdirectory(transform)
diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt
index 8da6af4e7613e9b777cfb092d9c805fa583282f5..96e47759aefee773095c388e50b3db42e08ac9b9 100644
--- a/testing/adios2/bindings/python/CMakeLists.txt
+++ b/testing/adios2/bindings/python/CMakeLists.txt
@@ -3,4 +3,10 @@
# accompanying file Copyright.txt for details.
#------------------------------------------------------------------------------#
-python_add_test(PythonBPWrite TestBPWriteTypes.py)
+if(NOT ADIOS2_HAVE_MPI)
+ python_add_test(PythonBPWrite TestBPWriteTypes_nompi.py)
+endif()
+
+if(ADIOS2_HAVE_MPI)
+ python_add_test(PythonBPWrite TestBPWriteTypes.py)
+endif()
\ No newline at end of file
diff --git a/testing/adios2/bindings/python/TestBPWriteTypes.py b/testing/adios2/bindings/python/TestBPWriteTypes.py
index 842adacd8cf1e9b87c53a66725ab50d7b3269aef..36cfe7243abae03f2aa12acf9556fec0c345d594 100644
--- a/testing/adios2/bindings/python/TestBPWriteTypes.py
+++ b/testing/adios2/bindings/python/TestBPWriteTypes.py
@@ -10,13 +10,15 @@
from adios2NPTypes import SmallTestData
+from mpi4py import MPI
import adios2
# Test data
data = SmallTestData()
-adios = adios2.ADIOS(adios2.DebugON)
+comm = MPI.COMM_WORLD
+adios = adios2.ADIOS(comm, adios2.DebugON)
bpIO = adios.DeclareIO("NPTypes")
diff --git a/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py b/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py
new file mode 100644
index 0000000000000000000000000000000000000000..842adacd8cf1e9b87c53a66725ab50d7b3269aef
--- /dev/null
+++ b/testing/adios2/bindings/python/TestBPWriteTypes_nompi.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+#
+# Distributed under the OSI-approved Apache License, Version 2.0. See
+# accompanying file Copyright.txt for details.
+#
+# TestBPWriteTypes.py: test Python numpy types in ADIOS2 File Write
+# Created on: Feb 2, 2017
+# Author: William F Godoy godoywf@ornl.gov
+
+
+from adios2NPTypes import SmallTestData
+import adios2
+
+
+# Test data
+data = SmallTestData()
+
+adios = adios2.ADIOS(adios2.DebugON)
+
+bpIO = adios.DeclareIO("NPTypes")
+
+# ADIOS Variable name, shape, start, offset, constant dims
+# All local variables
+varI8 = bpIO.DefineVariable(
+ "varI8", [], [], [data.I8.size], adios2.ConstantDims)
+varI16 = bpIO.DefineVariable(
+ "varI16", [], [], [data.I16.size], adios2.ConstantDims)
+varI32 = bpIO.DefineVariable(
+ "varI32", [], [], [data.I32.size], adios2.ConstantDims)
+varI64 = bpIO.DefineVariable(
+ "varI64", [], [], [data.I64.size], adios2.ConstantDims)
+
+varU8 = bpIO.DefineVariable(
+ "varUI8", [], [], [data.U8.size], adios2.ConstantDims)
+varU16 = bpIO.DefineVariable(
+ "varUI16", [], [], [data.U16.size], adios2.ConstantDims)
+varU32 = bpIO.DefineVariable(
+ "varUI32", [], [], [data.U32.size], adios2.ConstantDims)
+varU64 = bpIO.DefineVariable(
+ "varUI64", [], [], [data.U64.size], adios2.ConstantDims)
+
+varR32 = bpIO.DefineVariable(
+ "varR32", [], [], [data.R32.size], adios2.ConstantDims)
+
+varR64 = bpIO.DefineVariable(
+ "varR64", [], [], [data.R64.size], adios2.ConstantDims)
+
+
+# ADIOS Engine
+bpFileWriter = bpIO.Open("npTypes.bp", adios2.OpenModeWrite)
+
+bpFileWriter.Write(varI8, data.I8)
+bpFileWriter.Write(varI16, data.I16)
+bpFileWriter.Write(varI32, data.I32)
+bpFileWriter.Write(varI64, data.I64)
+
+bpFileWriter.Write(varU8, data.U8)
+bpFileWriter.Write(varU16, data.U16)
+bpFileWriter.Write(varU32, data.U32)
+bpFileWriter.Write(varU64, data.U64)
+
+bpFileWriter.Write(varR32, data.R32)
+bpFileWriter.Write(varR64, data.R64)
+
+bpFileWriter.Close()
diff --git a/testing/adios2/engine/SmallTestData.h b/testing/adios2/engine/SmallTestData.h
index d7f736582dabc3eedc9c377973461e5578cc1260..05de0729e8e7fdf1ba957942892a24a945722082 100644
--- a/testing/adios2/engine/SmallTestData.h
+++ b/testing/adios2/engine/SmallTestData.h
@@ -10,21 +10,21 @@
struct SmallTestData
{
std::array<char, 10> I8 = {{0, 1, -2, 3, -4, 5, -6, 7, -8, 9}};
- std::array<int16_t, 10> I16 = {
+ std::array<short, 10> I16 = {
{512, 513, -510, 515, -508, 517, -506, 519, -504, 521}};
- std::array<int32_t, 10> I32 = {{131072, 131073, -131070, 131075, -131068,
- 131077, -131066, 131079, -131064, 131081}};
- std::array<int64_t, 10> I64 = {
+ std::array<int, 10> I32 = {{131072, 131073, -131070, 131075, -131068,
+ 131077, -131066, 131079, -131064, 131081}};
+ std::array<long int, 10> I64 = {
{8589934592, 8589934593, -8589934590, 8589934595, -8589934588,
8589934597, -8589934586, 8589934599, -8589934584, 8589934601}};
std::array<unsigned char, 10> U8 = {
{128, 129, 130, 131, 132, 133, 134, 135, 136, 137}};
- std::array<uint16_t, 10> U16 = {
+ std::array<unsigned short, 10> U16 = {
{32768, 32769, 32770, 32771, 32772, 32773, 32774, 32775, 32776, 32777}};
- std::array<uint32_t, 10> U32 = {
+ std::array<unsigned int, 10> U32 = {
{2147483648, 2147483649, 2147483650, 2147483651, 2147483652, 2147483653,
2147483654, 2147483655, 2147483656, 2147483657}};
- std::array<uint64_t, 10> U64 = {
+ std::array<unsigned long int, 10> U64 = {
{9223372036854775808UL, 9223372036854775809UL, 9223372036854775810UL,
9223372036854775811UL, 9223372036854775812UL, 9223372036854775813UL,
9223372036854775814UL, 9223372036854775815UL, 9223372036854775816UL,
diff --git a/testing/adios2/interface/CMakeLists.txt b/testing/adios2/interface/CMakeLists.txt
index e229416de28347dbe46931805724e7d05bbdfdb2..e68ccfa6187ab17710e4bca93579ede6ded95079 100644
--- a/testing/adios2/interface/CMakeLists.txt
+++ b/testing/adios2/interface/CMakeLists.txt
@@ -6,4 +6,8 @@
add_executable(TestADIOSInterfaceWrite TestADIOSInterfaceWrite.cpp)
target_link_libraries(TestADIOSInterfaceWrite adios2 gtest gtest_main)
+add_executable(TestADIOSDefineVariable TestADIOSDefineVariable.cpp)
+target_link_libraries(TestADIOSDefineVariable adios2 gtest gtest_main)
+
gtest_add_tests(TARGET TestADIOSInterfaceWrite)
+gtest_add_tests(TARGET TestADIOSDefineVariable)
\ No newline at end of file
diff --git a/testing/adios2/interface/TestADIOSDefineVariable.cpp b/testing/adios2/interface/TestADIOSDefineVariable.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..100f05fc4e7df69a6512a749cf7b3cf1eb705260
--- /dev/null
+++ b/testing/adios2/interface/TestADIOSDefineVariable.cpp
@@ -0,0 +1,355 @@
+#include <cstdint>
+
+#include <iostream>
+#include <stdexcept>
+
+#include <adios2.h>
+
+#include <gtest/gtest.h>
+
+class ADIOSDefineVariableTest : public ::testing::Test
+{
+public:
+ ADIOSDefineVariableTest() : adios(true), io(adios.DeclareIO("TestIO")) {}
+
+protected:
+ // virtual void SetUp() { }
+
+ // virtual void TearDown() { }
+
+ adios2::ADIOS adios;
+ adios2::IO &io;
+};
+
+TEST_F(ADIOSDefineVariableTest, DefineGlobalValue)
+{
+ // Define ADIOS global value
+ auto &globalvalue = io.DefineVariable<int>("globalvalue");
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(globalvalue),
+ adios2::Variable<int> &>();
+
+ // Verify the dimensions, name, and type are correct
+ ASSERT_EQ(globalvalue.m_Shape.size(), 0);
+ EXPECT_EQ(globalvalue.m_Start.size(), 0);
+ EXPECT_EQ(globalvalue.m_Count.size(), 0);
+ EXPECT_EQ(globalvalue.m_Name, "globalvalue");
+ EXPECT_EQ(globalvalue.m_Type, "int");
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineLocalValue)
+{
+ // Define ADIOS local value (a value changing across processes)
+ auto &localvalue =
+ io.DefineVariable<int>("localvalue", {adios2::LocalValueDim});
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(localvalue),
+ adios2::Variable<int> &>();
+
+ // Verify the dimensions, name, and type are correct
+ ASSERT_EQ(localvalue.m_Shape.size(), 1);
+ EXPECT_EQ(localvalue.m_Shape[0], adios2::LocalValueDim);
+ EXPECT_EQ(localvalue.m_Start.size(), 0);
+ EXPECT_EQ(localvalue.m_Count.size(), 0);
+ EXPECT_EQ(localvalue.m_Name, "localvalue");
+ EXPECT_EQ(localvalue.m_Type, "int");
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineGlobalArray)
+{
+ // Define ADIOS global array
+ std::size_t n = 50;
+ auto &globalarray = io.DefineVariable<int>("globalarray", {100, n, 30},
+ {50, n / 2, 0}, {10, n / 2, 30});
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(globalarray),
+ adios2::Variable<int> &>();
+
+ // Verify the dimensions, name, and type are correct
+ ASSERT_EQ(globalarray.m_Shape.size(), 3);
+ EXPECT_EQ(globalarray.m_Shape[0], 100);
+ EXPECT_EQ(globalarray.m_Shape[1], n);
+ EXPECT_EQ(globalarray.m_Shape[2], 30);
+ EXPECT_EQ(globalarray.m_Start.size(), 3);
+ EXPECT_EQ(globalarray.m_Start[0], 50);
+ EXPECT_EQ(globalarray.m_Start[1], n / 2);
+ EXPECT_EQ(globalarray.m_Start[2], 0);
+ EXPECT_EQ(globalarray.m_Count.size(), 3);
+ EXPECT_EQ(globalarray.m_Count[0], 10);
+ EXPECT_EQ(globalarray.m_Count[1], n / 2);
+ EXPECT_EQ(globalarray.m_Count[2], 30);
+ EXPECT_EQ(globalarray.m_Name, "globalarray");
+ EXPECT_EQ(globalarray.m_Type, "int");
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineGlobalArrayWithSelections)
+{
+ // Define ADIOS global array with postponed size definition in SetSelection
+ std::size_t n = 50;
+ auto &globalarray = io.DefineVariable<int>("globalarray", {100, n, 30});
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(globalarray),
+ adios2::Variable<int> &>();
+
+ // Make a 3D selection to describe the local dimensions of the
+ // variable we write and its offsets in the global spaces
+ adios2::SelectionBoundingBox sel({50, n / 2, 0}, {10, n / 2, 30});
+ globalarray.SetSelection(sel);
+
+ // Verify the dimensions, name, and type are correct
+ ASSERT_EQ(globalarray.m_Shape.size(), 3);
+ EXPECT_EQ(globalarray.m_Shape[0], 100);
+ EXPECT_EQ(globalarray.m_Shape[1], n);
+ EXPECT_EQ(globalarray.m_Shape[2], 30);
+ EXPECT_EQ(globalarray.m_Start.size(), 3);
+ EXPECT_EQ(globalarray.m_Start[0], 50);
+ EXPECT_EQ(globalarray.m_Start[1], n / 2);
+ EXPECT_EQ(globalarray.m_Start[2], 0);
+ EXPECT_EQ(globalarray.m_Count.size(), 3);
+ EXPECT_EQ(globalarray.m_Count[0], 10);
+ EXPECT_EQ(globalarray.m_Count[1], n / 2);
+ EXPECT_EQ(globalarray.m_Count[2], 30);
+ EXPECT_EQ(globalarray.m_Name, "globalarray");
+ EXPECT_EQ(globalarray.m_Type, "int");
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineGlobalArrayConstantDims)
+{
+ // Define ADIOS global array with locked-down dimensions
+ std::size_t n = 50;
+ auto &globalarray = io.DefineVariable<int>(
+ "globalarray", {100, n, 30}, {50, n / 2, 0}, {10, n / 2, 30}, true);
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(globalarray),
+ adios2::Variable<int> &>();
+
+ adios2::SelectionBoundingBox sel({50, n / 2, 0}, {10, n / 2, 30});
+ EXPECT_THROW(globalarray.SetSelection(sel), std::invalid_argument);
+
+ // Verify the dimensions, name, and type are correct
+ ASSERT_EQ(globalarray.m_Shape.size(), 3);
+ EXPECT_EQ(globalarray.m_Shape[0], 100);
+ EXPECT_EQ(globalarray.m_Shape[1], n);
+ EXPECT_EQ(globalarray.m_Shape[2], 30);
+ EXPECT_EQ(globalarray.m_Start.size(), 3);
+ EXPECT_EQ(globalarray.m_Start[0], 50);
+ EXPECT_EQ(globalarray.m_Start[1], n / 2);
+ EXPECT_EQ(globalarray.m_Start[2], 0);
+ EXPECT_EQ(globalarray.m_Count.size(), 3);
+ EXPECT_EQ(globalarray.m_Count[0], 10);
+ EXPECT_EQ(globalarray.m_Count[1], n / 2);
+ EXPECT_EQ(globalarray.m_Count[2], 30);
+ EXPECT_EQ(globalarray.m_Name, "globalarray");
+ EXPECT_EQ(globalarray.m_Type, "int");
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineGlobalArrayInvalidLocalValueDim)
+{
+ // Define ADIOS global array
+ std::size_t n = 50;
+ adios2::Variable<int> *globalarray;
+ EXPECT_THROW(globalarray = &io.DefineVariable<int>(
+ "globalarray", {100, adios2::LocalValueDim, 30},
+ {50, n / 2, 0}, {10, n / 2, 30}),
+ std::invalid_argument);
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineLocalArray)
+{
+ // Define ADIOS local array (no global dimensions, no offsets)
+ std::size_t n = 50;
+ auto &localarray =
+ io.DefineVariable<int>("localarray", {}, {}, {10, n / 2, 30});
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(localarray),
+ adios2::Variable<int> &>();
+
+ // Verify the dimensions, name, and type are correct
+ ASSERT_EQ(localarray.m_Shape.size(), 0);
+ EXPECT_EQ(localarray.m_Start.size(), 0);
+ EXPECT_EQ(localarray.m_Count.size(), 3);
+ EXPECT_EQ(localarray.m_Count[0], 10);
+ EXPECT_EQ(localarray.m_Count[1], n / 2);
+ EXPECT_EQ(localarray.m_Count[2], 30);
+ EXPECT_EQ(localarray.m_Name, "localarray");
+ EXPECT_EQ(localarray.m_Type, "int");
+ EXPECT_EQ(localarray.m_ShapeID, adios2::ShapeID::LocalArray);
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineLocalArrayWithSelection)
+{
+ // Define ADIOS local array with postponed size definition in SetSelection
+ std::size_t n = 50;
+ auto &localarray = io.DefineVariable<int>(
+ "localarray", {}, {},
+ {adios2::UnknownDim, adios2::UnknownDim, adios2::UnknownDim});
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(localarray),
+ adios2::Variable<int> &>();
+
+ ASSERT_EQ(localarray.m_Shape.size(), 0);
+ EXPECT_EQ(localarray.m_Start.size(), 0);
+ EXPECT_EQ(localarray.m_Count.size(), 3);
+ EXPECT_EQ(localarray.m_Count[0], 0);
+ EXPECT_EQ(localarray.m_Count[1], 0);
+ EXPECT_EQ(localarray.m_Count[2], 0);
+ EXPECT_EQ(localarray.m_Name, "localarray");
+ EXPECT_EQ(localarray.m_Type, "int");
+ EXPECT_EQ(localarray.m_ShapeID, adios2::ShapeID::LocalArray);
+
+ // Make a 3D selection to describe the local dimensions of the
+ // variable we write
+ adios2::SelectionBoundingBox sel({}, {10, n / 2, 30});
+ localarray.SetSelection(sel);
+
+ adios2::SelectionBoundingBox selbad({50, n / 2, 0}, {10, n / 2, 30});
+ EXPECT_THROW(localarray.SetSelection(selbad), std::invalid_argument);
+
+ // Verify the dimensions, name, and type are correct
+ ASSERT_EQ(localarray.m_Shape.size(), 0);
+ EXPECT_EQ(localarray.m_Start.size(), 0);
+ EXPECT_EQ(localarray.m_Count.size(), 3);
+ EXPECT_EQ(localarray.m_Count[0], 10);
+ EXPECT_EQ(localarray.m_Count[1], n / 2);
+ EXPECT_EQ(localarray.m_Count[2], 30);
+ EXPECT_EQ(localarray.m_Name, "localarray");
+ EXPECT_EQ(localarray.m_Type, "int");
+ EXPECT_EQ(localarray.m_ShapeID, adios2::ShapeID::LocalArray);
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineLocalArrayConstantDims)
+{
+ // Define ADIOS local array with locked down dimensions
+ std::size_t n = 50;
+ auto &localarray =
+ io.DefineVariable<int>("localarray", {}, {}, {10, n / 2, 30}, true);
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(localarray),
+ adios2::Variable<int> &>();
+
+ adios2::SelectionBoundingBox sel({}, {10, n / 2, 30});
+ EXPECT_THROW(localarray.SetSelection(sel), std::invalid_argument);
+
+ // Verify the dimensions, name, and type are correct
+ ASSERT_EQ(localarray.m_Shape.size(), 0);
+ EXPECT_EQ(localarray.m_Start.size(), 0);
+ EXPECT_EQ(localarray.m_Count.size(), 3);
+ EXPECT_EQ(localarray.m_Count[0], 10);
+ EXPECT_EQ(localarray.m_Count[1], n / 2);
+ EXPECT_EQ(localarray.m_Count[2], 30);
+ EXPECT_EQ(localarray.m_Name, "localarray");
+ EXPECT_EQ(localarray.m_Type, "int");
+ EXPECT_EQ(localarray.m_ShapeID, adios2::ShapeID::LocalArray);
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineLocalArrayInvalidOffsets)
+{
+ // Define ADIOS local array but try to add offsets
+ std::size_t n = 50;
+
+ adios2::Variable<int> *localarray;
+ EXPECT_THROW(localarray = &io.DefineVariable<int>(
+ "localarray", {}, {50, n / 2, 0}, {10, n / 2, 30}),
+ std::invalid_argument);
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineJoinedArrayFirstDim)
+{
+ // Define ADIOS joined array
+ std::size_t n = 50;
+ auto &joinedarray = io.DefineVariable<int>(
+ "joinedarray", {adios2::JoinedDim, n, 30}, {}, {10, n, 30});
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(joinedarray),
+ adios2::Variable<int> &>();
+
+ // Verify the dimensions, name, and type are correct
+ ASSERT_EQ(joinedarray.m_Shape.size(), 3);
+ EXPECT_EQ(joinedarray.m_Shape[0], adios2::JoinedDim);
+ EXPECT_EQ(joinedarray.m_Shape[1], n);
+ EXPECT_EQ(joinedarray.m_Shape[2], 30);
+ EXPECT_EQ(joinedarray.m_Start.size(), 0);
+ EXPECT_EQ(joinedarray.m_Count.size(), 3);
+ EXPECT_EQ(joinedarray.m_Count[0], 10);
+ EXPECT_EQ(joinedarray.m_Count[1], n);
+ EXPECT_EQ(joinedarray.m_Count[2], 30);
+ EXPECT_EQ(joinedarray.m_Name, "joinedarray");
+ EXPECT_EQ(joinedarray.m_Type, "int");
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineJoinedArraySecondDim)
+{
+ // Define ADIOS joined array
+ std::size_t n = 50;
+ auto &joinedarray = io.DefineVariable<int>(
+ "joinedarray", {n, adios2::JoinedDim, 30}, {0, 0, 0}, {n, 10, 30});
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(joinedarray),
+ adios2::Variable<int> &>();
+
+ // Verify the dimensions, name, and type are correct
+ ASSERT_EQ(joinedarray.m_Shape.size(), 3);
+ EXPECT_EQ(joinedarray.m_Shape[0], n);
+ EXPECT_EQ(joinedarray.m_Shape[1], adios2::JoinedDim);
+ EXPECT_EQ(joinedarray.m_Shape[2], 30);
+ EXPECT_EQ(joinedarray.m_Start.size(), 3);
+ EXPECT_EQ(joinedarray.m_Start[0], 0);
+ EXPECT_EQ(joinedarray.m_Start[1], 0);
+ EXPECT_EQ(joinedarray.m_Start[2], 0);
+ EXPECT_EQ(joinedarray.m_Count.size(), 3);
+ EXPECT_EQ(joinedarray.m_Count[0], n);
+ EXPECT_EQ(joinedarray.m_Count[1], 10);
+ EXPECT_EQ(joinedarray.m_Count[2], 30);
+ EXPECT_EQ(joinedarray.m_Name, "joinedarray");
+ EXPECT_EQ(joinedarray.m_Type, "int");
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineJoinedArrayTooManyJoinedDims)
+{
+ // Define ADIOS joined array
+ std::size_t n = 50;
+ adios2::Variable<int> *joinedarray;
+ EXPECT_THROW(joinedarray = &io.DefineVariable<int>(
+ "joinedarray", {n, adios2::JoinedDim, adios2::JoinedDim},
+ {}, {n, 50, 30}),
+ std::invalid_argument);
+}
+
+TEST_F(ADIOSDefineVariableTest, DefineJoinedArrayInvalidStart)
+{
+ // Define ADIOS joined array
+ std::size_t n = 10;
+ std::size_t WrongValue = 1;
+ adios2::Variable<int> *joinedarray;
+ // Start must be empty or full zero array
+ EXPECT_THROW(
+ joinedarray = &io.DefineVariable<int>(
+ "joinedarray", {adios2::JoinedDim, 50}, {0, WrongValue}, {n, 50}),
+ std::invalid_argument);
+}
+
+int main(int argc, char **argv)
+{
+#ifdef ADIOS2_HAVE_MPI
+ MPI_Init(nullptr, nullptr);
+#endif
+
+ ::testing::InitGoogleTest(&argc, argv);
+ int result = RUN_ALL_TESTS();
+
+#ifdef ADIOS2_HAVE_MPI
+ MPI_Finalize();
+#endif
+
+ return result;
+}
diff --git a/testing/adios2/transform/CMakeLists.txt b/testing/adios2/transform/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1f8111162515c3dbc40533f6ecd31bcc61031413
--- /dev/null
+++ b/testing/adios2/transform/CMakeLists.txt
@@ -0,0 +1,19 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0. See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+
+if(ADIOS2_HAVE_BZip2)
+ add_executable(TestBZip2Wrapper TestBZip2Wrapper.cpp)
+ target_link_libraries(TestBZip2Wrapper adios2 gtest gtest_main)
+
+ gtest_add_tests(TARGET TestBZip2Wrapper)
+endif()
+
+if(ADIOS2_HAVE_ZFP)
+ add_executable(TestZfpWrapper TestZfpWrapper.cpp)
+ target_link_libraries(TestZfpWrapper adios2 gtest gtest_main)
+
+ gtest_add_tests(TARGET TestZfpWrapper)
+endif()
\ No newline at end of file
diff --git a/testing/adios2/transform/TestBZip2Wrapper.cpp b/testing/adios2/transform/TestBZip2Wrapper.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..57f506ded62083af2155cf8588df67d454d4db7b
--- /dev/null
+++ b/testing/adios2/transform/TestBZip2Wrapper.cpp
@@ -0,0 +1,125 @@
+#include <cstdint>
+#include <iostream>
+#include <numeric> //std::iota
+#include <stdexcept>
+
+#include <adios2.h>
+
+#include <gtest/gtest.h>
+
+class ADIOSBZip2Wrapper : public ::testing::Test
+{
+public:
+ ADIOSBZip2Wrapper() : adios(true), io(adios.DeclareIO("TestADIOSBZip2")) {}
+
+protected:
+ adios2::ADIOS adios;
+ adios2::IO &io;
+};
+
+TEST_F(ADIOSBZip2Wrapper, UInt100)
+{
+ /** Application variable uints from 0 to 1000 */
+ std::vector<unsigned int> myUInts(100);
+ std::iota(myUInts.begin(), myUInts.end(), 0.f);
+ const std::size_t Nx = myUInts.size();
+ const std::size_t inputBytes = Nx * sizeof(unsigned int);
+
+ // Define ADIOS variable
+ auto &var_UInt = io.DefineVariable<unsigned int>("myUInts", {}, {}, {Nx},
+ adios2::ConstantDims);
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(var_UInt),
+ adios2::Variable<unsigned int> &>();
+
+ // Define bzip2 transform
+ adios2::Transform &adiosBZip2 = adios.GetTransform("BZip2");
+
+ const unsigned int bzip2ID =
+ var_UInt.AddTransform(adiosBZip2, {{"BlockSize100K", "2"}});
+
+ const std::size_t estimatedSize =
+ adiosBZip2.BufferMaxSize(Nx * var_UInt.m_ElementSize);
+ std::vector<char> compressedBuffer(estimatedSize);
+ size_t compressedSize = adiosBZip2.Compress(
+ myUInts.data(), var_UInt.m_Count, var_UInt.m_ElementSize,
+ var_UInt.m_Type, compressedBuffer.data(),
+ var_UInt.m_TransformsInfo[bzip2ID].Parameters);
+
+ EXPECT_LE(compressedSize, estimatedSize);
+
+ compressedBuffer.resize(compressedSize);
+
+ // Allocate original data size
+ std::vector<unsigned int> decompressedBuffer(Nx);
+ size_t decompressedSize = adiosBZip2.Decompress(
+ compressedBuffer.data(), compressedSize, decompressedBuffer.data(),
+ decompressedBuffer.size() * sizeof(unsigned int));
+
+ // testing data recovery
+ for (size_t i = 0; i < Nx; ++i)
+ {
+ ASSERT_EQ(decompressedBuffer[i], myUInts[i]);
+ }
+}
+
+TEST_F(ADIOSBZip2Wrapper, WrongParameterValue)
+{
+ /** Application variable uints from 0 to 1000 */
+ std::vector<unsigned int> myUInts(100);
+ std::iota(myUInts.begin(), myUInts.end(), 0.f);
+ const std::size_t Nx = myUInts.size();
+ const std::size_t inputBytes = Nx * sizeof(unsigned int);
+
+ // Define ADIOS variable
+ auto &var_UInt = io.DefineVariable<unsigned int>("myUInts", {}, {}, {Nx},
+ adios2::ConstantDims);
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(var_UInt),
+ adios2::Variable<unsigned int> &>();
+
+ // Define bzip2 transform
+ adios2::Transform &adiosBZip2 = adios.GetTransform("BZip2");
+
+ const unsigned int bzip2ID =
+ var_UInt.AddTransform(adiosBZip2, {{"BlockSize100K", "10"}});
+
+ const std::size_t estimatedSize =
+ adiosBZip2.BufferMaxSize(Nx * var_UInt.m_ElementSize);
+ std::vector<char> compressedBuffer(estimatedSize);
+
+ EXPECT_THROW(size_t compressedSize = adiosBZip2.Compress(
+ myUInts.data(), var_UInt.m_Count, var_UInt.m_ElementSize,
+ var_UInt.m_Type, compressedBuffer.data(),
+ var_UInt.m_TransformsInfo[bzip2ID].Parameters),
+ std::invalid_argument);
+}
+
+TEST_F(ADIOSBZip2Wrapper, WrongBZip2Name)
+{
+ /** Application variable uints from 0 to 1000 */
+ std::vector<unsigned int> myUInts(100);
+ std::iota(myUInts.begin(), myUInts.end(), 0.f);
+ const std::size_t Nx = myUInts.size();
+ const std::size_t inputBytes = Nx * sizeof(unsigned int);
+
+ // Define ADIOS variable
+ auto &var_UInt = io.DefineVariable<unsigned int>("myUInts", {}, {}, {Nx},
+ adios2::ConstantDims);
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(var_UInt),
+ adios2::Variable<unsigned int> &>();
+
+ // Check bzip2 lower case and camel case
+ EXPECT_NO_THROW(adios2::Transform &adiosBZip2 =
+ adios.GetTransform("bzip2"));
+ EXPECT_NO_THROW(adios2::Transform &adiosBZip2 =
+ adios.GetTransform("BZip2"));
+ EXPECT_THROW(adios2::Transform &adiosBZip2 = adios.GetTransform("bzip"),
+ std::invalid_argument);
+ EXPECT_THROW(adios2::Transform &adiosBZip2 = adios.GetTransform("BZIP2"),
+ std::invalid_argument);
+}
diff --git a/testing/adios2/transform/TestZfpWrapper.cpp b/testing/adios2/transform/TestZfpWrapper.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..db2a2d2b09a00e91939feb977253952003348c91
--- /dev/null
+++ b/testing/adios2/transform/TestZfpWrapper.cpp
@@ -0,0 +1,123 @@
+#include <cmath> //std::abs
+#include <iostream>
+#include <numeric> //std::iota
+#include <stdexcept>
+
+#include <adios2.h>
+
+#include <gtest/gtest.h>
+
+class ADIOSZfpWrapper : public ::testing::Test
+{
+public:
+ ADIOSZfpWrapper() : adios(true), io(adios.DeclareIO("TestADIOSZfp")) {}
+
+protected:
+ adios2::ADIOS adios;
+ adios2::IO &io;
+};
+
+TEST_F(ADIOSZfpWrapper, Float100)
+{
+ /** Application variable uints from 0 to 1000 */
+ std::vector<float> myFloats(100);
+ std::iota(myFloats.begin(), myFloats.end(), 0.f);
+ const std::size_t Nx = myFloats.size();
+ const std::size_t inputBytes = Nx * sizeof(float);
+
+ // Define ADIOS variable
+ auto &var_Floats = io.DefineVariable<float>("myFloats", {}, {}, {Nx},
+ adios2::ConstantDims);
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(var_Floats),
+ adios2::Variable<float> &>();
+
+ // Define bzip2 transform
+ adios2::Transform &adiosZfp = adios.GetTransform("Zfp");
+
+ const unsigned int zfpID =
+ var_Floats.AddTransform(adiosZfp, {{"Rate", "8"}});
+
+ const std::size_t estimatedSize =
+ adiosZfp.BufferMaxSize(myFloats.data(), var_Floats.m_Count,
+ var_Floats.m_TransformsInfo[zfpID].Parameters);
+
+ std::vector<char> compressedBuffer(estimatedSize);
+
+ size_t compressedSize = adiosZfp.Compress(
+ myFloats.data(), var_Floats.m_Count, var_Floats.m_ElementSize,
+ var_Floats.m_Type, compressedBuffer.data(),
+ var_Floats.m_TransformsInfo[zfpID].Parameters);
+
+ compressedBuffer.resize(compressedSize);
+
+ // Allocate original data size
+ std::vector<float> decompressedBuffer(Nx);
+
+ size_t decompressedSize = adiosZfp.Decompress(
+ compressedBuffer.data(), compressedBuffer.size(),
+ decompressedBuffer.data(), var_Floats.m_Count, var_Floats.m_Type,
+ var_Floats.m_TransformsInfo[zfpID].Parameters);
+
+ // testing data recovery for rate = 8
+ for (size_t i = 0; i < Nx; ++i)
+ {
+ ASSERT_LT(std::abs(decompressedBuffer[i] - myFloats[i]), 1E-6);
+ }
+}
+
+TEST_F(ADIOSZfpWrapper, UnsupportedCall)
+{
+ /** Application variable uints from 0 to 1000 */
+ std::vector<float> myFloats(100);
+ std::iota(myFloats.begin(), myFloats.end(), 0.f);
+ const std::size_t Nx = myFloats.size();
+ const std::size_t inputBytes = Nx * sizeof(float);
+
+ // Define ADIOS variable
+ auto &var_Floats = io.DefineVariable<float>("myFloats", {}, {}, {Nx},
+ adios2::ConstantDims);
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(var_Floats),
+ adios2::Variable<float> &>();
+
+ // Define bzip2 transform
+ adios2::Transform &adiosZfp = adios.GetTransform("zfp");
+
+ const unsigned int zfpID =
+ var_Floats.AddTransform(adiosZfp, {{"Rate", "8"}});
+
+ // Wrong signature for Zfp
+ EXPECT_THROW(const std::size_t estimatedSize =
+ adiosZfp.BufferMaxSize(Nx * var_Floats.m_ElementSize),
+ std::invalid_argument);
+}
+
+TEST_F(ADIOSZfpWrapper, MissingMandatoryParameter)
+{
+ /** Application variable uints from 0 to 1000 */
+ std::vector<float> myFloats(100);
+ std::iota(myFloats.begin(), myFloats.end(), 0.f);
+ const std::size_t Nx = myFloats.size();
+ const std::size_t inputBytes = Nx * sizeof(float);
+
+ // Define ADIOS variable
+ auto &var_Floats = io.DefineVariable<float>("myFloats", {}, {}, {Nx},
+ adios2::ConstantDims);
+
+ // Verify the return type is as expected
+ ::testing::StaticAssertTypeEq<decltype(var_Floats),
+ adios2::Variable<float> &>();
+
+ // Define bzip2 transform
+ adios2::Transform &adiosZfp = adios.GetTransform("zfp");
+
+ const unsigned int zfpID = var_Floats.AddTransform(adiosZfp);
+
+ EXPECT_THROW(const std::size_t estimatedSize = adiosZfp.BufferMaxSize(
+ myFloats.data(), var_Floats.m_Count,
+ var_Floats.m_TransformsInfo[zfpID].Parameters),
+ std::invalid_argument);
+}
diff --git a/testing/adios2/xml/TestXMLConfig.cpp b/testing/adios2/xml/TestXMLConfig.cpp
index 16b45d2f3f551ddc930097dad64f09b9b5e98f1d..5d3cb161b01080cd33a19083516deba70d3caaaa 100644
--- a/testing/adios2/xml/TestXMLConfig.cpp
+++ b/testing/adios2/xml/TestXMLConfig.cpp
@@ -51,6 +51,20 @@ TEST_F(XMLConfigTest, TwoIOs)
});
}
+TEST_F(XMLConfigTest, TwoEnginesException)
+{
+ std::string configFile = configDir + "/config2.xml";
+
+#ifdef ADIOS2_HAVE_MPI
+ EXPECT_THROW(
+ adios2::ADIOS adios(configFile, MPI_COMM_WORLD, adios2::DebugON),
+ std::invalid_argument);
+#else
+ EXPECT_THROW(adios2::ADIOS adios(configFile, adios2::DebugON),
+ std::invalid_argument);
+#endif
+}
+
int main(int argc, char **argv)
{
#ifdef ADIOS2_HAVE_MPI
diff --git a/testing/adios2/xml/config2.xml b/testing/adios2/xml/config2.xml
new file mode 100644
index 0000000000000000000000000000000000000000..c26d2ec852400e34cf12d287ad3cbdf2b2b18672
--- /dev/null
+++ b/testing/adios2/xml/config2.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<adios-config>
+ <io name="Test IO 2">
+ <engine type="BPFileWriter" />
+ <engine type="Other" /> <!-- std::invalid_argument exception -->
+ </io>
+</adios-config>