From a8e58b4869c8f18b7bedfe78d000279b17bada48 Mon Sep 17 00:00:00 2001
From: guj <jgu@lbl.gov>
Date: Mon, 10 Apr 2017 09:31:36 -0700
Subject: [PATCH 1/6] setting up for hdf5
---
cmake/FindPHDF5.cmake | 59 +++++
include/engine/hdf5/HDF5ReaderP.h | 4 +
include/engine/hdf5/HDF5WriterP.h | 224 +++++++++++++++++
source/engine/hdf5/HDF5ReaderP.cpp | 11 +
source/engine/hdf5/HDF5WriterP.cpp | 374 +++++++++++++++++++++++++++++
5 files changed, 672 insertions(+)
create mode 100644 cmake/FindPHDF5.cmake
create mode 100644 include/engine/hdf5/HDF5ReaderP.h
create mode 100644 include/engine/hdf5/HDF5WriterP.h
create mode 100644 source/engine/hdf5/HDF5ReaderP.cpp
create mode 100644 source/engine/hdf5/HDF5WriterP.cpp
diff --git a/cmake/FindPHDF5.cmake b/cmake/FindPHDF5.cmake
new file mode 100644
index 000000000..e357e7279
--- /dev/null
+++ b/cmake/FindPHDF5.cmake
@@ -0,0 +1,59 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0. See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+#
+# FindPHDF5
+# ---------
+#
+
+
+if(NOT PHDF5_FOUND)
+ if((NOT HDF5_DIR) AND (NOT (ENV{HDF5_DIR} STREQUAL "")))
+ set(HDF5_DIR "$ENV{HDF5_DIR}")
+ endif()
+
+ # Search for the core libraries
+ if(HDF5_DIR)
+ # If a root directory is specified, then don't look anywhere else
+ find_path(PHDF5_INCLUDE_DIR hdf5.h
+ HINTS ${HDF5_DIR}/include
+ NO_DEFAULT_PATHS
+ )
+ set(_PHDF5_LIBRARY_HINT HINTS ${HDF5_DIR}/lib NO_DEFAULT_PATHS)
+ else()
+ # Otherwise use the include dir as a basis to search for the lib
+ find_path(PHDF5_INCLUDE_DIR hdf5.h)
+ if(PHDF5_INCLUDE_DIR)
+ get_filename_component(_PHDF5_PREFIX "${PHDF5_INCLUDE_DIR}" PATH)
+ set(_PHDF5_LIBRARY_HINT HINTS ${_PHDF5_PREFIX}/lib)
+ unset(_PHDF5_PREFIX)
+ endif()
+ endif()
+ find_library(PHDF5_LIBRARY hdf5 ${_PHDF5_LIBRARY_HINT})
+ unset(_PHDF5_LIBRARY_HINT)
+
+
+ include(FindPackageHandleStandardArgs)
+ find_package_handle_standard_args(PHDF5 DEFAULT_MSG PHDF5_LIBRARY PHDF5_INCLUDE_DIR)
+
+ find_package_handle_standard_args(PHDF5
+ FOUND_VAR PHDF5_FOUND
+ REQUIRED_VARS
+ PHDF5_INCLUDE_DIR
+ PHDF5_LIBRARY
+ )
+
+ MESSAGE(STATUS "PHDF5_FOUND: " ${PHDF5_FOUND} "HDF5_DIR: " ${HDF5_DIR} "TARGET=" ${TARGET} )
+ if(PHDF5_FOUND)
+ set(PHDF5_INCLUDE_DIRS ${PHDF5_INCLUDE_DIR})
+ set(PHDF5_LIBRARIES ${PHDF5_LIBRARY})
+ if(PHDF5_FOUND AND NOT TARGET PHDF5::PHDF5)
+ add_library(PHDF5::PHDF5 UNKNOWN IMPORTED)
+ set_target_properties(PHDF5::PHDF5 PROPERTIES
+ IMPORTED_LOCATION "${PHDF5_LIBRARY}"
+ INTERFACE_INCLUDE_DIRECTORIES "${PHDF5_INCLUDE_DIR}"
+ )
+ endif()
+ endif()
+endif()
diff --git a/include/engine/hdf5/HDF5ReaderP.h b/include/engine/hdf5/HDF5ReaderP.h
new file mode 100644
index 000000000..28d688208
--- /dev/null
+++ b/include/engine/hdf5/HDF5ReaderP.h
@@ -0,0 +1,4 @@
+#ifndef HDF5_READER_P_H
+#define HDF5_READER_P_H
+
+#endif
diff --git a/include/engine/hdf5/HDF5WriterP.h b/include/engine/hdf5/HDF5WriterP.h
new file mode 100644
index 000000000..27d5e8d10
--- /dev/null
+++ b/include/engine/hdf5/HDF5WriterP.h
@@ -0,0 +1,224 @@
+/*
+ * HDF5WriterP.h
+ *
+ * Created on: March 20, 2017
+ * Author: Junmin
+ */
+
+#ifndef HDF5_WRITER_P_H_
+#define HDF5_WRITER_P_H_
+
+#include <iostream> //std::cout must be removed, only used for hello example
+#include <unistd.h> //sleep must be removed
+
+#include "core/Engine.h"
+#include "format/BP1Writer.h"
+
+// supported capsules
+#include "capsule/heap/STLVector.h"
+
+#include "ADIOS_MPI.h"
+
+#include "hdf5.h"
+
+namespace adios {
+
+typedef struct {
+ double _re; /*real part*/
+ double _im; /*imaginary part*/
+} ADIOS2_Complex_Double;
+
+typedef struct {
+ float _re; // real
+ float _im; // imaginary part
+} ADIOS2_Complex_Float;
+
+// using ADIOS2_Complex_Float = std::complex<float>;
+
+typedef struct {
+ long double _re; /*real part*/
+ long double _im; /*imaginary part*/
+} ADIOS2_Complex_LongDouble;
+
+class HDF5Writer : public Engine {
+
+public:
+ /**
+ * Constructor for single BP capsule engine, writes in BP format into a single
+ * heap capsule
+ * @param name unique name given to the engine
+ * @param accessMode
+ * @param mpiComm
+ * @param method
+ * @param debugMode
+ */
+
+ HDF5Writer(ADIOS &adios, const std::string name, const std::string accessMode,
+ MPI_Comm mpiComm, const Method &method,
+ const bool debugMode = false, const unsigned int cores = 1);
+
+ ~HDF5Writer();
+
+ void Write(Variable<char> &variable, const char *values);
+ void Write(Variable<unsigned char> &variable, const unsigned char *values);
+ void Write(Variable<short> &variable, const short *values);
+ void Write(Variable<unsigned short> &variable, const unsigned short *values);
+ void Write(Variable<int> &variable, const int *values);
+ void Write(Variable<unsigned int> &variable, const unsigned int *values);
+ void Write(Variable<long int> &variable, const long int *values);
+ void Write(Variable<unsigned long int> &variable,
+ const unsigned long int *values);
+ void Write(Variable<long long int> &variable, const long long int *values);
+ void Write(Variable<unsigned long long int> &variable,
+ const unsigned long long int *values);
+ void Write(Variable<float> &variable, const float *values);
+ void Write(Variable<double> &variable, const double *values);
+ void Write(Variable<long double> &variable, const long double *values);
+ void Write(Variable<std::complex<float>> &variable,
+ const std::complex<float> *values);
+ void Write(Variable<std::complex<double>> &variable,
+ const std::complex<double> *values);
+ void Write(Variable<std::complex<long double>> &variable,
+ const std::complex<long double> *values);
+
+ void Write(const std::string variableName, const char *values);
+ void Write(const std::string variableName, const unsigned char *values);
+ void Write(const std::string variableName, const short *values);
+ void Write(const std::string variableName, const unsigned short *values);
+ void Write(const std::string variableName, const int *values);
+ void Write(const std::string variableName, const unsigned int *values);
+ void Write(const std::string variableName, const long int *values);
+ void Write(const std::string variableName, const unsigned long int *values);
+ void Write(const std::string variableName, const long long int *values);
+ void Write(const std::string variableName,
+ const unsigned long long int *values);
+ void Write(const std::string variableName, const float *values);
+ void Write(const std::string variableName, const double *values);
+ void Write(const std::string variableName, const long double *values);
+ void Write(const std::string variableName, const std::complex<float> *values);
+ void Write(const std::string variableName,
+ const std::complex<double> *values);
+ void Write(const std::string variableName,
+ const std::complex<long double> *values);
+
+ void Close(const int transportIndex = -1);
+ /*
+ {
+ std::cout<<" ===> CLOSING HDF5 <===== "<<std::endl;
+ //H5Dclose(_dset_id);
+ //H5Sclose(_filespace);
+ //H5Sclose(_memspace);
+ //H5Pclose(_plist_id);
+ H5Fclose(_file_id);
+ }
+ */
+private:
+ capsule::STLVector
+ m_Buffer; ///< heap capsule, contains data and metadata buffers
+
+ void Init();
+ void clean();
+
+ hid_t _plist_id, _file_id, _dset_id;
+ hid_t _memspace, _filespace;
+
+ hid_t DefH5T_COMPLEX_DOUBLE;
+ hid_t DefH5T_COMPLEX_FLOAT;
+ hid_t DefH5T_COMPLEX_LongDOUBLE;
+
+ hid_t DefH5T_filetype_COMPLEX_DOUBLE;
+ hid_t DefH5T_filetype_COMPLEX_FLOAT;
+ hid_t DefH5T_filetype_COMPLEX_LongDOUBLE;
+
+ template <class T>
+ void UseHDFWrite(Variable<T> &variable, const T *values, hid_t h5type);
+
+ /*
+ template<class T>
+ void UseHDFWrite( Variable<T>& variable, const T* values, hid_t h5type )
+ {
+ //here comes your magic at Writing now variable.m_UserValues has the data
+ passed by the user
+ //set variable
+ variable.m_AppValues = values;
+ m_WrittenVariables.insert( variable.m_Name );
+
+ std::cout<<"writting : "<<variable.m_Name<<" dim
+ size:"<<variable.m_GlobalDimensions.size()<<std::endl;
+
+ int dimSize = variable.m_GlobalDimensions.size();
+ //hsize_t dimsf[dimSize], count[dimSize], offset[dimSize];
+ std::vector<hsize_t> dimsf, count, offset;
+
+ for(int i=0; i<dimSize; i++) {
+ dimsf.push_back(variable.m_GlobalDimensions[i]);
+ count.push_back(dimsf[i]);
+ offset.push_back(0);
+ }
+ count[0] = dimsf[0]/m_SizeMPI;
+ offset[0]= m_RankMPI*count[0];
+
+ _filespace = H5Screate_simple(dimSize, dimsf.data(), NULL);
+
+ _dset_id = H5Dcreate(_file_id, variable.m_Name.c_str(), h5type,
+ _filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Sclose(_filespace);
+
+ _memspace = H5Screate_simple(dimSize, count.data(), NULL);
+
+ //Select hyperslab
+
+ _filespace = H5Dget_space(_dset_id);
+ H5Sselect_hyperslab(_filespace, H5S_SELECT_SET, offset.data(), NULL,
+ count.data(), NULL);
+
+
+ // Create property list for collective dataset write.
+
+ _plist_id = H5Pcreate(H5P_DATASET_XFER);
+ H5Pset_dxpl_mpio(_plist_id, H5FD_MPIO_COLLECTIVE);
+
+ herr_t status;
+ if (m_SizeMPI == 1) {
+ status = H5Dwrite(_dset_id, h5type, _memspace, _filespace, _plist_id,
+ values);
+ } else {
+ std::vector<T> val;
+ for (int i=0; i<100; i++) {
+ //hsize_t c = count[i];
+ //hsize_t o = offset[i];
+ val.push_back(0);
+ }
+ status = H5Dwrite(_dset_id, h5type, _memspace, _filespace, _plist_id,
+ val.data());
+ }
+
+ if (status < 0) {
+ // error
+ std::cerr<<" Write failed. "<<std::endl;
+ }
+
+ std::cout <<" ==> User is responsible for freeing the data "<<std::endl;
+ //free(values);
+
+
+ //Close/release resources.
+
+ //H5Dclose(_dset_id);
+ //H5Sclose(_filespace);
+ //H5Sclose(_memspace);
+ //H5Pclose(_plist_id);
+ //H5Fclose(_file_id);
+
+ H5Dclose(_dset_id);
+ H5Sclose(_filespace);
+ H5Sclose(_memspace);
+ H5Pclose(_plist_id);
+
+ }
+ */
+};
+
+} // end namespace adios
+
+#endif /* HDF5_WRITER_P_H_ */
diff --git a/source/engine/hdf5/HDF5ReaderP.cpp b/source/engine/hdf5/HDF5ReaderP.cpp
new file mode 100644
index 000000000..2412944f4
--- /dev/null
+++ b/source/engine/hdf5/HDF5ReaderP.cpp
@@ -0,0 +1,11 @@
+/*
+ * HDF5ReaderP.cpp
+ *
+ * Created on: Mar 23, 2017
+ * Author: junmin
+ */
+
+#include <iostream> //needs to go away, this is just for demo purposes
+
+#include "engine/hdf5/HDF5ReaderP.h"
+
diff --git a/source/engine/hdf5/HDF5WriterP.cpp b/source/engine/hdf5/HDF5WriterP.cpp
new file mode 100644
index 000000000..5d07fd6e7
--- /dev/null
+++ b/source/engine/hdf5/HDF5WriterP.cpp
@@ -0,0 +1,374 @@
+/*
+ * HDF5WriterP.cpp
+ *
+ * Created on: Mar 23, 2017
+ * Author: junmin
+ */
+
+#include <iostream> //needs to go away, this is just for demo purposes
+
+#ifdef ADIOS_HAVE_MPI
+#include "engine/hdf5/HDF5WriterP.h"
+
+#include "core/Support.h"
+#include "functions/adiosFunctions.h" //CSVToVector
+
+// supported transports
+//#include "transport/file/FD.h" // uses POSIX
+//#include "transport/file/FP.h" // uses C FILE*
+//#include "transport/file/FStream.h" // uses C++ fstream
+
+namespace adios {
+
+HDF5Writer::HDF5Writer(ADIOS &adios, const std::string name,
+ const std::string accessMode, MPI_Comm mpiComm,
+ const Method &method, const bool debugMode,
+ const unsigned int cores)
+ : Engine(adios, "HDF5Writer", name, accessMode, mpiComm, method, debugMode,
+ cores, " HDF5Writer constructor (or call to ADIOS Open).\n"),
+ m_Buffer(accessMode, m_RankMPI, m_DebugMode) {
+ DefH5T_COMPLEX_DOUBLE =
+ H5Tcreate(H5T_COMPOUND, sizeof(ADIOS2_Complex_Double));
+ DefH5T_COMPLEX_FLOAT = H5Tcreate(H5T_COMPOUND, sizeof(ADIOS2_Complex_Float));
+ DefH5T_COMPLEX_LongDOUBLE =
+ H5Tcreate(H5T_COMPOUND, sizeof(ADIOS2_Complex_LongDouble));
+
+ // compound data type for memory
+ H5Tinsert(DefH5T_COMPLEX_DOUBLE, "double real",
+ HOFFSET(ADIOS2_Complex_Double, _re), H5T_NATIVE_DOUBLE);
+ H5Tinsert(DefH5T_COMPLEX_DOUBLE, "double img",
+ HOFFSET(ADIOS2_Complex_Double, _im), H5T_NATIVE_DOUBLE);
+ H5Tinsert(DefH5T_COMPLEX_FLOAT, "float real",
+ HOFFSET(ADIOS2_Complex_Float, _re), H5T_NATIVE_FLOAT);
+ H5Tinsert(DefH5T_COMPLEX_FLOAT, "float img",
+ HOFFSET(ADIOS2_Complex_Float, _im), H5T_NATIVE_FLOAT);
+ H5Tinsert(DefH5T_COMPLEX_LongDOUBLE, "ldouble real",
+ HOFFSET(ADIOS2_Complex_LongDouble, _re), H5T_NATIVE_LDOUBLE);
+ H5Tinsert(DefH5T_COMPLEX_LongDOUBLE, "ldouble img",
+ HOFFSET(ADIOS2_Complex_LongDouble, _im), H5T_NATIVE_LDOUBLE);
+
+ // data type for file
+ int ldsize = H5Tget_size(H5T_NATIVE_DOUBLE);
+ DefH5T_filetype_COMPLEX_DOUBLE = H5Tcreate(H5T_COMPOUND, ldsize + ldsize);
+ H5Tinsert(DefH5T_filetype_COMPLEX_DOUBLE, "double real", 0, H5T_IEEE_F64BE);
+ H5Tinsert(DefH5T_filetype_COMPLEX_DOUBLE, "double img", ldsize,
+ H5T_IEEE_F64BE);
+
+ ldsize = H5Tget_size(H5T_NATIVE_FLOAT);
+ DefH5T_filetype_COMPLEX_FLOAT = H5Tcreate(H5T_COMPOUND, ldsize + ldsize);
+ H5Tinsert(DefH5T_filetype_COMPLEX_FLOAT, "float real", 0, H5T_IEEE_F32BE);
+ H5Tinsert(DefH5T_filetype_COMPLEX_FLOAT, "float img", ldsize, H5T_IEEE_F32BE);
+
+ ldsize = H5Tget_size(
+ H5T_NATIVE_LDOUBLE); // note that sizeof(H5T_NATIVE_LDOUBLE) returned 4!!
+ DefH5T_filetype_COMPLEX_LongDOUBLE = H5Tcreate(H5T_COMPOUND, ldsize + ldsize);
+ H5Tinsert(DefH5T_filetype_COMPLEX_LongDOUBLE, "ldouble real", 0,
+ H5T_NATIVE_LDOUBLE);
+ H5Tinsert(DefH5T_filetype_COMPLEX_LongDOUBLE, "ldouble img", ldsize,
+ H5T_NATIVE_LDOUBLE);
+
+ Init();
+}
+
+HDF5Writer::~HDF5Writer() {}
+
+void HDF5Writer::Init() {
+ if (m_AccessMode != "w" && m_AccessMode != "write" && m_AccessMode != "a" &&
+ m_AccessMode != "append") {
+ throw std::invalid_argument(
+ "ERROR: HDF5Writer doesn't support access mode " + m_AccessMode +
+ ", in call to ADIOS Open or HDF5Writer constructor\n");
+ }
+ std::cout << "method: # of inputs:" << m_Method.m_Parameters.size()
+ << std::endl;
+ std::cout << "::Init hdf5 parallel writer. File name:" << m_Name << std::endl;
+
+ _plist_id = H5Pcreate(H5P_FILE_ACCESS);
+
+#ifdef HAVE_MPI
+ H5Pset_fapl_mpio(_plist_id, m_MPIComm, MPI_INFO_NULL);
+#endif
+
+ /*
+ * Create a new file collectively and release property list identifier.
+ */
+ _file_id = H5Fcreate(m_Name.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, _plist_id);
+ H5Pclose(_plist_id);
+}
+
+void HDF5Writer::Write(Variable<char> &variable, const char *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_CHAR);
+}
+
+void HDF5Writer::Write(Variable<unsigned char> &variable,
+ const unsigned char *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_UCHAR);
+}
+
+void HDF5Writer::Write(Variable<short> &variable, const short *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_SHORT);
+}
+
+void HDF5Writer::Write(Variable<unsigned short> &variable,
+ const unsigned short *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_USHORT);
+}
+
+void HDF5Writer::Write(Variable<int> &variable, const int *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_INT);
+}
+
+void HDF5Writer::Write(Variable<unsigned int> &variable,
+ const unsigned int *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_UINT);
+}
+
+void HDF5Writer::Write(Variable<long int> &variable, const long int *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_LONG);
+}
+
+void HDF5Writer::Write(Variable<unsigned long int> &variable,
+ const unsigned long int *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_ULONG);
+}
+
+void HDF5Writer::Write(Variable<long long int> &variable,
+ const long long int *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_LLONG);
+}
+
+void HDF5Writer::Write(Variable<unsigned long long int> &variable,
+ const unsigned long long int *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_ULLONG);
+}
+
+void HDF5Writer::Write(Variable<float> &variable, const float *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_FLOAT);
+}
+
+void HDF5Writer::Write(Variable<double> &variable, const double *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_DOUBLE);
+}
+
+void HDF5Writer::Write(Variable<long double> &variable,
+ const long double *values) {
+ UseHDFWrite(variable, values, H5T_NATIVE_LDOUBLE);
+}
+
+void HDF5Writer::Write(Variable<std::complex<float>> &variable,
+ const std::complex<float> *values) {
+ UseHDFWrite(variable, values, DefH5T_filetype_COMPLEX_FLOAT);
+}
+
+void HDF5Writer::Write(Variable<std::complex<double>> &variable,
+ const std::complex<double> *values) {
+ UseHDFWrite(variable, values, DefH5T_filetype_COMPLEX_DOUBLE);
+}
+
+void HDF5Writer::Write(Variable<std::complex<long double>> &variable,
+ const std::complex<long double> *values) {
+ UseHDFWrite(variable, values, DefH5T_filetype_COMPLEX_LongDOUBLE);
+}
+
+// String version
+void HDF5Writer::Write(const std::string variableName, const char *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<char>(variableName), values, H5T_NATIVE_CHAR);
+}
+
+void HDF5Writer::Write(const std::string variableName,
+ const unsigned char *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<unsigned char>(variableName), values,
+ H5T_NATIVE_UCHAR);
+}
+
+void HDF5Writer::Write(const std::string variableName, const short *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<short>(variableName), values,
+ H5T_NATIVE_SHORT);
+}
+
+void HDF5Writer::Write(const std::string variableName,
+ const unsigned short *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<unsigned short>(variableName), values,
+ H5T_NATIVE_USHORT);
+}
+
+void HDF5Writer::Write(const std::string variableName, const int *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<int>(variableName), values, H5T_NATIVE_INT);
+}
+
+void HDF5Writer::Write(const std::string variableName,
+ const unsigned int *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<unsigned int>(variableName), values,
+ H5T_NATIVE_UINT);
+}
+
+void HDF5Writer::Write(const std::string variableName, const long int *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<long int>(variableName), values,
+ H5T_NATIVE_LONG);
+}
+
+void HDF5Writer::Write(const std::string variableName,
+ const unsigned long int *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<unsigned long int>(variableName), values,
+ H5T_NATIVE_ULONG);
+}
+
+void HDF5Writer::Write(const std::string variableName,
+ const long long int *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<long long int>(variableName), values,
+ H5T_NATIVE_LLONG);
+}
+
+void HDF5Writer::Write(const std::string variableName,
+ const unsigned long long int *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<unsigned long long int>(variableName), values,
+ H5T_NATIVE_ULLONG);
+}
+
+void HDF5Writer::Write(const std::string variableName, const float *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<float>(variableName), values,
+ H5T_NATIVE_FLOAT);
+}
+
+void HDF5Writer::Write(const std::string variableName, const double *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<double>(variableName), values,
+ H5T_NATIVE_DOUBLE);
+}
+
+void HDF5Writer::Write(const std::string variableName,
+ const long double *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<long double>(variableName), values,
+ H5T_NATIVE_LDOUBLE);
+}
+
+void HDF5Writer::Write(const std::string variableName,
+ const std::complex<float> *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<std::complex<float>>(variableName), values,
+ DefH5T_COMPLEX_FLOAT);
+}
+
+void HDF5Writer::Write(const std::string variableName,
+ const std::complex<double> *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<std::complex<double>>(variableName), values,
+ DefH5T_COMPLEX_DOUBLE);
+}
+
+void HDF5Writer::Write(const std::string variableName,
+ const std::complex<long double> *values) {
+ UseHDFWrite(m_ADIOS.GetVariable<std::complex<long double>>(variableName),
+ values, DefH5T_COMPLEX_LongDOUBLE);
+}
+
+void HDF5Writer::Close(const int transportIndex) {
+ std::cout << " ===> CLOSING HDF5 <===== " << std::endl;
+ // H5Dclose(_dset_id);
+ // H5Sclose(_filespace);
+ // H5Sclose(_memspace);
+ // H5Pclose(_plist_id);
+ H5Fclose(_file_id);
+}
+
+template <class T>
+void HDF5Writer::UseHDFWrite(Variable<T> &variable, const T *values,
+ hid_t h5type) {
+ // here comes your magic at Writing now variable.m_UserValues has the data
+ // passed by the user
+ // set variable
+ variable.m_AppValues = values;
+ m_WrittenVariables.insert(variable.m_Name);
+
+ std::cout << "writting : " << variable.m_Name
+ << " dim size:" << variable.m_GlobalDimensions.size() << std::endl;
+
+ int dimSize = variable.m_GlobalDimensions.size();
+ for (int i = 0; i < dimSize; i++) {
+ std::cout << " dim: " << i << ", size:" << variable.m_GlobalDimensions[i]
+ << " offset=" << variable.m_GlobalOffsets[i]
+ << " count=" << variable.m_Dimensions[i] << std::endl;
+ }
+ std::vector<hsize_t> dimsf, count, offset;
+
+ for (int i = 0; i < dimSize; i++) {
+ dimsf.push_back(variable.m_GlobalDimensions[i]);
+ count.push_back(variable.m_Dimensions[i]);
+ offset.push_back(variable.m_GlobalOffsets[i]);
+ }
+
+ _filespace = H5Screate_simple(dimSize, dimsf.data(), NULL);
+
+ _dset_id = H5Dcreate(_file_id, variable.m_Name.c_str(), h5type, _filespace,
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ // H5Sclose(_filespace);
+
+ _memspace = H5Screate_simple(dimSize, count.data(), NULL);
+
+ // Select hyperslab
+ _filespace = H5Dget_space(_dset_id);
+ H5Sselect_hyperslab(_filespace, H5S_SELECT_SET, offset.data(), NULL,
+ count.data(), NULL);
+
+ // Create property list for collective dataset write.
+ _plist_id = H5Pcreate(H5P_DATASET_XFER);
+ H5Pset_dxpl_mpio(_plist_id, H5FD_MPIO_COLLECTIVE);
+
+ herr_t status;
+
+ if (h5type == DefH5T_filetype_COMPLEX_FLOAT) {
+ /*
+ ADIOS2_Complex_Float v[3];
+ v[0]._re = 1.1, v[0]._im = 1.2;
+ v[1]._re = 2.1, v[1]._im = 2.2;
+ v[2]._re = 3.1, v[2]._im = 3.2;
+ // status = H5Dwrite(_dset_id, DefH5T_COMPLEX_FLOAT, _memspace, _filespace,
+ // _plist_id, values);
+ status = H5Dwrite(_dset_id, DefH5T_COMPLEX_FLOAT, _memspace, _filespace,
+ _plist_id, v);
+ */
+ } else if (h5type == DefH5T_filetype_COMPLEX_DOUBLE) {
+ /*
+ ADIOS2_Complex_Double v[3];
+ v[0]._re = 1.1, v[0]._im = 1.2;
+ v[1]._re = 2.1, v[1]._im = 2.2;
+ v[2]._re = 3.1, v[2]._im = 3.2;
+ status = H5Dwrite(_dset_id, DefH5T_COMPLEX_DOUBLE, _memspace, _filespace,
+ _plist_id, v);
+ */
+ } else if (h5type == DefH5T_filetype_COMPLEX_LongDOUBLE) {
+ /*
+ ADIOS2_Complex_LongDouble v[3];
+ v[0]._re = 1.1, v[0]._im = 1.2;
+ v[1]._re = 2.1, v[1]._im = 2.2;
+ v[2]._re = 3.1, v[2]._im = 3.2;
+ status = H5Dwrite(_dset_id, DefH5T_COMPLEX_LongDOUBLE, _memspace,
+ _filespace, _plist_id, v);
+ */
+ } else {
+ status =
+ H5Dwrite(_dset_id, h5type, _memspace, _filespace, _plist_id, values);
+ }
+
+ if (status < 0) {
+ // error
+ std::cerr << " Write failed. " << std::endl;
+ }
+
+ std::cout << " ==> User is responsible for freeing the data " << std::endl;
+ // free(values);
+
+ // Close/release resources.
+
+ // H5Dclose(_dset_id);
+ // H5Sclose(_filespace);
+ // H5Sclose(_memspace);
+ // H5Pclose(_plist_id);
+ // H5Fclose(_file_id);
+
+ H5Dclose(_dset_id);
+ H5Sclose(_filespace);
+ H5Sclose(_memspace);
+ H5Pclose(_plist_id);
+}
+
+} // end namespace adios
+
+#endif //ADIOS_HAVE_MPI
--
GitLab
From 8f7389e982552abdaed233454a865976e788e53c Mon Sep 17 00:00:00 2001
From: guj <jgu@lbl.gov>
Date: Fri, 14 Apr 2017 12:24:44 -0700
Subject: [PATCH 2/6] hdf5 basic support
---
CMakeLists.txt | 2 +
cmake/FindPHDF5.cmake | 59 -------
examples/hello/CMakeLists.txt | 4 +
examples/hello/hdf5Writer/CMakeLists.txt | 30 ++++
examples/hello/hdf5Writer/helloHDF5Writer.cpp | 137 +++++++++++++++
.../hdf5Writer/helloHDF5Writer_nompi.cpp | 134 ++++++++++++++
include/engine/hdf5/HDF5ReaderP.h | 11 ++
include/engine/hdf5/HDF5WriterP.h | 33 ++--
source/ADIOS.cpp | 15 ++
source/CMakeLists.txt | 33 +++-
source/engine/hdf5/HDF5ReaderP.cpp | 11 +-
source/engine/hdf5/HDF5WriterP.cpp | 166 ++++++------------
12 files changed, 439 insertions(+), 196 deletions(-)
delete mode 100644 cmake/FindPHDF5.cmake
create mode 100644 examples/hello/hdf5Writer/CMakeLists.txt
create mode 100644 examples/hello/hdf5Writer/helloHDF5Writer.cpp
create mode 100644 examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9e267cc5b..0d70affa2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -73,6 +73,7 @@ endif()
option(ADIOS_USE_BZip2 "Enable support for BZip2 transforms" OFF)
option(ADIOS_USE_ADIOS1 "Enable support for the ADIOS 1 engine" OFF)
option(ADIOS_USE_DataMan "Enable support for the DataMan engine" OFF)
+option(ADIOS_USE_PHDF5 "Enable support for hdf5" OFF)
#------------------------------------------------------------------------------#
# Third party libraries
@@ -134,4 +135,5 @@ message(" MPI: ${ADIOS_USE_MPI}")
message(" BZip2: ${ADIOS_USE_BZip2}")
message(" ADIOS1: ${ADIOS_USE_ADIOS1}")
message(" DataMan: ${ADIOS_USE_DataMan}")
+message(" HDF5: ${ADIOS_USE_PHDF5}")
message("")
diff --git a/cmake/FindPHDF5.cmake b/cmake/FindPHDF5.cmake
deleted file mode 100644
index e357e7279..000000000
--- a/cmake/FindPHDF5.cmake
+++ /dev/null
@@ -1,59 +0,0 @@
-#------------------------------------------------------------------------------#
-# Distributed under the OSI-approved Apache License, Version 2.0. See
-# accompanying file Copyright.txt for details.
-#------------------------------------------------------------------------------#
-#
-# FindPHDF5
-# ---------
-#
-
-
-if(NOT PHDF5_FOUND)
- if((NOT HDF5_DIR) AND (NOT (ENV{HDF5_DIR} STREQUAL "")))
- set(HDF5_DIR "$ENV{HDF5_DIR}")
- endif()
-
- # Search for the core libraries
- if(HDF5_DIR)
- # If a root directory is specified, then don't look anywhere else
- find_path(PHDF5_INCLUDE_DIR hdf5.h
- HINTS ${HDF5_DIR}/include
- NO_DEFAULT_PATHS
- )
- set(_PHDF5_LIBRARY_HINT HINTS ${HDF5_DIR}/lib NO_DEFAULT_PATHS)
- else()
- # Otherwise use the include dir as a basis to search for the lib
- find_path(PHDF5_INCLUDE_DIR hdf5.h)
- if(PHDF5_INCLUDE_DIR)
- get_filename_component(_PHDF5_PREFIX "${PHDF5_INCLUDE_DIR}" PATH)
- set(_PHDF5_LIBRARY_HINT HINTS ${_PHDF5_PREFIX}/lib)
- unset(_PHDF5_PREFIX)
- endif()
- endif()
- find_library(PHDF5_LIBRARY hdf5 ${_PHDF5_LIBRARY_HINT})
- unset(_PHDF5_LIBRARY_HINT)
-
-
- include(FindPackageHandleStandardArgs)
- find_package_handle_standard_args(PHDF5 DEFAULT_MSG PHDF5_LIBRARY PHDF5_INCLUDE_DIR)
-
- find_package_handle_standard_args(PHDF5
- FOUND_VAR PHDF5_FOUND
- REQUIRED_VARS
- PHDF5_INCLUDE_DIR
- PHDF5_LIBRARY
- )
-
- MESSAGE(STATUS "PHDF5_FOUND: " ${PHDF5_FOUND} "HDF5_DIR: " ${HDF5_DIR} "TARGET=" ${TARGET} )
- if(PHDF5_FOUND)
- set(PHDF5_INCLUDE_DIRS ${PHDF5_INCLUDE_DIR})
- set(PHDF5_LIBRARIES ${PHDF5_LIBRARY})
- if(PHDF5_FOUND AND NOT TARGET PHDF5::PHDF5)
- add_library(PHDF5::PHDF5 UNKNOWN IMPORTED)
- set_target_properties(PHDF5::PHDF5 PROPERTIES
- IMPORTED_LOCATION "${PHDF5_LIBRARY}"
- INTERFACE_INCLUDE_DIRECTORIES "${PHDF5_INCLUDE_DIR}"
- )
- endif()
- endif()
-endif()
diff --git a/examples/hello/CMakeLists.txt b/examples/hello/CMakeLists.txt
index 0cf0a5df5..73fa0f560 100644
--- a/examples/hello/CMakeLists.txt
+++ b/examples/hello/CMakeLists.txt
@@ -14,3 +14,7 @@ if(ADIOS_USE_DataMan)
add_subdirectory(datamanReader)
add_subdirectory(datamanWriter)
endif()
+
+if(ADIOS_USE_PHDF5)
+ add_subdirectory(hdf5Writer)
+endif()
diff --git a/examples/hello/hdf5Writer/CMakeLists.txt b/examples/hello/hdf5Writer/CMakeLists.txt
new file mode 100644
index 000000000..742560057
--- /dev/null
+++ b/examples/hello/hdf5Writer/CMakeLists.txt
@@ -0,0 +1,30 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0. See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+add_executable(hello_hdf5Writer_nompi helloHDF5Writer_nompi.cpp)
+message(" HDF5: ${HDF5_INCLUDE_DIRS}")
+include_directories(${HDF5_INCLUDE_DIRS})
+target_link_libraries(hello_hdf5Writer_nompi adios2_nompi)
+
+if(ADIOS_BUILD_TESTING)
+ add_test(
+ NAME Example::hello::hdf5Writer_nompi
+ COMMAND hello_hdf5Writer_nompi
+ )
+endif()
+
+if(ADIOS_USE_MPI)
+ find_package(MPI COMPONENTS C REQUIRED)
+
+ add_executable(hello_hdf5Writer helloHDF5Writer.cpp)
+ target_link_libraries(hello_hdf5Writer adios2)
+ target_include_directories(hello_hdf5Writer PRIVATE ${MPI_C_INCLUDE_PATH})
+ target_link_libraries(hello_hdf5Writer adios2 ${MPI_C_LIBRARIES})
+
+ if(ADIOS_BUILD_TESTING)
+ add_test(NAME Example::hello::hdf5Writer COMMAND hello_hdf5Writer)
+ endif()
+endif()
+
diff --git a/examples/hello/hdf5Writer/helloHDF5Writer.cpp b/examples/hello/hdf5Writer/helloHDF5Writer.cpp
new file mode 100644
index 000000000..8dd6260bf
--- /dev/null
+++ b/examples/hello/hdf5Writer/helloHDF5Writer.cpp
@@ -0,0 +1,137 @@
+/*
+ * HDF5Writer.cpp
+ *
+ * Created on: March 20, 2017
+ * Author: Junmin
+ */
+
+
+
+#include <vector>
+#include <iostream>
+
+
+#include <mpi.h>
+
+
+#include "ADIOS_CPP.h"
+
+
+int main( int argc, char* argv [] )
+{
+ MPI_Init( &argc, &argv );
+ int rank, size;
+ MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+ MPI_Comm_size( MPI_COMM_WORLD, &size );
+
+ const bool adiosDebug = true;
+ adios::ADIOS adios(MPI_COMM_WORLD, adios::Verbose::INFO, adiosDebug);
+
+
+ //Application variable
+ const std::size_t intDim1 = 4;
+ const std::size_t intDim2 = 3;
+ std::vector<int> myInts = {10,11,12,13,14,15,16,17,18,19,20,21};
+
+ std::vector<double> myDoubles = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+ const std::size_t Nx = myDoubles.size();
+
+ std::vector<std::complex<float>> myCFloats;
+ const std::size_t ComplexDataSize = 3;
+ myCFloats.reserve( ComplexDataSize );
+ myCFloats.emplace_back( 1, 3 );
+ myCFloats.emplace_back( 2, 2 );
+ myCFloats.emplace_back( 3, 1 );
+
+
+ std::vector<std::complex<double>> myCDoubles;
+ myCDoubles.reserve( ComplexDataSize );
+ myCDoubles.emplace_back( 1.1, -3.3 );
+ myCDoubles.emplace_back( 2.1, -2.2 );
+ myCDoubles.emplace_back( 3.1, -1.1 );
+
+ std::vector<std::complex<long double>> myCLongDoubles;
+ myCLongDoubles.reserve( ComplexDataSize );
+ myCLongDoubles.emplace_back( 1.11, -3.33 );
+ myCLongDoubles.emplace_back( 2.11, -2.22 );
+ myCLongDoubles.emplace_back( 3.11, -1.11 );
+
+
+ std::size_t doubleVCount = Nx/size;
+ std::size_t complexCount = ComplexDataSize/size;
+ std::size_t intCountDim1 = intDim1/size;
+
+ std::size_t doubleVOffset = rank * doubleVCount;
+ std::size_t complexOffset = rank * complexCount;
+ std::size_t intOffsetDim1 = rank * intCountDim1;
+ std::size_t intOffsetDim2 = 0;
+
+ if ((size > 1) && (rank == size-1))
+ {
+ doubleVCount = Nx - rank*(Nx/size);
+ complexCount = ComplexDataSize - rank * (ComplexDataSize/size);
+ intCountDim1 = intDim1 - rank * (intDim1/size);
+ }
+
+ try
+ {
+ //Define variable and local size
+ auto& ioMyInts = adios.DefineVariable<int>( "myInts", {intCountDim1, intDim2}, {4,3}, {intOffsetDim1, intOffsetDim2} );
+ auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", {doubleVCount}, {Nx}, {doubleVOffset} );
+ auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>( "myCFloats", {complexCount}, {3}, {complexOffset} );
+ auto& ioMyCDoubles = adios.DefineVariable<std::complex<double>>( "myCDoubles", {complexCount}, {3}, {complexOffset} );
+ auto& ioMyCLongDoubles = adios.DefineVariable<std::complex<long double>>( "myCLongDoubles", {complexCount}, {3}, {complexOffset} );
+
+
+ //Define method for engine creation, it is basically straight-forward parameters
+ adios::Method& HDF5Settings = adios.DeclareMethod("w" );
+ HDF5Settings.SetEngine("HDF5Writer");
+ HDF5Settings.SetParameters( "chunck=yes", "collectiveIO=yes");
+ //HDF5Settings.AddTransport( "Mdtm", "localIP=128.0.0.0.1", "remoteIP=128.0.0.0.2", "tolerances=1,2,3" );
+
+ //Create engine smart pointer to HDF5 Engine due to polymorphism,
+ //Open returns a smart pointer to Engine containing the Derived class HDF5
+ auto HDF5Writer = adios.Open( "test.bp", "w", HDF5Settings );
+
+ if( HDF5Writer == nullptr )
+ throw std::ios_base::failure( "ERROR: failed to create HDF5 I/O engine at Open\n" );
+
+ HDF5Writer->Write( ioMyDoubles, myDoubles.data()+doubleVOffset ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
+ HDF5Writer->Write( ioMyInts, myInts.data()+(intOffsetDim1*intDim2*rank) );
+
+ HDF5Writer->Write( ioMyCFloats, myCFloats.data()+complexOffset );
+ HDF5Writer->Write( ioMyCDoubles, myCDoubles.data()+complexOffset );
+ HDF5Writer->Write( ioMyCLongDoubles, myCLongDoubles.data()+complexOffset );
+ HDF5Writer->Close( );
+
+ }
+ catch( std::invalid_argument& e )
+ {
+ if( rank == 0 )
+ {
+ std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ }
+ catch( std::ios_base::failure& e )
+ {
+ if( rank == 0 )
+ {
+ std::cout << "System exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ }
+ catch( std::exception& e )
+ {
+ if( rank == 0 )
+ {
+ std::cout << "Exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ }
+
+ MPI_Finalize( );
+
+ return 0;
+
+}
diff --git a/examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp b/examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp
new file mode 100644
index 000000000..e49794eae
--- /dev/null
+++ b/examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp
@@ -0,0 +1,134 @@
+/*
+ * HDF5Writer.cpp
+ *
+ * Created on: March 20, 2017
+ * Author: Junmin
+ */
+
+
+
+#include <vector>
+#include <iostream>
+
+
+//#include <mpi.h>
+#include "ADIOS_MPI.h"
+
+#ifndef HAVE_MPI
+// mpidummy has namespace adios
+using namespace adios;
+#endif
+#include "ADIOS_CPP.h"
+
+
+int main( int argc, char* argv [] )
+{
+ MPI_Init( &argc, &argv );
+ int rank, size;
+ MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+ MPI_Comm_size( MPI_COMM_WORLD, &size );
+
+ const bool adiosDebug = true;
+ adios::ADIOS adios(MPI_COMM_WORLD, adios::Verbose::INFO, adiosDebug);
+
+
+ //Application variable
+ const std::size_t intDim1 = 4;
+ const std::size_t intDim2 = 3;
+ std::vector<int> myInts = {10,11,12,13,14,15,16,17,18,19,20,21};
+
+ std::vector<double> myDoubles = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+ const std::size_t Nx = myDoubles.size();
+
+ std::vector<std::complex<float>> myCFloats;
+ const std::size_t CFloatSize = 3;
+ myCFloats.reserve( CFloatSize );
+ myCFloats.emplace_back( 1, 3 );
+ myCFloats.emplace_back( 2, 2 );
+ myCFloats.emplace_back( 3, 1 );
+
+
+ std::vector<std::complex<double>> myCDoubles;
+ const std::size_t CDoubleSize = 3;
+ myCDoubles.reserve( CDoubleSize );
+ myCDoubles.emplace_back( 1, 3 );
+ myCDoubles.emplace_back( 2, 2 );
+ myCDoubles.emplace_back( 3, 1 );
+
+
+ std::size_t doubleVCount = Nx/size;
+ std::size_t floatCount = CFloatSize/size;
+ std::size_t intCountDim1 = intDim1/size;
+
+ std::size_t doubleVOffset = rank * doubleVCount;
+ std::size_t floatOffset = rank * floatCount;
+ std::size_t intOffsetDim1 = rank * intCountDim1;
+ std::size_t intOffsetDim2 = 0;
+
+ if ((size > 1) && (rank == size-1)) {
+ doubleVCount = Nx - rank*(Nx/size);
+ floatCount = CFloatSize - rank * (CFloatSize/size);
+ intCountDim1 = intDim1 - rank * (intDim1/size);
+ }
+
+ try
+ {
+ //Define variable and local size
+ //auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", {Nx}, {Nx} );
+ //auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>( "myCFloats", {3}, {3} );
+ //auto& ioMyInts = adios.DefineVariable<int>( "myInts", {4,3}, {4,3} );
+
+ auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", {doubleVCount}, {Nx}, {doubleVOffset} );
+ auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>( "myCFloats", {floatCount}, {3}, {floatOffset} );
+ auto& ioMyCDoubles = adios.DefineVariable<std::complex<double>>( "myCDoubles", {floatCount}, {3}, {floatOffset} );
+ auto& ioMyInts = adios.DefineVariable<int>( "myInts", {intCountDim1, intDim2}, {4,3}, {intOffsetDim1, intOffsetDim2} );
+
+ //Define method for engine creation, it is basically straight-forward parameters
+ adios::Method& HDF5Settings = adios.DeclareMethod("HDF5Writer" ); //default method type is Writer
+ HDF5Settings.SetParameters( "chunck=yes", "collectiveIO=yes");
+ //HDF5Settings.AddTransport( "Mdtm", "localIP=128.0.0.0.1", "remoteIP=128.0.0.0.2", "tolerances=1,2,3" );
+
+ //Create engine smart pointer to HDF5 Engine due to polymorphism,
+ //Open returns a smart pointer to Engine containing the Derived class HDF5
+ auto HDF5Writer = adios.Open( "test.bp", "w", HDF5Settings );
+
+ if( HDF5Writer == nullptr )
+ throw std::ios_base::failure( "ERROR: failed to create HDF5 I/O engine at Open\n" );
+
+ HDF5Writer->Write( ioMyDoubles, myDoubles.data()+doubleVOffset ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
+ HDF5Writer->Write( ioMyInts, myInts.data()+(intOffsetDim1*intDim2*rank) );
+ HDF5Writer->Write( ioMyCFloats, myCFloats.data() );
+ HDF5Writer->Write( ioMyCDoubles, myCDoubles.data() );
+ HDF5Writer->Close( );
+
+ }
+ catch( std::invalid_argument& e )
+ {
+ if( rank == 0 )
+ {
+ std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ }
+ catch( std::ios_base::failure& e )
+ {
+ if( rank == 0 )
+ {
+ std::cout << "System exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ }
+ catch( std::exception& e )
+ {
+ if( rank == 0 )
+ {
+ std::cout << "Exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ }
+
+ MPI_Finalize( );
+
+ return 0;
+
+}
diff --git a/include/engine/hdf5/HDF5ReaderP.h b/include/engine/hdf5/HDF5ReaderP.h
index 28d688208..959b08e91 100644
--- a/include/engine/hdf5/HDF5ReaderP.h
+++ b/include/engine/hdf5/HDF5ReaderP.h
@@ -1,3 +1,14 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * HDF5ReaderP.h
+ *
+ * Created on: March 20, 2017
+ * Author: Junmin
+ */
+
+
#ifndef HDF5_READER_P_H
#define HDF5_READER_P_H
diff --git a/include/engine/hdf5/HDF5WriterP.h b/include/engine/hdf5/HDF5WriterP.h
index 27d5e8d10..098a56c45 100644
--- a/include/engine/hdf5/HDF5WriterP.h
+++ b/include/engine/hdf5/HDF5WriterP.h
@@ -1,4 +1,8 @@
+
/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
* HDF5WriterP.h
*
* Created on: March 20, 2017
@@ -8,24 +12,23 @@
#ifndef HDF5_WRITER_P_H_
#define HDF5_WRITER_P_H_
-#include <iostream> //std::cout must be removed, only used for hello example
#include <unistd.h> //sleep must be removed
#include "core/Engine.h"
-#include "format/BP1Writer.h"
+
// supported capsules
#include "capsule/heap/STLVector.h"
#include "ADIOS_MPI.h"
-#include "hdf5.h"
+#include <hdf5.h>
namespace adios {
-
+ /*
typedef struct {
- double _re; /*real part*/
- double _im; /*imaginary part*/
+ double _re; //real part
+ double _im; //imaginary part
} ADIOS2_Complex_Double;
typedef struct {
@@ -36,10 +39,10 @@ typedef struct {
// using ADIOS2_Complex_Float = std::complex<float>;
typedef struct {
- long double _re; /*real part*/
- long double _im; /*imaginary part*/
+ long double _re; //real part
+ long double _im; //imaginary part
} ADIOS2_Complex_LongDouble;
-
+*/
class HDF5Writer : public Engine {
public:
@@ -52,12 +55,10 @@ public:
* @param method
* @param debugMode
*/
-
HDF5Writer(ADIOS &adios, const std::string name, const std::string accessMode,
- MPI_Comm mpiComm, const Method &method,
- const bool debugMode = false, const unsigned int cores = 1);
+ MPI_Comm mpiComm, const Method &method);
- ~HDF5Writer();
+ virtual ~HDF5Writer();
void Write(Variable<char> &variable, const char *values);
void Write(Variable<unsigned char> &variable, const unsigned char *values);
@@ -126,9 +127,9 @@ private:
hid_t DefH5T_COMPLEX_FLOAT;
hid_t DefH5T_COMPLEX_LongDOUBLE;
- hid_t DefH5T_filetype_COMPLEX_DOUBLE;
- hid_t DefH5T_filetype_COMPLEX_FLOAT;
- hid_t DefH5T_filetype_COMPLEX_LongDOUBLE;
+ //hid_t DefH5T_filetype_COMPLEX_DOUBLE;
+ //hid_t DefH5T_filetype_COMPLEX_FLOAT;
+ //hid_t DefH5T_filetype_COMPLEX_LongDOUBLE;
template <class T>
void UseHDFWrite(Variable<T> &variable, const T *values, hid_t h5type);
diff --git a/source/ADIOS.cpp b/source/ADIOS.cpp
index 729e6c8ee..47e3c333f 100644
--- a/source/ADIOS.cpp
+++ b/source/ADIOS.cpp
@@ -35,6 +35,12 @@
#include "engine/adios1/ADIOS1Writer.h"
#endif
+#ifdef ADIOS_HAVE_PHDF5 // external dependencies
+#ifdef ADIOS_HAVE_MPI
+#include "engine/hdf5/HDF5WriterP.h"
+#include "engine/hdf5/HDF5ReaderP.h"
+#endif
+#endif
namespace adios
{
@@ -185,6 +191,15 @@ std::shared_ptr<Engine> ADIOS::Open(const std::string &name,
// method,
// iomode, timeout_sec, m_DebugMode, method.m_nThreads );
}
+ else if (type == "HDF5Writer") // -junmin
+ {
+ #if defined(ADIOS_HAVE_PHDF5) && defined(ADIOS_HAVE_MPI)
+ return std::make_shared<HDF5Writer>(*this, name, accessMode, mpiComm, method);
+ #else
+ throw std::invalid_argument( "ERROR: this version didn't compile with HDF5 library, can't use HDF5\n" );
+ #endif
+ }
+
else
{
if (m_DebugMode == true)
diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt
index 2ccf94700..4c5afc475 100644
--- a/source/CMakeLists.txt
+++ b/source/CMakeLists.txt
@@ -25,18 +25,18 @@ foreach(adios2_target IN LISTS adios2_targets)
engine/bp/BPFileReader.cpp
engine/bp/BPFileWriter.cpp
+
+ utilities/format/bp1/BP1Base.cpp
+ utilities/format/bp1/BP1Aggregator.cpp
+ utilities/format/bp1/BP1Writer.cpp
+
+ utilities/profiling/iochrono/Timer.cpp
functions/adiosFunctions.cpp
transport/file/FStream.cpp
transport/file/FileDescriptor.cpp
transport/file/FilePointer.cpp
-
- utilities/format/bp1/BP1Base.cpp
- utilities/format/bp1/BP1Aggregator.cpp
- utilities/format/bp1/BP1Writer.cpp
-
- utilities/profiling/iochrono/Timer.cpp
)
target_include_directories(${adios2_target}
PUBLIC ${ADIOS_SOURCE_DIR}/include
@@ -52,6 +52,7 @@ foreach(adios2_target IN LISTS adios2_targets)
target_compile_definitions(${adios2_target} PRIVATE ADIOS_HAVE_DATAMAN)
target_link_libraries(${adios2_target} PRIVATE DataMan::DataMan)
endif()
+
if(ADIOS_USE_BZip2)
find_package(BZip2 REQUIRED)
@@ -81,4 +82,24 @@ if(ADIOS_USE_MPI)
target_compile_definitions(adios2 PRIVATE ADIOS_HAVE_ADIOS1)
target_link_libraries(adios2 PRIVATE adios1::adios)
endif()
+
+ if(ADIOS_USE_PHDF5)
+ find_package(HDF5 REQUIRED)
+ message(" HDF5 ROOT: ${HDF5_ROOT}")
+ message(" HDF5 include: ${HDF5_INCLUDE_DIRS} is paralle? ${HDF5_IS_PARALLEL}")
+
+ get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
+ foreach(dir ${dirs})
+ message(STATUS "dir='${dir}'")
+ endforeach()
+ include_directories(${HDF5_INCLUDE_DIRS})
+
+ target_sources(adios2 PRIVATE
+ engine/hdf5/HDF5ReaderP.cpp
+ engine/hdf5/HDF5WriterP.cpp
+ )
+ target_compile_definitions(adios2 PRIVATE ADIOS_HAVE_PHDF5)
+ target_link_libraries(adios2 PRIVATE hdf5)
+ endif()
+
endif()
diff --git a/source/engine/hdf5/HDF5ReaderP.cpp b/source/engine/hdf5/HDF5ReaderP.cpp
index 2412944f4..3716bfea5 100644
--- a/source/engine/hdf5/HDF5ReaderP.cpp
+++ b/source/engine/hdf5/HDF5ReaderP.cpp
@@ -1,10 +1,15 @@
/*
- * HDF5ReaderP.cpp
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
*
- * Created on: Mar 23, 2017
- * Author: junmin
+ * HDF5WriterP.cpp
+ *
+ * Created on: March 20, 2017
+ * Author: Junmin
*/
+
+
#include <iostream> //needs to go away, this is just for demo purposes
#include "engine/hdf5/HDF5ReaderP.h"
diff --git a/source/engine/hdf5/HDF5WriterP.cpp b/source/engine/hdf5/HDF5WriterP.cpp
index 5d07fd6e7..196744ac2 100644
--- a/source/engine/hdf5/HDF5WriterP.cpp
+++ b/source/engine/hdf5/HDF5WriterP.cpp
@@ -1,10 +1,14 @@
/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
* HDF5WriterP.cpp
*
- * Created on: Mar 23, 2017
- * Author: junmin
+ * Created on: March 20, 2017
+ * Author: Junmin
*/
+
#include <iostream> //needs to go away, this is just for demo purposes
#ifdef ADIOS_HAVE_MPI
@@ -13,59 +17,34 @@
#include "core/Support.h"
#include "functions/adiosFunctions.h" //CSVToVector
-// supported transports
-//#include "transport/file/FD.h" // uses POSIX
-//#include "transport/file/FP.h" // uses C FILE*
-//#include "transport/file/FStream.h" // uses C++ fstream
-
namespace adios {
HDF5Writer::HDF5Writer(ADIOS &adios, const std::string name,
const std::string accessMode, MPI_Comm mpiComm,
- const Method &method, const bool debugMode,
- const unsigned int cores)
- : Engine(adios, "HDF5Writer", name, accessMode, mpiComm, method, debugMode,
- cores, " HDF5Writer constructor (or call to ADIOS Open).\n"),
- m_Buffer(accessMode, m_RankMPI, m_DebugMode) {
- DefH5T_COMPLEX_DOUBLE =
- H5Tcreate(H5T_COMPOUND, sizeof(ADIOS2_Complex_Double));
- DefH5T_COMPLEX_FLOAT = H5Tcreate(H5T_COMPOUND, sizeof(ADIOS2_Complex_Float));
- DefH5T_COMPLEX_LongDOUBLE =
- H5Tcreate(H5T_COMPOUND, sizeof(ADIOS2_Complex_LongDouble));
-
- // compound data type for memory
- H5Tinsert(DefH5T_COMPLEX_DOUBLE, "double real",
- HOFFSET(ADIOS2_Complex_Double, _re), H5T_NATIVE_DOUBLE);
- H5Tinsert(DefH5T_COMPLEX_DOUBLE, "double img",
- HOFFSET(ADIOS2_Complex_Double, _im), H5T_NATIVE_DOUBLE);
- H5Tinsert(DefH5T_COMPLEX_FLOAT, "float real",
- HOFFSET(ADIOS2_Complex_Float, _re), H5T_NATIVE_FLOAT);
- H5Tinsert(DefH5T_COMPLEX_FLOAT, "float img",
- HOFFSET(ADIOS2_Complex_Float, _im), H5T_NATIVE_FLOAT);
- H5Tinsert(DefH5T_COMPLEX_LongDOUBLE, "ldouble real",
- HOFFSET(ADIOS2_Complex_LongDouble, _re), H5T_NATIVE_LDOUBLE);
- H5Tinsert(DefH5T_COMPLEX_LongDOUBLE, "ldouble img",
- HOFFSET(ADIOS2_Complex_LongDouble, _im), H5T_NATIVE_LDOUBLE);
-
- // data type for file
- int ldsize = H5Tget_size(H5T_NATIVE_DOUBLE);
- DefH5T_filetype_COMPLEX_DOUBLE = H5Tcreate(H5T_COMPOUND, ldsize + ldsize);
- H5Tinsert(DefH5T_filetype_COMPLEX_DOUBLE, "double real", 0, H5T_IEEE_F64BE);
- H5Tinsert(DefH5T_filetype_COMPLEX_DOUBLE, "double img", ldsize,
- H5T_IEEE_F64BE);
-
- ldsize = H5Tget_size(H5T_NATIVE_FLOAT);
- DefH5T_filetype_COMPLEX_FLOAT = H5Tcreate(H5T_COMPOUND, ldsize + ldsize);
- H5Tinsert(DefH5T_filetype_COMPLEX_FLOAT, "float real", 0, H5T_IEEE_F32BE);
- H5Tinsert(DefH5T_filetype_COMPLEX_FLOAT, "float img", ldsize, H5T_IEEE_F32BE);
-
- ldsize = H5Tget_size(
- H5T_NATIVE_LDOUBLE); // note that sizeof(H5T_NATIVE_LDOUBLE) returned 4!!
- DefH5T_filetype_COMPLEX_LongDOUBLE = H5Tcreate(H5T_COMPOUND, ldsize + ldsize);
- H5Tinsert(DefH5T_filetype_COMPLEX_LongDOUBLE, "ldouble real", 0,
- H5T_NATIVE_LDOUBLE);
- H5Tinsert(DefH5T_filetype_COMPLEX_LongDOUBLE, "ldouble img", ldsize,
- H5T_NATIVE_LDOUBLE);
+ const Method &method)
+ : Engine(adios, "HDF5Writer", name, accessMode, mpiComm, method, /*debugMode, cores,*/
+ " HDF5Writer constructor (or call to ADIOS Open).\n"),
+ m_Buffer(accessMode, m_RankMPI, m_DebugMode)
+{
+ //
+ // 16, 4 vs: 8
+ //std::cout<<sizeof(std::complex<double>)<<", "<<sizeof(H5T_NATIVE_DOUBLE)<<" vs: "<<H5Tget_size(H5T_NATIVE_DOUBLE)<<std::endl;
+ // 8, 4 vs: 4
+ //std::cout<<sizeof(std::complex<float>)<<", "<<sizeof(H5T_NATIVE_FLOAT)<<" vs: "<<H5Tget_size(H5T_NATIVE_FLOAT)<<std::endl;
+ // 32, 4 vs: 16
+ //std::cout<<sizeof(std::complex<long double>)<<", "<<sizeof(H5T_NATIVE_LDOUBLE)<<" vs: "<<H5Tget_size(H5T_NATIVE_LDOUBLE)<<std::endl;
+
+ DefH5T_COMPLEX_FLOAT = H5Tcreate(H5T_COMPOUND, sizeof(std::complex<float>));
+ H5Tinsert(DefH5T_COMPLEX_FLOAT, "freal", 0, H5T_NATIVE_FLOAT);
+ H5Tinsert(DefH5T_COMPLEX_FLOAT, "fimg", H5Tget_size(H5T_NATIVE_FLOAT), H5T_NATIVE_FLOAT);
+
+ DefH5T_COMPLEX_DOUBLE = H5Tcreate(H5T_COMPOUND, sizeof(std::complex<double>));
+ H5Tinsert(DefH5T_COMPLEX_DOUBLE, "dreal", 0, H5T_NATIVE_DOUBLE);
+ H5Tinsert(DefH5T_COMPLEX_DOUBLE, "dimg", H5Tget_size(H5T_NATIVE_DOUBLE), H5T_NATIVE_DOUBLE);
+
+ DefH5T_COMPLEX_LongDOUBLE = H5Tcreate(H5T_COMPOUND, sizeof(std::complex<long double>));
+ H5Tinsert(DefH5T_COMPLEX_LongDOUBLE, "ldouble real", 0, H5T_NATIVE_LDOUBLE);
+ H5Tinsert(DefH5T_COMPLEX_LongDOUBLE, "ldouble img", H5Tget_size(H5T_NATIVE_LDOUBLE), H5T_NATIVE_LDOUBLE);
Init();
}
@@ -74,20 +53,20 @@ HDF5Writer::~HDF5Writer() {}
void HDF5Writer::Init() {
if (m_AccessMode != "w" && m_AccessMode != "write" && m_AccessMode != "a" &&
- m_AccessMode != "append") {
- throw std::invalid_argument(
- "ERROR: HDF5Writer doesn't support access mode " + m_AccessMode +
- ", in call to ADIOS Open or HDF5Writer constructor\n");
+ m_AccessMode != "append")
+ {
+ throw std::invalid_argument("ERROR: HDF5Writer doesn't support access mode " + m_AccessMode +
+ ", in call to ADIOS Open or HDF5Writer constructor\n");
}
- std::cout << "method: # of inputs:" << m_Method.m_Parameters.size()
- << std::endl;
- std::cout << "::Init hdf5 parallel writer. File name:" << m_Name << std::endl;
+ //std::cout << "method: # of inputs:" << m_Method.m_Parameters.size() << std::endl;
+
+ //std::cout << "::Init hdf5 parallel writer. File name:" << m_Name << std::endl;
_plist_id = H5Pcreate(H5P_FILE_ACCESS);
-#ifdef HAVE_MPI
+
H5Pset_fapl_mpio(_plist_id, m_MPIComm, MPI_INFO_NULL);
-#endif
+
/*
* Create a new file collectively and release property list identifier.
@@ -157,17 +136,17 @@ void HDF5Writer::Write(Variable<long double> &variable,
void HDF5Writer::Write(Variable<std::complex<float>> &variable,
const std::complex<float> *values) {
- UseHDFWrite(variable, values, DefH5T_filetype_COMPLEX_FLOAT);
+ UseHDFWrite(variable, values, DefH5T_COMPLEX_FLOAT);
}
void HDF5Writer::Write(Variable<std::complex<double>> &variable,
const std::complex<double> *values) {
- UseHDFWrite(variable, values, DefH5T_filetype_COMPLEX_DOUBLE);
+ UseHDFWrite(variable, values, DefH5T_COMPLEX_DOUBLE);
}
void HDF5Writer::Write(Variable<std::complex<long double>> &variable,
const std::complex<long double> *values) {
- UseHDFWrite(variable, values, DefH5T_filetype_COMPLEX_LongDOUBLE);
+ UseHDFWrite(variable, values, DefH5T_COMPLEX_LongDOUBLE);
}
// String version
@@ -260,7 +239,7 @@ void HDF5Writer::Write(const std::string variableName,
}
void HDF5Writer::Close(const int transportIndex) {
- std::cout << " ===> CLOSING HDF5 <===== " << std::endl;
+ //std::cout << " ===> CLOSING HDF5 <===== " << std::endl;
// H5Dclose(_dset_id);
// H5Sclose(_filespace);
// H5Sclose(_memspace);
@@ -277,21 +256,23 @@ void HDF5Writer::UseHDFWrite(Variable<T> &variable, const T *values,
variable.m_AppValues = values;
m_WrittenVariables.insert(variable.m_Name);
+ int dimSize = variable.m_GlobalDimensions.size();
+ /*
std::cout << "writting : " << variable.m_Name
<< " dim size:" << variable.m_GlobalDimensions.size() << std::endl;
- int dimSize = variable.m_GlobalDimensions.size();
for (int i = 0; i < dimSize; i++) {
std::cout << " dim: " << i << ", size:" << variable.m_GlobalDimensions[i]
- << " offset=" << variable.m_GlobalOffsets[i]
- << " count=" << variable.m_Dimensions[i] << std::endl;
+ << " offset=" << variable.m_Offsets[i]
+ << " count=" << variable.m_LocalDimensions[i] << std::endl;
}
+ */
std::vector<hsize_t> dimsf, count, offset;
for (int i = 0; i < dimSize; i++) {
dimsf.push_back(variable.m_GlobalDimensions[i]);
- count.push_back(variable.m_Dimensions[i]);
- offset.push_back(variable.m_GlobalOffsets[i]);
+ count.push_back(variable.m_LocalDimensions[i]);
+ offset.push_back(variable.m_Offsets[i]);
}
_filespace = H5Screate_simple(dimSize, dimsf.data(), NULL);
@@ -308,60 +289,21 @@ void HDF5Writer::UseHDFWrite(Variable<T> &variable, const T *values,
count.data(), NULL);
// Create property list for collective dataset write.
+
_plist_id = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(_plist_id, H5FD_MPIO_COLLECTIVE);
herr_t status;
- if (h5type == DefH5T_filetype_COMPLEX_FLOAT) {
- /*
- ADIOS2_Complex_Float v[3];
- v[0]._re = 1.1, v[0]._im = 1.2;
- v[1]._re = 2.1, v[1]._im = 2.2;
- v[2]._re = 3.1, v[2]._im = 3.2;
- // status = H5Dwrite(_dset_id, DefH5T_COMPLEX_FLOAT, _memspace, _filespace,
- // _plist_id, values);
- status = H5Dwrite(_dset_id, DefH5T_COMPLEX_FLOAT, _memspace, _filespace,
- _plist_id, v);
- */
- } else if (h5type == DefH5T_filetype_COMPLEX_DOUBLE) {
- /*
- ADIOS2_Complex_Double v[3];
- v[0]._re = 1.1, v[0]._im = 1.2;
- v[1]._re = 2.1, v[1]._im = 2.2;
- v[2]._re = 3.1, v[2]._im = 3.2;
- status = H5Dwrite(_dset_id, DefH5T_COMPLEX_DOUBLE, _memspace, _filespace,
- _plist_id, v);
- */
- } else if (h5type == DefH5T_filetype_COMPLEX_LongDOUBLE) {
- /*
- ADIOS2_Complex_LongDouble v[3];
- v[0]._re = 1.1, v[0]._im = 1.2;
- v[1]._re = 2.1, v[1]._im = 2.2;
- v[2]._re = 3.1, v[2]._im = 3.2;
- status = H5Dwrite(_dset_id, DefH5T_COMPLEX_LongDOUBLE, _memspace,
- _filespace, _plist_id, v);
- */
- } else {
- status =
- H5Dwrite(_dset_id, h5type, _memspace, _filespace, _plist_id, values);
- }
+ status =
+ H5Dwrite(_dset_id, h5type, _memspace, _filespace, _plist_id, values);
if (status < 0) {
// error
std::cerr << " Write failed. " << std::endl;
}
- std::cout << " ==> User is responsible for freeing the data " << std::endl;
- // free(values);
-
- // Close/release resources.
-
- // H5Dclose(_dset_id);
- // H5Sclose(_filespace);
- // H5Sclose(_memspace);
- // H5Pclose(_plist_id);
- // H5Fclose(_file_id);
+ //std::cout << " ==> User is responsible for freeing the data " << std::endl;
H5Dclose(_dset_id);
H5Sclose(_filespace);
--
GitLab
From c606e13c6c88c46e3d14a3d7c9694eb775765639 Mon Sep 17 00:00:00 2001
From: guj <jgu@lbl.gov>
Date: Fri, 14 Apr 2017 12:30:09 -0700
Subject: [PATCH 3/6] removed comments
---
include/engine/hdf5/HDF5WriterP.h | 269 +++++++++---------------------
1 file changed, 80 insertions(+), 189 deletions(-)
diff --git a/include/engine/hdf5/HDF5WriterP.h b/include/engine/hdf5/HDF5WriterP.h
index 098a56c45..9dc00a329 100644
--- a/include/engine/hdf5/HDF5WriterP.h
+++ b/include/engine/hdf5/HDF5WriterP.h
@@ -16,7 +16,6 @@
#include "core/Engine.h"
-
// supported capsules
#include "capsule/heap/STLVector.h"
@@ -24,200 +23,92 @@
#include <hdf5.h>
-namespace adios {
- /*
-typedef struct {
- double _re; //real part
- double _im; //imaginary part
-} ADIOS2_Complex_Double;
-
-typedef struct {
- float _re; // real
- float _im; // imaginary part
-} ADIOS2_Complex_Float;
+namespace adios
+{
-// using ADIOS2_Complex_Float = std::complex<float>;
-
-typedef struct {
- long double _re; //real part
- long double _im; //imaginary part
-} ADIOS2_Complex_LongDouble;
-*/
-class HDF5Writer : public Engine {
+class HDF5Writer : public Engine
+{
public:
- /**
- * Constructor for single BP capsule engine, writes in BP format into a single
- * heap capsule
- * @param name unique name given to the engine
- * @param accessMode
- * @param mpiComm
- * @param method
- * @param debugMode
- */
- HDF5Writer(ADIOS &adios, const std::string name, const std::string accessMode,
- MPI_Comm mpiComm, const Method &method);
-
- virtual ~HDF5Writer();
-
- void Write(Variable<char> &variable, const char *values);
- void Write(Variable<unsigned char> &variable, const unsigned char *values);
- void Write(Variable<short> &variable, const short *values);
- void Write(Variable<unsigned short> &variable, const unsigned short *values);
- void Write(Variable<int> &variable, const int *values);
- void Write(Variable<unsigned int> &variable, const unsigned int *values);
- void Write(Variable<long int> &variable, const long int *values);
- void Write(Variable<unsigned long int> &variable,
- const unsigned long int *values);
- void Write(Variable<long long int> &variable, const long long int *values);
- void Write(Variable<unsigned long long int> &variable,
- const unsigned long long int *values);
- void Write(Variable<float> &variable, const float *values);
- void Write(Variable<double> &variable, const double *values);
- void Write(Variable<long double> &variable, const long double *values);
- void Write(Variable<std::complex<float>> &variable,
- const std::complex<float> *values);
- void Write(Variable<std::complex<double>> &variable,
- const std::complex<double> *values);
- void Write(Variable<std::complex<long double>> &variable,
- const std::complex<long double> *values);
-
- void Write(const std::string variableName, const char *values);
- void Write(const std::string variableName, const unsigned char *values);
- void Write(const std::string variableName, const short *values);
- void Write(const std::string variableName, const unsigned short *values);
- void Write(const std::string variableName, const int *values);
- void Write(const std::string variableName, const unsigned int *values);
- void Write(const std::string variableName, const long int *values);
- void Write(const std::string variableName, const unsigned long int *values);
- void Write(const std::string variableName, const long long int *values);
- void Write(const std::string variableName,
- const unsigned long long int *values);
- void Write(const std::string variableName, const float *values);
- void Write(const std::string variableName, const double *values);
- void Write(const std::string variableName, const long double *values);
- void Write(const std::string variableName, const std::complex<float> *values);
- void Write(const std::string variableName,
- const std::complex<double> *values);
- void Write(const std::string variableName,
- const std::complex<long double> *values);
-
- void Close(const int transportIndex = -1);
- /*
- {
- std::cout<<" ===> CLOSING HDF5 <===== "<<std::endl;
- //H5Dclose(_dset_id);
- //H5Sclose(_filespace);
- //H5Sclose(_memspace);
- //H5Pclose(_plist_id);
- H5Fclose(_file_id);
- }
- */
+ /**
+ * Constructor for single BP capsule engine, writes in BP format into a
+ * single
+ * heap capsule
+ * @param name unique name given to the engine
+ * @param accessMode
+ * @param mpiComm
+ * @param method
+ * @param debugMode
+ */
+ HDF5Writer(ADIOS &adios, const std::string name,
+ const std::string accessMode, MPI_Comm mpiComm,
+ const Method &method);
+
+ virtual ~HDF5Writer();
+
+ void Write(Variable<char> &variable, const char *values);
+ void Write(Variable<unsigned char> &variable, const unsigned char *values);
+ void Write(Variable<short> &variable, const short *values);
+ void Write(Variable<unsigned short> &variable,
+ const unsigned short *values);
+ void Write(Variable<int> &variable, const int *values);
+ void Write(Variable<unsigned int> &variable, const unsigned int *values);
+ void Write(Variable<long int> &variable, const long int *values);
+ void Write(Variable<unsigned long int> &variable,
+ const unsigned long int *values);
+ void Write(Variable<long long int> &variable, const long long int *values);
+ void Write(Variable<unsigned long long int> &variable,
+ const unsigned long long int *values);
+ void Write(Variable<float> &variable, const float *values);
+ void Write(Variable<double> &variable, const double *values);
+ void Write(Variable<long double> &variable, const long double *values);
+ void Write(Variable<std::complex<float>> &variable,
+ const std::complex<float> *values);
+ void Write(Variable<std::complex<double>> &variable,
+ const std::complex<double> *values);
+ void Write(Variable<std::complex<long double>> &variable,
+ const std::complex<long double> *values);
+
+ void Write(const std::string variableName, const char *values);
+ void Write(const std::string variableName, const unsigned char *values);
+ void Write(const std::string variableName, const short *values);
+ void Write(const std::string variableName, const unsigned short *values);
+ void Write(const std::string variableName, const int *values);
+ void Write(const std::string variableName, const unsigned int *values);
+ void Write(const std::string variableName, const long int *values);
+ void Write(const std::string variableName, const unsigned long int *values);
+ void Write(const std::string variableName, const long long int *values);
+ void Write(const std::string variableName,
+ const unsigned long long int *values);
+ void Write(const std::string variableName, const float *values);
+ void Write(const std::string variableName, const double *values);
+ void Write(const std::string variableName, const long double *values);
+ void Write(const std::string variableName,
+ const std::complex<float> *values);
+ void Write(const std::string variableName,
+ const std::complex<double> *values);
+ void Write(const std::string variableName,
+ const std::complex<long double> *values);
+
+ void Close(const int transportIndex = -1);
+
private:
- capsule::STLVector
- m_Buffer; ///< heap capsule, contains data and metadata buffers
+ capsule::STLVector
+ m_Buffer; ///< heap capsule, contains data and metadata buffers
+
+ void Init();
+ void clean();
+
+ hid_t _plist_id, _file_id, _dset_id;
+ hid_t _memspace, _filespace;
+
+ hid_t DefH5T_COMPLEX_DOUBLE;
+ hid_t DefH5T_COMPLEX_FLOAT;
+ hid_t DefH5T_COMPLEX_LongDOUBLE;
- void Init();
- void clean();
-
- hid_t _plist_id, _file_id, _dset_id;
- hid_t _memspace, _filespace;
-
- hid_t DefH5T_COMPLEX_DOUBLE;
- hid_t DefH5T_COMPLEX_FLOAT;
- hid_t DefH5T_COMPLEX_LongDOUBLE;
-
- //hid_t DefH5T_filetype_COMPLEX_DOUBLE;
- //hid_t DefH5T_filetype_COMPLEX_FLOAT;
- //hid_t DefH5T_filetype_COMPLEX_LongDOUBLE;
-
- template <class T>
- void UseHDFWrite(Variable<T> &variable, const T *values, hid_t h5type);
-
- /*
- template<class T>
- void UseHDFWrite( Variable<T>& variable, const T* values, hid_t h5type )
- {
- //here comes your magic at Writing now variable.m_UserValues has the data
- passed by the user
- //set variable
- variable.m_AppValues = values;
- m_WrittenVariables.insert( variable.m_Name );
-
- std::cout<<"writting : "<<variable.m_Name<<" dim
- size:"<<variable.m_GlobalDimensions.size()<<std::endl;
-
- int dimSize = variable.m_GlobalDimensions.size();
- //hsize_t dimsf[dimSize], count[dimSize], offset[dimSize];
- std::vector<hsize_t> dimsf, count, offset;
-
- for(int i=0; i<dimSize; i++) {
- dimsf.push_back(variable.m_GlobalDimensions[i]);
- count.push_back(dimsf[i]);
- offset.push_back(0);
- }
- count[0] = dimsf[0]/m_SizeMPI;
- offset[0]= m_RankMPI*count[0];
-
- _filespace = H5Screate_simple(dimSize, dimsf.data(), NULL);
-
- _dset_id = H5Dcreate(_file_id, variable.m_Name.c_str(), h5type,
- _filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- H5Sclose(_filespace);
-
- _memspace = H5Screate_simple(dimSize, count.data(), NULL);
-
- //Select hyperslab
-
- _filespace = H5Dget_space(_dset_id);
- H5Sselect_hyperslab(_filespace, H5S_SELECT_SET, offset.data(), NULL,
- count.data(), NULL);
-
-
- // Create property list for collective dataset write.
-
- _plist_id = H5Pcreate(H5P_DATASET_XFER);
- H5Pset_dxpl_mpio(_plist_id, H5FD_MPIO_COLLECTIVE);
-
- herr_t status;
- if (m_SizeMPI == 1) {
- status = H5Dwrite(_dset_id, h5type, _memspace, _filespace, _plist_id,
- values);
- } else {
- std::vector<T> val;
- for (int i=0; i<100; i++) {
- //hsize_t c = count[i];
- //hsize_t o = offset[i];
- val.push_back(0);
- }
- status = H5Dwrite(_dset_id, h5type, _memspace, _filespace, _plist_id,
- val.data());
- }
-
- if (status < 0) {
- // error
- std::cerr<<" Write failed. "<<std::endl;
- }
-
- std::cout <<" ==> User is responsible for freeing the data "<<std::endl;
- //free(values);
-
-
- //Close/release resources.
-
- //H5Dclose(_dset_id);
- //H5Sclose(_filespace);
- //H5Sclose(_memspace);
- //H5Pclose(_plist_id);
- //H5Fclose(_file_id);
-
- H5Dclose(_dset_id);
- H5Sclose(_filespace);
- H5Sclose(_memspace);
- H5Pclose(_plist_id);
+ template <class T>
+ void UseHDFWrite(Variable<T> &variable, const T *values, hid_t h5type);
- }
- */
};
} // end namespace adios
--
GitLab
From d13d6731cea0fb7a2fb7568bdab220e6d2d5e380 Mon Sep 17 00:00:00 2001
From: guj <jgu@lbl.gov>
Date: Fri, 14 Apr 2017 13:42:34 -0700
Subject: [PATCH 4/6] minor changes
---
examples/hello/hdf5Writer/CMakeLists.txt | 8 +-
examples/hello/hdf5Writer/helloHDF5Writer.cpp | 157 +++----
.../hdf5Writer/helloHDF5Writer_nompi.cpp | 150 ++++---
include/engine/hdf5/HDF5ReaderP.h | 1 -
include/engine/hdf5/HDF5WriterP.h | 2 -
source/ADIOS.cpp | 18 +-
source/CMakeLists.txt | 8 +-
source/engine/hdf5/HDF5ReaderP.cpp | 3 -
source/engine/hdf5/HDF5WriterP.cpp | 399 ++++++++++--------
9 files changed, 403 insertions(+), 343 deletions(-)
diff --git a/examples/hello/hdf5Writer/CMakeLists.txt b/examples/hello/hdf5Writer/CMakeLists.txt
index 742560057..7f4197c0b 100644
--- a/examples/hello/hdf5Writer/CMakeLists.txt
+++ b/examples/hello/hdf5Writer/CMakeLists.txt
@@ -9,10 +9,10 @@ include_directories(${HDF5_INCLUDE_DIRS})
target_link_libraries(hello_hdf5Writer_nompi adios2_nompi)
if(ADIOS_BUILD_TESTING)
- add_test(
- NAME Example::hello::hdf5Writer_nompi
- COMMAND hello_hdf5Writer_nompi
- )
+ #add_test(
+ # NAME Example::hello::hdf5Writer_nompi
+ # COMMAND hello_hdf5Writer_nompi
+ #)
endif()
if(ADIOS_USE_MPI)
diff --git a/examples/hello/hdf5Writer/helloHDF5Writer.cpp b/examples/hello/hdf5Writer/helloHDF5Writer.cpp
index 8dd6260bf..85c5a910a 100644
--- a/examples/hello/hdf5Writer/helloHDF5Writer.cpp
+++ b/examples/hello/hdf5Writer/helloHDF5Writer.cpp
@@ -5,133 +5,140 @@
* Author: Junmin
*/
-
-
#include <vector>
#include <iostream>
-
#include <mpi.h>
-
#include "ADIOS_CPP.h"
-
-int main( int argc, char* argv [] )
+int main(int argc, char *argv[])
{
- MPI_Init( &argc, &argv );
+ MPI_Init(&argc, &argv);
int rank, size;
- MPI_Comm_rank( MPI_COMM_WORLD, &rank );
- MPI_Comm_size( MPI_COMM_WORLD, &size );
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
const bool adiosDebug = true;
adios::ADIOS adios(MPI_COMM_WORLD, adios::Verbose::INFO, adiosDebug);
-
- //Application variable
+ // Application variable
const std::size_t intDim1 = 4;
const std::size_t intDim2 = 3;
- std::vector<int> myInts = {10,11,12,13,14,15,16,17,18,19,20,21};
+ std::vector<int> myInts = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21};
- std::vector<double> myDoubles = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+ std::vector<double> myDoubles = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
const std::size_t Nx = myDoubles.size();
std::vector<std::complex<float>> myCFloats;
const std::size_t ComplexDataSize = 3;
- myCFloats.reserve( ComplexDataSize );
- myCFloats.emplace_back( 1, 3 );
- myCFloats.emplace_back( 2, 2 );
- myCFloats.emplace_back( 3, 1 );
-
+ myCFloats.reserve(ComplexDataSize);
+ myCFloats.emplace_back(1, 3);
+ myCFloats.emplace_back(2, 2);
+ myCFloats.emplace_back(3, 1);
std::vector<std::complex<double>> myCDoubles;
- myCDoubles.reserve( ComplexDataSize );
- myCDoubles.emplace_back( 1.1, -3.3 );
- myCDoubles.emplace_back( 2.1, -2.2 );
- myCDoubles.emplace_back( 3.1, -1.1 );
+ myCDoubles.reserve(ComplexDataSize);
+ myCDoubles.emplace_back(1.1, -3.3);
+ myCDoubles.emplace_back(2.1, -2.2);
+ myCDoubles.emplace_back(3.1, -1.1);
std::vector<std::complex<long double>> myCLongDoubles;
- myCLongDoubles.reserve( ComplexDataSize );
- myCLongDoubles.emplace_back( 1.11, -3.33 );
- myCLongDoubles.emplace_back( 2.11, -2.22 );
- myCLongDoubles.emplace_back( 3.11, -1.11 );
-
+ myCLongDoubles.reserve(ComplexDataSize);
+ myCLongDoubles.emplace_back(1.11, -3.33);
+ myCLongDoubles.emplace_back(2.11, -2.22);
+ myCLongDoubles.emplace_back(3.11, -1.11);
- std::size_t doubleVCount = Nx/size;
- std::size_t complexCount = ComplexDataSize/size;
- std::size_t intCountDim1 = intDim1/size;
+ std::size_t doubleVCount = Nx / size;
+ std::size_t complexCount = ComplexDataSize / size;
+ std::size_t intCountDim1 = intDim1 / size;
std::size_t doubleVOffset = rank * doubleVCount;
- std::size_t complexOffset = rank * complexCount;
+ std::size_t complexOffset = rank * complexCount;
std::size_t intOffsetDim1 = rank * intCountDim1;
std::size_t intOffsetDim2 = 0;
- if ((size > 1) && (rank == size-1))
+ if ((size > 1) && (rank == size - 1))
{
- doubleVCount = Nx - rank*(Nx/size);
- complexCount = ComplexDataSize - rank * (ComplexDataSize/size);
- intCountDim1 = intDim1 - rank * (intDim1/size);
- }
-
+ doubleVCount = Nx - rank * (Nx / size);
+ complexCount = ComplexDataSize - rank * (ComplexDataSize / size);
+ intCountDim1 = intDim1 - rank * (intDim1 / size);
+ }
+
try
{
- //Define variable and local size
- auto& ioMyInts = adios.DefineVariable<int>( "myInts", {intCountDim1, intDim2}, {4,3}, {intOffsetDim1, intOffsetDim2} );
- auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", {doubleVCount}, {Nx}, {doubleVOffset} );
- auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>( "myCFloats", {complexCount}, {3}, {complexOffset} );
- auto& ioMyCDoubles = adios.DefineVariable<std::complex<double>>( "myCDoubles", {complexCount}, {3}, {complexOffset} );
- auto& ioMyCLongDoubles = adios.DefineVariable<std::complex<long double>>( "myCLongDoubles", {complexCount}, {3}, {complexOffset} );
-
-
- //Define method for engine creation, it is basically straight-forward parameters
- adios::Method& HDF5Settings = adios.DeclareMethod("w" );
- HDF5Settings.SetEngine("HDF5Writer");
- HDF5Settings.SetParameters( "chunck=yes", "collectiveIO=yes");
- //HDF5Settings.AddTransport( "Mdtm", "localIP=128.0.0.0.1", "remoteIP=128.0.0.0.2", "tolerances=1,2,3" );
-
- //Create engine smart pointer to HDF5 Engine due to polymorphism,
- //Open returns a smart pointer to Engine containing the Derived class HDF5
- auto HDF5Writer = adios.Open( "test.bp", "w", HDF5Settings );
-
- if( HDF5Writer == nullptr )
- throw std::ios_base::failure( "ERROR: failed to create HDF5 I/O engine at Open\n" );
-
- HDF5Writer->Write( ioMyDoubles, myDoubles.data()+doubleVOffset ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
- HDF5Writer->Write( ioMyInts, myInts.data()+(intOffsetDim1*intDim2*rank) );
-
- HDF5Writer->Write( ioMyCFloats, myCFloats.data()+complexOffset );
- HDF5Writer->Write( ioMyCDoubles, myCDoubles.data()+complexOffset );
- HDF5Writer->Write( ioMyCLongDoubles, myCLongDoubles.data()+complexOffset );
- HDF5Writer->Close( );
-
- }
- catch( std::invalid_argument& e )
+ // Define variable and local size
+ auto &ioMyInts =
+ adios.DefineVariable<int>("myInts", {intCountDim1, intDim2}, {4, 3},
+ {intOffsetDim1, intOffsetDim2});
+ auto &ioMyDoubles = adios.DefineVariable<double>(
+ "myDoubles", {doubleVCount}, {Nx}, {doubleVOffset});
+ auto &ioMyCFloats = adios.DefineVariable<std::complex<float>>(
+ "myCFloats", {complexCount}, {3}, {complexOffset});
+ auto &ioMyCDoubles = adios.DefineVariable<std::complex<double>>(
+ "myCDoubles", {complexCount}, {3}, {complexOffset});
+ auto &ioMyCLongDoubles =
+ adios.DefineVariable<std::complex<long double>>(
+ "myCLongDoubles", {complexCount}, {3}, {complexOffset});
+
+ // Define method for engine creation, it is basically straight-forward
+ // parameters
+ adios::Method &HDF5Settings = adios.DeclareMethod("w");
+ HDF5Settings.SetEngine("HDF5Writer");
+ HDF5Settings.SetParameters("chunck=yes", "collectiveIO=yes");
+ // HDF5Settings.AddTransport( "Mdtm", "localIP=128.0.0.0.1",
+ // "remoteIP=128.0.0.0.2", "tolerances=1,2,3" );
+
+ // Create engine smart pointer to HDF5 Engine due to polymorphism,
+ // Open returns a smart pointer to Engine containing the Derived class
+ // HDF5
+ auto HDF5Writer = adios.Open("test.h5", "w", HDF5Settings);
+
+ if (HDF5Writer == nullptr)
+ throw std::ios_base::failure(
+ "ERROR: failed to create HDF5 I/O engine at Open\n");
+
+ HDF5Writer->Write(ioMyDoubles, myDoubles.data() +
+ doubleVOffset); // Base class Engine
+ // own the Write<T>
+ // that will call
+ // overloaded Write
+ // from Derived
+ HDF5Writer->Write(ioMyInts,
+ myInts.data() + (intOffsetDim1 * intDim2 * rank));
+
+ HDF5Writer->Write(ioMyCFloats, myCFloats.data() + complexOffset);
+ HDF5Writer->Write(ioMyCDoubles, myCDoubles.data() + complexOffset);
+ HDF5Writer->Write(ioMyCLongDoubles,
+ myCLongDoubles.data() + complexOffset);
+ HDF5Writer->Close();
+ }
+ catch (std::invalid_argument &e)
{
- if( rank == 0 )
+ if (rank == 0)
{
std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
std::cout << e.what() << "\n";
}
}
- catch( std::ios_base::failure& e )
+ catch (std::ios_base::failure &e)
{
- if( rank == 0 )
+ if (rank == 0)
{
std::cout << "System exception, STOPPING PROGRAM\n";
std::cout << e.what() << "\n";
}
}
- catch( std::exception& e )
+ catch (std::exception &e)
{
- if( rank == 0 )
+ if (rank == 0)
{
std::cout << "Exception, STOPPING PROGRAM\n";
std::cout << e.what() << "\n";
}
}
- MPI_Finalize( );
+ MPI_Finalize();
return 0;
-
}
diff --git a/examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp b/examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp
index e49794eae..87211cb13 100644
--- a/examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp
+++ b/examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp
@@ -5,130 +5,140 @@
* Author: Junmin
*/
-
-
#include <vector>
#include <iostream>
-
//#include <mpi.h>
#include "ADIOS_MPI.h"
#ifndef HAVE_MPI
// mpidummy has namespace adios
using namespace adios;
-#endif
+#endif
#include "ADIOS_CPP.h"
-
-int main( int argc, char* argv [] )
+int main(int argc, char *argv[])
{
- MPI_Init( &argc, &argv );
+ MPI_Init(&argc, &argv);
int rank, size;
- MPI_Comm_rank( MPI_COMM_WORLD, &rank );
- MPI_Comm_size( MPI_COMM_WORLD, &size );
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
const bool adiosDebug = true;
adios::ADIOS adios(MPI_COMM_WORLD, adios::Verbose::INFO, adiosDebug);
-
- //Application variable
+ // Application variable
const std::size_t intDim1 = 4;
const std::size_t intDim2 = 3;
- std::vector<int> myInts = {10,11,12,13,14,15,16,17,18,19,20,21};
+ std::vector<int> myInts = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21};
- std::vector<double> myDoubles = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+ std::vector<double> myDoubles = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
const std::size_t Nx = myDoubles.size();
std::vector<std::complex<float>> myCFloats;
const std::size_t CFloatSize = 3;
- myCFloats.reserve( CFloatSize );
- myCFloats.emplace_back( 1, 3 );
- myCFloats.emplace_back( 2, 2 );
- myCFloats.emplace_back( 3, 1 );
-
+ myCFloats.reserve(CFloatSize);
+ myCFloats.emplace_back(1, 3);
+ myCFloats.emplace_back(2, 2);
+ myCFloats.emplace_back(3, 1);
std::vector<std::complex<double>> myCDoubles;
const std::size_t CDoubleSize = 3;
- myCDoubles.reserve( CDoubleSize );
- myCDoubles.emplace_back( 1, 3 );
- myCDoubles.emplace_back( 2, 2 );
- myCDoubles.emplace_back( 3, 1 );
+ myCDoubles.reserve(CDoubleSize);
+ myCDoubles.emplace_back(1, 3);
+ myCDoubles.emplace_back(2, 2);
+ myCDoubles.emplace_back(3, 1);
-
- std::size_t doubleVCount = Nx/size;
- std::size_t floatCount = CFloatSize/size;
- std::size_t intCountDim1 = intDim1/size;
+ std::size_t doubleVCount = Nx / size;
+ std::size_t floatCount = CFloatSize / size;
+ std::size_t intCountDim1 = intDim1 / size;
std::size_t doubleVOffset = rank * doubleVCount;
- std::size_t floatOffset = rank * floatCount;
+ std::size_t floatOffset = rank * floatCount;
std::size_t intOffsetDim1 = rank * intCountDim1;
std::size_t intOffsetDim2 = 0;
- if ((size > 1) && (rank == size-1)) {
- doubleVCount = Nx - rank*(Nx/size);
- floatCount = CFloatSize - rank * (CFloatSize/size);
- intCountDim1 = intDim1 - rank * (intDim1/size);
- }
-
+ if ((size > 1) && (rank == size - 1))
+ {
+ doubleVCount = Nx - rank * (Nx / size);
+ floatCount = CFloatSize - rank * (CFloatSize / size);
+ intCountDim1 = intDim1 - rank * (intDim1 / size);
+ }
+
try
{
- //Define variable and local size
- //auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", {Nx}, {Nx} );
- //auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>( "myCFloats", {3}, {3} );
- //auto& ioMyInts = adios.DefineVariable<int>( "myInts", {4,3}, {4,3} );
-
- auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", {doubleVCount}, {Nx}, {doubleVOffset} );
- auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>( "myCFloats", {floatCount}, {3}, {floatOffset} );
- auto& ioMyCDoubles = adios.DefineVariable<std::complex<double>>( "myCDoubles", {floatCount}, {3}, {floatOffset} );
- auto& ioMyInts = adios.DefineVariable<int>( "myInts", {intCountDim1, intDim2}, {4,3}, {intOffsetDim1, intOffsetDim2} );
-
- //Define method for engine creation, it is basically straight-forward parameters
- adios::Method& HDF5Settings = adios.DeclareMethod("HDF5Writer" ); //default method type is Writer
- HDF5Settings.SetParameters( "chunck=yes", "collectiveIO=yes");
- //HDF5Settings.AddTransport( "Mdtm", "localIP=128.0.0.0.1", "remoteIP=128.0.0.0.2", "tolerances=1,2,3" );
-
- //Create engine smart pointer to HDF5 Engine due to polymorphism,
- //Open returns a smart pointer to Engine containing the Derived class HDF5
- auto HDF5Writer = adios.Open( "test.bp", "w", HDF5Settings );
-
- if( HDF5Writer == nullptr )
- throw std::ios_base::failure( "ERROR: failed to create HDF5 I/O engine at Open\n" );
-
- HDF5Writer->Write( ioMyDoubles, myDoubles.data()+doubleVOffset ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
- HDF5Writer->Write( ioMyInts, myInts.data()+(intOffsetDim1*intDim2*rank) );
- HDF5Writer->Write( ioMyCFloats, myCFloats.data() );
- HDF5Writer->Write( ioMyCDoubles, myCDoubles.data() );
- HDF5Writer->Close( );
-
- }
- catch( std::invalid_argument& e )
+ // Define variable and local size
+ // auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", {Nx},
+ // {Nx} );
+ // auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>(
+ // "myCFloats", {3}, {3} );
+ // auto& ioMyInts = adios.DefineVariable<int>( "myInts", {4,3}, {4,3} );
+
+ auto &ioMyDoubles = adios.DefineVariable<double>(
+ "myDoubles", {doubleVCount}, {Nx}, {doubleVOffset});
+ auto &ioMyCFloats = adios.DefineVariable<std::complex<float>>(
+ "myCFloats", {floatCount}, {3}, {floatOffset});
+ auto &ioMyCDoubles = adios.DefineVariable<std::complex<double>>(
+ "myCDoubles", {floatCount}, {3}, {floatOffset});
+ auto &ioMyInts =
+ adios.DefineVariable<int>("myInts", {intCountDim1, intDim2}, {4, 3},
+ {intOffsetDim1, intOffsetDim2});
+
+ // Define method for engine creation, it is basically straight-forward
+ // parameters
+ adios::Method &HDF5Settings =
+ adios.DeclareMethod("HDF5Writer"); // default method type is Writer
+ HDF5Settings.SetParameters("chunck=yes", "collectiveIO=yes");
+ // HDF5Settings.AddTransport( "Mdtm", "localIP=128.0.0.0.1",
+ // "remoteIP=128.0.0.0.2", "tolerances=1,2,3" );
+
+ // Create engine smart pointer to HDF5 Engine due to polymorphism,
+ // Open returns a smart pointer to Engine containing the Derived class
+ // HDF5
+ auto HDF5Writer = adios.Open("test.bp", "w", HDF5Settings);
+
+ if (HDF5Writer == nullptr)
+ throw std::ios_base::failure(
+ "ERROR: failed to create HDF5 I/O engine at Open\n");
+
+ HDF5Writer->Write(ioMyDoubles, myDoubles.data() +
+ doubleVOffset); // Base class Engine
+ // own the Write<T>
+ // that will call
+ // overloaded Write
+ // from Derived
+ HDF5Writer->Write(ioMyInts,
+ myInts.data() + (intOffsetDim1 * intDim2 * rank));
+ HDF5Writer->Write(ioMyCFloats, myCFloats.data());
+ HDF5Writer->Write(ioMyCDoubles, myCDoubles.data());
+ HDF5Writer->Close();
+ }
+ catch (std::invalid_argument &e)
{
- if( rank == 0 )
+ if (rank == 0)
{
std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
std::cout << e.what() << "\n";
}
}
- catch( std::ios_base::failure& e )
+ catch (std::ios_base::failure &e)
{
- if( rank == 0 )
+ if (rank == 0)
{
std::cout << "System exception, STOPPING PROGRAM\n";
std::cout << e.what() << "\n";
}
}
- catch( std::exception& e )
+ catch (std::exception &e)
{
- if( rank == 0 )
+ if (rank == 0)
{
std::cout << "Exception, STOPPING PROGRAM\n";
std::cout << e.what() << "\n";
}
}
- MPI_Finalize( );
+ MPI_Finalize();
return 0;
-
}
diff --git a/include/engine/hdf5/HDF5ReaderP.h b/include/engine/hdf5/HDF5ReaderP.h
index 959b08e91..ec5d910eb 100644
--- a/include/engine/hdf5/HDF5ReaderP.h
+++ b/include/engine/hdf5/HDF5ReaderP.h
@@ -8,7 +8,6 @@
* Author: Junmin
*/
-
#ifndef HDF5_READER_P_H
#define HDF5_READER_P_H
diff --git a/include/engine/hdf5/HDF5WriterP.h b/include/engine/hdf5/HDF5WriterP.h
index 9dc00a329..b0d703065 100644
--- a/include/engine/hdf5/HDF5WriterP.h
+++ b/include/engine/hdf5/HDF5WriterP.h
@@ -12,8 +12,6 @@
#ifndef HDF5_WRITER_P_H_
#define HDF5_WRITER_P_H_
-#include <unistd.h> //sleep must be removed
-
#include "core/Engine.h"
// supported capsules
diff --git a/source/ADIOS.cpp b/source/ADIOS.cpp
index 47e3c333f..b3aee87e3 100644
--- a/source/ADIOS.cpp
+++ b/source/ADIOS.cpp
@@ -191,14 +191,16 @@ std::shared_ptr<Engine> ADIOS::Open(const std::string &name,
// method,
// iomode, timeout_sec, m_DebugMode, method.m_nThreads );
}
- else if (type == "HDF5Writer") // -junmin
- {
- #if defined(ADIOS_HAVE_PHDF5) && defined(ADIOS_HAVE_MPI)
- return std::make_shared<HDF5Writer>(*this, name, accessMode, mpiComm, method);
- #else
- throw std::invalid_argument( "ERROR: this version didn't compile with HDF5 library, can't use HDF5\n" );
- #endif
- }
+ else if (type == "HDF5Writer") // -junmin
+ {
+#if defined(ADIOS_HAVE_PHDF5) && defined(ADIOS_HAVE_MPI)
+ return std::make_shared<HDF5Writer>(*this, name, accessMode, mpiComm,
+ method);
+#else
+ throw std::invalid_argument("ERROR: this version didn't compile with "
+ "HDF5 library, can't use HDF5\n");
+#endif
+ }
else
{
diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt
index 4c5afc475..975ae6dcc 100644
--- a/source/CMakeLists.txt
+++ b/source/CMakeLists.txt
@@ -88,18 +88,14 @@ if(ADIOS_USE_MPI)
message(" HDF5 ROOT: ${HDF5_ROOT}")
message(" HDF5 include: ${HDF5_INCLUDE_DIRS} is paralle? ${HDF5_IS_PARALLEL}")
- get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
- foreach(dir ${dirs})
- message(STATUS "dir='${dir}'")
- endforeach()
- include_directories(${HDF5_INCLUDE_DIRS})
+ target_include_directories(adios2 PRIVATE ${HDF5_INCLUDE_DIRS})
target_sources(adios2 PRIVATE
engine/hdf5/HDF5ReaderP.cpp
engine/hdf5/HDF5WriterP.cpp
)
target_compile_definitions(adios2 PRIVATE ADIOS_HAVE_PHDF5)
- target_link_libraries(adios2 PRIVATE hdf5)
+ target_link_libraries(adios2 PRIVATE ${HDF5_C_LIBRARIES})
endif()
endif()
diff --git a/source/engine/hdf5/HDF5ReaderP.cpp b/source/engine/hdf5/HDF5ReaderP.cpp
index 3716bfea5..00ec0a8fb 100644
--- a/source/engine/hdf5/HDF5ReaderP.cpp
+++ b/source/engine/hdf5/HDF5ReaderP.cpp
@@ -8,9 +8,6 @@
* Author: Junmin
*/
-
-
#include <iostream> //needs to go away, this is just for demo purposes
#include "engine/hdf5/HDF5ReaderP.h"
-
diff --git a/source/engine/hdf5/HDF5WriterP.cpp b/source/engine/hdf5/HDF5WriterP.cpp
index 196744ac2..0ce36bf57 100644
--- a/source/engine/hdf5/HDF5WriterP.cpp
+++ b/source/engine/hdf5/HDF5WriterP.cpp
@@ -8,309 +8,360 @@
* Author: Junmin
*/
-
#include <iostream> //needs to go away, this is just for demo purposes
-#ifdef ADIOS_HAVE_MPI
#include "engine/hdf5/HDF5WriterP.h"
#include "core/Support.h"
#include "functions/adiosFunctions.h" //CSVToVector
-namespace adios {
+namespace adios
+{
HDF5Writer::HDF5Writer(ADIOS &adios, const std::string name,
const std::string accessMode, MPI_Comm mpiComm,
const Method &method)
- : Engine(adios, "HDF5Writer", name, accessMode, mpiComm, method, /*debugMode, cores,*/
- " HDF5Writer constructor (or call to ADIOS Open).\n"),
- m_Buffer(accessMode, m_RankMPI, m_DebugMode)
+: Engine(adios, "HDF5Writer", name, accessMode, mpiComm,
+ method, /*debugMode, cores,*/
+ " HDF5Writer constructor (or call to ADIOS Open).\n"),
+ m_Buffer(accessMode, m_RankMPI, m_DebugMode)
{
- //
- // 16, 4 vs: 8
- //std::cout<<sizeof(std::complex<double>)<<", "<<sizeof(H5T_NATIVE_DOUBLE)<<" vs: "<<H5Tget_size(H5T_NATIVE_DOUBLE)<<std::endl;
- // 8, 4 vs: 4
- //std::cout<<sizeof(std::complex<float>)<<", "<<sizeof(H5T_NATIVE_FLOAT)<<" vs: "<<H5Tget_size(H5T_NATIVE_FLOAT)<<std::endl;
- // 32, 4 vs: 16
- //std::cout<<sizeof(std::complex<long double>)<<", "<<sizeof(H5T_NATIVE_LDOUBLE)<<" vs: "<<H5Tget_size(H5T_NATIVE_LDOUBLE)<<std::endl;
-
- DefH5T_COMPLEX_FLOAT = H5Tcreate(H5T_COMPOUND, sizeof(std::complex<float>));
- H5Tinsert(DefH5T_COMPLEX_FLOAT, "freal", 0, H5T_NATIVE_FLOAT);
- H5Tinsert(DefH5T_COMPLEX_FLOAT, "fimg", H5Tget_size(H5T_NATIVE_FLOAT), H5T_NATIVE_FLOAT);
+ //
+ // 16, 4 vs: 8
+ // std::cout<<sizeof(std::complex<double>)<<",
+ // "<<sizeof(H5T_NATIVE_DOUBLE)<<" vs:
+ // "<<H5Tget_size(H5T_NATIVE_DOUBLE)<<std::endl;
+ // 8, 4 vs: 4
+ // std::cout<<sizeof(std::complex<float>)<<", "<<sizeof(H5T_NATIVE_FLOAT)<<"
+ // vs: "<<H5Tget_size(H5T_NATIVE_FLOAT)<<std::endl;
+ // 32, 4 vs: 16
+ // std::cout<<sizeof(std::complex<long double>)<<",
+ // "<<sizeof(H5T_NATIVE_LDOUBLE)<<" vs:
+ // "<<H5Tget_size(H5T_NATIVE_LDOUBLE)<<std::endl;
+
+ DefH5T_COMPLEX_FLOAT = H5Tcreate(H5T_COMPOUND, sizeof(std::complex<float>));
+ H5Tinsert(DefH5T_COMPLEX_FLOAT, "freal", 0, H5T_NATIVE_FLOAT);
+ H5Tinsert(DefH5T_COMPLEX_FLOAT, "fimg", H5Tget_size(H5T_NATIVE_FLOAT),
+ H5T_NATIVE_FLOAT);
- DefH5T_COMPLEX_DOUBLE = H5Tcreate(H5T_COMPOUND, sizeof(std::complex<double>));
- H5Tinsert(DefH5T_COMPLEX_DOUBLE, "dreal", 0, H5T_NATIVE_DOUBLE);
- H5Tinsert(DefH5T_COMPLEX_DOUBLE, "dimg", H5Tget_size(H5T_NATIVE_DOUBLE), H5T_NATIVE_DOUBLE);
+ DefH5T_COMPLEX_DOUBLE =
+ H5Tcreate(H5T_COMPOUND, sizeof(std::complex<double>));
+ H5Tinsert(DefH5T_COMPLEX_DOUBLE, "dreal", 0, H5T_NATIVE_DOUBLE);
+ H5Tinsert(DefH5T_COMPLEX_DOUBLE, "dimg", H5Tget_size(H5T_NATIVE_DOUBLE),
+ H5T_NATIVE_DOUBLE);
- DefH5T_COMPLEX_LongDOUBLE = H5Tcreate(H5T_COMPOUND, sizeof(std::complex<long double>));
- H5Tinsert(DefH5T_COMPLEX_LongDOUBLE, "ldouble real", 0, H5T_NATIVE_LDOUBLE);
- H5Tinsert(DefH5T_COMPLEX_LongDOUBLE, "ldouble img", H5Tget_size(H5T_NATIVE_LDOUBLE), H5T_NATIVE_LDOUBLE);
+ DefH5T_COMPLEX_LongDOUBLE =
+ H5Tcreate(H5T_COMPOUND, sizeof(std::complex<long double>));
+ H5Tinsert(DefH5T_COMPLEX_LongDOUBLE, "ldouble real", 0, H5T_NATIVE_LDOUBLE);
+ H5Tinsert(DefH5T_COMPLEX_LongDOUBLE, "ldouble img",
+ H5Tget_size(H5T_NATIVE_LDOUBLE), H5T_NATIVE_LDOUBLE);
- Init();
+ Init();
}
HDF5Writer::~HDF5Writer() {}
-void HDF5Writer::Init() {
- if (m_AccessMode != "w" && m_AccessMode != "write" && m_AccessMode != "a" &&
- m_AccessMode != "append")
- {
- throw std::invalid_argument("ERROR: HDF5Writer doesn't support access mode " + m_AccessMode +
- ", in call to ADIOS Open or HDF5Writer constructor\n");
- }
- //std::cout << "method: # of inputs:" << m_Method.m_Parameters.size() << std::endl;
-
- //std::cout << "::Init hdf5 parallel writer. File name:" << m_Name << std::endl;
-
- _plist_id = H5Pcreate(H5P_FILE_ACCESS);
+void HDF5Writer::Init()
+{
+ if (m_AccessMode != "w" && m_AccessMode != "write" && m_AccessMode != "a" &&
+ m_AccessMode != "append")
+ {
+ throw std::invalid_argument(
+ "ERROR: HDF5Writer doesn't support access mode " + m_AccessMode +
+ ", in call to ADIOS Open or HDF5Writer constructor\n");
+ }
+ // std::cout << "method: # of inputs:" << m_Method.m_Parameters.size() <<
+ // std::endl;
+ // std::cout << "::Init hdf5 parallel writer. File name:" << m_Name <<
+ // std::endl;
- H5Pset_fapl_mpio(_plist_id, m_MPIComm, MPI_INFO_NULL);
+ _plist_id = H5Pcreate(H5P_FILE_ACCESS);
+ H5Pset_fapl_mpio(_plist_id, m_MPIComm, MPI_INFO_NULL);
- /*
- * Create a new file collectively and release property list identifier.
- */
- _file_id = H5Fcreate(m_Name.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, _plist_id);
- H5Pclose(_plist_id);
+ /*
+ * Create a new file collectively and release property list identifier.
+ */
+ _file_id = H5Fcreate(m_Name.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, _plist_id);
+ H5Pclose(_plist_id);
}
-void HDF5Writer::Write(Variable<char> &variable, const char *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_CHAR);
+void HDF5Writer::Write(Variable<char> &variable, const char *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_CHAR);
}
void HDF5Writer::Write(Variable<unsigned char> &variable,
- const unsigned char *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_UCHAR);
+ const unsigned char *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_UCHAR);
}
-void HDF5Writer::Write(Variable<short> &variable, const short *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_SHORT);
+void HDF5Writer::Write(Variable<short> &variable, const short *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_SHORT);
}
void HDF5Writer::Write(Variable<unsigned short> &variable,
- const unsigned short *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_USHORT);
+ const unsigned short *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_USHORT);
}
-void HDF5Writer::Write(Variable<int> &variable, const int *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_INT);
+void HDF5Writer::Write(Variable<int> &variable, const int *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_INT);
}
void HDF5Writer::Write(Variable<unsigned int> &variable,
- const unsigned int *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_UINT);
+ const unsigned int *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_UINT);
}
-void HDF5Writer::Write(Variable<long int> &variable, const long int *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_LONG);
+void HDF5Writer::Write(Variable<long int> &variable, const long int *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_LONG);
}
void HDF5Writer::Write(Variable<unsigned long int> &variable,
- const unsigned long int *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_ULONG);
+ const unsigned long int *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_ULONG);
}
void HDF5Writer::Write(Variable<long long int> &variable,
- const long long int *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_LLONG);
+ const long long int *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_LLONG);
}
void HDF5Writer::Write(Variable<unsigned long long int> &variable,
- const unsigned long long int *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_ULLONG);
+ const unsigned long long int *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_ULLONG);
}
-void HDF5Writer::Write(Variable<float> &variable, const float *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_FLOAT);
+void HDF5Writer::Write(Variable<float> &variable, const float *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_FLOAT);
}
-void HDF5Writer::Write(Variable<double> &variable, const double *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_DOUBLE);
+void HDF5Writer::Write(Variable<double> &variable, const double *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_DOUBLE);
}
void HDF5Writer::Write(Variable<long double> &variable,
- const long double *values) {
- UseHDFWrite(variable, values, H5T_NATIVE_LDOUBLE);
+ const long double *values)
+{
+ UseHDFWrite(variable, values, H5T_NATIVE_LDOUBLE);
}
void HDF5Writer::Write(Variable<std::complex<float>> &variable,
- const std::complex<float> *values) {
- UseHDFWrite(variable, values, DefH5T_COMPLEX_FLOAT);
+ const std::complex<float> *values)
+{
+ UseHDFWrite(variable, values, DefH5T_COMPLEX_FLOAT);
}
void HDF5Writer::Write(Variable<std::complex<double>> &variable,
- const std::complex<double> *values) {
- UseHDFWrite(variable, values, DefH5T_COMPLEX_DOUBLE);
+ const std::complex<double> *values)
+{
+ UseHDFWrite(variable, values, DefH5T_COMPLEX_DOUBLE);
}
void HDF5Writer::Write(Variable<std::complex<long double>> &variable,
- const std::complex<long double> *values) {
- UseHDFWrite(variable, values, DefH5T_COMPLEX_LongDOUBLE);
+ const std::complex<long double> *values)
+{
+ UseHDFWrite(variable, values, DefH5T_COMPLEX_LongDOUBLE);
}
// String version
-void HDF5Writer::Write(const std::string variableName, const char *values) {
- UseHDFWrite(m_ADIOS.GetVariable<char>(variableName), values, H5T_NATIVE_CHAR);
+void HDF5Writer::Write(const std::string variableName, const char *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<char>(variableName), values,
+ H5T_NATIVE_CHAR);
}
void HDF5Writer::Write(const std::string variableName,
- const unsigned char *values) {
- UseHDFWrite(m_ADIOS.GetVariable<unsigned char>(variableName), values,
- H5T_NATIVE_UCHAR);
+ const unsigned char *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<unsigned char>(variableName), values,
+ H5T_NATIVE_UCHAR);
}
-void HDF5Writer::Write(const std::string variableName, const short *values) {
- UseHDFWrite(m_ADIOS.GetVariable<short>(variableName), values,
- H5T_NATIVE_SHORT);
+void HDF5Writer::Write(const std::string variableName, const short *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<short>(variableName), values,
+ H5T_NATIVE_SHORT);
}
void HDF5Writer::Write(const std::string variableName,
- const unsigned short *values) {
- UseHDFWrite(m_ADIOS.GetVariable<unsigned short>(variableName), values,
- H5T_NATIVE_USHORT);
+ const unsigned short *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<unsigned short>(variableName), values,
+ H5T_NATIVE_USHORT);
}
-void HDF5Writer::Write(const std::string variableName, const int *values) {
- UseHDFWrite(m_ADIOS.GetVariable<int>(variableName), values, H5T_NATIVE_INT);
+void HDF5Writer::Write(const std::string variableName, const int *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<int>(variableName), values, H5T_NATIVE_INT);
}
void HDF5Writer::Write(const std::string variableName,
- const unsigned int *values) {
- UseHDFWrite(m_ADIOS.GetVariable<unsigned int>(variableName), values,
- H5T_NATIVE_UINT);
+ const unsigned int *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<unsigned int>(variableName), values,
+ H5T_NATIVE_UINT);
}
-void HDF5Writer::Write(const std::string variableName, const long int *values) {
- UseHDFWrite(m_ADIOS.GetVariable<long int>(variableName), values,
- H5T_NATIVE_LONG);
+void HDF5Writer::Write(const std::string variableName, const long int *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<long int>(variableName), values,
+ H5T_NATIVE_LONG);
}
void HDF5Writer::Write(const std::string variableName,
- const unsigned long int *values) {
- UseHDFWrite(m_ADIOS.GetVariable<unsigned long int>(variableName), values,
- H5T_NATIVE_ULONG);
+ const unsigned long int *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<unsigned long int>(variableName), values,
+ H5T_NATIVE_ULONG);
}
void HDF5Writer::Write(const std::string variableName,
- const long long int *values) {
- UseHDFWrite(m_ADIOS.GetVariable<long long int>(variableName), values,
- H5T_NATIVE_LLONG);
+ const long long int *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<long long int>(variableName), values,
+ H5T_NATIVE_LLONG);
}
void HDF5Writer::Write(const std::string variableName,
- const unsigned long long int *values) {
- UseHDFWrite(m_ADIOS.GetVariable<unsigned long long int>(variableName), values,
- H5T_NATIVE_ULLONG);
+ const unsigned long long int *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<unsigned long long int>(variableName),
+ values, H5T_NATIVE_ULLONG);
}
-void HDF5Writer::Write(const std::string variableName, const float *values) {
- UseHDFWrite(m_ADIOS.GetVariable<float>(variableName), values,
- H5T_NATIVE_FLOAT);
+void HDF5Writer::Write(const std::string variableName, const float *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<float>(variableName), values,
+ H5T_NATIVE_FLOAT);
}
-void HDF5Writer::Write(const std::string variableName, const double *values) {
- UseHDFWrite(m_ADIOS.GetVariable<double>(variableName), values,
- H5T_NATIVE_DOUBLE);
+void HDF5Writer::Write(const std::string variableName, const double *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<double>(variableName), values,
+ H5T_NATIVE_DOUBLE);
}
void HDF5Writer::Write(const std::string variableName,
- const long double *values) {
- UseHDFWrite(m_ADIOS.GetVariable<long double>(variableName), values,
- H5T_NATIVE_LDOUBLE);
+ const long double *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<long double>(variableName), values,
+ H5T_NATIVE_LDOUBLE);
}
void HDF5Writer::Write(const std::string variableName,
- const std::complex<float> *values) {
- UseHDFWrite(m_ADIOS.GetVariable<std::complex<float>>(variableName), values,
- DefH5T_COMPLEX_FLOAT);
+ const std::complex<float> *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<std::complex<float>>(variableName), values,
+ DefH5T_COMPLEX_FLOAT);
}
void HDF5Writer::Write(const std::string variableName,
- const std::complex<double> *values) {
- UseHDFWrite(m_ADIOS.GetVariable<std::complex<double>>(variableName), values,
- DefH5T_COMPLEX_DOUBLE);
+ const std::complex<double> *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<std::complex<double>>(variableName), values,
+ DefH5T_COMPLEX_DOUBLE);
}
void HDF5Writer::Write(const std::string variableName,
- const std::complex<long double> *values) {
- UseHDFWrite(m_ADIOS.GetVariable<std::complex<long double>>(variableName),
- values, DefH5T_COMPLEX_LongDOUBLE);
+ const std::complex<long double> *values)
+{
+ UseHDFWrite(m_ADIOS.GetVariable<std::complex<long double>>(variableName),
+ values, DefH5T_COMPLEX_LongDOUBLE);
}
-void HDF5Writer::Close(const int transportIndex) {
- //std::cout << " ===> CLOSING HDF5 <===== " << std::endl;
- // H5Dclose(_dset_id);
- // H5Sclose(_filespace);
- // H5Sclose(_memspace);
- // H5Pclose(_plist_id);
- H5Fclose(_file_id);
+void HDF5Writer::Close(const int transportIndex)
+{
+ // std::cout << " ===> CLOSING HDF5 <===== " << std::endl;
+ // H5Dclose(_dset_id);
+ // H5Sclose(_filespace);
+ // H5Sclose(_memspace);
+ // H5Pclose(_plist_id);
+ H5Fclose(_file_id);
}
template <class T>
void HDF5Writer::UseHDFWrite(Variable<T> &variable, const T *values,
- hid_t h5type) {
- // here comes your magic at Writing now variable.m_UserValues has the data
- // passed by the user
- // set variable
- variable.m_AppValues = values;
- m_WrittenVariables.insert(variable.m_Name);
+ hid_t h5type)
+{
+ // here comes your magic at Writing now variable.m_UserValues has the data
+ // passed by the user
+ // set variable
+ variable.m_AppValues = values;
+ m_WrittenVariables.insert(variable.m_Name);
- int dimSize = variable.m_GlobalDimensions.size();
- /*
- std::cout << "writting : " << variable.m_Name
- << " dim size:" << variable.m_GlobalDimensions.size() << std::endl;
+ int dimSize = variable.m_GlobalDimensions.size();
+ /*
+ std::cout << "writting : " << variable.m_Name
+ << " dim size:" << variable.m_GlobalDimensions.size() <<
+ std::endl;
- for (int i = 0; i < dimSize; i++) {
- std::cout << " dim: " << i << ", size:" << variable.m_GlobalDimensions[i]
- << " offset=" << variable.m_Offsets[i]
- << " count=" << variable.m_LocalDimensions[i] << std::endl;
- }
- */
- std::vector<hsize_t> dimsf, count, offset;
+ for (int i = 0; i < dimSize; i++) {
+ std::cout << " dim: " << i << ", size:" << variable.m_GlobalDimensions[i]
+ << " offset=" << variable.m_Offsets[i]
+ << " count=" << variable.m_LocalDimensions[i] << std::endl;
+ }
+ */
+ std::vector<hsize_t> dimsf, count, offset;
- for (int i = 0; i < dimSize; i++) {
- dimsf.push_back(variable.m_GlobalDimensions[i]);
- count.push_back(variable.m_LocalDimensions[i]);
- offset.push_back(variable.m_Offsets[i]);
- }
+ for (int i = 0; i < dimSize; i++)
+ {
+ dimsf.push_back(variable.m_GlobalDimensions[i]);
+ count.push_back(variable.m_LocalDimensions[i]);
+ offset.push_back(variable.m_Offsets[i]);
+ }
- _filespace = H5Screate_simple(dimSize, dimsf.data(), NULL);
+ _filespace = H5Screate_simple(dimSize, dimsf.data(), NULL);
- _dset_id = H5Dcreate(_file_id, variable.m_Name.c_str(), h5type, _filespace,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- // H5Sclose(_filespace);
+ _dset_id = H5Dcreate(_file_id, variable.m_Name.c_str(), h5type, _filespace,
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ // H5Sclose(_filespace);
- _memspace = H5Screate_simple(dimSize, count.data(), NULL);
+ _memspace = H5Screate_simple(dimSize, count.data(), NULL);
- // Select hyperslab
- _filespace = H5Dget_space(_dset_id);
- H5Sselect_hyperslab(_filespace, H5S_SELECT_SET, offset.data(), NULL,
- count.data(), NULL);
+ // Select hyperslab
+ _filespace = H5Dget_space(_dset_id);
+ H5Sselect_hyperslab(_filespace, H5S_SELECT_SET, offset.data(), NULL,
+ count.data(), NULL);
- // Create property list for collective dataset write.
+ // Create property list for collective dataset write.
- _plist_id = H5Pcreate(H5P_DATASET_XFER);
- H5Pset_dxpl_mpio(_plist_id, H5FD_MPIO_COLLECTIVE);
+ _plist_id = H5Pcreate(H5P_DATASET_XFER);
+ H5Pset_dxpl_mpio(_plist_id, H5FD_MPIO_COLLECTIVE);
- herr_t status;
+ herr_t status;
- status =
- H5Dwrite(_dset_id, h5type, _memspace, _filespace, _plist_id, values);
+ status =
+ H5Dwrite(_dset_id, h5type, _memspace, _filespace, _plist_id, values);
- if (status < 0) {
- // error
- std::cerr << " Write failed. " << std::endl;
- }
+ if (status < 0)
+ {
+ // error
+ std::cerr << " Write failed. " << std::endl;
+ }
- //std::cout << " ==> User is responsible for freeing the data " << std::endl;
+ // std::cout << " ==> User is responsible for freeing the data " <<
+ // std::endl;
- H5Dclose(_dset_id);
- H5Sclose(_filespace);
- H5Sclose(_memspace);
- H5Pclose(_plist_id);
+ H5Dclose(_dset_id);
+ H5Sclose(_filespace);
+ H5Sclose(_memspace);
+ H5Pclose(_plist_id);
}
} // end namespace adios
-#endif //ADIOS_HAVE_MPI
+
--
GitLab
From 83a4796a811d5bce8425b0f36eeeb64dd34c7bb3 Mon Sep 17 00:00:00 2001
From: guj <jgu@lbl.gov>
Date: Fri, 14 Apr 2017 15:21:10 -0700
Subject: [PATCH 5/6] removed helloHDF5Writer_nompi* in CMake and in dir
---
examples/hello/hdf5Writer/CMakeLists.txt | 4 +-
.../hdf5Writer/helloHDF5Writer_nompi.cpp | 144 ------------------
2 files changed, 2 insertions(+), 146 deletions(-)
delete mode 100644 examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp
diff --git a/examples/hello/hdf5Writer/CMakeLists.txt b/examples/hello/hdf5Writer/CMakeLists.txt
index 7f4197c0b..2f7d7fc0c 100644
--- a/examples/hello/hdf5Writer/CMakeLists.txt
+++ b/examples/hello/hdf5Writer/CMakeLists.txt
@@ -3,10 +3,10 @@
# accompanying file Copyright.txt for details.
#------------------------------------------------------------------------------#
-add_executable(hello_hdf5Writer_nompi helloHDF5Writer_nompi.cpp)
+#add_executable(hello_hdf5Writer_nompi helloHDF5Writer_nompi.cpp)
message(" HDF5: ${HDF5_INCLUDE_DIRS}")
include_directories(${HDF5_INCLUDE_DIRS})
-target_link_libraries(hello_hdf5Writer_nompi adios2_nompi)
+#target_link_libraries(hello_hdf5Writer_nompi adios2_nompi)
if(ADIOS_BUILD_TESTING)
#add_test(
diff --git a/examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp b/examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp
deleted file mode 100644
index 87211cb13..000000000
--- a/examples/hello/hdf5Writer/helloHDF5Writer_nompi.cpp
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * HDF5Writer.cpp
- *
- * Created on: March 20, 2017
- * Author: Junmin
- */
-
-#include <vector>
-#include <iostream>
-
-//#include <mpi.h>
-#include "ADIOS_MPI.h"
-
-#ifndef HAVE_MPI
-// mpidummy has namespace adios
-using namespace adios;
-#endif
-#include "ADIOS_CPP.h"
-
-int main(int argc, char *argv[])
-{
- MPI_Init(&argc, &argv);
- int rank, size;
- MPI_Comm_rank(MPI_COMM_WORLD, &rank);
- MPI_Comm_size(MPI_COMM_WORLD, &size);
-
- const bool adiosDebug = true;
- adios::ADIOS adios(MPI_COMM_WORLD, adios::Verbose::INFO, adiosDebug);
-
- // Application variable
- const std::size_t intDim1 = 4;
- const std::size_t intDim2 = 3;
- std::vector<int> myInts = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21};
-
- std::vector<double> myDoubles = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
- const std::size_t Nx = myDoubles.size();
-
- std::vector<std::complex<float>> myCFloats;
- const std::size_t CFloatSize = 3;
- myCFloats.reserve(CFloatSize);
- myCFloats.emplace_back(1, 3);
- myCFloats.emplace_back(2, 2);
- myCFloats.emplace_back(3, 1);
-
- std::vector<std::complex<double>> myCDoubles;
- const std::size_t CDoubleSize = 3;
- myCDoubles.reserve(CDoubleSize);
- myCDoubles.emplace_back(1, 3);
- myCDoubles.emplace_back(2, 2);
- myCDoubles.emplace_back(3, 1);
-
- std::size_t doubleVCount = Nx / size;
- std::size_t floatCount = CFloatSize / size;
- std::size_t intCountDim1 = intDim1 / size;
-
- std::size_t doubleVOffset = rank * doubleVCount;
- std::size_t floatOffset = rank * floatCount;
- std::size_t intOffsetDim1 = rank * intCountDim1;
- std::size_t intOffsetDim2 = 0;
-
- if ((size > 1) && (rank == size - 1))
- {
- doubleVCount = Nx - rank * (Nx / size);
- floatCount = CFloatSize - rank * (CFloatSize / size);
- intCountDim1 = intDim1 - rank * (intDim1 / size);
- }
-
- try
- {
- // Define variable and local size
- // auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", {Nx},
- // {Nx} );
- // auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>(
- // "myCFloats", {3}, {3} );
- // auto& ioMyInts = adios.DefineVariable<int>( "myInts", {4,3}, {4,3} );
-
- auto &ioMyDoubles = adios.DefineVariable<double>(
- "myDoubles", {doubleVCount}, {Nx}, {doubleVOffset});
- auto &ioMyCFloats = adios.DefineVariable<std::complex<float>>(
- "myCFloats", {floatCount}, {3}, {floatOffset});
- auto &ioMyCDoubles = adios.DefineVariable<std::complex<double>>(
- "myCDoubles", {floatCount}, {3}, {floatOffset});
- auto &ioMyInts =
- adios.DefineVariable<int>("myInts", {intCountDim1, intDim2}, {4, 3},
- {intOffsetDim1, intOffsetDim2});
-
- // Define method for engine creation, it is basically straight-forward
- // parameters
- adios::Method &HDF5Settings =
- adios.DeclareMethod("HDF5Writer"); // default method type is Writer
- HDF5Settings.SetParameters("chunck=yes", "collectiveIO=yes");
- // HDF5Settings.AddTransport( "Mdtm", "localIP=128.0.0.0.1",
- // "remoteIP=128.0.0.0.2", "tolerances=1,2,3" );
-
- // Create engine smart pointer to HDF5 Engine due to polymorphism,
- // Open returns a smart pointer to Engine containing the Derived class
- // HDF5
- auto HDF5Writer = adios.Open("test.bp", "w", HDF5Settings);
-
- if (HDF5Writer == nullptr)
- throw std::ios_base::failure(
- "ERROR: failed to create HDF5 I/O engine at Open\n");
-
- HDF5Writer->Write(ioMyDoubles, myDoubles.data() +
- doubleVOffset); // Base class Engine
- // own the Write<T>
- // that will call
- // overloaded Write
- // from Derived
- HDF5Writer->Write(ioMyInts,
- myInts.data() + (intOffsetDim1 * intDim2 * rank));
- HDF5Writer->Write(ioMyCFloats, myCFloats.data());
- HDF5Writer->Write(ioMyCDoubles, myCDoubles.data());
- HDF5Writer->Close();
- }
- catch (std::invalid_argument &e)
- {
- if (rank == 0)
- {
- std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
- std::cout << e.what() << "\n";
- }
- }
- catch (std::ios_base::failure &e)
- {
- if (rank == 0)
- {
- std::cout << "System exception, STOPPING PROGRAM\n";
- std::cout << e.what() << "\n";
- }
- }
- catch (std::exception &e)
- {
- if (rank == 0)
- {
- std::cout << "Exception, STOPPING PROGRAM\n";
- std::cout << e.what() << "\n";
- }
- }
-
- MPI_Finalize();
-
- return 0;
-}
--
GitLab
From e6bfd90c02a78f8826c81977da3e6dc6cb692101 Mon Sep 17 00:00:00 2001
From: Chuck Atkins <chuck.atkins@kitware.com>
Date: Mon, 17 Apr 2017 10:26:52 -0400
Subject: [PATCH 6/6] Fixed formatting
---
examples/hello/hdf5Writer/helloHDF5Writer.cpp | 2 +-
include/engine/hdf5/HDF5WriterP.h | 1 -
source/ADIOS.cpp | 2 +-
source/engine/hdf5/HDF5WriterP.cpp | 2 --
4 files changed, 2 insertions(+), 5 deletions(-)
diff --git a/examples/hello/hdf5Writer/helloHDF5Writer.cpp b/examples/hello/hdf5Writer/helloHDF5Writer.cpp
index 85c5a910a..3b3282840 100644
--- a/examples/hello/hdf5Writer/helloHDF5Writer.cpp
+++ b/examples/hello/hdf5Writer/helloHDF5Writer.cpp
@@ -5,8 +5,8 @@
* Author: Junmin
*/
-#include <vector>
#include <iostream>
+#include <vector>
#include <mpi.h>
diff --git a/include/engine/hdf5/HDF5WriterP.h b/include/engine/hdf5/HDF5WriterP.h
index b0d703065..cbdbf1c7d 100644
--- a/include/engine/hdf5/HDF5WriterP.h
+++ b/include/engine/hdf5/HDF5WriterP.h
@@ -106,7 +106,6 @@ private:
template <class T>
void UseHDFWrite(Variable<T> &variable, const T *values, hid_t h5type);
-
};
} // end namespace adios
diff --git a/source/ADIOS.cpp b/source/ADIOS.cpp
index b3aee87e3..5925f5e30 100644
--- a/source/ADIOS.cpp
+++ b/source/ADIOS.cpp
@@ -37,8 +37,8 @@
#ifdef ADIOS_HAVE_PHDF5 // external dependencies
#ifdef ADIOS_HAVE_MPI
-#include "engine/hdf5/HDF5WriterP.h"
#include "engine/hdf5/HDF5ReaderP.h"
+#include "engine/hdf5/HDF5WriterP.h"
#endif
#endif
namespace adios
diff --git a/source/engine/hdf5/HDF5WriterP.cpp b/source/engine/hdf5/HDF5WriterP.cpp
index 0ce36bf57..05656e2ba 100644
--- a/source/engine/hdf5/HDF5WriterP.cpp
+++ b/source/engine/hdf5/HDF5WriterP.cpp
@@ -363,5 +363,3 @@ void HDF5Writer::UseHDFWrite(Variable<T> &variable, const T *values,
}
} // end namespace adios
-
-
--
GitLab