diff --git a/examples/heatTransfer/write/IO_ph5.cpp b/examples/heatTransfer/write/IO_ph5.cpp index 674b5388761cefa4c4203374522f16105199038a..def806785636b46182ea8bb2738988ce9c2bf1f4 100644 --- a/examples/heatTransfer/write/IO_ph5.cpp +++ b/examples/heatTransfer/write/IO_ph5.cpp @@ -34,8 +34,14 @@ public: hid_t h5Type, const hsize_t *shape, const hsize_t *offset, const hsize_t *count); + void applyMetadataCacheEviction(); + void WriteSimpleWithChunking(const std::string &varName, int dimSize, + const void *data, hid_t h5Type, + const hsize_t *shape, const hsize_t *offset, + const hsize_t *count); int m_CurrentTimeStep; unsigned int m_TotalTimeSteps; + bool m_Chunking; private: hid_t m_FilePropertyListId; @@ -44,7 +50,7 @@ private: }; HDF5NativeWriter::HDF5NativeWriter(const std::string &fileName) -: m_CurrentTimeStep(0), m_TotalTimeSteps(0) +: m_CurrentTimeStep(0), m_TotalTimeSteps(0), m_Chunking(false) { m_FilePropertyListId = H5Pcreate(H5P_FILE_ACCESS); @@ -59,6 +65,8 @@ HDF5NativeWriter::HDF5NativeWriter(const std::string &fileName) throw std::runtime_error("Unable to open " + fileName + " for reading"); } + applyMetadataCacheEviction(); + std::string ts0 = "/TimeStep0"; m_GroupId = H5Gcreate2(m_FileId, ts0.c_str(), H5P_DEFAULT, H5P_DEFAULT, @@ -67,10 +75,35 @@ HDF5NativeWriter::HDF5NativeWriter(const std::string &fileName) { throw std::runtime_error("HDF5: Unable to create group " + ts0); } + + std::string prefix = "chunking"; + if (fileName.substr(0, prefix.size()) == prefix) + { + m_Chunking = true; + } } HDF5NativeWriter::~HDF5NativeWriter() { Close(); } +void HDF5NativeWriter::applyMetadataCacheEviction() +{ +#ifdef NEVER + /* + see + https://lists.hdfgroup.org/pipermail/hdf-forum_lists.hdfgroup.org/2011-February/004201.html + John said the code below worked for the paper but not anymore after + updates + */ + H5AC_cache_config_t mdc_config; + mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + H5Pget_mdc_config(m_FilePropertyListId, &mdc_config); + mdc_config.evictions_enabled = 0; // FALSE + mdc_config.incr_mode = H5C_incr__off; + mdc_config.decr_mode = H5C_decr__off; + H5Pset_mdc_config(m_FilePropertyListId, &mdc_config); +#endif +} + void HDF5NativeWriter::Close() { if (m_FileId < 0) @@ -138,8 +171,10 @@ void HDF5NativeWriter::WriteScalar(const std::string &varName, const void *data, hid_t filespaceID = H5Screate(H5S_SCALAR); hid_t dsetID = H5Dcreate(m_GroupId, varName.c_str(), h5Type, filespaceID, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - herr_t status = - H5Dwrite(dsetID, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + hid_t plistID = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plistID, H5FD_MPIO_COLLECTIVE); + + herr_t status = H5Dwrite(dsetID, h5Type, H5S_ALL, H5S_ALL, plistID, data); H5Sclose(filespaceID); H5Dclose(dsetID); @@ -183,6 +218,57 @@ void HDF5NativeWriter::WriteSimple(const std::string &varName, int dimSize, H5Pclose(plistID); } +void HDF5NativeWriter::WriteSimpleWithChunking( + const std::string &varName, int dimSize, const void *data, hid_t h5Type, + const hsize_t *shape, const hsize_t *offset, const hsize_t *count) +{ + CheckWriteGroup(); + hid_t fileSpace = H5Screate_simple(dimSize, shape, NULL); + + hid_t dsetPid = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_chunk(dsetPid, dimSize, count); + + size_t bytes = H5Tget_size(h5Type); + for (int i = 0; i < dimSize; i++) + { + bytes *= count[i]; + } + hid_t access_plistid = H5Pcreate(H5P_DATASET_ACCESS); + H5Pset_chunk_cache(access_plistid, 101, bytes, 1); + + hid_t dsetID = H5Dcreate(m_GroupId, varName.c_str(), h5Type, fileSpace, + H5P_DEFAULT, dsetPid, access_plistid); + + hid_t memSpace = H5Screate_simple(dimSize, count, NULL); + + // Select hyperslab + fileSpace = H5Dget_space(dsetID); + H5Sselect_hyperslab(fileSpace, H5S_SELECT_SET, offset, NULL, count, NULL); + + // Create property list for collective dataset write. + + hid_t plistID = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(plistID, H5FD_MPIO_COLLECTIVE); + + herr_t status; + + status = H5Dwrite(dsetID, h5Type, memSpace, fileSpace, plistID, data); + + if (status < 0) + { + // error + std::cerr << " Write failed. " << std::endl; + } + + H5Dclose(dsetID); + H5Sclose(fileSpace); + H5Sclose(memSpace); + H5Pclose(plistID); + + H5Pclose(dsetPid); + H5Pclose(access_plistid); +} + // // std::shared_ptr<HDF5NativeWriter> h5writer; @@ -224,8 +310,19 @@ void IO::write(int step, const HeatTransfer &ht, const Settings &s, std::vector<hsize_t> offset = {s.offsx, s.offsy}; std::vector<hsize_t> count = {s.ndx, s.ndy}; - h5writer->WriteSimple("T", 2, ht.data_noghost().data(), H5T_NATIVE_DOUBLE, - dims.data(), offset.data(), count.data()); + if (h5writer->m_Chunking) + { + h5writer->WriteSimpleWithChunking("T", 2, ht.data_noghost().data(), + H5T_NATIVE_DOUBLE, dims.data(), + offset.data(), count.data()); + } + else + { + h5writer->WriteSimple("T", 2, ht.data_noghost().data(), + H5T_NATIVE_DOUBLE, dims.data(), offset.data(), + count.data()); + } + h5writer->WriteScalar("gndy", &(s.gndy), H5T_NATIVE_UINT); h5writer->WriteScalar("gndx", &(s.gndx), H5T_NATIVE_UINT); diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.h b/source/adios2/toolkit/interop/hdf5/HDF5Common.h index 84151a2ec9d569f7bf5d3b474dd173cceb4b5907..546587a708bdf79cd8fc2ae1f28118063c555d57 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.h +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.h @@ -20,6 +20,8 @@ #include "adios2/ADIOSTypes.h" #include "adios2/core/Variable.h" +#include <stdexcept> // for Intel Compiler + namespace adios { namespace interop