IO_ph5.cpp

/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * IO_ph5.cpp
 *
 *  Created on: Feb 2017
 *      Author: Norbert Podhorszki
 */

#include "IO.h"

#include <hdf5.h>
#include <ios>
#include <iostream>
#include <memory>
#include <stdexcept>
#include <string>

class HDF5NativeWriter
{

public:
    HDF5NativeWriter(const std::string &fileName);
    ~HDF5NativeWriter();

    bool Advance();
    void Close();
    void CheckWriteGroup();

    void WriteScalar(const std::string &varName, const void *data,
                     hid_t h5Type);
    void WriteSimple(const std::string &varName, int dimSize, const void *data,
                     hid_t h5Type, const hsize_t *shape, const hsize_t *offset,
                     const hsize_t *count);

    void applyMetadataCacheEviction();
    void WriteSimpleWithChunking(const std::string &varName, int dimSize,
                                 const void *data, hid_t h5Type,
                                 const hsize_t *shape, const hsize_t *offset,
                                 const hsize_t *count);
    int m_CurrentTimeStep;
    unsigned int m_TotalTimeSteps;
    bool m_Chunking;

private:
    hid_t m_FilePropertyListId;
    hid_t m_FileId;
    hid_t m_GroupId;
};

HDF5NativeWriter::HDF5NativeWriter(const std::string &fileName)
: m_CurrentTimeStep(0), m_TotalTimeSteps(0), m_Chunking(false)
{
    m_FilePropertyListId = H5Pcreate(H5P_FILE_ACCESS);

    // read a file collectively
    H5Pset_fapl_mpio(m_FilePropertyListId, MPI_COMM_WORLD, MPI_INFO_NULL);

    m_FileId = H5Fcreate(fileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT,
                         m_FilePropertyListId);

    if (m_FileId < 0)
    {
        throw std::runtime_error("Unable to open " + fileName + " for reading");
    }

    applyMetadataCacheEviction();

    std::string ts0 = "/TimeStep0";

    m_GroupId = H5Gcreate2(m_FileId, ts0.c_str(), H5P_DEFAULT, H5P_DEFAULT,
                           H5P_DEFAULT);
    if (m_GroupId < 0)
    {
        throw std::runtime_error("HDF5: Unable to create group " + ts0);
    }

    std::string prefix = "chunking";
    if (fileName.substr(0, prefix.size()) == prefix)
    {
        m_Chunking = true;
    }
}

HDF5NativeWriter::~HDF5NativeWriter() { Close(); }

void HDF5NativeWriter::applyMetadataCacheEviction()
{
#ifdef NEVER
    /*
      see
      https://lists.hdfgroup.org/pipermail/hdf-forum_lists.hdfgroup.org/2011-February/004201.html
      John said the code below worked for the paper but not anymore after
      updates
    */
    H5AC_cache_config_t mdc_config;
    mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
    H5Pget_mdc_config(m_FilePropertyListId, &mdc_config);
    mdc_config.evictions_enabled = 0; // FALSE
    mdc_config.incr_mode = H5C_incr__off;
    mdc_config.decr_mode = H5C_decr__off;
    H5Pset_mdc_config(m_FilePropertyListId, &mdc_config);
#endif
}

void HDF5NativeWriter::Close()
{
    if (m_FileId < 0)
        return;

    hid_t s = H5Screate(H5S_SCALAR);
    hid_t attr = H5Acreate(m_FileId, "NumTimeSteps", H5T_NATIVE_UINT, s,
                           H5P_DEFAULT, H5P_DEFAULT);
    uint totalTimeSteps = m_CurrentTimeStep + 1;

    if (m_GroupId < 0)
    {
        totalTimeSteps = m_CurrentTimeStep;
    }
    H5Awrite(attr, H5T_NATIVE_UINT, &totalTimeSteps);
    H5Sclose(s);
    H5Aclose(attr);

    if (m_GroupId >= 0)
    {
        H5Gclose(m_GroupId);
        m_GroupId = -1;
    }

    H5Fclose(m_FileId);
    m_FileId = -1;
    H5Pclose(m_FilePropertyListId);
}

bool HDF5NativeWriter::Advance()
{
    if (m_GroupId >= 0)
    {
        H5Gclose(m_GroupId);
        m_GroupId = -1;
    }

    ++m_CurrentTimeStep;

    return true;
}

void HDF5NativeWriter::CheckWriteGroup()
{
    if (m_GroupId >= 0)
    {
        return;
    }

    std::string timeStepName = "/TimeStep" + std::to_string(m_CurrentTimeStep);
    m_GroupId = H5Gcreate2(m_FileId, timeStepName.c_str(), H5P_DEFAULT,
                           H5P_DEFAULT, H5P_DEFAULT);
    if (m_GroupId < 0)
    {
        throw std::runtime_error("HDF5: Unable to create group " +
                                 timeStepName);
    }
}

void HDF5NativeWriter::WriteScalar(const std::string &varName, const void *data,
                                   hid_t h5Type)
{
    CheckWriteGroup();
    // scalar
    hid_t filespaceID = H5Screate(H5S_SCALAR);
    hid_t dsetID = H5Dcreate(m_GroupId, varName.c_str(), h5Type, filespaceID,
                             H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
    hid_t plistID = H5Pcreate(H5P_DATASET_XFER);
    H5Pset_dxpl_mpio(plistID, H5FD_MPIO_COLLECTIVE);

    herr_t status = H5Dwrite(dsetID, h5Type, H5S_ALL, H5S_ALL, plistID, data);

    H5Sclose(filespaceID);
    H5Dclose(dsetID);
}

void HDF5NativeWriter::WriteSimple(const std::string &varName, int dimSize,
                                   const void *data, hid_t h5Type,
                                   const hsize_t *shape, const hsize_t *offset,
                                   const hsize_t *count)
{
    CheckWriteGroup();
    hid_t fileSpace = H5Screate_simple(dimSize, shape, NULL);

    hid_t dsetID = H5Dcreate(m_GroupId, varName.c_str(), h5Type, fileSpace,
                             H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);

    hid_t memSpace = H5Screate_simple(dimSize, count, NULL);

    // Select hyperslab
    fileSpace = H5Dget_space(dsetID);
    H5Sselect_hyperslab(fileSpace, H5S_SELECT_SET, offset, NULL, count, NULL);

    //  Create property list for collective dataset write.

    hid_t plistID = H5Pcreate(H5P_DATASET_XFER);
    H5Pset_dxpl_mpio(plistID, H5FD_MPIO_COLLECTIVE);

    herr_t status;

    status = H5Dwrite(dsetID, h5Type, memSpace, fileSpace, plistID, data);

    if (status < 0)
    {
        // error
        std::cerr << " Write failed. " << std::endl;
    }

    H5Dclose(dsetID);
    H5Sclose(fileSpace);
    H5Sclose(memSpace);
    H5Pclose(plistID);
}

void HDF5NativeWriter::WriteSimpleWithChunking(
    const std::string &varName, int dimSize, const void *data, hid_t h5Type,
    const hsize_t *shape, const hsize_t *offset, const hsize_t *count)
{
    CheckWriteGroup();
    hid_t fileSpace = H5Screate_simple(dimSize, shape, NULL);

    hid_t dsetPid = H5Pcreate(H5P_DATASET_CREATE);
    H5Pset_chunk(dsetPid, dimSize, count);

    size_t bytes = H5Tget_size(h5Type);
    for (int i = 0; i < dimSize; i++)
    {
        bytes *= count[i];
    }
    hid_t access_plistid = H5Pcreate(H5P_DATASET_ACCESS);
    H5Pset_chunk_cache(access_plistid, 101, bytes, 1);

    hid_t dsetID = H5Dcreate(m_GroupId, varName.c_str(), h5Type, fileSpace,
                             H5P_DEFAULT, dsetPid, access_plistid);

    hid_t memSpace = H5Screate_simple(dimSize, count, NULL);

    // Select hyperslab
    fileSpace = H5Dget_space(dsetID);
    H5Sselect_hyperslab(fileSpace, H5S_SELECT_SET, offset, NULL, count, NULL);

    //  Create property list for collective dataset write.

    hid_t plistID = H5Pcreate(H5P_DATASET_XFER);
    H5Pset_dxpl_mpio(plistID, H5FD_MPIO_COLLECTIVE);

    herr_t status;

    status = H5Dwrite(dsetID, h5Type, memSpace, fileSpace, plistID, data);

    if (status < 0)
    {
        // error
        std::cerr << " Write failed. " << std::endl;
    }

    H5Dclose(dsetID);
    H5Sclose(fileSpace);
    H5Sclose(memSpace);
    H5Pclose(plistID);

    H5Pclose(dsetPid);
    H5Pclose(access_plistid);
}

//
//
std::shared_ptr<HDF5NativeWriter> h5writer;
// HDF5NativeWriter* h5writer;

IO::IO(const Settings &s, MPI_Comm comm)
{
    m_outputfilename = MakeFilename(s.outputfile, ".h5");

    if (s.outputfile[0] == '0')
    {
        std::cout << " no writer. " << std::endl;
        h5writer = nullptr;
        return;
    }
    h5writer = std::make_shared<HDF5NativeWriter>(m_outputfilename);

    if (h5writer == nullptr)
        throw std::ios_base::failure("ERROR: failed to open ADIOS h5writer\n");
}

IO::~IO()
{
    if (h5writer != nullptr)
    {
        h5writer->Close();
    }
    // delete h5writer;
}

void IO::write(int step, const HeatTransfer &ht, const Settings &s,
               MPI_Comm comm)
{
    if (h5writer == nullptr)
    {
        return;
    }
    std::vector<hsize_t> dims = {s.gndx, s.gndy};
    std::vector<hsize_t> offset = {s.offsx, s.offsy};
    std::vector<hsize_t> count = {s.ndx, s.ndy};

    if (h5writer->m_Chunking)
    {
        h5writer->WriteSimpleWithChunking("T", 2, ht.data_noghost().data(),
                                          H5T_NATIVE_DOUBLE, dims.data(),
                                          offset.data(), count.data());
    }
    else
    {
        h5writer->WriteSimple("T", 2, ht.data_noghost().data(),
                              H5T_NATIVE_DOUBLE, dims.data(), offset.data(),
                              count.data());
    }

    h5writer->WriteScalar("gndy", &(s.gndy), H5T_NATIVE_UINT);
    h5writer->WriteScalar("gndx", &(s.gndx), H5T_NATIVE_UINT);

    h5writer->Advance();
}