BP3Deserializer.cpp

/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * BP3Deserializer.cpp
 *
 *  Created on: Sep 7, 2017
 *      Author: William F Godoy godoywf@ornl.gov
 */

#include "BP3Deserializer.h"
#include "BP3Deserializer.tcc"

#include <future>
#include <vector>

#include "adios2/helper/adiosFunctions.h" //ReadValue<T>

#ifdef _WIN32
#pragma warning(disable : 4503) // Windows complains about SubFileInfoMap levels
#endif

namespace adios2
{
namespace format
{

std::mutex BP3Deserializer::m_Mutex;

BP3Deserializer::BP3Deserializer(MPI_Comm mpiComm, const bool debugMode)
: BP3Base(mpiComm, debugMode)
{
}

void BP3Deserializer::ParseMetadata(const BufferSTL &bufferSTL, IO &io)
{
    ParseMinifooter(bufferSTL);
    ParsePGIndex(bufferSTL);
    ParseVariablesIndex(bufferSTL, io);
    // ParseAttributesIndex(io);
}

void BP3Deserializer::ClipContiguousMemory(
    const std::string &variableName, IO &io,
    const std::vector<char> &contiguousMemory, const Box<Dims> &blockBox,
    const Box<Dims> &intersectionBox) const
{
    // get variable pointer and set data in it with local dimensions
    const std::string type(io.InquireVariableType(variableName));

    if (type == "compound")
    {
    }
#define declare_type(T)                                                        \
    else if (type == GetType<T>())                                             \
    {                                                                          \
        Variable<T> *variable = io.InquireVariable<T>(variableName);           \
        if (variable != nullptr)                                               \
        {                                                                      \
            ClipContiguousMemoryCommon(*variable, contiguousMemory, blockBox,  \
                                       intersectionBox);                       \
        }                                                                      \
    }
    ADIOS2_FOREACH_TYPE_1ARG(declare_type)
#undef declare_type
}

// PRIVATE
void BP3Deserializer::ParseMinifooter(const BufferSTL &bufferSTL)
{
    auto lf_GetEndianness = [](const uint8_t endianness, bool &isLittleEndian) {

        switch (endianness)
        {
        case 0:
            isLittleEndian = true;
            break;
        case 1:
            isLittleEndian = false;
            break;
        }
    };

    const auto &buffer = bufferSTL.m_Buffer;
    const size_t bufferSize = buffer.size();

    size_t position = bufferSize - 4;
    const uint8_t endianess = ReadValue<uint8_t>(buffer, position);
    lf_GetEndianness(endianess, m_Minifooter.IsLittleEndian);
    position += 1;

    const uint8_t subFilesIndex = ReadValue<uint8_t>(buffer, position);
    if (subFilesIndex > 0)
    {
        m_Minifooter.HasSubFiles = true;
    }

    m_Minifooter.Version = ReadValue<uint8_t>(buffer, position);
    if (m_Minifooter.Version < 3)
    {
        throw std::runtime_error("ERROR: ADIOS2 only supports bp format "
                                 "version 3 and above, found " +
                                 std::to_string(m_Minifooter.Version) +
                                 " version \n");
    }

    position = bufferSize - m_MetadataSet.MiniFooterSize;
    m_Minifooter.PGIndexStart = ReadValue<uint64_t>(buffer, position);
    m_Minifooter.VarsIndexStart = ReadValue<uint64_t>(buffer, position);
    m_Minifooter.AttributesIndexStart = ReadValue<uint64_t>(buffer, position);
}

void BP3Deserializer::ParsePGIndex(const BufferSTL &bufferSTL)
{
    const auto &buffer = bufferSTL.m_Buffer;
    auto &position = m_Metadata.m_Position;
    position = m_Minifooter.PGIndexStart;

    m_MetadataSet.DataPGCount = ReadValue<uint64_t>(buffer, position);
    position += 10; // skipping lengths
    const uint16_t nameLength = ReadValue<uint16_t>(buffer, position);
    position += static_cast<size_t>(nameLength); // skipping name
    const char isFortran = ReadValue<char>(buffer, position);

    if (isFortran == 'y')
    {
        m_IsRowMajor = false;
    }
}

void BP3Deserializer::ParseVariablesIndex(const BufferSTL &bufferSTL, IO &io)
{
    auto lf_ReadElementIndex = [&](IO &io, const std::vector<char> &buffer,
                                   size_t position) {

        const ElementIndexHeader header =
            ReadElementIndexHeader(buffer, position);

        switch (header.DataType)
        {

        case (type_byte):
        {
            DefineVariableInIO<signed char>(header, io, buffer, position);
            break;
        }

        case (type_short):
        {
            DefineVariableInIO<short>(header, io, buffer, position);
            break;
        }

        case (type_integer):
        {
            DefineVariableInIO<int>(header, io, buffer, position);
            break;
        }

        case (type_long):
        {
#ifdef _WIN32
            DefineVariableInIO<long long int>(header, io, buffer, position);
#else
            DefineVariableInIO<long int>(header, io, buffer, position);
#endif
            break;
        }

        case (type_unsigned_byte):
        {
            DefineVariableInIO<unsigned char>(header, io, buffer, position);
            break;
        }

        case (type_unsigned_short):
        {
            DefineVariableInIO<unsigned short>(header, io, buffer, position);
            break;
        }

        case (type_unsigned_integer):
        {
            DefineVariableInIO<unsigned int>(header, io, buffer, position);
            break;
        }

        case (type_unsigned_long):
        {
#ifdef _WIN32
            DefineVariableInIO<unsigned long long int>(header, io, buffer,
                                                       position);
#else
            DefineVariableInIO<unsigned long int>(header, io, buffer, position);
#endif
            break;
        }

        case (type_real):
        {
            DefineVariableInIO<float>(header, io, buffer, position);
            break;
        }

        case (type_double):
        {
            DefineVariableInIO<double>(header, io, buffer, position);
            break;
        }

        case (type_long_double):
        {
            DefineVariableInIO<long double>(header, io, buffer, position);
            break;
        }

        case (type_complex):
        {
            DefineVariableInIO<std::complex<float>>(header, io, buffer,
                                                    position);
            break;
        }

        case (type_double_complex):
        {
            DefineVariableInIO<std::complex<double>>(header, io, buffer,
                                                     position);
            break;
        }

        case (type_long_double_complex):
        {
            DefineVariableInIO<std::complex<long double>>(header, io, buffer,
                                                          position);
            break;
        }
            // TODO: string
        } // end switch
    };

    const auto &buffer = bufferSTL.m_Buffer;
    size_t position = m_Minifooter.VarsIndexStart;

    const uint32_t count = ReadValue<uint32_t>(buffer, position);
    const uint64_t length = ReadValue<uint64_t>(buffer, position);

    const size_t startPosition = position;
    size_t localPosition = 0;

    // threads for reading Variables
    std::vector<std::future<void>> asyncs(m_Threads);
    std::vector<size_t> asyncPositions(m_Threads);

    bool launched = false;

    while (localPosition < length)
    {
        // extract async positions
        for (unsigned int t = 0; t < m_Threads; ++t)
        {
            asyncPositions[t] = position;
            const size_t elementIndexSize =
                static_cast<size_t>(ReadValue<uint32_t>(buffer, position));
            position += elementIndexSize;
            localPosition = position - startPosition;

            if (launched)
            {
                asyncs[t].get();
            }

            if (localPosition <= length)
            {
                asyncs[t] = std::async(std::launch::async, lf_ReadElementIndex,
                                       std::ref(io), std::ref(buffer),
                                       asyncPositions[t]);
            }
        }
        launched = true;
    }

    for (auto &async : asyncs)
    {
        if (async.valid())
        {
            async.wait();
        }
    }
}

void BP3Deserializer::ParseAttributesIndex(IO &io) {}

std::map<std::string, SubFileInfoMap>
BP3Deserializer::PerformGetsVariablesSubFileInfo(IO &io)
{
    if (m_DeferredVariables.empty())
    {
        return m_DeferredVariables;
    }

    for (auto &subFileInfoPair : m_DeferredVariables)
    {
        const std::string variableName(subFileInfoPair.first);
        const std::string type(io.InquireVariableType(variableName));

        if (type == "compound")
        {
        }
#define declare_type(T)                                                        \
    else if (type == GetType<T>())                                             \
    {                                                                          \
        subFileInfoPair.second =                                               \
            GetSubFileInfo(*io.InquireVariable<T>(variableName));              \
    }
        ADIOS2_FOREACH_TYPE_1ARG(declare_type)
#undef declare_type
    }
    return m_DeferredVariables;
}

#define declare_template_instantiation(T)                                      \
    template std::map<std::string, SubFileInfoMap>                             \
    BP3Deserializer::GetSyncVariableSubFileInfo(const Variable<T> &variable)   \
        const;                                                                 \
                                                                               \
    template void BP3Deserializer::GetDeferredVariable(Variable<T> &variable,  \
                                                       T *data);

ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation)
#undef declare_template_instantiation

} // end namespace format
} // end namespace adios2