Unverified Commit 7185c0d0 authored by Podhorszki, Norbert's avatar Podhorszki, Norbert Committed by GitHub
Browse files

Merge pull request #3275 from pnorbert/backward-compatibility-compression 

Add backward compatible read with BLOSC compression to BP4 files crea…
parents 499d2686 62f0cca4

source/adios2/CMakeLists.txt

+1 −0
Original line number Diff line number Diff line
@@ -76,6 +76,7 @@ add_library(adios2_core 
  toolkit/format/bp/bp4/BP4Base.cpp

  toolkit/format/bp/bp4/BP4Serializer.cpp toolkit/format/bp/bp4/BP4Serializer.tcc

  toolkit/format/bp/bp4/BP4Deserializer.cpp toolkit/format/bp/bp4/BP4Deserializer.tcc

  toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.cpp



  toolkit/profiling/iochrono/Timer.cpp

  toolkit/profiling/iochrono/IOChrono.cpp

source/adios2/toolkit/format/bp/BPBase.cpp

+13 −0
Original line number Diff line number Diff line
@@ -13,6 +13,8 @@ 


#include "adios2/helper/adiosFunctions.h"



#include "adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.h"



namespace adios2

{

namespace format
@@ -487,5 +489,16 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) 


size_t BPBase::DebugGetDataBufferSize() const { return m_Data.DebugGetSize(); }



std::shared_ptr<BPBackCompatOperation>

BPBase::SetBPBackCompatOperation(const std::string type) const noexcept

{

    std::shared_ptr<BPBackCompatOperation> bpOp;

    if (type == "blosc")

    {

        bpOp = std::make_shared<BPBackCompatBlosc>();

    }

    return bpOp;

}



} // end namespace format

} // end namespace adios2

source/adios2/toolkit/format/bp/BPBase.h

+13 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 
#include "adios2/common/ADIOSTypes.h"

#include "adios2/helper/adiosFunctions.h"

#include "adios2/toolkit/aggregator/mpi/MPIChain.h"

#include "adios2/toolkit/format/bp/bpBackCompatOperation/BPBackCompatOperation.h"

#include "adios2/toolkit/format/buffer/Buffer.h"

#include "adios2/toolkit/format/buffer/heap/BufferSTL.h"

#include "adios2/toolkit/profiling/iochrono/IOChrono.h"
@@ -144,6 +145,10 @@ public: 
        uint64_t VarsIndexStart = 0;

        uint64_t AttributesIndexStart = 0;

        int8_t Version = -1;

        uint8_t ADIOSVersionMajor = 0;

        uint8_t ADIOSVersionMinor = 0;

        uint8_t ADIOSVersionPatch = 0;

        uint32_t ADIOSVersion = 0; // major*1M + minor*1k + patch e.g. 2007001

        bool IsLittleEndian = true;

        bool HasSubFiles = false;
@@ -472,6 +477,14 @@ protected: 
    TransformTypes TransformTypeEnum(const std::string transformType) const

        noexcept;



    /**

     * Returns the proper derived class for BPOperation based on type

     * @param type input, must be a supported type under BPOperation

     * @return derived class if supported, false pointer if type not supported

     */

    std::shared_ptr<BPBackCompatOperation>

    SetBPBackCompatOperation(const std::string type) const noexcept;



    struct ProcessGroupIndex

    {

        uint64_t Offset;

source/adios2/toolkit/format/bp/bp4/BP4Base.h

+3 −0
Original line number Diff line number Diff line
@@ -61,6 +61,9 @@ public: 
    static constexpr size_t m_ActiveFlagPosition = 38;

    static constexpr size_t m_VersionTagPosition = 0;

    static constexpr size_t m_VersionTagLength = 32;

    static constexpr size_t m_VersionMajorPosition = 32;

    static constexpr size_t m_VersionMinorPosition = 33;

    static constexpr size_t m_VersionPatchPosition = 34;



    /**

     * Unique constructor

source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp

+17 −0
Original line number Diff line number Diff line
@@ -114,6 +114,23 @@ void BP4Deserializer::ParseMetadataIndex(BufferSTL &bufferSTL, 
        // This has no flag in BP4 header. Always true

        m_Minifooter.HasSubFiles = true;



        // Writer's ADIOS version

        position = m_VersionMajorPosition;

        uint8_t ascii = helper::ReadValue<uint8_t>(buffer, position,

                                                   m_Minifooter.IsLittleEndian);

        m_Minifooter.ADIOSVersionMajor = ascii - (uint8_t)'0';

        position = m_VersionMinorPosition;

        ascii = helper::ReadValue<uint8_t>(buffer, position,

                                           m_Minifooter.IsLittleEndian);

        m_Minifooter.ADIOSVersionMinor = ascii - (uint8_t)'0';

        position = m_VersionPatchPosition;

        ascii = helper::ReadValue<uint8_t>(buffer, position,

                                           m_Minifooter.IsLittleEndian);

        m_Minifooter.ADIOSVersionPatch = ascii - (uint8_t)'0';

        m_Minifooter.ADIOSVersion = m_Minifooter.ADIOSVersionMajor * 1000000 +

                                    m_Minifooter.ADIOSVersionMinor * 1000 +

                                    m_Minifooter.ADIOSVersionPatch;



        // BP version

        position = m_BPVersionPosition;

        m_Minifooter.Version = helper::ReadValue<uint8_t>(