BP1Base.h

/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * BP1Base.h  base class for BP1Writer and BP1Reader
 *
 *  Created on: Feb 2, 2017
 *      Author: William F Godoy godoywf@ornl.gov
 */

#ifndef ADIOS2_TOOLKIT_FORMAT_BP1_BP1BASE_H_
#define ADIOS2_TOOLKIT_FORMAT_BP1_BP1BASE_H_

/// \cond EXCLUDE_FROM_DOXYGEN
#include <memory> //std::shared_ptr
#include <vector>
/// \endcond

#include "adios2/ADIOSConfig.h"
#include "adios2/ADIOSMPICommOnly.h"
#include "adios2/ADIOSMacros.h"
#include "adios2/ADIOSTypes.h"
#include "adios2/core/Variable.h"
#include "adios2/toolkit/capsule/heap/STLVector.h"
#include "adios2/toolkit/format/bp1/BP1Aggregator.h"
#include "adios2/toolkit/format/bp1/BP1Structs.h"
#include "adios2/toolkit/profiling/iochrono/IOChrono.h"

namespace adios
{
namespace format
{

/**
 * Base class for BP1Writer and BP1Reader format
 */
class BP1Base
{

public:
    /** statistics verbosity, only 0 is supported */
    unsigned int m_Verbosity = 0;

    /** contains data buffer and position */
    capsule::STLVector m_HeapBuffer;

    /** memory growth factor,s set by the user */
    float m_GrowthFactor = DefaultBufferGrowthFactor;

    /** max buffer size, set by the user */
    size_t m_MaxBufferSize = DefaultMaxBufferSize;

    /** contains bp1 format metadata indices*/
    BP1MetadataSet m_MetadataSet;

    /** object that takes care of all MPI aggregation tasks */
    BP1Aggregator m_BP1Aggregator;

    /** true: Close was called, Engine will call this many times for different
     * transports */
    bool m_IsClosed = false;

    /** buffering and MPI aggregation profiling info, set by user */
    profiling::IOChrono m_Profiler;

    /**
     * Unique constructor
     * @param mpiComm for m_BP1Aggregator
     * @param debugMode true: exceptions checks
     */
    BP1Base(MPI_Comm mpiComm, const bool debugMode);

    virtual ~BP1Base() = default;

    void InitParameters(const Params &parameters);

    /**
     * Vector version of BPBaseName
     * @param names
     * @return vector of base (name.bp) names
     */
    std::vector<std::string>
    GetBPBaseNames(const std::vector<std::string> &names) const noexcept;

    /**
     * Checks if input name has .bp extension and returns a .bp directory
     * name
     * @param name input (might or not have .bp)
     * @return either name.bp (name has no .bp) or name (name has .bp
     * extension)
     */
    std::string GetBPBaseName(const std::string &name) const noexcept;

    /**
     * Vector version of GetBPName
     * @param names
     * @return
     */
    std::vector<std::string>
    GetBPNames(const std::vector<std::string> &names) const noexcept;

    /**
     * Gets the name for bp buffer based on rank
     * @param name
     * @return name.bp/name.bp.rank
     */
    std::string GetBPName(const std::string &name) const noexcept;

    /** Return type of the CheckAllocation function. */
    enum class ResizeResult
    {
        Failure,   //!< FAILURE, caught a std::bad_alloc
        Unchanged, //!< UNCHANGED, no need to resize (sufficient capacity)
        Success,   //!< SUCCESS, resize was successful
        Flush      //!< FLUSH, need to flush to transports for current variable
    };

    /**
     * @param variable
     * @return
     * -1: allocation failed,
     *  0: no allocation needed,
     *  1: reallocation is sucessful
     *  2: need a transport flush
     */
    template <class T>
    ResizeResult ResizeBuffer(const Variable<T> &variable);

protected:
    /** might be used in large payload copies to buffer */
    unsigned int m_Threads = 1;
    const bool m_DebugMode = false;

    /** method type for file I/O */
    enum IO_METHOD
    {
        METHOD_UNKNOWN = -2,
        METHOD_NULL = -1,
        METHOD_MPI = 0,
        METHOD_DATATAP = 1 // OBSOLETE
        ,
        METHOD_POSIX = 2,
        METHOD_DATASPACES = 3,
        METHOD_VTK = 4 // non-existent
        ,
        METHOD_POSIX_ASCII = 5 // non-existent
        ,
        METHOD_MPI_CIO = 6 // OBSOLETE
        ,
        METHOD_PHDF5 = 7,
        METHOD_PROVENANCE = 8 // OBSOLETE
        ,
        METHOD_MPI_STRIPE = 9 // OBSOLETE
        ,
        METHOD_MPI_LUSTRE = 10,
        METHOD_MPI_STAGGER = 11 // OBSOLETE
        ,
        METHOD_MPI_AGG = 12 // OBSOLETE
        ,
        METHOD_ADAPTIVE = 13 // OBSOLETE
        ,
        METHOD_POSIX1 = 14 // OBSOLETE
        ,
        METHOD_NC4 = 15,
        METHOD_MPI_AMR = 16,
        METHOD_MPI_AMR1 = 17 // OBSOLETE
        ,
        METHOD_FLEXPATH = 18,
        METHOD_NSSI_STAGING = 19,
        METHOD_NSSI_FILTER = 20,
        METHOD_DIMES = 21,
        METHOD_VAR_MERGE = 22,
        METHOD_MPI_BGQ = 23,
        METHOD_ICEE = 24,
        METHOD_COUNT = 25,
        METHOD_FSTREAM = 26,
        METHOD_FILE = 27,
        METHOD_ZMQ = 28,
        METHOD_MDTM = 29
    };

    /**
     * DataTypes mapping in BP Format
     */
    enum DataTypes
    {
        type_unknown = -1, //!< type_unknown
        type_byte = 0,     //!< type_byte
        type_short = 1,    //!< type_short
        type_integer = 2,  //!< type_integer
        type_long = 4,     //!< type_long

        type_unsigned_byte = 50,    //!< type_unsigned_byte
        type_unsigned_short = 51,   //!< type_unsigned_short
        type_unsigned_integer = 52, //!< type_unsigned_integer
        type_unsigned_long = 54,    //!< type_unsigned_long

        type_real = 5,        //!< type_real or float
        type_double = 6,      //!< type_double
        type_long_double = 7, //!< type_long_double

        type_string = 9,              //!< type_string
        type_complex = 10,            //!< type_complex
        type_double_complex = 11,     //!< type_double_complex
        type_string_array = 12,       //!< type_string_array
        type_long_double_complex = 13 //!< type_double_complex
    };

    /**
     * Characteristic ID in variable metadata
     */
    enum VariableCharacteristicID
    {
        characteristic_value = 0,      //!< characteristic_value
        characteristic_min = 1,        //!< Used to read in older bp file format
        characteristic_max = 2,        //!< Used to read in older bp file format
        characteristic_offset = 3,     //!< characteristic_offset
        characteristic_dimensions = 4, //!< characteristic_dimensions
        characteristic_var_id = 5,     //!< characteristic_var_id
        characteristic_payload_offset = 6, //!< characteristic_payload_offset
        characteristic_file_index = 7,     //!< characteristic_file_index
        characteristic_time_index = 8,     //!< characteristic_time_index
        characteristic_bitmap = 9,         //!< characteristic_bitmap
        characteristic_stat = 10,          //!< characteristic_stat
        characteristic_transform_type = 11 //!< characteristic_transform_type
    };

    /** Define statistics type for characteristic ID = 10 in bp1 format */
    enum VariableStatistics
    {
        statistic_min = 0,
        statistic_max = 1,
        statistic_cnt = 2,
        statistic_sum = 3,
        statistic_sum_square = 4,
        statistic_hist = 5,
        statistic_finite = 6
    };

    template <class T>
    struct Stats
    {
        T Min;
        T Max;
        uint64_t Offset;
        uint64_t PayloadOffset;
        uint32_t TimeIndex;
        uint32_t MemberID;

        //      unsigned long int count;
        //      long double sum;
        //      long double sumSquare;
        // unsigned long int histogram
        // bool finite??
    };

    /** profile=on (default) generate profiling.log
         *  profile=off */
    void InitParameterProfile(const std::string value);

    /** profile_units=s (default) (mus, ms, s,m,h) from ADIOSTypes.h TimeUnit */
    void InitParameterProfileUnits(const std::string value);

    /** growth_factor=1.5 (default), must be > 1.0 */
    void InitParameterBufferGrowth(const std::string value);

    /** set initial buffer size */
    void InitParameterInitBufferSize(const std::string value);

    /** unlimited (default), set max buffer size in Gb or Mb
     *  max_buffer_size=100Mb or  max_buffer_size=1Gb */
    void InitParameterMaxBufferSize(const std::string value);

    /** verbose file level=0 (default) */
    void InitParameterVerbose(const std::string value);

    /**
     * Returns data type index from enum Datatypes
     * @param variable input variable
     * @return data type
     */
    template <class T>
    int8_t GetDataType() const noexcept;

    std::vector<uint8_t>
    GetTransportIDs(const std::vector<std::string> &transportsTypes) const
        noexcept;

    /**
     * Calculates the Process Index size in bytes according to the BP
     * format,
     * including list of method with no parameters (for now)
     * @param name
     * @param timeStepName
     * @param transportsSize
     * @return size of pg index
     */
    size_t GetProcessGroupIndexSize(const std::string name,
                                    const std::string timeStepName,
                                    const size_t transportsSize) const noexcept;

    /**
     * Returns the estimated variable index size
     * @param variable
     */
    template <class T>
    size_t GetVariableIndexSize(const Variable<T> &variable) const noexcept;
};

#define declare_template_instantiation(T)                                      \
    extern template BP1Base::ResizeResult BP1Base::ResizeBuffer(               \
        const Variable<T> &variable);

ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation)
#undef declare_template_instantiation

} // end namespace format
} // end namespace adios

#endif /* ADIOS2_TOOLKIT_FORMAT_BP1_BP1BASE_H_ */