adiosFunctions.cpp

/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * adiosFunctions.cpp
 *
 *  Created on: Oct 10, 2016
 *      Author: wfg
 */

#include "adiosFunctions.h"

#include <sys/stat.h>  //stat
#include <sys/types.h> //CreateDirectory
#include <unistd.h>    //CreateDirectory

#include <cmath>   // std::ceil, std::pow, std::log
#include <cstring> //std::memcpy

#include <algorithm> //std::count
#include <fstream>
#include <ios> //std::ios_base::failure
#include <sstream>
#include <stdexcept>
#include <thread> //std::thread


#include "adios2/ADIOSTypes.h"
#include "adios2/ADIOSMPI.h"

#include "adios2/core/Support.h"

#ifdef ADIOS2_HAVE_BZIP2
#include "adios2/transform/BZip2.h"
#endif

namespace adios
{

void DumpFileToString(const std::string fileName, std::string &fileContent)
{
    std::ifstream fileStream(fileName);

    if (fileStream.good() == false)
    { // check file
        throw std::ios_base::failure(
            "ERROR: file " + fileName +
            " could not be opened. Check permissions or file existence\n");
    }

    std::ostringstream fileSS;
    fileSS << fileStream.rdbuf();
    fileStream.close();
    fileContent = fileSS.str(); // convert to string and check

    if (fileContent.empty())
    {
        throw std::invalid_argument("ERROR: file " + fileName + " is empty\n");
    }
}

void GetSubString(const std::string initialTag, const std::string finalTag,
                  const std::string content, std::string &subString,
                  std::string::size_type &currentPosition)
{
    auto lf_Wipe = [](std::string &subString,
                      std::string::size_type &currentPosition) {
        subString.clear();
        currentPosition = std::string::npos;
    };

    auto lf_SetPositions =
        [](const char quote, const std::string::size_type quotePosition,
           const std::string &content, std::string::size_type &currentPosition,
           std::string::size_type &closingQuotePosition) {
            currentPosition = quotePosition;
            closingQuotePosition = content.find(quote, currentPosition + 1);
        };

    // BODY OF FUNCTION STARTS HERE
    std::string::size_type start(content.find(initialTag, currentPosition));
    if (start == content.npos)
    {
        lf_Wipe(subString, currentPosition);
        return;
    }
    currentPosition = start;

    std::string::size_type end(content.find(finalTag, currentPosition));
    if (end == content.npos)
    {
        lf_Wipe(subString, currentPosition);
        return;
    }

    // here make sure the finalTag is not a value surrounded by " " or ' ', if
    // so
    // find next
    bool isValue = true;

    while (isValue == true)
    {
        std::string::size_type singleQuotePosition =
            content.find('\'', currentPosition);
        std::string::size_type doubleQuotePosition =
            content.find('\"', currentPosition);

        if ((singleQuotePosition == content.npos &&
             doubleQuotePosition == content.npos) ||
            (singleQuotePosition == content.npos &&
             end < doubleQuotePosition) ||
            (doubleQuotePosition == content.npos &&
             end < singleQuotePosition) ||
            (end < singleQuotePosition && end < doubleQuotePosition))
        {
            break;
        }

        // find the closing corresponding quote
        std::string::size_type closingQuotePosition;

        if (singleQuotePosition == content.npos)
        { // no ' anywhere
            lf_SetPositions('\"', doubleQuotePosition, content, currentPosition,
                            closingQuotePosition);
        }
        else if (doubleQuotePosition == content.npos)
        { // no " anywhere
            lf_SetPositions('\'', singleQuotePosition, content, currentPosition,
                            closingQuotePosition);
        }
        else
        {
            if (singleQuotePosition < doubleQuotePosition)
            {
                lf_SetPositions('\'', singleQuotePosition, content,
                                currentPosition, closingQuotePosition);
            }
            else
            { // find the closing "
                lf_SetPositions('\"', doubleQuotePosition, content,
                                currentPosition, closingQuotePosition);
            }
        }

        if (closingQuotePosition ==
            content.npos) // if can't find closing it's open until the end
        {
            lf_Wipe(subString, currentPosition);
            return;
        }

        currentPosition = closingQuotePosition + 1;

        if (closingQuotePosition < end)
        {
            continue;
        }

        // if this point is reached it means it's a value inside " " or ' ',
        // move to
        // the next end
        end = content.find(finalTag, currentPosition);
    }

    subString = content.substr(start, end - start + finalTag.size());
    currentPosition = end;
}

void GetQuotedValue(const char quote,
                    const std::string::size_type &quotePosition,
                    std::string &currentTag, std::string &value)
{
    currentTag = currentTag.substr(quotePosition + 1);
    auto nextQuotePosition = currentTag.find(quote);

    if (nextQuotePosition == currentTag.npos)
    {
        throw std::invalid_argument("ERROR: Invalid attribute in..." +
                                    currentTag + "...check XML file\n");
    }

    value = currentTag.substr(0, nextQuotePosition);
    currentTag = currentTag.substr(nextQuotePosition + 1);
}

void GetPairs(const std::string tag,
              std::vector<std::pair<const std::string, const std::string>>
                  &pairs) noexcept
{
    std::string currentTag(
        tag.substr(tag.find_first_of(" \t\n"))); // initialize current tag

    while (currentTag.find('=') != currentTag.npos) // equalPosition
    {
        currentTag = currentTag.substr(currentTag.find_first_not_of(" \t\n"));
        auto equalPosition = currentTag.find('=');
        const std::string field(
            currentTag.substr(0, equalPosition)); // get field
        std::string value;

        const char quote = currentTag[equalPosition + 1];
        if (quote == '\'' || quote == '"') // single quotes
        {
            GetQuotedValue(quote, equalPosition + 1, currentTag, value);
        }

        pairs.push_back(
            std::pair<const std::string, const std::string>(field, value));
    }
}

void GetPairsFromTag(
    const std::string &fileContent, const std::string tag,
    std::vector<std::pair<const std::string, const std::string>> &pairs)
{
    if (tag.back() == '/') // last char is / --> "XML empty tag"
    {
        GetPairs(tag, pairs);
    }
    else if (tag[0] == '/') // first char is / ---> closing tag
    {
    }
    else // opening tag
    {
        const std::string tagName(tag.substr(0, tag.find_first_of(" \t\n\r")));
        const std::string closingTagName("</" + tagName +
                                         ">"); // check for closing tagName

        if (fileContent.find(closingTagName) == fileContent.npos)
        {
            throw std::invalid_argument("ERROR: closing tag " + closingTagName +
                                        " missing, check XML file\n");
        }

        GetPairs(tag, pairs);
    }
}

// void SetMembers( const std::string& fileContent, const MPI_Comm mpiComm,
// const bool debugMode,
//                 std::string& hostLanguage, std::vector<
//                 std::shared_ptr<Transform> >& transforms,
//                 std::map< std::string, Group >& groups )
//{
//    //adios-config
//    std::string currentContent;
//    std::string::size_type currentPosition( 0 );
//    GetSubString( "<adios-config ", "</adios-config>", fileContent,
//    currentContent, currentPosition );
//
//    //remove comment sections
//    std::string::size_type startComment ( currentContent.find( "<!--" ) );
//
//    while( startComment != currentContent.npos )
//    {
//        std::string::size_type endComment( currentContent.find( "-->") );
//        currentContent.erase( startComment, endComment-startComment+3 );
//        startComment = currentContent.find( "<!--" );
//    }
//
//    //Tag <adios-config
//    currentPosition = 0;
//
//    std::string tag; //use for < > tags
//    GetSubString( "<adios-config", ">", currentContent, tag, currentPosition
//    );
//    tag = tag.substr( 1, tag.size() - 2 ); //eliminate < >
//
//    std::vector< std::pair<const std::string, const std::string> > pairs; //
//    pairs in tag
//    GetPairsFromTag( currentContent, tag, pairs );
//
//    for( auto& pair : pairs )
//        if( pair.first == "host-language" )
//            hostLanguage = pair.second;
//
//    if( debugMode == true )
//    {
//        if( Support::HostLanguages.count( hostLanguage ) == 0 )
//            throw std::invalid_argument("ERROR: host language " + hostLanguage
//            + " not supported.\n" );
//
//        if( hostLanguage.empty() == true )
//            throw std::invalid_argument("ERROR: host language is empty.\n" );
//    }
//
//    //adios-group
//    currentPosition = 0;
//
//    while( currentPosition != std::string::npos )
//    {
//        std::string xmlGroup;
//        GetSubString("<adios-group ", "</adios-group>", currentContent,
//        xmlGroup, currentPosition ); //Get all group contents
//
//        if( xmlGroup.empty() ) //no more groups to find
//            break;
//
//        //get group name
//        std::string::size_type groupPosition( 0 );
//        GetSubString( "<adios-group ", ">", xmlGroup, tag, groupPosition );
//        if( debugMode == true )
//        {
//            if( tag.size() < 2 )
//                throw std::invalid_argument( "ERROR: wrong tag " + tag + " in
//                adios-group\n" ); //check < or <=
//        }
//
//        tag = tag.substr( 1, tag.size() - 2 ); //eliminate < >
//        GetPairsFromTag( xmlGroup, tag, pairs );
//        std::string groupName;
//
//        for( auto& pair : pairs )
//        {
//            if( pair.first == "name")
//                groupName = pair.second;
//        }
//
//        if( debugMode == true )
//        {
//            if( groupName.empty() )
//                throw std::invalid_argument( "ERROR: group name not found. \n"
//                );
//
//            if( groups.count( groupName ) == 1 ) //group exists
//                throw std::invalid_argument( "ERROR: group " + groupName + "
//                defined twice.\n" );
//        }
//
//        groups.emplace( groupName, Group( groupName, xmlGroup, transforms,
//        debugMode ) );
//
//        currentContent.erase( currentContent.find( xmlGroup ), xmlGroup.size()
//        );
//        currentPosition = 0;
//    }
//
//    //transport
//    //lambda function to check priority and iteration casting to unsigned int
//    auto lf_UIntCheck = []( const std::string method, const std::string
//    fieldStr, const std::string fieldName,
//                            const bool debugMode, int& field )
//    {
//        field = 0;
//        if( fieldStr.empty() == false )
//        {
//            field = std::stoi( fieldStr ); //throws invalid_argument
//
//            if( debugMode == true )
//            {
//                if( field < 0 )
//                    throw std::invalid_argument("ERROR: " + fieldName + " in
//                    transport " + method + " can't be negative\n" );
//            }
//        }
//    };
//
//    //this section will have to change, doing nothing for now
//    currentPosition = 0;
//    while( currentPosition != std::string::npos )
//    {
//        GetSubString( "<transport ", ">", currentContent, tag, currentPosition
//        );
//        if( tag.empty() ) break;
//        tag = tag.substr( 1, tag.size() - 2 ); //eliminate < >
//        pairs.clear();
//        GetPairsFromTag( currentContent, tag, pairs );
//
//        std::string groupName, method, priorityStr, iterationStr;
//        for( auto& pair : pairs )
//        {
//            if( pair.first == "group" )  groupName = pair.second;
//            else if( pair.first == "method" ) method = pair.second;
//            else if( pair.first == "priority" ) priorityStr = pair.second;
//            else if( pair.first == "iteration" ) iterationStr = pair.second;
//        }
//
//        auto itGroup = groups.find( groupName );
//        if( debugMode == true )
//        {
//            if( itGroup == groups.end() ) //not found
//                throw std::invalid_argument( "ERROR: in transport " + method +
//                " group " + groupName + " not found.\n" );
//        }
//
//        int priority, iteration;
//        lf_UIntCheck( method, priorityStr, "priority", debugMode, priority );
//        lf_UIntCheck( method, iterationStr, "iteration", debugMode, iteration
//        );
//        //here do something with the capsule
//    }
//}

// void InitXML( const std::string xmlConfigFile, const MPI_Comm mpiComm, const
// bool debugMode,
//              std::string& hostLanguage, std::vector<
//              std::shared_ptr<Transform> >& transforms,
//              std::map< std::string, Group >& groups )
//{
//    int xmlFileContentSize;
//    std::string xmlFileContent;
//
//    int rank;
//    MPI_Comm_rank( mpiComm, &rank );
//
//    if( rank == 0 ) //serial part
//    {
//        DumpFileToString( xmlConfigFile, xmlFileContent ); //in
//        ADIOSFunctions.h dumps all XML Config File to xmlFileContent
//        xmlFileContentSize = xmlFileContent.size( ) + 1; // add one for the
//        null character
//
//        MPI_Bcast( &xmlFileContentSize, 1, MPI_INT, 0, mpiComm ); //broadcast
//        size for allocation
//        MPI_Bcast( (char*)xmlFileContent.c_str(), xmlFileContentSize,
//        MPI_CHAR, 0, mpiComm ); //broadcast contents
//    }
//    else
//    {
//        MPI_Bcast( &xmlFileContentSize, 1, MPI_INT, 0, mpiComm  ); //receive
//        size
//
//        char* xmlFileContentMPI = new char[ xmlFileContentSize ]; //allocate
//        xml C-char
//        MPI_Bcast( xmlFileContentMPI, xmlFileContentSize, MPI_CHAR, 0, mpiComm
//        ); //receive xml C-char
//        xmlFileContent.assign( xmlFileContentMPI ); //copy to a string
//
//        delete []( xmlFileContentMPI ); //delete char* needed for MPI, might
//        add size is moving to C++14 for optimization, avoid memory leak
//    }
//
//    SetMembers( xmlFileContent,  mpiComm, debugMode, hostLanguage, transforms,
//    groups );
//}

std::size_t GetTotalSize(const std::vector<std::size_t> &dimensions)
{
    std::size_t product = 1;

    for (const auto dimension : dimensions)
    {
        product *= dimension;
    }

    return product;
}

void CreateDirectory(const std::string fullPath) noexcept
{
    auto lf_Mkdir = [](const std::string directory, struct stat &st) {
        if (stat(directory.c_str(), &st) == -1)
        {
            mkdir(directory.c_str(), 0777);
        }
    };

    auto directoryPosition = fullPath.find("/");

    if (fullPath[0] == '/' || fullPath[0] == '.')
    { // find the second '/'
        directoryPosition = fullPath.find("/", directoryPosition + 1);
    }

    struct stat st = {0};
    if (directoryPosition == fullPath.npos) // no subdirectories
    {
        lf_Mkdir(fullPath.c_str(), st);
        return;
    }

    std::string directory(fullPath.substr(0, directoryPosition));
    lf_Mkdir(directory.c_str(), st);

    while (directoryPosition != fullPath.npos)
    {
        directoryPosition = fullPath.find("/", directoryPosition + 1);
        directory = fullPath.substr(0, directoryPosition);
        lf_Mkdir(directory.c_str(), st);
    }
}

void SetTransformsHelper(const std::vector<std::string> &transformNames,
                         std::vector<std::shared_ptr<Transform>> &transforms,
                         const bool debugMode,
                         std::vector<short> &transformIndices,
                         std::vector<short> &parameters)
{
    // function to get a parameter from "method:parameter"
    auto lf_GetParameter = [](const std::string transformName,
                              std::string &transformMethod,
                              const bool debugMode) -> short {
        short parameter = -1;
        auto colonPosition = transformName.find(":");

        if (colonPosition != transformName.npos)
        {
            if (debugMode == true)
            {
                if (colonPosition == transformName.size() - 1)
                {
                    throw std::invalid_argument(
                        "ERROR: wrong format for transform " + transformName +
                        ", in call to SetTransform\n");
                }
            }

            transformMethod = transformName.substr(0, colonPosition);
            parameter = std::stoi(
                transformName.substr(colonPosition + 1)); // need to test
        }
        return parameter;
    };

    // Get transform index from transforms, if not found return -1
    auto lf_GetTransformIndex =
        [](const std::string transformMethod,
           const std::vector<std::shared_ptr<Transform>> &transforms) -> short {
        short transformIndex = -1;
        for (unsigned int i = 0; i < transforms.size(); ++i)
        {
            if (transforms[i]->m_Method == transformMethod)
            {
                transformIndex = i;
                break;
            }
        }
        return transformIndex;
    };

    // BODY of FUNCTION STARTS HERE
    for (const std::string transformName : transformNames)
    {
        std::string transformMethod(transformName);
        short parameter =
            lf_GetParameter(transformName, transformMethod,
                            debugMode); // from transform = "method:parameter"
        short transformIndex =
            lf_GetTransformIndex(transformMethod, transforms);

        if (transformIndex == -1) // not found, then create a new transform
        {
            if (transformMethod == "bzip2")
            {
#ifdef ADIOS2_HAVE_BZIP2
                transforms.push_back(
                    std::make_shared<adios::transform::BZip2>());
#endif
            }

            transformIndex = static_cast<short>(transforms.size() - 1);
        }
        transformIndices.push_back(transformIndex);
        parameters.push_back(parameter);
    }
}

std::map<std::string, std::string>
BuildParametersMap(const std::vector<std::string> &parameters,
                   const bool debugMode)
{
    auto lf_GetFieldValue = [](const std::string parameter, std::string &field,
                               std::string &value, const bool debugMode) {
        auto equalPosition = parameter.find("=");

        if (debugMode == true)
        {
            if (equalPosition == parameter.npos)
            {
                throw std::invalid_argument(
                    "ERROR: wrong format for parameter " + parameter +
                    ", format must be field=value \n");
            }

            if (equalPosition == parameter.size() - 1)
            {
                throw std::invalid_argument("ERROR: empty value in parameter " +
                                            parameter +
                                            ", format must be field=value \n");
            }
        }

        field = parameter.substr(0, equalPosition);
        value = parameter.substr(equalPosition + 1); // need to test
    };

    // BODY OF FUNCTION STARTS HERE
    std::map<std::string, std::string> parametersOutput;

    for (const auto parameter : parameters)
    {
        std::string field, value;
        lf_GetFieldValue(parameter, field, value, debugMode);

        if (debugMode == true)
        {
            if (parametersOutput.count(field) == 1)
            {
                throw std::invalid_argument(
                    "ERROR: parameter " + field +
                    " already exists, must be unique\n");
            }
        }

        parametersOutput[field] = value;
    }

    return parametersOutput;
}

std::vector<int> CSVToVectorInt(const std::string csv)
{
    std::vector<int> numbers;
    if (csv.empty())
    {
        return numbers;
    }

    if (csv.find(",") == csv.npos) // check if no commas, one int
    {
        numbers.push_back(std::stoi(csv)); // might need to be checked
    }
    else
    {
        int count = std::count(csv.begin(), csv.end(), ',');
        numbers.reserve(count);

        std::istringstream csvSS(csv);
        std::string value;
        while (std::getline(csvSS, value, ',')) // need to test
        {
            numbers.push_back(std::stoi(csv));
        }
    }

    return numbers;
}

std::string DimsToCSV(const std::vector<std::size_t> &dims)
{
    std::string dimsCSV;

    for (const auto dim : dims)
    {
        dimsCSV += std::to_string(dim) + ",";
    }

    if (dimsCSV.empty() == false)
    {
        dimsCSV.pop_back();
    }

    return dimsCSV;
}

void ConvertUint64VectorToSizetVector(const std::vector<std::uint64_t> &in,
                                      std::vector<std::size_t> &out)
{
    out.clear();
    out.reserve(in.size());
    for (const auto inElement : in)
    {
        out.push_back(static_cast<std::size_t>(inElement));
    }
}

int GrowBuffer(const size_t incomingDataSize, const float growthFactor,
               std::vector<char> &buffer, const size_t position)
{
    const size_t currentCapacity = buffer.capacity();
    const size_t availableSpace = currentCapacity - position;
    const double gf = static_cast<double>(growthFactor);

    if (incomingDataSize < availableSpace)
    {
        return 0;
    }

    const size_t neededCapacity = incomingDataSize + position;
    const double numerator = std::log(static_cast<double>(neededCapacity) /
                                      static_cast<double>(currentCapacity));
    const double denominator = std::log(gf);

    const double n = std::ceil(numerator / denominator);
    const size_t newSize =
        static_cast<size_t>(std::ceil(std::pow(gf, n) * currentCapacity));

    try
    {
        buffer.resize(newSize);
    }
    catch (std::bad_alloc &e)
    {
        return -1;
    }

    return 1;
}

bool IsLittleEndian() noexcept
{
    uint16_t hexa = 0x1234;
    return *reinterpret_cast<uint8_t *>(&hexa) != 0x12; // NOLINT
}

} // namespace adios