Unverified Commit def25d6a authored by Bolea Sanchez, Vicente Adolfo's avatar Bolea Sanchez, Vicente Adolfo Committed by GitHub
Browse files

Merge pull request #5001 from vicentebolea/backport-from-master 

backport from master
parents 0c7a8d73 66a13b2b

REUSE.toml

+6 −0
Original line number Diff line number Diff line
@@ -102,6 +102,12 @@ precedence = "aggregate" 
SPDX-FileCopyrightText = "mingw-w64 contributors"

SPDX-License-Identifier = "LicenseRef-public-domain"



[[annotations]]

path = "source/adios2/toolkit/derived/parser/pregen-source/*"

precedence = "aggregate"

SPDX-FileCopyrightText = "Oak Ridge National Laboratory and Contributors"

SPDX-License-Identifier = "Apache-2.0"



[[annotations]]

path = ["thirdparty/CMakeLists.txt", "thirdparty/atl/CMakeLists.txt", "thirdparty/dill/CMakeLists.txt", "thirdparty/enet/CMakeLists.txt", "thirdparty/EVPath/CMakeLists.txt", "thirdparty/ffs/CMakeLists.txt", "thirdparty/GTest/CMakeLists.txt", "thirdparty/KWSys/CMakeLists.txt", "thirdparty/mingw-w64/CMakeLists.txt", "thirdparty/nanobind/CMakeLists.txt", "thirdparty/nlohmann_json/CMakeLists.txt", "thirdparty/perfstubs/CMakeLists.txt", "thirdparty/pugixml/CMakeLists.txt", "thirdparty/yaml-cpp/CMakeLists.txt"]

precedence = "aggregate"

source/adios2/CMakeLists.txt

+5 −4
Original line number Diff line number Diff line
@@ -149,7 +149,7 @@ set(maybe_adios2_core_derived) 
if(ADIOS2_HAVE_Derived_Variable)

  target_sources(adios2_core PRIVATE

    core/VariableDerived.cpp

    toolkit/derived/Expression.cpp

    toolkit/derived/ExprCodeStream.cpp

    toolkit/derived/Function.cpp)

  set_target_properties(adios2_core PROPERTIES

    INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>")
@@ -182,17 +182,18 @@ if(ADIOS2_HAVE_Derived_Variable) 
  elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "IntelLLVM")

    SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable)

  elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")

    SET_SOURCE_FILES_PROPERTIES(toolkit/derived/Expression.cpp toolkit/derived/Function.cpp PROPERTIES COMPILE_FLAGS "/wd4005 /wd4065 /wd4267 -DYY_NO_UNISTD_H")

    SET_SOURCE_FILES_PROPERTIES(toolkit/derived/ExprCodeStream.cpp toolkit/derived/Function.cpp PROPERTIES COMPILE_FLAGS "/wd4005 /wd4065 /wd4267 -DYY_NO_UNISTD_H")

  endif()



  add_library(adios2_core_derived

    ${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp

    ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp

    toolkit/derived/parser/ASTDriver.cpp

    toolkit/derived/parser/ASTNode.cpp)

    toolkit/derived/parser/ASTNode.cpp

    toolkit/derived/parser/ASTToExprNode.cpp)

  set_target_properties(adios2_core_derived PROPERTIES

    VISIBILITY_INLINES_HIDDEN ON

    INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>"

    INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source>;$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>"

    EXPORT_NAME core_derived

    OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_derived)

  target_link_libraries(adios2_core PRIVATE adios2_core_derived)

source/adios2/core/IO.cpp

+26 −14
Original line number Diff line number Diff line
@@ -7,6 +7,8 @@ 
#include "IO.h"

#include "IO.tcc"



#include <cstdlib>

#include <iostream>

#include <memory>

#include <mutex>

#include <sstream>
@@ -948,9 +950,9 @@ VariableDerived &IO::DefineDerivedVariable(const std::string &name, const std::s 
        }

    }



    derived::Expression derived_exp(exp_string);

    std::vector<std::string> var_list = derived_exp.VariableNameList();

    bool isConstant = true;

    // Parse expression string into ExprNode tree

    derived::ExprNode exprTree = detail::ParseToExprNode(exp_string);

    std::vector<std::string> var_list = derived::VariableNameList(exprTree);

    std::map<std::string, DataType> name_to_type;

    std::map<std::string, std::tuple<Dims, Dims, Dims>> name_to_dims;

    // check correctness for the variable names and types within the expression
@@ -963,22 +965,32 @@ VariableDerived &IO::DefineDerivedVariable(const std::string &name, const std::s 
                                                     " in defining the derived variable " + name);

        DataType var_type = InquireVariableType(var_name);

        name_to_type.insert({var_name, var_type});

        if ((itVariable->second)->IsConstantDims() == false)

            isConstant = false;

        name_to_dims.insert({var_name,

                             {(itVariable->second)->m_Start, (itVariable->second)->m_Count,

                              (itVariable->second)->m_Shape}});

    }

    // set the type of the expression and check correcness

    DataType expressionType = derived_exp.GetType(name_to_type);

    // set the initial shape of the expression and check correcness

    derived_exp.SetDims(name_to_dims);

    // std::cout << "Derived variable " << name << ": PASS : initial variable dimensions are valid"

    //          << std::endl;



    // create derived variable with the expression

    // Build the expression code stream: GenerateCode -> ResolveTypes -> ConstantFold -> PlanBuffers

    derived::ExprCodeStream codeStream = derived::GenerateCode(exprTree);

    derived::ResolveTypes(codeStream, name_to_type);

    derived::ConstantFold(codeStream);

    derived::PlanBuffers(codeStream);

    codeStream.ExprString = exp_string;

    DataType expressionType = codeStream.OutputType;



    {

        static bool verbose = (getenv("DerivedVerbose") != nullptr);

        if (verbose)

            std::cerr << derived::DumpCodeStream(codeStream);

    }



    // Compute output dims before construction (VariableBase validates in InitShapeType)

    auto outDims = derived::GetDims(codeStream, name_to_dims);



    auto itVariablePair = m_VariablesDerived.emplace(

        name, std::make_unique<VariableDerived>(name, derived_exp, expressionType, isConstant,

        name, std::make_unique<VariableDerived>(name, std::move(exprTree), std::move(codeStream),

                                                exp_string, expressionType, std::get<2>(outDims),

                                                std::get<0>(outDims), std::get<1>(outDims), false,

                                                varType, name_to_type));

    VariableDerived &variable = static_cast<VariableDerived &>(*itVariablePair.first->second);

source/adios2/core/VariableDerived.cpp

+18 −13
Original line number Diff line number Diff line
@@ -12,13 +12,15 @@ namespace adios2 
namespace core

{



VariableDerived::VariableDerived(const std::string &name, adios2::derived::Expression expr,

                                 const DataType exprType, const bool isConstant,

                                 const DerivedVarType varType,

VariableDerived::VariableDerived(const std::string &name, adios2::derived::ExprNode exprTree,

                                 adios2::derived::ExprCodeStream codeStream,

                                 const std::string &exprString, const DataType exprType,

                                 const Dims &shape, const Dims &start, const Dims &count,

                                 const bool isConstant, const DerivedVarType varType,

                                 const std::map<std::string, DataType> nameToType)

: VariableBase(name, exprType, helper::GetDataTypeSize(exprType), expr.GetShape(), expr.GetStart(),

               expr.GetCount(), isConstant),

  m_DerivedType(varType), m_NameToType(nameToType), m_Expr(expr)

: VariableBase(name, exprType, helper::GetDataTypeSize(exprType), shape, start, count, isConstant),

  m_DerivedType(varType), m_NameToType(nameToType), m_ExprTree(std::move(exprTree)),

  m_CodeStream(std::move(codeStream)), m_ExprString(exprString)

{

    if (varType != DerivedVarType::StoreData)

        m_WriteData = false;
@@ -26,13 +28,16 @@ VariableDerived::VariableDerived(const std::string &name, adios2::derived::Expre 


DerivedVarType VariableDerived::GetDerivedType() { return m_DerivedType; }



std::vector<std::string> VariableDerived::VariableNameList() { return m_Expr.VariableNameList(); }

std::vector<std::string> VariableDerived::VariableNameList() { return m_CodeStream.InputVarNames; }



void VariableDerived::UpdateExprDim(std::map<std::string, std::tuple<Dims, Dims, Dims>> NameToDims)

{

    m_Expr.SetDims(NameToDims);

    m_Shape = m_Expr.GetShape();

    m_Start = m_Expr.GetStart();

    m_Count = m_Expr.GetCount();

    auto outDims = adios2::derived::GetDims(m_CodeStream, NameToDims);

    m_Shape = std::get<2>(outDims);

    m_Start = std::get<0>(outDims);

    m_Count = std::get<1>(outDims);

    if (!m_Shape.empty())

        m_ShapeID = ShapeID::GlobalArray;

}



std::vector<std::tuple<void *, Dims, Dims>>
@@ -80,7 +85,7 @@ VariableDerived::ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInf 
        inputData.insert({variable.first, varData});

    }

    std::vector<adios2::derived::DerivedData> outputData =

        m_Expr.ApplyExpression(numBlocks, inputData);

        adios2::derived::Execute(m_CodeStream, numBlocks, inputData);



    std::vector<std::tuple<void *, Dims, Dims>> blockData;

    for (size_t i = 0; i < numBlocks; i++)
@@ -115,7 +120,7 @@ VariableDerived::CreateEmptyData(std::map<std::string, std::unique_ptr<MinVarInf 
            nameToDims.insert({variable.first, varDims});

        }



        auto outputDims = m_Expr.GetDims(nameToDims);

        auto outputDims = adios2::derived::GetDims(m_CodeStream, nameToDims);



        blockData.push_back({nullptr, std::get<0>(outputDims), std::get<1>(outputDims)});

    }

source/adios2/core/VariableDerived.h

+10 −4
Original line number Diff line number Diff line
@@ -8,7 +8,8 @@ 
#define ADIOS2_CORE_VARIABLE_DERIVED_H_



#include "adios2/core/VariableBase.h"

#include "adios2/toolkit/derived/Expression.h"

#include "adios2/toolkit/derived/ExprCodeStream.h"

#include "adios2/toolkit/derived/ExprNode.h"



namespace adios2

{
@@ -28,9 +29,14 @@ class VariableDerived : public VariableBase 
                    size_t numBlocks);



public:

    adios2::derived::Expression m_Expr;

    VariableDerived(const std::string &name, adios2::derived::Expression expr,

                    const DataType exprType, const bool isConstant, const DerivedVarType varType,

    adios2::derived::ExprNode m_ExprTree;

    adios2::derived::ExprCodeStream m_CodeStream;

    std::string m_ExprString;



    VariableDerived(const std::string &name, adios2::derived::ExprNode exprTree,

                    adios2::derived::ExprCodeStream codeStream, const std::string &exprString,

                    const DataType exprType, const Dims &shape, const Dims &start,

                    const Dims &count, const bool isConstant, const DerivedVarType varType,

                    const std::map<std::string, DataType> nameToType);

    ~VariableDerived() = default;