Commit 377a7d55 authored by Eisenhauer, Greg's avatar Eisenhauer, Greg Committed by Bolea Sanchez, Vicente Adolfo
Browse files

Merge pull request #3823 from eisenhauer/SstMemSel 

Add Memory Selection to SST, and prototype a mechanism for running st…

(cherry picked from commit c503940b)
parent 15326e65

source/adios2/engine/sst/SstWriter.tcc

+51 −11
Original line number Diff line number Diff line
@@ -70,22 +70,62 @@ void SstWriter::PutSyncCommon(Variable<T> &variable, const T *values) 
                          DimCount, Shape, Count, Start, values);

        }

        else

        {

            if (!variable.m_MemoryCount.empty())

            {

                size_t ObjSize;

                if (variable.m_Type == DataType::Struct)

                {

                    ObjSize = variable.m_ElementSize;

                }

                else

                {

                    ObjSize = helper::GetDataTypeSize(variable.m_Type);

                }



                const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage);

                helper::DimsArray MemoryStart(variable.m_MemoryStart);

                helper::DimsArray MemoryCount(variable.m_MemoryCount);

                helper::DimsArray varCount(variable.m_Count);



                int DimCount = (int)variable.m_Count.size();

                helper::DimsArray ZeroDims(DimCount, (size_t)0);

                // get a temporary span then fill with memselection now

                format::BufferV::BufferPos bp5span(0, 0, 0);



                m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(),

                                         variable.m_Type, variable.m_ElementSize, DimCount, Shape,

                                         Count, Start, nullptr, false, &bp5span);

                void *ptr = m_BP5Serializer->GetPtr(bp5span.bufferIdx, bp5span.posInBuffer);



                if (!sourceRowMajor)

                {

                    std::reverse(MemoryStart.begin(), MemoryStart.end());

                    std::reverse(MemoryCount.begin(), MemoryCount.end());

                    std::reverse(varCount.begin(), varCount.end());

                }

                helper::NdCopy((const char *)values, helper::CoreDims(ZeroDims), MemoryCount,

                               sourceRowMajor, false, (char *)ptr, MemoryStart, varCount,

                               sourceRowMajor, false, (int)ObjSize, helper::CoreDims(),

                               helper::CoreDims(), helper::CoreDims(), helper::CoreDims(),

                               false /* safemode */, variable.m_MemSpace);

            }

            else

            {

                if (variable.m_Type == DataType::String)

                {

                    std::string &source = *(std::string *)values;

                    void *p = &(source[0]);

                m_BP5Serializer->Marshal(

                    (void *)&variable, variable.m_Name.c_str(), variable.m_Type,

                    variable.m_ElementSize, DimCount, Shape, Count, Start, &p,

                    true, nullptr);

                    m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(),

                                             variable.m_Type, variable.m_ElementSize, DimCount,

                                             Shape, Count, Start, &p, true, nullptr);

                }

                else

                {

                m_BP5Serializer->Marshal(

                    (void *)&variable, variable.m_Name.c_str(), variable.m_Type,

                    variable.m_ElementSize, DimCount, Shape, Count, Start,

                    values, true, nullptr);

                    m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(),

                                             variable.m_Type, variable.m_ElementSize, DimCount,

                                             Shape, Count, Start, values, true, nullptr);

                }

            }

        }

    }

testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp

+91 −25
Original line number Diff line number Diff line
@@ -15,6 +15,8 @@ 
#include "../SmallTestData.h"



std::string engineName; // comes from command line

bool DoWrite = true;

bool DoRead = true;



namespace

{
@@ -178,9 +180,13 @@ void AssignStep3D(const size_t step, std::vector<std::complex<double>> &vector, 


} // end anonymous namespace



#if ADIOS2_USE_MPI

MPI_Comm testComm;

#endif



void BPSteps1D(const size_t ghostCells)

{

    const std::string fname("BPSteps1D_" + std::to_string(ghostCells) + ".bp");

    const std::string fname("BPSteps1D_" + std::to_string(ghostCells));



    int mpiRank = 0, mpiSize = 1;

    // Number of rows
@@ -190,15 +196,16 @@ void BPSteps1D(const size_t ghostCells) 
    const size_t NSteps = 3;



#if ADIOS2_USE_MPI

    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);

    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);

    MPI_Comm_rank(testComm, &mpiRank);

    MPI_Comm_size(testComm, &mpiSize);

#endif



#if ADIOS2_USE_MPI

    adios2::ADIOS adios(MPI_COMM_WORLD);

    adios2::ADIOS adios(testComm);

#else

    adios2::ADIOS adios;

#endif

    if (DoWrite)

    {

        adios2::IO io = adios.DeclareIO("WriteIO");
@@ -207,6 +214,7 @@ void BPSteps1D(const size_t ghostCells) 
            io.SetEngine(engineName);

        }



        io.SetParameters("StatsLevel=1");

        const adios2::Dims shape{static_cast<size_t>(Nx * mpiSize)};

        const adios2::Dims start{static_cast<size_t>(Nx * mpiRank)};

        const adios2::Dims count{Nx};
@@ -272,9 +280,10 @@ void BPSteps1D(const size_t ghostCells) 
        bpWriter.Close();

    }

#if ADIOS2_USE_MPI

    MPI_Barrier(MPI_COMM_WORLD);

    MPI_Barrier(testComm);

#endif

    // Reader

    if (DoRead)

    {

        adios2::IO io = adios.DeclareIO("ReadIO");
@@ -403,8 +412,7 @@ void BPSteps1D(const size_t ghostCells) 


void BPSteps2D4x2(const size_t ghostCells)

{

    const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells) +

                            ".bp");

    const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells));



    int mpiRank = 0, mpiSize = 1;

    // Number of rows
@@ -418,15 +426,16 @@ void BPSteps2D4x2(const size_t ghostCells) 
    const size_t NSteps = 3;



#if ADIOS2_USE_MPI

    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);

    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);

    MPI_Comm_rank(testComm, &mpiRank);

    MPI_Comm_size(testComm, &mpiSize);

#endif



#if ADIOS2_USE_MPI

    adios2::ADIOS adios(MPI_COMM_WORLD);

    adios2::ADIOS adios(testComm);

#else

    adios2::ADIOS adios;

#endif

    if (DoWrite)

    {

        adios2::IO io = adios.DeclareIO("WriteIO");
@@ -500,9 +509,10 @@ void BPSteps2D4x2(const size_t ghostCells) 
        bpWriter.Close();

    }

#if ADIOS2_USE_MPI

    MPI_Barrier(MPI_COMM_WORLD);

    MPI_Barrier(testComm);

#endif

    // Reader

    if (DoRead)

    {

        adios2::IO io = adios.DeclareIO("ReadIO");
@@ -641,8 +651,7 @@ void BPSteps2D4x2(const size_t ghostCells) 


void BPSteps3D8x2x4(const size_t ghostCells)

{

    const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells) +

                            ".bp");

    const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells));



    int mpiRank = 0, mpiSize = 1;

    // Number of rows
@@ -658,15 +667,16 @@ void BPSteps3D8x2x4(const size_t ghostCells) 
    const size_t NSteps = 3;



#if ADIOS2_USE_MPI

    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);

    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);

    MPI_Comm_rank(testComm, &mpiRank);

    MPI_Comm_size(testComm, &mpiSize);

#endif



#if ADIOS2_USE_MPI

    adios2::ADIOS adios(MPI_COMM_WORLD);

    adios2::ADIOS adios(testComm);

#else

    adios2::ADIOS adios;

#endif

    if (DoWrite)

    {

        adios2::IO io = adios.DeclareIO("WriteIO");
@@ -750,9 +760,10 @@ void BPSteps3D8x2x4(const size_t ghostCells) 
        bpWriter.Close();

    }

#if ADIOS2_USE_MPI

    MPI_Barrier(MPI_COMM_WORLD);

    MPI_Barrier(testComm);

#endif

    // Reader

    if (DoRead)

    {

        adios2::IO io = adios.DeclareIO("ReadIO");
@@ -950,25 +961,80 @@ INSTANTIATE_TEST_SUITE_P(ghostCells, BPWriteMemSelReadVector, 


int main(int argc, char **argv)

{

#if ADIOS2_USE_MPI

    int provided;



    // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP

    MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided);

#endif



    int result;

    ::testing::InitGoogleTest(&argc, argv);

    int bare_arg = 0;



    if (argc > 1)

    for (int i = 1; i < argc; i++)

    {

        if (strcmp(argv[i], "-do_write") == 0)

        {

            DoWrite = true;

            DoRead = false;

        }

        else if (strcmp(argv[i], "-do_read") == 0)

        {

            DoWrite = false;

            DoRead = true;

        }

        else if (argv[i][0] == '-')

        {

            std::cerr << "Unknown argument: " << argv[i] << std::endl;

            exit(1);

        }

        else

        {

            std::string fname;

            std::string engineParams;

            if (bare_arg == 0)

            {

                /* first arg without -- is engine */

                engineName = std::string(argv[1]);

                bare_arg++;

            }

            else if (bare_arg == 1)

            {

                /* second arg without -- is filename */

                //                fname = std::string(argv[1]);

                bare_arg++;

            }

            else if (bare_arg == 2)

            {

                //                engineParams = ParseEngineParams(argv[1]);

                bare_arg++;

            }

            else

            {



                throw std::invalid_argument("Unknown argument \"" + std::string(argv[1]) + "\"");

            }

        }

    }



#if ADIOS2_USE_MPI

    int provided;

    int thread_support_level =

        (engineName == "SST" || engineName == "sst") ? MPI_THREAD_MULTIPLE : MPI_THREAD_SINGLE;



    // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP

    MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided);



    int key;

    MPI_Comm_rank(MPI_COMM_WORLD, &key);



    const unsigned int color = (DoRead & !DoWrite) ? 1 : 0;



    MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm);

#endif



    result = RUN_ALL_TESTS();



#if ADIOS2_USE_MPI

#ifdef CRAY_MPICH_VERSION

    MPI_Barrier(MPI_COMM_WORLD);

#else

    MPI_Finalize();

#endif

#endif



    return result;

testing/adios2/engine/staging-common/CMakeLists.txt

+5 −0
Original line number Diff line number Diff line
@@ -149,8 +149,13 @@ set (ALL_SIMPLE_TESTS "") 
list (APPEND ALL_SIMPLE_TESTS ${SIMPLE_TESTS} ${SIMPLE_FORTRAN_TESTS} ${SIMPLE_MPI_TESTS} ${SIMPLE_ZFP_TESTS})



set (SST_SPECIFIC_TESTS  "")

import_bp_test(WriteMemorySelectionRead 1 1)

list (APPEND SST_SPECIFIC_TESTS  "WriteMemorySelectionRead.1x1")

list (APPEND SST_SPECIFIC_TESTS  "1x1.SstRUDP;1x1.LocalMultiblock;RoundRobinDistribution.1x1x3;AllToAllDistribution.1x1x3;OnDemandSingle.1x1")



if (ADIOS2_HAVE_MPI)

  import_bp_test(WriteMemorySelectionRead 3 3)

  list (APPEND SST_SPECIFIC_TESTS  "WriteMemorySelectionRead.3x3")

  list (APPEND SST_SPECIFIC_TESTS  "2x3.SstRUDP;2x1.LocalMultiblock;5x3.LocalMultiblock;")

endif()

testing/adios2/engine/staging-common/TestSupp.cmake

+12 −0
Original line number Diff line number Diff line
@@ -321,3 +321,15 @@ function(from_hex HEX DEC) 
    set(${DEC} ${_res} PARENT_SCOPE)

endfunction()



function(import_bp_test BASENAME WRITE_SCALE READ_SCALE)

    set (WRITER_POSTFIX "Serial")

    set (READER_POSTFIX "Serial")

    if(ADIOS2_HAVE_MPI)

        set (WRITER_POSTFIX "MPI")

    endif()

    if(ADIOS2_HAVE_MPI)

        set (READER_POSTFIX "MPI")

    endif()

  set (${BASENAME}.${WRITE_SCALE}x${READ_SCALE}_CMD "run_test.py.$<CONFIG> -nw ${WRITE_SCALE} -nr ${READ_SCALE} --warg=-do_write --rarg=-do_read -w $<TARGET_FILE:Test.Engine.BP.${BASENAME}.${WRITER_POSTFIX}> -r $<TARGET_FILE:Test.Engine.BP.${BASENAME}.${READER_POSTFIX}>" PARENT_SCOPE)



endfunction()