Unverified Commit ae4b1dae authored by pnorbert's avatar pnorbert Committed by GitHub
Browse files

Merge pull request #989 from keichi/reorganize-params 

Allow specifying read/write method parameters in adios_reorganize
parents 79e162f1 3e5b97d6

source/utils/adios_reorganize/Reorganize.cpp

+26 −6
Original line number Diff line number Diff line
@@ -30,6 +30,7 @@ 
#include "adios2/core/Engine.h"

#include "adios2/core/IO.h"

#include "adios2/helper/adiosFunctions.h"

#include "adios2/helper/adiosString.h"



// C headers

#include <cerrno>
@@ -56,9 +57,9 @@ Reorganize::Reorganize(int argc, char *argv[]) 
    infilename = std::string(argv[1]);

    outfilename = std::string(argv[2]);

    rmethodname = std::string(argv[3]);

    rmethodparams = std::string(argv[4]);

    rmethodparam_str = std::string(argv[4]);

    wmethodname = std::string(argv[5]);

    wmethodparams = std::string(argv[6]);

    wmethodparam_str = std::string(argv[6]);



    int nd = 0;

    int j = 7;
@@ -113,9 +114,9 @@ void Reorganize::Run() 
    print0("Input stream            = ", infilename);

    print0("Output stream           = ", outfilename);

    print0("Read method             = ", rmethodname);

    print0("Read method parameters  = ", rmethodparams);

    print0("Read method parameters  = ", rmethodparam_str);

    print0("Write method            = ", wmethodname);

    print0("Write method parameters = ", wmethodparams);

    print0("Write method parameters = ", wmethodparam_str);



#ifdef ADIOS2_HAVE_MPI

    core::ADIOS adios(comm, true, "C++");
@@ -127,11 +128,12 @@ void Reorganize::Run() 
    print0("Waiting to open stream ", infilename, "...");



    io.SetEngine(rmethodname);

    io.SetParameter("verbose", "5");

    io.SetParameters(rmethodparams);

    core::Engine &rStream = io.Open(infilename, adios2::Mode::Read);

    // rStream.FixedSchedule();



    io.SetEngine(wmethodname);

    io.SetParameters(wmethodparams);

    core::Engine &wStream = io.Open(outfilename, adios2::Mode::Write);



    int steps = 0;
@@ -205,7 +207,25 @@ void Reorganize::print0(Arg &&arg, Args &&... args) 
    }

}



void Reorganize::ParseArguments() {}

Params Reorganize::parseParams(const std::string &param_str)

{

    std::istringstream ss(param_str);

    std::vector<std::string> kvs;

    std::string kv;



    while (std::getline(ss, kv, ','))

    {

        kvs.push_back(kv);

    }



    return helper::BuildParametersMap(kvs, true);

}



void Reorganize::ParseArguments()

{

    rmethodparams = parseParams(rmethodparam_str);

    wmethodparams = parseParams(wmethodparam_str);

}



void Reorganize::ProcessParameters() const {}

source/utils/adios_reorganize/Reorganize.h

+11 −6
Original line number Diff line number Diff line
@@ -66,14 +66,15 @@ private: 
                        const core::DataMap &attributes, int step);

    int ReadWrite(core::Engine &rStream, core::Engine &wStream, core::IO &io,

                  const core::DataMap &variables, int step);

    Params parseParams(const std::string &param_str);



    // Input arguments

    std::string infilename;       // File/stream to read

    std::string outfilename;      // File to write

    std::string wmethodname;      // ADIOS write method

    std::string wmethodparams; // ADIOS write method

    std::string wmethodparam_str; // ADIOS write method parameter string

    std::string rmethodname;      // ADIOS read method

    std::string rmethodparams; // ADIOS read method

    std::string rmethodparam_str; // ADIOS read method parameter string



    static const int max_read_buffer_size = 1024 * 1024 * 1024;

    static const int max_write_buffer_size = 1024 * 1024 * 1024;
@@ -86,6 +87,10 @@ private: 
    int numproc = 1;

    MPI_Comm comm;



    // Read/write method parameters

    Params rmethodparams;

    Params wmethodparams;



    uint64_t write_total = 0;   // data size read/written by one processor

    uint64_t largest_block = 0; // the largest variable block one process reads