Refactored heat transfer example. A config file is now command line argument....

Refactored heat transfer example. A config file is now command line argument. Reader's decomposition is also command line argument.

examples/heatTransfer/ReadMe.md

 examples/heatTransfer
 
-Test that solves a 2D Poisson equation for temperature in homogeneous media
+This example solves a 2D Poisson equation for temperature in homogeneous media
 using finite differences. This examples shows a straight-forward way to hook 
 an application to the ADIOS2 library for its IO.
 
 
-1. read: illustrates the Read API that allows running the reader either as
-
-    * post-mortem to read all output steps
-    * in situ to read step by step as the writer outputs them
-
-2. write: illustrates the Write API as well as has implementations of other IO libraries
+1. write: illustrates the Write API as well as has implementations of other IO libraries
   
     * adios 1.x    
     * hdf5
     * phdf5
 
+2. read: illustrates the Read API that allows running the reader either as
+
+    * post-mortem to read all output steps
+    * in situ to read step by step as the writer outputs them 
+       (need to run with a suitable engine)
+
 3. read_fileonly: illustrates reading all output steps at once (a single read 
    statement) into a single contiguous memory block. This approach only works 
    for post-mortem processing. 
+
+
+
+Example
+
+
+1. Produce an output
+
+Writer usage:  heatTransfer  config output  N  M   nx  ny   steps iterations
+  config: XML config file to use
+  output: name of output data file/stream
+  N:      number of processes in X dimension
+  M:      number of processes in Y dimension
+  nx:     local array size in X dimension per processor
+  ny:     local array size in Y dimension per processor
+  steps:  the total number of steps to output
+  iterations: one step consist of this many iterations
+
+$  mpirun -np 12 ./bin/heatTransfer_write_adios2 ../examples/heatTransfer/heat.xml  heat  4 3  5 10 10 10
+
+
+2. Read the output step-by-step and print data into text files (data.<rank> per reader process)
+
+Reader Usage:   heatRead  config  input  N  M 
+  config: XML config file to use
+  input:  name of input data file/stream
+  N:      number of processes in X dimension
+  M:      number of processes in Y dimension
+
+
+$ mpirun -np 2 ./bin/heatTransfer_read ../examples/heatTransfer/heat.xml heat 2 1 
+
+
+Notes:
+1. 	Engines for file-based output and post-mortem reading: i
+
+   * BPFileWriter/BPFileReader
+   * HDF5Writer/HDF5Reader
+   * ADIOS1Writer/ADIOS1Reader
+
+2. Engines for in situ execution
+
+   * DataManWriter/DataManReader
+     (Must run writer and reader with the same number of processes and same decomposition)
+
+
+

examples/heatTransfer/read/CMakeLists.txt

@@ -4,7 +4,11 @@
 #------------------------------------------------------------------------------#
 
 if(ADIOS2_HAVE_MPI)
-  add_executable(heatTransfer_read heatRead.cpp PrintDataStep.h)
+  add_executable(heatTransfer_read 
+     heatRead.cpp 
+     PrintDataStep.h 
+     ReadSettings.cpp
+  )
   target_link_libraries(heatTransfer_read adios2 MPI::MPI_C)
   target_compile_definitions(heatTransfer_read PRIVATE
    -DDEFAULT_CONFIG=${CMAKE_CURRENT_SOURCE_DIR}/../heat.xml)

examples/heatTransfer/read/heatRead.cpp

@@ -22,29 +22,26 @@
 #include <vector>
 
 #include "PrintDataStep.h"
+#include "ReadSettings.h"
 
-#define str_helper(X) #X
-#define str(X) str_helper(X)
-#ifndef DEFAULT_CONFIG
-#define DEFAULT_CONFIG "../heat.xml"
-#endif
-#define DEFAULT_CONFIG_STR str(DEFAULT_CONFIG)
+void printUsage()
+{
+    std::cout << "Usage: heatRead  config  input  N  M \n"
+              << "  config: XML config file to use\n"
+              << "  input:  name of input data file/stream\n"
+              << "  N:      number of processes in X dimension\n"
+              << "  M:      number of processes in Y dimension\n\n";
+}
 
 int main(int argc, char *argv[])
 {
     MPI_Init(&argc, &argv);
 
-    if (argc < 2)
-    {
-        std::cout << "Not enough arguments: need an input file\n";
-        return 1;
-    }
-    const char *inputfile = argv[1];
-
-    /* World comm spans all applications started with the same aprun command
-     on a Cray XK6. So we have to split and create the local
-     'world' communicator for the reader only.
-     In normal start-up, the communicator will just equal the MPI_COMM_WORLD.
+    /* When writer and reader is launched together with a single mpirun command,
+       the world comm spans all applications. We have to split and create the
+       local 'world' communicator for the reader only.
+       When writer and reader is launched separately, the mpiReaderComm
+       communicator will just equal the MPI_COMM_WORLD.
      */
 
     int wrank, wnproc;
@@ -60,100 +57,87 @@ int main(int argc, char *argv[])
     MPI_Comm_rank(mpiReaderComm, &rank);
     MPI_Comm_size(mpiReaderComm, &nproc);
 
-    adios2::ADIOS ad(std::string(DEFAULT_CONFIG_STR), mpiReaderComm,
-                     adios2::DebugON);
-
-    // Define method for engine creation
-    // 1. Get method def from config file or define new one
-
-    adios2::IO &bpReaderIO = ad.DeclareIO("reader");
-    if (!bpReaderIO.InConfigFile())
+    try
     {
-        // if not defined by user, we can change the default settings
-        // BPFileWriter is the default engine
-        bpReaderIO.SetEngine("ADIOS1Reader");
-        bpReaderIO.SetParameters({{"num_threads", "2"}});
-
-        // ISO-POSIX file output is the default transport (called "File")
-        // Passing parameters to the transport
-        bpReaderIO.AddTransport("File", {{"verbose", "4"}});
-    }
+        ReadSettings settings(argc, argv, rank, nproc);
+        adios2::ADIOS ad(settings.configfile, mpiReaderComm, adios2::DebugON);
 
-    adios2::Engine &bpReader =
-        bpReaderIO.Open(inputfile, adios2::Mode::Read, mpiReaderComm);
+        // Define method for engine creation
+        // 1. Get method def from config file or define new one
 
-    unsigned int gndx;
-    unsigned int gndy;
-    double *T;
-    adios2::Dims readsize;
-    adios2::Dims offset;
-    adios2::Variable<double> *vT = nullptr;
-    bool firstStep = true;
-    int step = 0;
+        adios2::IO &bpReaderIO = ad.DeclareIO("reader");
+        if (!bpReaderIO.InConfigFile())
+        {
+            // if not defined by user, we can change the default settings
+            // BPFileWriter is the default engine
+            bpReaderIO.SetEngine("ADIOS1Reader");
+            bpReaderIO.SetParameters({{"num_threads", "2"}});
+
+            // ISO-POSIX file output is the default transport (called "File")
+            // Passing parameters to the transport
+            bpReaderIO.AddTransport("File", {{"verbose", "4"}});
+        }
 
-    while (true)
-    {
-        adios2::StepStatus status =
-            bpReader.BeginStep(adios2::StepMode::NextAvailable);
-        if (status != adios2::StepStatus::OK)
-            break;
+        adios2::Engine &bpReader = bpReaderIO.Open(
+            settings.inputfile, adios2::Mode::Read, mpiReaderComm);
 
-        if (firstStep)
-        {
-            adios2::Variable<unsigned int> *vgndx =
-                bpReaderIO.InquireVariable<unsigned int>("gndx");
-            gndx = vgndx->m_Value;
-            // bpReader.GetSync<unsigned int>("gndx", gndx);
+        double *T;
+        adios2::Variable<double> *vT = nullptr;
+        bool firstStep = true;
+        int step = 0;
 
-            adios2::Variable<unsigned int> *vgndy =
-                bpReaderIO.InquireVariable<unsigned int>("gndy");
-            gndy = vgndy->m_Value;
-            // bpReader.GetSync<unsigned int>("gndy", gndy);
+        while (true)
+        {
+            adios2::StepStatus status =
+                bpReader.BeginStep(adios2::StepMode::NextAvailable);
+            if (status != adios2::StepStatus::OK)
+                break;
 
-            if (rank == 0)
+            if (firstStep)
             {
-                std::cout << "gndx       = " << gndx << std::endl;
-                std::cout << "gndy       = " << gndy << std::endl;
+                vT = bpReaderIO.InquireVariable<double>("T");
+                unsigned int gndx = vT->m_Shape[0];
+                unsigned int gndy = vT->m_Shape[1];
+
+                if (rank == 0)
+                {
+                    std::cout << "gndx       = " << gndx << std::endl;
+                    std::cout << "gndy       = " << gndy << std::endl;
+                }
+
+                settings.DecomposeArray(gndx, gndy);
+                T = new double[settings.readsize[0] * settings.readsize[1]];
+
+                // Create a 2D selection for the subset
+                vT->SetSelection(adios2::Box<adios2::Dims>(settings.offset,
+                                                           settings.readsize));
+                firstStep = false;
+                MPI_Barrier(mpiReaderComm); // sync processes just for stdout
             }
 
-            // 1D decomposition of the columns, which is inefficient for
-            // reading!
-            readsize.push_back(gndx);
-            readsize.push_back(gndy / nproc);
-            offset.push_back(0LL);
-            offset.push_back(rank * readsize[1]);
-            if (rank == nproc - 1)
+            if (!rank)
             {
-                // last process should read all the rest of columns
-                readsize[1] = gndy - readsize[1] * (nproc - 1);
+                std::cout << "Processing step " << step << std::endl;
             }
-
-            std::cout << "rank " << rank << " reads " << readsize[1]
-                      << " columns from offset " << offset[1] << std::endl;
-
-            vT = bpReaderIO.InquireVariable<double>("T");
-            T = new double[readsize[0] * readsize[1]];
-
-            // Create a 2D selection for the subset
-            vT->SetSelection(adios2::Box<adios2::Dims>(offset, readsize));
-            firstStep = false;
+            // Arrays are read by scheduling one or more of them
+            // and performing the reads at once
+            bpReader.GetDeferred<double>(*vT, T);
+            bpReader.PerformGets();
+
+            printDataStep(T, settings.readsize.data(), settings.offset.data(),
+                          rank, step);
+            bpReader.EndStep();
+            step++;
         }
-
-        if (!rank)
-        {
-            std::cout << "Processing step " << step << std::endl;
-        }
-        // Arrays are read by scheduling one or more of them
-        // and performing the reads at once
-        bpReader.GetDeferred<double>(*vT, T);
-        bpReader.PerformGets();
-
-        printDataStep(T, readsize.data(), offset.data(), rank, step);
-        bpReader.EndStep();
-        step++;
+        bpReader.Close();
+        delete[] T;
     }
-    bpReader.Close();
-    delete[] T;
+    catch (std::invalid_argument &e) // command-line argument errors
+    {
+        std::cout << e.what() << std::endl;
+        printUsage();
+    }
+
     MPI_Finalize();
     return 0;
 }

examples/heatTransfer/write/IO_adios2.cpp

@@ -34,7 +34,7 @@ IO::IO(const Settings &s, MPI_Comm comm)
 
     // Define method for engine creation
 
-    adios2::IO &bpio = *ad->InquireIO("output");
+    adios2::IO &bpio = ad->DeclareIO("output");
     if (!bpio.InConfigFile())
     {
         // if not defined by user, we can change the default settings
@@ -75,8 +75,6 @@ IO::~IO()
 void IO::write(int step, const HeatTransfer &ht, const Settings &s,
                MPI_Comm comm)
 {
-#if 1
-
     bpWriter->BeginStep();
     /* This selection is redundant and not required, since we defined
      * the selection already in DefineVariable(). It is here just as an example.
@@ -87,29 +85,17 @@ void IO::write(int step, const HeatTransfer &ht, const Settings &s,
     varT->SetSelection(
         adios2::Box<adios2::Dims>({s.offsx, s.offsy}, {s.ndx, s.ndy}));
 
-    /* Select the area that we want to write from the data pointer we pass to
-       the
-       writer.
-       Think HDF5 memspace, just not hyperslabs, only a bounding box selection.
-       Engine will copy this bounding box from the data pointer into the output
-       buffer.
-       Size of the bounding box should match the "space" selection which was
-       given
-       above.
-       Default memspace is always the full selection.
-    */
-    varT->SetMemorySelection(adios2::Box<adios2::Dims>({1, 1}, {s.ndx, s.ndy}));
-
-    bpWriter->PutSync<unsigned int>(*varGndx, s.gndx);
-    bpWriter->PutSync<unsigned int>("gndy", s.gndy);
-    bpWriter->PutSync<double>(*varT, ht.data_noghost().data());
-
-    bpWriter->EndStep();
-
-#else
-
+    if (!step)
+    {
+        int rank;
+        MPI_Comm_rank(comm, &rank);
+        if (!rank)
+        {
+            bpWriter->PutSync<unsigned int>(*varGndx, s.gndx);
+            bpWriter->PutSync<unsigned int>("gndy", s.gndy);
+        }
+    }
     bpWriter->PutSync<double>(*varT, ht.data_noghost().data());
+    // bpWriter->PerformPuts();
     bpWriter->EndStep();
-
-#endif
 }

examples/heatTransfer/write/Settings.cpp

@@ -35,37 +35,20 @@ static unsigned int convertToUint(std::string varName, char *arg)
 
 Settings::Settings(int argc, char *argv[], int rank, int nproc) : rank{rank}
 {
-    if (argc < 8)
+    if (argc < 9)
     {
         throw std::invalid_argument("Not enough arguments");
     }
     this->nproc = (unsigned int)nproc;
 
-    outputfile = argv[1];
-    npx = convertToUint("N", argv[2]);
-    npy = convertToUint("M", argv[3]);
-    ndx = convertToUint("nx", argv[4]);
-    ndy = convertToUint("ny", argv[5]);
-    steps = convertToUint("steps", argv[6]);
-    iterations = convertToUint("iterations", argv[7]);
-
-    if (argc == 9)
-    {
-        const std::string asyncArg(argv[8]);
-        if (asyncArg == "ON" || asyncArg == "on")
-        {
-            async = true;
-        }
-        else if (asyncArg == "OFF" || asyncArg == "off")
-        {
-            // nothing off is default
-        }
-        else
-        {
-            throw std::invalid_argument("ERROR: wrong async argument " +
-                                        asyncArg + " must be on or off\n");
-        }
-    }
+    configfile = argv[1];
+    outputfile = argv[2];
+    npx = convertToUint("N", argv[3]);
+    npy = convertToUint("M", argv[4]);
+    ndx = convertToUint("nx", argv[5]);
+    ndy = convertToUint("ny", argv[6]);
+    steps = convertToUint("steps", argv[7]);
+    iterations = convertToUint("iterations", argv[8]);
 
     if (npx * npy != this->nproc)
     {

examples/heatTransfer/write/Settings.h

@@ -18,6 +18,7 @@ class Settings
 
 public:
     // user arguments
+    std::string configfile;
     std::string outputfile;
     unsigned int npx;        // Number of processes in X (slow) dimension
     unsigned int npy;        // Number of processes in Y (fast) dimension

examples/heatTransfer/write/main.cpp

@@ -12,7 +12,6 @@
  */
 #include <mpi.h>
 
-#include <future> //std::future, std::async
 #include <iostream>
 #include <memory>
 #include <stdexcept>
@@ -24,26 +23,28 @@
 
 void printUsage()
 {
-    std::cout << "Usage: heatTransfer  output  N  M   nx  ny   steps "
-                 "iterations async\n"
-              << "  output: name of output file\n"
+    std::cout << "Usage: heatTransfer  config   output  N  M   nx  ny   steps "
+                 "iterations\n"
+              << "  config: XML config file to use\n"
+              << "  output: name of output data file/stream\n"
               << "  N:      number of processes in X dimension\n"
               << "  M:      number of processes in Y dimension\n"
               << "  nx:     local array size in X dimension per processor\n"
               << "  ny:     local array size in Y dimension per processor\n"
               << "  steps:  the total number of steps to output\n"
-              << "  iterations: one step consist of this many iterations\n"
-              << "  async: on or off (default) \n\n";
+              << "  iterations: one step consist of this many iterations\n\n";
 }
 
 int main(int argc, char *argv[])
 {
     MPI_Init(&argc, &argv);
-    /* World comm spans all applications started with the same aprun command
-       on a Cray XK6. So we have to split and create the local
-       'world' communicator for heat_transfer only.
-       In normal start-up, the communicator will just equal the MPI_COMM_WORLD.
-    */
+
+    /* When writer and reader is launched together with a single mpirun command,
+       the world comm spans all applications. We have to split and create the
+       local 'world' communicator mpiHeatTransferComm for the writer only.
+       When writer and reader is launched separately, the mpiHeatTransferComm
+       communicator will just equal the MPI_COMM_WORLD.
+     */
 
     int wrank, wnproc;
     MPI_Comm_rank(MPI_COMM_WORLD, &wrank);
@@ -71,17 +72,7 @@ int main(int argc, char *argv[])
         ht.exchange(mpiHeatTransferComm);
         // ht.printT("Heated T:", mpiHeatTransferComm);
 
-        std::future<void> futureWrite;
-        if (settings.async)
-        {
-            futureWrite =
-                std::async(std::launch::async, &IO::write, &io, 0, std::ref(ht),
-                           std::ref(settings), mpiHeatTransferComm);
-        }
-        else
-        {
-            io.write(0, ht, settings, mpiHeatTransferComm);
-        }
+        io.write(0, ht, settings, mpiHeatTransferComm);
 
         for (unsigned int t = 1; t <= settings.steps; ++t)
         {
@@ -94,21 +85,7 @@ int main(int argc, char *argv[])
                 ht.heatEdges();
             }
 
-            if (settings.async)
-            {
-                futureWrite.get();
-                futureWrite = std::async(std::launch::async, &IO::write, &io, t,
-                                         std::ref(ht), std::ref(settings),
-                                         mpiHeatTransferComm);
-            }
-            else
-            {
-                io.write(t, ht, settings, mpiHeatTransferComm);
-            }
-        }
-        if (settings.async)
-        {
-            futureWrite.get();
+            io.write(t, ht, settings, mpiHeatTransferComm);
         }
         MPI_Barrier(mpiHeatTransferComm);