Merge pull request #207 from williamfgc/profiling_json

Moved profiling.log to profiling.json format

source/adios2/engine/bp/BPFileWriter.cpp

@@ -71,37 +71,12 @@ void BPFileWriter::Close(const int transportIndex)
                                    m_BP1Writer.m_HeapBuffer.m_DataPosition,
                                    transportIndex);
 
-    // close them
     m_TransportsManager.CloseFiles(transportIndex);
 
-    if (m_BP1Writer.m_Profiler.IsActive)
+    if (m_BP1Writer.m_Profiler.IsActive &&
+        m_TransportsManager.AllTransportsClosed())
     {
-        // aggregate and write profiling.log
-        if (m_TransportsManager.AllTransportsClosed())
-        {
-            auto transportTypes = m_TransportsManager.GetTransportsTypes();
-            auto transportProfilers =
-                m_TransportsManager.GetTransportsProfilers();
-
-            const std::string log(m_BP1Writer.GetRankProfilingLog(
-                transportTypes, transportProfilers));
-            // TODO profiling.log from rank0
-
-            const std::string profilingLog(
-                m_BP1Writer.AggregateProfilingLog(log));
-
-            if (m_BP1Writer.m_BP1Aggregator.m_RankMPI == 0)
-            {
-                transport::FileStream profilingLogStream(m_MPIComm,
-                                                         m_DebugMode);
-                auto bpBaseNames = m_BP1Writer.GetBPBaseNames({m_Name});
-                profilingLogStream.Open(bpBaseNames[0] + "/profiling.log",
-                                        OpenMode::Write);
-                profilingLogStream.Write(profilingLog.c_str(),
-                                         profilingLog.size());
-                profilingLogStream.Close();
-            }
-        }
+        WriteProfilingJSONFile();
     }
 }
 
@@ -147,4 +122,26 @@ void BPFileWriter::InitBPBuffer()
     }
 }
 
+void BPFileWriter::WriteProfilingJSONFile()
+{
+    auto transportTypes = m_TransportsManager.GetTransportsTypes();
+    auto transportProfilers = m_TransportsManager.GetTransportsProfilers();
+
+    const std::string lineJSON(
+        m_BP1Writer.GetRankProfilingJSON(transportTypes, transportProfilers));
+
+    const std::string profilingJSON(
+        m_BP1Writer.AggregateProfilingJSON(lineJSON));
+
+    if (m_BP1Writer.m_BP1Aggregator.m_RankMPI == 0)
+    {
+        transport::FileStream profilingJSONStream(m_MPIComm, m_DebugMode);
+        auto bpBaseNames = m_BP1Writer.GetBPBaseNames({m_Name});
+        profilingJSONStream.Open(bpBaseNames[0] + "/profiling.json",
+                                 OpenMode::Write);
+        profilingJSONStream.Write(profilingJSON.c_str(), profilingJSON.size());
+        profilingJSONStream.Close();
+    }
+}
+
 } // end namespace adios

source/adios2/engine/bp/BPFileWriter.h

@@ -77,6 +77,10 @@ private:
      */
     template <class T>
     void DoWriteCommon(Variable<T> &variable, const T *values);
+
+    /** Write a profiling.json file from m_BP1Writer and m_TransportsManager
+     * profilers*/
+    void WriteProfilingJSONFile();
 };
 
 } // end namespace adios

source/adios2/toolkit/format/bp1/BP1Aggregator.cpp

@@ -33,9 +33,9 @@ BP1Aggregator::BP1Aggregator(MPI_Comm mpiComm, const bool debugMode)
     MPI_Comm_size(m_MPIComm, &m_SizeMPI);
 }
 
-std::string BP1Aggregator::GetGlobalProfilingLog(const std::string &rankLog)
+std::string BP1Aggregator::GetGlobalProfilingJSON(const std::string &rankLog)
 {
-    std::string profilingLog;
+    std::string profilingJSON;
 
     if (m_RankMPI == 0)
     {
@@ -80,38 +80,17 @@ std::string BP1Aggregator::GetGlobalProfilingLog(const std::string &rankLog)
 
         // write global string
         // key is to reserve memory first
-        profilingLog.reserve(rankLog.size() * m_SizeMPI);
+        profilingJSON.reserve(rankLog.size() * m_SizeMPI);
 
-        profilingLog += "{\n";
-        profilingLog += rankLog + ",\n";
+        profilingJSON += "[\n";
+        profilingJSON += rankLog;
         for (unsigned int i = 1; i < sizeMPI; ++i)
         {
-            const std::string rankLogStr(rankLogs[i - 1].data(),
-                                         rankLogs[i - 1].size());
-            profilingLog += rankLogStr + ",";
-            if (i < sizeMPI - 1)
-            {
-                profilingLog += "\n";
-            }
+            profilingJSON += ",\n";
+            profilingJSON.append(rankLogs[i - 1].data(),
+                                 rankLogs[i - 1].size());
         }
-        profilingLog.pop_back(); // eliminate trailing comma
-        profilingLog += "\n";
-        profilingLog += "}\n";
-
-        //        // write to file
-        //        std::ofstream logStream(fileName);
-        //        if (m_DebugMode)
-        //        {
-        //            if (!logStream)
-        //            {
-        //                throw std::ios_base::failure(
-        //                    "ERROR: couldn't open profiling file " + fileName
-        //                    + "\n");
-        //            }
-        //        }
-        //
-        //        logStream.write(logFile.c_str(), logFile.size());
-        //        logStream.close();
+        profilingJSON += "\n]\n"; // close json
     }
     else
     {
@@ -127,7 +106,7 @@ std::string BP1Aggregator::GetGlobalProfilingLog(const std::string &rankLog)
 
     MPI_Barrier(m_MPIComm); // Barrier here?
 
-    return profilingLog;
+    return profilingJSON;
 }
 
 } // end namespace format

source/adios2/toolkit/format/bp1/BP1Aggregator.h

@@ -46,7 +46,7 @@ public:
      * python dictionary format
      * @param rankLog contain rank profiling info to be aggregated
      */
-    std::string GetGlobalProfilingLog(const std::string &rankLog);
+    std::string GetGlobalProfilingJSON(const std::string &rankLog);
 
 private:
     const bool m_DebugMode = false;

source/adios2/toolkit/format/bp1/BP1Writer.cpp

@@ -182,36 +182,39 @@ void BP1Writer::Close() noexcept
     }
 }
 
-std::string BP1Writer::GetRankProfilingLog(
+std::string BP1Writer::GetRankProfilingJSON(
     const std::vector<std::string> &transportsTypes,
     const std::vector<profiling::IOChrono *> &transportsProfilers) noexcept
 {
     auto lf_WriterTimer = [](std::string &rankLog,
                              const profiling::Timer &timer) {
-        rankLog += "'" + timer.m_Process + "_" + timer.GetShortUnits() + "': " +
-                   std::to_string(timer.m_ProcessTime) + ", ";
+        rankLog += "\"" + timer.m_Process + "_" + timer.GetShortUnits() +
+                   "\": " + std::to_string(timer.m_ProcessTime) + ", ";
     };
 
     // prepare string dictionary per rank
-    std::string rankLog("'rank_" + std::to_string(m_BP1Aggregator.m_RankMPI) +
-                        "': { ");
+    std::string rankLog("{ \"rank\": " +
+                        std::to_string(m_BP1Aggregator.m_RankMPI) + ", ");
 
     auto &profiler = m_Profiler;
 
     std::string timeDate(profiler.Timers.at("buffering").m_LocalTimeDate);
     timeDate.pop_back();
+    // avoid whitespace
+    std::replace(timeDate.begin(), timeDate.end(), ' ', '_');
 
-    rankLog += "'date_and_time': '" + timeDate + "', 'threads': " +
-               std::to_string(m_Threads) + ", 'bytes': " +
-               std::to_string(profiler.Bytes.at("buffering")) + ", ";
+    rankLog += "\"start\": \"" + timeDate + "\", ";
+    rankLog += "\"threads\": " + std::to_string(m_Threads) + ", ";
+    rankLog +=
+        "\"bytes\": " + std::to_string(profiler.Bytes.at("buffering")) + ", ";
     lf_WriterTimer(rankLog, profiler.Timers.at("buffering"));
 
     const size_t transportsSize = transportsTypes.size();
 
     for (unsigned int t = 0; t < transportsSize; ++t)
     {
-        rankLog += "'transport_" + std::to_string(t) + "': { ";
-        rankLog += "'type': '" + transportsTypes[t] + "', ";
+        rankLog += "\"transport_" + std::to_string(t) + "\": { ";
+        rankLog += "\"type\": \"" + transportsTypes[t] + "\", ";
 
         for (const auto &transportTimerPair : transportsProfilers[t]->Timers)
         {
@@ -231,15 +234,15 @@ std::string BP1Writer::GetRankProfilingLog(
             rankLog += "},";
         }
     }
-    rankLog += " }";
+    rankLog += " }"; // end rank entry
 
     return rankLog;
 }
 
 std::string
-BP1Writer::AggregateProfilingLog(const std::string &rankProfilingLog) noexcept
+BP1Writer::AggregateProfilingJSON(const std::string &rankProfilingLog) noexcept
 {
-    return m_BP1Aggregator.GetGlobalProfilingLog(rankProfilingLog);
+    return m_BP1Aggregator.GetGlobalProfilingJSON(rankProfilingLog);
 }
 
 // PRIVATE FUNCTIONS

source/adios2/toolkit/format/bp1/BP1Writer.h

@@ -86,12 +86,17 @@ public:
      * @param transportsTypes list of transport types
      * @param transportsProfilers list of references to transport profilers
      */
-    std::string GetRankProfilingLog(
+    std::string GetRankProfilingJSON(
         const std::vector<std::string> &transportsTypes,
         const std::vector<profiling::IOChrono *> &transportsProfilers) noexcept;
 
+    /**
+     * Forms the final profiling.json string aggregating from all ranks
+     * @param rankProfilingJSON
+     * @return profiling.json
+     */
     std::string
-    AggregateProfilingLog(const std::string &rankProfilingLog) noexcept;
+    AggregateProfilingJSON(const std::string &rankProfilingJSON) noexcept;
 
 private:
     /** BP format version */

testing/adios2/engine/CMakeLists.txt

@@ -5,9 +5,7 @@
 
 # We currently require ADIOS1 to test ADIOS v2 bp functionality since the read
 # API is not available yet
-if(ADIOS2_HAVE_ADIOS1)
-  add_subdirectory(bp)
-endif()
+add_subdirectory(bp)
 
 if(ADIOS2_HAVE_ADIOS1)
   add_subdirectory(adios1)

testing/adios2/engine/bp/CMakeLists.txt

@@ -5,19 +5,27 @@
 
 # MPI versions of the test are not properly implemented at the moment
 if(NOT ADIOS2_HAVE_MPI)
-  find_package(ADIOS1 COMPONENTS sequential REQUIRED)
-
-  add_executable(TestBPWriteRead TestBPWriteRead.cpp)
-  target_link_libraries(TestBPWriteRead adios2 gtest adios1::adios)
   
-  add_executable(TestBPWriteReadstdio TestBPWriteReadstdio.cpp)
-  target_link_libraries(TestBPWriteReadstdio adios2 gtest adios1::adios)
+  if(ADIOS2_HAVE_ADIOS1)
+    find_package(ADIOS1 COMPONENTS sequential REQUIRED)
 
-  add_executable(TestBPWriteReadfstream TestBPWriteReadfstream.cpp)
-  target_link_libraries(TestBPWriteReadfstream adios2 gtest adios1::adios)
+    add_executable(TestBPWriteRead TestBPWriteRead.cpp)
+    target_link_libraries(TestBPWriteRead adios2 gtest adios1::adios)
   
+    add_executable(TestBPWriteReadstdio TestBPWriteReadstdio.cpp)
+    target_link_libraries(TestBPWriteReadstdio adios2 gtest adios1::adios)
 
-  gtest_add_tests(TARGET TestBPWriteRead)
-  gtest_add_tests(TARGET TestBPWriteReadstdio)
-  gtest_add_tests(TARGET TestBPWriteReadfstream)
+    add_executable(TestBPWriteReadfstream TestBPWriteReadfstream.cpp)
+    target_link_libraries(TestBPWriteReadfstream adios2 gtest adios1::adios)
+    
+    gtest_add_tests(TARGET TestBPWriteRead)
+    gtest_add_tests(TARGET TestBPWriteReadstdio)
+    gtest_add_tests(TARGET TestBPWriteReadfstream)
+    
+  endif()
+    
+  add_executable(TestBPWriteProfilingJSON TestBPWriteProfilingJSON.cpp)
+  target_link_libraries(TestBPWriteProfilingJSON adios2 gtest gtest_main NLohmannJson)
+  
+  gtest_add_tests(TARGET TestBPWriteProfilingJSON)
 endif()

testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp

+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * TestBPWriteProfilingJSON.cpp
+ *
+ *  Created on: Jul 18, 2017
+ *      Author: William F Godoy godoywf@ornl.gov
+ */
+
+#include <cstdint>
+#include <cstring>
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+
+#include <adios2.h>
+
+#include <gtest/gtest.h>
+#include <json.hpp> //This fails to be included
+
+#include "../SmallTestData.h"
+
+using json = nlohmann::json;
+
+class BPWriteProfilingJSONTest : public ::testing::Test
+{
+public:
+    BPWriteProfilingJSONTest() = default;
+
+    SmallTestData m_TestData;
+};
+
+//******************************************************************************
+// 1D 1x8 test data
+//******************************************************************************
+
+// ADIOS2 write, native ADIOS1 read
+TEST_F(BPWriteProfilingJSONTest, ADIOS2BPWriteProfilingJSON)
+{
+    std::string fname = "ADIOS2BPWriteProfilingJSON.bp";
+
+    // Write test data and profiling.json using ADIOS2
+    {
+        adios2::ADIOS adios(true);
+        adios2::IO &io = adios.DeclareIO("TestIO");
+
+        // Declare 1D variables
+        {
+            auto &var_i8 =
+                io.DefineVariable<char>("i8", {}, {}, adios2::Dims{8});
+            auto &var_i16 =
+                io.DefineVariable<short>("i16", {}, {}, adios2::Dims{8});
+            auto &var_i32 =
+                io.DefineVariable<int>("i32", {}, {}, adios2::Dims{8});
+            auto &var_i64 =
+                io.DefineVariable<long>("i64", {}, {}, adios2::Dims{8});
+            auto &var_u8 =
+                io.DefineVariable<unsigned char>("u8", {}, {}, adios2::Dims{8});
+            auto &var_u16 = io.DefineVariable<unsigned short>("u16", {}, {},
+                                                              adios2::Dims{8});
+            auto &var_u32 =
+                io.DefineVariable<unsigned int>("u32", {}, {}, adios2::Dims{8});
+            auto &var_u64 = io.DefineVariable<unsigned long>("u64", {}, {},
+                                                             adios2::Dims{8});
+            auto &var_r32 =
+                io.DefineVariable<float>("r32", {}, {}, adios2::Dims{8});
+            auto &var_r64 =
+                io.DefineVariable<double>("r64", {}, {}, adios2::Dims{8});
+        }
+
+        // Create the BP Engine
+        io.SetEngine("BPFileWriter");
+        io.SetParameters({{"Threads", "2"}});
+        io.AddTransport("File", {{"Library", "POSIX"}});
+
+        auto engine = io.Open(fname, adios2::OpenMode::Write);
+        ASSERT_NE(engine.get(), nullptr);
+
+        for (size_t step = 0; step < 3; ++step)
+        {
+            // Retrieve the variables that previously went out of scope
+            auto &var_i8 = io.GetVariable<char>("i8");
+            auto &var_i16 = io.GetVariable<short>("i16");
+            auto &var_i32 = io.GetVariable<int>("i32");
+            auto &var_i64 = io.GetVariable<long>("i64");
+            auto &var_u8 = io.GetVariable<unsigned char>("u8");
+            auto &var_u16 = io.GetVariable<unsigned short>("u16");
+            auto &var_u32 = io.GetVariable<unsigned int>("u32");
+            auto &var_u64 = io.GetVariable<unsigned long>("u64");
+            auto &var_r32 = io.GetVariable<float>("r32");
+            auto &var_r64 = io.GetVariable<double>("r64");
+
+            // Write each one
+            engine->Write(var_i8, m_TestData.I8.data() + step);
+            engine->Write(var_i16, m_TestData.I16.data() + step);
+            engine->Write(var_i32, m_TestData.I32.data() + step);
+            engine->Write(var_i64, m_TestData.I64.data() + step);
+            engine->Write(var_u8, m_TestData.U8.data() + step);
+            engine->Write(var_u16, m_TestData.U16.data() + step);
+            engine->Write(var_u32, m_TestData.U32.data() + step);
+            engine->Write(var_u64, m_TestData.U64.data() + step);
+            engine->Write(var_r32, m_TestData.R32.data() + step);
+            engine->Write(var_r64, m_TestData.R64.data() + step);
+
+            // Advance to the next time step
+            engine->Advance();
+        }
+
+        // Close the file
+        engine->Close();
+    }
+
+    // open json file, parse it to a json structure, and verify a few things
+    {
+        std::ifstream profilingJSONFile(fname + ".dir/profiling.json");
+        std::stringstream buffer;
+        buffer << profilingJSONFile.rdbuf();
+
+        const json profilingJSON = json::parse(buffer);
+
+        // check rank is zero
+        const int rank = profilingJSON[0].value("rank", -1);
+        ASSERT_EQ(rank, 0);
+
+        // check threads
+        const int threads = profilingJSON[0].value("threads", 0);
+        ASSERT_EQ(threads, 2);
+
+        // check bytes
+        const unsigned long int bytes = profilingJSON[0].value("bytes", 0UL);
+        ASSERT_EQ(bytes, 6536);
+
+        const auto transportType =
+            profilingJSON[0]["transport_0"].value("type", "0");
+        ASSERT_EQ(transportType, "File_POSIX");
+    }
+}