From 1d610267bfc1b601451af882259df471e1b59eee Mon Sep 17 00:00:00 2001
From: William F Godoy <williamfgc@yahoo.com>
Date: Mon, 17 Jul 2017 17:55:55 -0400
Subject: [PATCH] Moved profiling.log to profiling.json format Added test for
generating profiling.json Fixed inclusion for NLohmannJson json.hpp in Test
Tested resulting profiling.json on R (jsonlite), Python (json) and C++ (json
for modern C++)
---
source/adios2/engine/bp/BPFileWriter.cpp | 53 ++++---
source/adios2/engine/bp/BPFileWriter.h | 4 +
.../toolkit/format/bp1/BP1Aggregator.cpp | 41 ++---
.../adios2/toolkit/format/bp1/BP1Aggregator.h | 2 +-
.../adios2/toolkit/format/bp1/BP1Writer.cpp | 29 ++--
source/adios2/toolkit/format/bp1/BP1Writer.h | 9 +-
testing/adios2/engine/CMakeLists.txt | 4 +-
testing/adios2/engine/bp/CMakeLists.txt | 30 ++--
.../engine/bp/TestBPWriteProfilingJSON.cpp | 140 ++++++++++++++++++
9 files changed, 223 insertions(+), 89 deletions(-)
create mode 100644 testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp
diff --git a/source/adios2/engine/bp/BPFileWriter.cpp b/source/adios2/engine/bp/BPFileWriter.cpp
index c8e6fcef4..a37b69c68 100644
--- a/source/adios2/engine/bp/BPFileWriter.cpp
+++ b/source/adios2/engine/bp/BPFileWriter.cpp
@@ -71,37 +71,12 @@ void BPFileWriter::Close(const int transportIndex)
m_BP1Writer.m_HeapBuffer.m_DataPosition,
transportIndex);
- // close them
m_TransportsManager.CloseFiles(transportIndex);
- if (m_BP1Writer.m_Profiler.IsActive)
+ if (m_BP1Writer.m_Profiler.IsActive &&
+ m_TransportsManager.AllTransportsClosed())
{
- // aggregate and write profiling.log
- if (m_TransportsManager.AllTransportsClosed())
- {
- auto transportTypes = m_TransportsManager.GetTransportsTypes();
- auto transportProfilers =
- m_TransportsManager.GetTransportsProfilers();
-
- const std::string log(m_BP1Writer.GetRankProfilingLog(
- transportTypes, transportProfilers));
- // TODO profiling.log from rank0
-
- const std::string profilingLog(
- m_BP1Writer.AggregateProfilingLog(log));
-
- if (m_BP1Writer.m_BP1Aggregator.m_RankMPI == 0)
- {
- transport::FileStream profilingLogStream(m_MPIComm,
- m_DebugMode);
- auto bpBaseNames = m_BP1Writer.GetBPBaseNames({m_Name});
- profilingLogStream.Open(bpBaseNames[0] + "/profiling.log",
- OpenMode::Write);
- profilingLogStream.Write(profilingLog.c_str(),
- profilingLog.size());
- profilingLogStream.Close();
- }
- }
+ WriteProfilingJSONFile();
}
}
@@ -147,4 +122,26 @@ void BPFileWriter::InitBPBuffer()
}
}
+void BPFileWriter::WriteProfilingJSONFile()
+{
+ auto transportTypes = m_TransportsManager.GetTransportsTypes();
+ auto transportProfilers = m_TransportsManager.GetTransportsProfilers();
+
+ const std::string lineJSON(
+ m_BP1Writer.GetRankProfilingJSON(transportTypes, transportProfilers));
+
+ const std::string profilingJSON(
+ m_BP1Writer.AggregateProfilingJSON(lineJSON));
+
+ if (m_BP1Writer.m_BP1Aggregator.m_RankMPI == 0)
+ {
+ transport::FileStream profilingJSONStream(m_MPIComm, m_DebugMode);
+ auto bpBaseNames = m_BP1Writer.GetBPBaseNames({m_Name});
+ profilingJSONStream.Open(bpBaseNames[0] + "/profiling.json",
+ OpenMode::Write);
+ profilingJSONStream.Write(profilingJSON.c_str(), profilingJSON.size());
+ profilingJSONStream.Close();
+ }
+}
+
} // end namespace adios
diff --git a/source/adios2/engine/bp/BPFileWriter.h b/source/adios2/engine/bp/BPFileWriter.h
index e38cd1050..bb7d55265 100644
--- a/source/adios2/engine/bp/BPFileWriter.h
+++ b/source/adios2/engine/bp/BPFileWriter.h
@@ -77,6 +77,10 @@ private:
*/
template <class T>
void DoWriteCommon(Variable<T> &variable, const T *values);
+
+ /** Write a profiling.json file from m_BP1Writer and m_TransportsManager
+ * profilers*/
+ void WriteProfilingJSONFile();
};
} // end namespace adios
diff --git a/source/adios2/toolkit/format/bp1/BP1Aggregator.cpp b/source/adios2/toolkit/format/bp1/BP1Aggregator.cpp
index 9c63f7258..4962e7007 100644
--- a/source/adios2/toolkit/format/bp1/BP1Aggregator.cpp
+++ b/source/adios2/toolkit/format/bp1/BP1Aggregator.cpp
@@ -33,9 +33,9 @@ BP1Aggregator::BP1Aggregator(MPI_Comm mpiComm, const bool debugMode)
MPI_Comm_size(m_MPIComm, &m_SizeMPI);
}
-std::string BP1Aggregator::GetGlobalProfilingLog(const std::string &rankLog)
+std::string BP1Aggregator::GetGlobalProfilingJSON(const std::string &rankLog)
{
- std::string profilingLog;
+ std::string profilingJSON;
if (m_RankMPI == 0)
{
@@ -80,38 +80,17 @@ std::string BP1Aggregator::GetGlobalProfilingLog(const std::string &rankLog)
// write global string
// key is to reserve memory first
- profilingLog.reserve(rankLog.size() * m_SizeMPI);
+ profilingJSON.reserve(rankLog.size() * m_SizeMPI);
- profilingLog += "{\n";
- profilingLog += rankLog + ",\n";
+ profilingJSON += "[\n";
+ profilingJSON += rankLog;
for (unsigned int i = 1; i < sizeMPI; ++i)
{
- const std::string rankLogStr(rankLogs[i - 1].data(),
- rankLogs[i - 1].size());
- profilingLog += rankLogStr + ",";
- if (i < sizeMPI - 1)
- {
- profilingLog += "\n";
- }
+ profilingJSON += ",\n";
+ profilingJSON.append(rankLogs[i - 1].data(),
+ rankLogs[i - 1].size());
}
- profilingLog.pop_back(); // eliminate trailing comma
- profilingLog += "\n";
- profilingLog += "}\n";
-
- // // write to file
- // std::ofstream logStream(fileName);
- // if (m_DebugMode)
- // {
- // if (!logStream)
- // {
- // throw std::ios_base::failure(
- // "ERROR: couldn't open profiling file " + fileName
- // + "\n");
- // }
- // }
- //
- // logStream.write(logFile.c_str(), logFile.size());
- // logStream.close();
+ profilingJSON += "\n]\n"; // close json
}
else
{
@@ -127,7 +106,7 @@ std::string BP1Aggregator::GetGlobalProfilingLog(const std::string &rankLog)
MPI_Barrier(m_MPIComm); // Barrier here?
- return profilingLog;
+ return profilingJSON;
}
} // end namespace format
diff --git a/source/adios2/toolkit/format/bp1/BP1Aggregator.h b/source/adios2/toolkit/format/bp1/BP1Aggregator.h
index 854a66d0c..f18dcf47b 100644
--- a/source/adios2/toolkit/format/bp1/BP1Aggregator.h
+++ b/source/adios2/toolkit/format/bp1/BP1Aggregator.h
@@ -46,7 +46,7 @@ public:
* python dictionary format
* @param rankLog contain rank profiling info to be aggregated
*/
- std::string GetGlobalProfilingLog(const std::string &rankLog);
+ std::string GetGlobalProfilingJSON(const std::string &rankLog);
private:
const bool m_DebugMode = false;
diff --git a/source/adios2/toolkit/format/bp1/BP1Writer.cpp b/source/adios2/toolkit/format/bp1/BP1Writer.cpp
index da9361fc8..c7b9210b3 100644
--- a/source/adios2/toolkit/format/bp1/BP1Writer.cpp
+++ b/source/adios2/toolkit/format/bp1/BP1Writer.cpp
@@ -182,36 +182,39 @@ void BP1Writer::Close() noexcept
}
}
-std::string BP1Writer::GetRankProfilingLog(
+std::string BP1Writer::GetRankProfilingJSON(
const std::vector<std::string> &transportsTypes,
const std::vector<profiling::IOChrono *> &transportsProfilers) noexcept
{
auto lf_WriterTimer = [](std::string &rankLog,
const profiling::Timer &timer) {
- rankLog += "'" + timer.m_Process + "_" + timer.GetShortUnits() + "': " +
- std::to_string(timer.m_ProcessTime) + ", ";
+ rankLog += "\"" + timer.m_Process + "_" + timer.GetShortUnits() +
+ "\": " + std::to_string(timer.m_ProcessTime) + ", ";
};
// prepare string dictionary per rank
- std::string rankLog("'rank_" + std::to_string(m_BP1Aggregator.m_RankMPI) +
- "': { ");
+ std::string rankLog("{ \"rank\": " +
+ std::to_string(m_BP1Aggregator.m_RankMPI) + ", ");
auto &profiler = m_Profiler;
std::string timeDate(profiler.Timers.at("buffering").m_LocalTimeDate);
timeDate.pop_back();
+ // avoid whitespace
+ std::replace(timeDate.begin(), timeDate.end(), ' ', '_');
- rankLog += "'date_and_time': '" + timeDate + "', 'threads': " +
- std::to_string(m_Threads) + ", 'bytes': " +
- std::to_string(profiler.Bytes.at("buffering")) + ", ";
+ rankLog += "\"start\": \"" + timeDate + "\", ";
+ rankLog += "\"threads\": " + std::to_string(m_Threads) + ", ";
+ rankLog +=
+ "\"bytes\": " + std::to_string(profiler.Bytes.at("buffering")) + ", ";
lf_WriterTimer(rankLog, profiler.Timers.at("buffering"));
const size_t transportsSize = transportsTypes.size();
for (unsigned int t = 0; t < transportsSize; ++t)
{
- rankLog += "'transport_" + std::to_string(t) + "': { ";
- rankLog += "'type': '" + transportsTypes[t] + "', ";
+ rankLog += "\"transport_" + std::to_string(t) + "\": { ";
+ rankLog += "\"type\": \"" + transportsTypes[t] + "\", ";
for (const auto &transportTimerPair : transportsProfilers[t]->Timers)
{
@@ -231,15 +234,15 @@ std::string BP1Writer::GetRankProfilingLog(
rankLog += "},";
}
}
- rankLog += " }";
+ rankLog += " }"; // end rank entry
return rankLog;
}
std::string
-BP1Writer::AggregateProfilingLog(const std::string &rankProfilingLog) noexcept
+BP1Writer::AggregateProfilingJSON(const std::string &rankProfilingLog) noexcept
{
- return m_BP1Aggregator.GetGlobalProfilingLog(rankProfilingLog);
+ return m_BP1Aggregator.GetGlobalProfilingJSON(rankProfilingLog);
}
// PRIVATE FUNCTIONS
diff --git a/source/adios2/toolkit/format/bp1/BP1Writer.h b/source/adios2/toolkit/format/bp1/BP1Writer.h
index 1a35644b2..44d306c3f 100644
--- a/source/adios2/toolkit/format/bp1/BP1Writer.h
+++ b/source/adios2/toolkit/format/bp1/BP1Writer.h
@@ -86,12 +86,17 @@ public:
* @param transportsTypes list of transport types
* @param transportsProfilers list of references to transport profilers
*/
- std::string GetRankProfilingLog(
+ std::string GetRankProfilingJSON(
const std::vector<std::string> &transportsTypes,
const std::vector<profiling::IOChrono *> &transportsProfilers) noexcept;
+ /**
+ * Forms the final profiling.json string aggregating from all ranks
+ * @param rankProfilingJSON
+ * @return profiling.json
+ */
std::string
- AggregateProfilingLog(const std::string &rankProfilingLog) noexcept;
+ AggregateProfilingJSON(const std::string &rankProfilingJSON) noexcept;
private:
/** BP format version */
diff --git a/testing/adios2/engine/CMakeLists.txt b/testing/adios2/engine/CMakeLists.txt
index 1992d27f7..1fac5d460 100644
--- a/testing/adios2/engine/CMakeLists.txt
+++ b/testing/adios2/engine/CMakeLists.txt
@@ -5,9 +5,7 @@
# We currently require ADIOS1 to test ADIOS v2 bp functionality since the read
# API is not available yet
-if(ADIOS2_HAVE_ADIOS1)
- add_subdirectory(bp)
-endif()
+add_subdirectory(bp)
if(ADIOS2_HAVE_ADIOS1)
add_subdirectory(adios1)
diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt
index d1d99e71e..3153fd2ab 100644
--- a/testing/adios2/engine/bp/CMakeLists.txt
+++ b/testing/adios2/engine/bp/CMakeLists.txt
@@ -5,19 +5,27 @@
# MPI versions of the test are not properly implemented at the moment
if(NOT ADIOS2_HAVE_MPI)
- find_package(ADIOS1 COMPONENTS sequential REQUIRED)
-
- add_executable(TestBPWriteRead TestBPWriteRead.cpp)
- target_link_libraries(TestBPWriteRead adios2 gtest adios1::adios)
- add_executable(TestBPWriteReadstdio TestBPWriteReadstdio.cpp)
- target_link_libraries(TestBPWriteReadstdio adios2 gtest adios1::adios)
+ if(ADIOS2_HAVE_ADIOS1)
+ find_package(ADIOS1 COMPONENTS sequential REQUIRED)
- add_executable(TestBPWriteReadfstream TestBPWriteReadfstream.cpp)
- target_link_libraries(TestBPWriteReadfstream adios2 gtest adios1::adios)
+ add_executable(TestBPWriteRead TestBPWriteRead.cpp)
+ target_link_libraries(TestBPWriteRead adios2 gtest adios1::adios)
+ add_executable(TestBPWriteReadstdio TestBPWriteReadstdio.cpp)
+ target_link_libraries(TestBPWriteReadstdio adios2 gtest adios1::adios)
- gtest_add_tests(TARGET TestBPWriteRead)
- gtest_add_tests(TARGET TestBPWriteReadstdio)
- gtest_add_tests(TARGET TestBPWriteReadfstream)
+ add_executable(TestBPWriteReadfstream TestBPWriteReadfstream.cpp)
+ target_link_libraries(TestBPWriteReadfstream adios2 gtest adios1::adios)
+
+ gtest_add_tests(TARGET TestBPWriteRead)
+ gtest_add_tests(TARGET TestBPWriteReadstdio)
+ gtest_add_tests(TARGET TestBPWriteReadfstream)
+
+ endif()
+
+ add_executable(TestBPWriteProfilingJSON TestBPWriteProfilingJSON.cpp)
+ target_link_libraries(TestBPWriteProfilingJSON adios2 gtest gtest_main NLohmannJson)
+
+ gtest_add_tests(TARGET TestBPWriteProfilingJSON)
endif()
diff --git a/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp b/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp
new file mode 100644
index 000000000..501062dbc
--- /dev/null
+++ b/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp
@@ -0,0 +1,140 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * TestBPWriteProfilingJSON.cpp
+ *
+ * Created on: Jul 18, 2017
+ * Author: William F Godoy godoywf@ornl.gov
+ */
+
+#include <cstdint>
+#include <cstring>
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+
+#include <adios2.h>
+
+#include <gtest/gtest.h>
+#include <json.hpp> //This fails to be included
+
+#include "../SmallTestData.h"
+
+using json = nlohmann::json;
+
+class BPWriteProfilingJSONTest : public ::testing::Test
+{
+public:
+ BPWriteProfilingJSONTest() = default;
+
+ SmallTestData m_TestData;
+};
+
+//******************************************************************************
+// 1D 1x8 test data
+//******************************************************************************
+
+// ADIOS2 write, native ADIOS1 read
+TEST_F(BPWriteProfilingJSONTest, ADIOS2BPWriteProfilingJSON)
+{
+ std::string fname = "ADIOS2BPWriteProfilingJSON.bp";
+
+ // Write test data and profiling.json using ADIOS2
+ {
+ adios2::ADIOS adios(true);
+ adios2::IO &io = adios.DeclareIO("TestIO");
+
+ // Declare 1D variables
+ {
+ auto &var_i8 =
+ io.DefineVariable<char>("i8", {}, {}, adios2::Dims{8});
+ auto &var_i16 =
+ io.DefineVariable<short>("i16", {}, {}, adios2::Dims{8});
+ auto &var_i32 =
+ io.DefineVariable<int>("i32", {}, {}, adios2::Dims{8});
+ auto &var_i64 =
+ io.DefineVariable<long>("i64", {}, {}, adios2::Dims{8});
+ auto &var_u8 =
+ io.DefineVariable<unsigned char>("u8", {}, {}, adios2::Dims{8});
+ auto &var_u16 = io.DefineVariable<unsigned short>("u16", {}, {},
+ adios2::Dims{8});
+ auto &var_u32 =
+ io.DefineVariable<unsigned int>("u32", {}, {}, adios2::Dims{8});
+ auto &var_u64 = io.DefineVariable<unsigned long>("u64", {}, {},
+ adios2::Dims{8});
+ auto &var_r32 =
+ io.DefineVariable<float>("r32", {}, {}, adios2::Dims{8});
+ auto &var_r64 =
+ io.DefineVariable<double>("r64", {}, {}, adios2::Dims{8});
+ }
+
+ // Create the BP Engine
+ io.SetEngine("BPFileWriter");
+ io.SetParameters({{"Threads", "2"}});
+ io.AddTransport("File", {{"Library", "POSIX"}});
+
+ auto engine = io.Open(fname, adios2::OpenMode::Write);
+ ASSERT_NE(engine.get(), nullptr);
+
+ for (size_t step = 0; step < 3; ++step)
+ {
+ // Retrieve the variables that previously went out of scope
+ auto &var_i8 = io.GetVariable<char>("i8");
+ auto &var_i16 = io.GetVariable<short>("i16");
+ auto &var_i32 = io.GetVariable<int>("i32");
+ auto &var_i64 = io.GetVariable<long>("i64");
+ auto &var_u8 = io.GetVariable<unsigned char>("u8");
+ auto &var_u16 = io.GetVariable<unsigned short>("u16");
+ auto &var_u32 = io.GetVariable<unsigned int>("u32");
+ auto &var_u64 = io.GetVariable<unsigned long>("u64");
+ auto &var_r32 = io.GetVariable<float>("r32");
+ auto &var_r64 = io.GetVariable<double>("r64");
+
+ // Write each one
+ engine->Write(var_i8, m_TestData.I8.data() + step);
+ engine->Write(var_i16, m_TestData.I16.data() + step);
+ engine->Write(var_i32, m_TestData.I32.data() + step);
+ engine->Write(var_i64, m_TestData.I64.data() + step);
+ engine->Write(var_u8, m_TestData.U8.data() + step);
+ engine->Write(var_u16, m_TestData.U16.data() + step);
+ engine->Write(var_u32, m_TestData.U32.data() + step);
+ engine->Write(var_u64, m_TestData.U64.data() + step);
+ engine->Write(var_r32, m_TestData.R32.data() + step);
+ engine->Write(var_r64, m_TestData.R64.data() + step);
+
+ // Advance to the next time step
+ engine->Advance();
+ }
+
+ // Close the file
+ engine->Close();
+ }
+
+ // open json file, parse it to a json structure, and verify a few things
+ {
+ std::ifstream profilingJSONFile(fname + ".dir/profiling.json");
+ std::stringstream buffer;
+ buffer << profilingJSONFile.rdbuf();
+
+ const json profilingJSON = json::parse(buffer);
+
+ // check rank is zero
+ const int rank = profilingJSON[0].value("rank", -1);
+ ASSERT_EQ(rank, 0);
+
+ // check threads
+ const int threads = profilingJSON[0].value("threads", 0);
+ ASSERT_EQ(threads, 2);
+
+ // check bytes
+ const unsigned long int bytes = profilingJSON[0].value("bytes", 0UL);
+ ASSERT_EQ(bytes, 6536);
+
+ const auto transportType =
+ profilingJSON[0]["transport_0"].value("type", "0");
+ ASSERT_EQ(transportType, "File_POSIX");
+ }
+}
--
GitLab