From f2ea5efc88329969c3f97ec759fde2e54353e764 Mon Sep 17 00:00:00 2001
From: wfg <wfg@pc0098504.ornl.gov>
Date: Tue, 4 Apr 2017 11:16:40 -0400
Subject: [PATCH] Added packages directory to support reusable functionality in
Engines
added:
./packages/format/bp1
./packages/profiling/iochrono
./packages/wan/dataman (for DataMan integration in ADIOS)
---
include/core/Engine.h | 68 +++--
include/core/Method.h | 8 +-
include/core/Profiler.h | 109 --------
include/core/Transport.h | 13 +-
include/engine/bp/BPFileReader.h | 4 +-
include/engine/bp/BPFileWriter.h | 19 +-
include/engine/dataman/DataManReader.h | 4 +-
include/engine/dataman/DataManWriter.h | 8 +-
include/packages/format/bp1/BP1.h | 19 ++
.../format/bp1}/BP1Aggregator.h | 4 +
.../BP1.h => packages/format/bp1/BP1Base.h} | 98 ++-----
include/packages/format/bp1/BP1Structs.h | 82 ++++++
.../format/bp1}/BP1Writer.h | 26 +-
.../packages/profiling/iochrono/IOChrono.h | 40 +++
include/packages/profiling/iochrono/Timer.h | 65 +++++
source/ADIOS.cpp | 15 +-
source/CMakeLists.txt | 8 +-
source/core/Engine.cpp | 256 ++++++------------
source/core/Method.cpp | 2 +-
source/core/Transport.cpp | 14 +-
source/engine/bp/BPFileReader.cpp | 8 +-
source/engine/bp/BPFileWriter.cpp | 32 +--
source/engine/dataman/DataManReader.cpp | 18 +-
source/engine/dataman/DataManWriter.cpp | 4 +-
.../format/bp1}/BP1Aggregator.cpp | 7 +-
.../format/bp1/BP1Base.cpp} | 11 +-
.../format/bp1}/BP1Writer.cpp | 29 +-
source/packages/profiling/iochrono/Timer.cpp | 100 +++++++
source/transport/file/FileDescriptor.cpp | 40 +--
29 files changed, 580 insertions(+), 531 deletions(-)
delete mode 100644 include/core/Profiler.h
create mode 100644 include/packages/format/bp1/BP1.h
rename include/{format => packages/format/bp1}/BP1Aggregator.h (94%)
rename include/{format/BP1.h => packages/format/bp1/BP1Base.h} (68%)
create mode 100644 include/packages/format/bp1/BP1Structs.h
rename include/{format => packages/format/bp1}/BP1Writer.h (95%)
create mode 100644 include/packages/profiling/iochrono/IOChrono.h
create mode 100644 include/packages/profiling/iochrono/Timer.h
rename source/{format => packages/format/bp1}/BP1Aggregator.cpp (91%)
rename source/{format/BP1.cpp => packages/format/bp1/BP1Base.cpp} (83%)
rename source/{format => packages/format/bp1}/BP1Writer.cpp (94%)
create mode 100644 source/packages/profiling/iochrono/Timer.cpp
diff --git a/include/core/Engine.h b/include/core/Engine.h
index d536873c7..eb943b070 100644
--- a/include/core/Engine.h
+++ b/include/core/Engine.h
@@ -26,7 +26,6 @@
#include "ADIOSTypes.h"
#include "core/Capsule.h"
#include "core/Method.h"
-#include "core/Profiler.h"
#include "core/Transform.h"
#include "core/Transport.h"
#include "core/Variable.h"
@@ -56,8 +55,7 @@ public:
const std::string m_EngineType; ///< from derived class
const std::string m_Name; ///< name used for this engine
- const std::string
- m_AccessMode; ///< accessMode for buffers used by this engine
+ const std::string m_AccessMode; ///< "w", "r", "a"
const Method &m_Method; ///< associated method containing engine metadata
int m_RankMPI = 0; ///< current MPI rank process
@@ -73,13 +71,11 @@ public:
* @param accessMode
* @param mpiComm
* @param method
- * @param debugMode
- * @param nthreads
* @param endMessage
*/
- Engine(ADIOS &adios, std::string engineType, std::string name,
- std::string accessMode, MPI_Comm mpiComm, const Method &method,
- bool debugMode, unsigned int nthreads, std::string endMessage);
+ Engine(ADIOS &adios, const std::string engineType, const std::string name,
+ const std::string accessMode, MPI_Comm mpiComm, const Method &method,
+ std::string endMessage);
virtual ~Engine() = default;
@@ -136,8 +132,7 @@ public:
* @param variableName
* @param values
*/
- template <class T>
- void Write(const std::string &variableName, const T *values)
+ template <class T> void Write(const std::string variableName, const T *values)
{
Write(variableName, values);
}
@@ -159,7 +154,7 @@ public:
* @param variableName
* @param values
*/
- template <class T> void Write(const std::string &variableName, const T values)
+ template <class T> void Write(const std::string variableName, const T values)
{
const T val = values;
Write(variableName, &val);
@@ -203,33 +198,32 @@ public:
* @param variableName
* @param values coming from user app
*/
- virtual void Write(const std::string &variableName, const char *values);
- virtual void Write(const std::string &variableName,
+ virtual void Write(const std::string variableName, const char *values);
+ virtual void Write(const std::string variableName,
const unsigned char *values);
- virtual void Write(const std::string &variableName, const short *values);
- virtual void Write(const std::string &variableName,
+ virtual void Write(const std::string variableName, const short *values);
+ virtual void Write(const std::string variableName,
const unsigned short *values);
- virtual void Write(const std::string &variableName, const int *values);
- virtual void Write(const std::string &variableName,
+ virtual void Write(const std::string variableName, const int *values);
+ virtual void Write(const std::string variableName,
const unsigned int *values);
- virtual void Write(const std::string &variableName, const long int *values);
- virtual void Write(const std::string &variableName,
+ virtual void Write(const std::string variableName, const long int *values);
+ virtual void Write(const std::string variableName,
const unsigned long int *values);
- virtual void Write(const std::string &variableName,
+ virtual void Write(const std::string variableName,
const long long int *values);
- virtual void Write(const std::string &variableName,
+ virtual void Write(const std::string variableName,
const unsigned long long int *values);
- virtual void Write(const std::string &variableName, const float *values);
- virtual void Write(const std::string &variableName, const double *values);
- virtual void Write(const std::string &variableName,
- const long double *values);
- virtual void Write(const std::string &variableName,
+ virtual void Write(const std::string variableName, const float *values);
+ virtual void Write(const std::string variableName, const double *values);
+ virtual void Write(const std::string variableName, const long double *values);
+ virtual void Write(const std::string variableName,
const std::complex<float> *values);
- virtual void Write(const std::string &variableName,
+ virtual void Write(const std::string variableName,
const std::complex<double> *values);
- virtual void Write(const std::string &variableName,
+ virtual void Write(const std::string variableName,
const std::complex<long double> *values);
- virtual void Write(const std::string &variableName, const void *values);
+ virtual void Write(const std::string variableName, const void *values);
/**
* Read function that adds static checking on the variable to be passed by
@@ -452,17 +446,17 @@ public:
*/
std::vector<std::string> VariableNames();
- virtual void
- Close(const int transportIndex =
- -1) = 0; ///< Closes a particular transport, or all if -1
+ /**
+ * Closes a particular transport, or all if -1
+ * @param transportIndex order from Method AddTransport
+ */
+ virtual void Close(const int transportIndex = -1) = 0;
protected:
- ADIOS
- &m_ADIOS; ///< reference to ADIOS object that creates this Engine at Open
+ ADIOS &m_ADIOS; ///< creates Engine at Open
std::vector<std::shared_ptr<Transport>> m_Transports; ///< transports managed
- const bool m_DebugMode =
- false; ///< true: additional checks, false: by-pass checks
- unsigned int m_nThreads = 0;
+ const bool m_DebugMode = false; ///< true: additional exceptions checks
+ unsigned int m_nThreads = 0; ///< from Method nthreads
const std::string m_EndMessage; ///< added to exceptions to improve debugging
std::set<std::string> m_WrittenVariables; ///< contains the names of the
/// variables that are being written
diff --git a/include/core/Method.h b/include/core/Method.h
index 686391f96..37eae8f74 100644
--- a/include/core/Method.h
+++ b/include/core/Method.h
@@ -46,8 +46,8 @@ public:
const bool m_DebugMode = false; ///< true: on, throws exceptions and do
/// additional checks, false: off, faster, but
/// unsafe
- int m_nThreads;
- std::string m_Type; ///< Method's engine type
+ std::string m_Type; ///< Method's engine type
+ unsigned int m_nThreads;
std::map<std::string, std::string> m_Parameters; ///< method parameters
std::vector<std::map<std::string, std::string>>
m_TransportParameters; ///< each is a separate Transport containing their
@@ -86,9 +86,9 @@ public:
* Set this parameter like you set it for OpenMP, i.e. count one thread for
* the main process that calls
* ADIOS functions.
- * @param number of threads, minimum 1 is required
+ * @param nThreads, minimum 1 is required
*/
- void AllowThreads(const int nThreads);
+ void AllowThreads(const unsigned int nThreads);
/**
* Sets parameters for the method in "parameter=value" format
diff --git a/include/core/Profiler.h b/include/core/Profiler.h
deleted file mode 100644
index eee51ebee..000000000
--- a/include/core/Profiler.h
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Distributed under the OSI-approved Apache License, Version 2.0. See
- * accompanying file Copyright.txt for details.
- *
- * Profiler.h
- *
- * Created on: Mar 9, 2017
- * Author: wfg
- */
-
-#ifndef PROFILER_H_
-#define PROFILER_H_
-
-#include <chrono>
-
-#include "Support.h"
-
-namespace adios
-{
-
-class Timer
-{
-
-public:
- const std::string Process;
- unsigned long long int ProcessTime = 0;
-
- Timer(const std::string process, const Support::Resolutions resolution)
- : Process{process}, Resolution{resolution}
- {
- }
-
- void SetInitialTime()
- {
- InitialTime = std::chrono::high_resolution_clock::now();
- }
-
- void SetTime()
- {
- ElapsedTime = std::chrono::high_resolution_clock::now();
- ProcessTime += GetTime();
- }
-
- long long int GetTime()
- {
- if (Resolution == Support::Resolutions::mus)
- return std::chrono::duration_cast<std::chrono::microseconds>(ElapsedTime -
- InitialTime)
- .count();
-
- else if (Resolution == Support::Resolutions::ms)
- return std::chrono::duration_cast<std::chrono::milliseconds>(ElapsedTime -
- InitialTime)
- .count();
-
- else if (Resolution == Support::Resolutions::s)
- return std::chrono::duration_cast<std::chrono::seconds>(ElapsedTime -
- InitialTime)
- .count();
-
- else if (Resolution == Support::Resolutions::m)
- return std::chrono::duration_cast<std::chrono::minutes>(ElapsedTime -
- InitialTime)
- .count();
-
- else if (Resolution == Support::Resolutions::h)
- return std::chrono::duration_cast<std::chrono::hours>(ElapsedTime -
- InitialTime)
- .count();
-
- return -1; // failure
- }
-
- std::string GetUnits() const
- {
- std::string units;
- if (Resolution == Support::Resolutions::mus)
- units = "mus";
- else if (Resolution == Support::Resolutions::ms)
- units = "ms";
- else if (Resolution == Support::Resolutions::s)
- units = "s";
- else if (Resolution == Support::Resolutions::m)
- units = "m";
- else if (Resolution == Support::Resolutions::h)
- units = "h";
- return units;
- }
-
-private:
- const Support::Resolutions Resolution;
- std::chrono::time_point<std::chrono::high_resolution_clock> InitialTime;
- std::chrono::time_point<std::chrono::high_resolution_clock> ElapsedTime;
- bool InitialTimeSet = false;
-};
-
-/**
- * Utilities for profiling using the chrono header in C++11
- */
-struct Profiler
-{
- std::vector<Timer> m_Timers;
- std::vector<unsigned long long int> m_TotalBytes;
- bool m_IsActive = false;
-};
-
-} // end namespace
-
-#endif /* PROFILER_H_ */
diff --git a/include/core/Transport.h b/include/core/Transport.h
index 8b768825a..cc803a6da 100644
--- a/include/core/Transport.h
+++ b/include/core/Transport.h
@@ -18,7 +18,7 @@
#include "ADIOS_MPI.h"
-#include "core/Profiler.h"
+#include "packages/profiling/iochrono/IOChrono.h"
namespace adios
{
@@ -34,9 +34,9 @@ public:
MPI_Comm m_MPIComm = MPI_COMM_SELF;
- int m_RankMPI = 0; ///< current MPI rank process
- int m_SizeMPI = 1; ///< current MPI processes size
- Profiler m_Profiler; ///< collects information about Open and bytes Transport
+ int m_RankMPI = 0; ///< current MPI rank process
+ int m_SizeMPI = 1; ///< current MPI processes size
+ profiling::IOChrono m_Profiler; ///< profiles Open, Write/Read, Close
/**
* Base constructor that all derived classes pass
@@ -75,6 +75,11 @@ public:
virtual void Close(); ///< closes current transport and flushes everything,
/// transport becomes unreachable
+ /**
+ * Inits the profiler
+ * @param accessMode
+ * @param resolution
+ */
virtual void InitProfiler(const std::string accessMode,
const Support::Resolutions resolution);
diff --git a/include/engine/bp/BPFileReader.h b/include/engine/bp/BPFileReader.h
index 3613ef283..2b4f756af 100644
--- a/include/engine/bp/BPFileReader.h
+++ b/include/engine/bp/BPFileReader.h
@@ -37,9 +37,7 @@ public:
*/
BPFileReader(ADIOS &adios, const std::string name,
const std::string accessMode, MPI_Comm mpiComm,
- const Method &method, const IOMode iomode,
- const float timeout_sec, const bool debugMode = false,
- const unsigned int nthreads = 1);
+ const Method &method);
~BPFileReader();
diff --git a/include/engine/bp/BPFileWriter.h b/include/engine/bp/BPFileWriter.h
index 0465333b1..3f04ee7be 100644
--- a/include/engine/bp/BPFileWriter.h
+++ b/include/engine/bp/BPFileWriter.h
@@ -12,8 +12,7 @@
#define BPFILEWRITER_H_
#include "core/Engine.h"
-#include "format/BP1Aggregator.h"
-#include "format/BP1Writer.h"
+#include "packages/format/bp1/BP1.h"
// supported capsules
#include "capsule/heap/STLVector.h"
@@ -36,9 +35,7 @@ public:
*/
BPFileWriter(ADIOS &adios, const std::string name,
const std::string accessMode, MPI_Comm mpiComm,
- const Method &method, const IOMode iomode = IOMode::INDEPENDENT,
- const float timeout_sec = 0., const bool debugMode = false,
- const unsigned int nthreads = 1);
+ const Method &method);
~BPFileWriter();
@@ -113,8 +110,8 @@ private:
float m_GrowthFactor = 1.5; ///< capsule memory growth factor, new_memory =
/// m_GrowthFactor * current_memory
- bool m_TransportFlush =
- false; ///< true: transport flush happened, buffer must be reset
+ bool m_TransportFlush = false; ///< true: due to buffer overflow
+
bool m_CloseProcessGroup = false; ///< set to true if advance is called, this
/// prevents flattening the data and metadata
/// in Close
@@ -135,8 +132,8 @@ private:
template <class T>
void WriteVariableCommon(Variable<T> &variable, const T *values)
{
- if (m_MetadataSet.Log.m_IsActive == true)
- m_MetadataSet.Log.m_Timers[0].SetInitialTime();
+ if (m_MetadataSet.Log.IsActive == true)
+ m_MetadataSet.Log.Timers[0].SetInitialTime();
// set variable
variable.m_AppValues = values;
@@ -173,8 +170,8 @@ private:
variable.m_AppValues =
nullptr; // setting pointer to null as not needed after write
- if (m_MetadataSet.Log.m_IsActive == true)
- m_MetadataSet.Log.m_Timers[0].SetTime();
+ if (m_MetadataSet.Log.IsActive == true)
+ m_MetadataSet.Log.Timers[0].SetTime();
}
};
diff --git a/include/engine/dataman/DataManReader.h b/include/engine/dataman/DataManReader.h
index 88ba7b65c..e82fbfda4 100644
--- a/include/engine/dataman/DataManReader.h
+++ b/include/engine/dataman/DataManReader.h
@@ -40,9 +40,7 @@ public:
*/
DataManReader(ADIOS &adios, const std::string name,
const std::string accessMode, MPI_Comm mpiComm,
- const Method &method, const IOMode iomode,
- const float timeout_sec, const bool debugMode = false,
- const unsigned int nthreads = 1);
+ const Method &method);
~DataManReader();
diff --git a/include/engine/dataman/DataManWriter.h b/include/engine/dataman/DataManWriter.h
index b60399fdc..8245c00a4 100644
--- a/include/engine/dataman/DataManWriter.h
+++ b/include/engine/dataman/DataManWriter.h
@@ -15,7 +15,7 @@
#include <unistd.h> //sleep must be removed
#include "core/Engine.h"
-#include "format/BP1Writer.h"
+#include "packages/format/bp1/BP1Writer.h"
// supported capsules
#include "capsule/heap/STLVector.h"
@@ -153,9 +153,9 @@ private:
{
MPI_Barrier(m_MPIComm);
std::cout << "I am hooked to the DataMan library\n";
- std::cout << "putshape " << variable.m_Dimensions.size() << endl;
- std::cout << "varshape " << variable.m_GlobalDimensions.size() << endl;
- std::cout << "offset " << variable.m_GlobalOffsets.size() << endl;
+ std::cout << "putshape " << variable.m_Dimensions.size() << "\n";
+ std::cout << "varshape " << variable.m_GlobalDimensions.size() << "\n";
+ std::cout << "offset " << variable.m_GlobalOffsets.size() << "\n";
for (int i = 0; i < m_SizeMPI; ++i)
{
if (i == m_RankMPI)
diff --git a/include/packages/format/bp1/BP1.h b/include/packages/format/bp1/BP1.h
new file mode 100644
index 000000000..1e0aebd1b
--- /dev/null
+++ b/include/packages/format/bp1/BP1.h
@@ -0,0 +1,19 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * BP1.h
+ *
+ * Created on: Apr 4, 2017
+ * Author: wfg
+ */
+
+#ifndef BP1_H_
+#define BP1_H_
+
+#include "packages/format/bp1/BP1Structs.h"
+#include "packages/format/bp1/BP1Writer.h"
+// will add Reader later
+#include "packages/format/bp1/BP1Aggregator.h"
+
+#endif /* BP1_H_ */
diff --git a/include/format/BP1Aggregator.h b/include/packages/format/bp1/BP1Aggregator.h
similarity index 94%
rename from include/format/BP1Aggregator.h
rename to include/packages/format/bp1/BP1Aggregator.h
index a8cf13372..13f07844b 100644
--- a/include/format/BP1Aggregator.h
+++ b/include/packages/format/bp1/BP1Aggregator.h
@@ -11,6 +11,10 @@
#ifndef BP1AGGREGATOR_H_
#define BP1AGGREGATOR_H_
+/// \cond EXCLUDE_FROM_DOXYGEN
+#include <string>
+/// \endcond
+
#include "ADIOS_MPI.h"
namespace adios
diff --git a/include/format/BP1.h b/include/packages/format/bp1/BP1Base.h
similarity index 68%
rename from include/format/BP1.h
rename to include/packages/format/bp1/BP1Base.h
index fd2e16a4a..bea6ccfda 100644
--- a/include/format/BP1.h
+++ b/include/packages/format/bp1/BP1Base.h
@@ -8,20 +8,18 @@
* Author: wfg
*/
-#ifndef BP1_H_
-#define BP1_H_
+#ifndef BP1BASE_H_
+#define BP1BASE_H_
/// \cond EXCLUDE_FROM_DOXYGEN
#include <cstdint> //std::uintX_t
#include <memory> //std::shared_ptr
#include <unordered_map>
#include <vector>
-//#include <queue> //std::priority_queue to be added later
/// \endcond
+//#include <queue> //std::priority_queue to be added later
#include "ADIOS_MPI.h"
-
-#include "core/Profiler.h"
#include "core/Transport.h"
namespace adios
@@ -29,64 +27,10 @@ namespace adios
namespace format
{
-/**
- * Used for Variables and Attributes, needed in a container for characteristic
- * sets merge independently for each Variable or Attribute
- */
-struct BP1Index
-{
- std::vector<char> Buffer; ///< metadata variable index, start with 100Kb
- std::uint64_t Count =
- 0; ///< number of characteristics sets (time and spatial aggregation)
- const std::uint32_t MemberID;
-
- BP1Index(const std::uint32_t memberID) : MemberID{memberID}
- {
- Buffer.reserve(500);
- }
-};
-
-/**
- * Single struct that tracks metadata indices in bp format
- */
-struct BP1MetadataSet
-{
- std::uint32_t TimeStep; ///< current time step, updated with advance step, if
- /// append it will be updated to last, starts with one
- /// in ADIOS1
-
- BP1Index PGIndex = BP1Index(0); ///< single buffer for PGIndex
-
- // no priority for now
- std::unordered_map<std::string, BP1Index>
- VarsIndices; ///< key: variable name, value: bp metadata variable index
- std::unordered_map<std::string, BP1Index> AttributesIndices; ///< key:
- /// attribute
- /// name, value:
- /// bp metadata
- /// attribute
- /// index
-
- const unsigned int MiniFooterSize = 28; ///< from bpls reader
-
- // PG (relative) positions in Data buffer to be updated
- std::uint64_t DataPGCount = 0;
- std::size_t DataPGLengthPosition = 0; ///< current PG initial ( relative )
- /// position, needs to be updated in
- /// every advance step or init
- std::uint32_t DataPGVarsCount = 0; ///< variables in current PG
- std::size_t DataPGVarsCountPosition =
- 0; ///< current PG variable count ( relative ) position, needs to be
- /// updated in every advance step or init
- bool DataPGIsOpen = false;
-
- Profiler Log; ///< object that takes buffering profiling info
-};
-
/**
* Base class for BP1Writer and BP1Reader format
*/
-class BP1
+class BP1Base
{
public:
@@ -106,7 +50,7 @@ public:
* @param transport file I/O transport
*/
void OpenRankFiles(const std::string name, const std::string accessMode,
- Transport &transport) const;
+ Transport &file) const;
protected:
/**
@@ -266,50 +210,58 @@ protected:
// Moving template BP1Writer::GetDataType template specializations outside of
// the class
-template <> inline std::int8_t BP1::GetDataType<char>() const noexcept
+template <> inline std::int8_t BP1Base::GetDataType<char>() const noexcept
{
return type_byte;
}
-template <> inline std::int8_t BP1::GetDataType<short>() const noexcept
+
+template <> inline std::int8_t BP1Base::GetDataType<short>() const noexcept
{
return type_short;
}
-template <> inline std::int8_t BP1::GetDataType<int>() const noexcept
+
+template <> inline std::int8_t BP1Base::GetDataType<int>() const noexcept
{
return type_integer;
}
-template <> inline std::int8_t BP1::GetDataType<long int>() const noexcept
+template <> inline std::int8_t BP1Base::GetDataType<long int>() const noexcept
{
return type_long;
}
-template <> inline std::int8_t BP1::GetDataType<unsigned char>() const noexcept
+template <>
+inline std::int8_t BP1Base::GetDataType<unsigned char>() const noexcept
{
return type_unsigned_byte;
}
-template <> inline std::int8_t BP1::GetDataType<unsigned short>() const noexcept
+template <>
+inline std::int8_t BP1Base::GetDataType<unsigned short>() const noexcept
{
return type_unsigned_short;
}
-template <> inline std::int8_t BP1::GetDataType<unsigned int>() const noexcept
+template <>
+inline std::int8_t BP1Base::GetDataType<unsigned int>() const noexcept
{
return type_unsigned_integer;
}
template <>
-inline std::int8_t BP1::GetDataType<unsigned long int>() const noexcept
+inline std::int8_t BP1Base::GetDataType<unsigned long int>() const noexcept
{
return type_unsigned_long;
}
-template <> inline std::int8_t BP1::GetDataType<float>() const noexcept
+template <> inline std::int8_t BP1Base::GetDataType<float>() const noexcept
{
return type_real;
}
-template <> inline std::int8_t BP1::GetDataType<double>() const noexcept
+
+template <> inline std::int8_t BP1Base::GetDataType<double>() const noexcept
{
return type_double;
}
-template <> inline std::int8_t BP1::GetDataType<long double>() const noexcept
+
+template <>
+inline std::int8_t BP1Base::GetDataType<long double>() const noexcept
{
return type_long_double;
}
@@ -317,4 +269,4 @@ template <> inline std::int8_t BP1::GetDataType<long double>() const noexcept
} // end namespace format
} // end namespace adios
-#endif /* BP1_H_ */
+#endif /* BP1BASE_H_ */
diff --git a/include/packages/format/bp1/BP1Structs.h b/include/packages/format/bp1/BP1Structs.h
new file mode 100644
index 000000000..8c6365933
--- /dev/null
+++ b/include/packages/format/bp1/BP1Structs.h
@@ -0,0 +1,82 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * BP1Structs.h
+ *
+ * Created on: Apr 3, 2017
+ * Author: wfg
+ */
+
+#ifndef BP1STRUCTS_H_
+#define BP1STRUCTS_H_
+
+/// \cond EXCLUDE_FROM_DOXYGEN
+#include <cstdint>
+#include <string>
+#include <unordered_map>
+#include <vector>
+/// \endcond
+
+#include "packages/profiling/iochrono/IOChrono.h"
+
+namespace adios
+{
+namespace format
+{
+
+/**
+ * Metadata index used for Variables and Attributes, needed in a container for
+ * characteristic
+ * sets merge independently for each Variable or Attribute
+ */
+struct BP1Index
+{
+ std::vector<char> Buffer; ///< metadata variable index, start with 100Kb
+ std::uint64_t Count =
+ 0; ///< number of characteristics sets (time and spatial aggregation)
+ const std::uint32_t MemberID;
+
+ BP1Index(const std::uint32_t memberID) : MemberID{memberID}
+ {
+ Buffer.reserve(500);
+ }
+};
+
+/**
+ * Single struct that tracks metadata indices in bp format
+ */
+struct BP1MetadataSet
+{
+ std::uint32_t TimeStep; ///< current time step, updated with advance step, if
+ /// append it will be updated to last, starts with one in ADIOS1
+
+ BP1Index PGIndex = BP1Index(0); ///< single buffer for PGIndex
+
+ // no priority for now
+ std::unordered_map<std::string, BP1Index>
+ VarsIndices; ///< key: variable name, value: bp metadata variable index
+
+ std::unordered_map<std::string, BP1Index>
+ AttributesIndices; ///< key: name, value: attribute bp index
+
+ const unsigned int MiniFooterSize = 28; ///< from bpls reader
+
+ // PG (relative) positions in Data buffer to be updated
+ std::uint64_t DataPGCount = 0;
+ std::size_t DataPGLengthPosition = 0; ///< current PG initial ( relative )
+ /// position, needs to be updated in
+ /// every advance step or init
+ std::uint32_t DataPGVarsCount = 0; ///< variables in current PG
+ std::size_t DataPGVarsCountPosition = 0;
+ ///< current PG variable count ( relative ) position, needs to be
+ /// updated in every advance step or init
+ bool DataPGIsOpen = false;
+
+ profiling::IOChrono Log; ///< object that takes buffering profiling info
+};
+
+} // end namespace format
+} // end namespace adios
+
+#endif /* BP1STRUCTS_H_ */
diff --git a/include/format/BP1Writer.h b/include/packages/format/bp1/BP1Writer.h
similarity index 95%
rename from include/format/BP1Writer.h
rename to include/packages/format/bp1/BP1Writer.h
index 1a08598a4..9295251db 100644
--- a/include/format/BP1Writer.h
+++ b/include/packages/format/bp1/BP1Writer.h
@@ -17,29 +17,25 @@
#include <cstring> //std::memcpy
/// \endcond
-#include "BP1.h"
#include "capsule/heap/STLVector.h"
-#include "core/Capsule.h"
-#include "core/Profiler.h"
#include "core/Variable.h"
#include "functions/adiosFunctions.h"
#include "functions/adiosTemplates.h"
+#include "packages/format/bp1/BP1Base.h"
+#include "packages/format/bp1/BP1Structs.h"
namespace adios
{
namespace format
{
-class BP1Writer : public BP1
+class BP1Writer : public BP1Base
{
public:
- unsigned int m_Threads =
- 1; ///< number of threads for thread operations in large array (min,max)
- unsigned int m_Verbosity =
- 0; ///< statistics verbosity, can change if redefined in Engine method.
- float m_GrowthFactor =
- 1.5; ///< memory growth factor, can change if redefined in Engine method.
+ unsigned int m_Threads = 1; ///< thread operations in large array (min,max)
+ unsigned int m_Verbosity = 0; ///< statistics verbosity, only 0 is supported
+ float m_GrowthFactor = 1.5; ///< memory growth factor
const std::uint8_t m_Version = 3; ///< BP format version
/**
@@ -90,7 +86,7 @@ public:
// characteristics 3 and 4, check variable number of dimensions
const std::size_t dimensions =
- variable.DimensionsSize(); // number of commas in CSV + 1
+ variable.DimensionsSize(); // commas in CSV + 1
indexSize += 28 * dimensions; // 28 bytes per dimension
indexSize += 1; // id
@@ -124,9 +120,9 @@ public:
* @param metadataSet
*/
template <class T>
- inline void WriteVariableMetadata(const Variable<T> &variable,
- capsule::STLVector &heap,
- BP1MetadataSet &metadataSet) const noexcept
+ void WriteVariableMetadata(const Variable<T> &variable,
+ capsule::STLVector &heap,
+ BP1MetadataSet &metadataSet) const noexcept
{
Stats<T> stats = GetStats(variable);
WriteVariableMetadataCommon(variable, stats, heap, metadataSet);
@@ -425,7 +421,7 @@ private:
// //here we can make decisions for threads
// based on valuesSize
GetMinMax(variable.m_AppValues, valuesSize, stats.Min, stats.Max,
- m_Threads); // here we can add cores from constructor
+ m_Threads);
else
GetMinMax(variable.m_AppValues, valuesSize, stats.Min, stats.Max);
}
diff --git a/include/packages/profiling/iochrono/IOChrono.h b/include/packages/profiling/iochrono/IOChrono.h
new file mode 100644
index 000000000..442a8df8c
--- /dev/null
+++ b/include/packages/profiling/iochrono/IOChrono.h
@@ -0,0 +1,40 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * IOChrono.h
+ *
+ * Created on: Mar 9, 2017
+ * Author: wfg
+ */
+
+#ifndef IOCHRONO_H_
+#define IOCHRONO_H_
+
+/// \cond EXCLUDE_FROM_DOXYGEN
+#include <vector>
+/// \endcond
+
+#include "packages/profiling/iochrono/Timer.h"
+
+namespace adios
+{
+namespace profiling
+{
+
+/**
+ * Struct used to track
+ */
+struct IOChrono
+{
+ std::vector<Timer>
+ Timers; ///< one timer for each process (open, write, buffering, etc.)
+ std::vector<unsigned long long int>
+ TotalBytes; ///< tracks bytes for buffering
+ bool IsActive = false; ///< flag to determine if profiling is used
+};
+
+} // end namespace
+} // end namespace
+
+#endif /* IOCHRONO_H_ */
diff --git a/include/packages/profiling/iochrono/Timer.h b/include/packages/profiling/iochrono/Timer.h
new file mode 100644
index 000000000..1ded96cb0
--- /dev/null
+++ b/include/packages/profiling/iochrono/Timer.h
@@ -0,0 +1,65 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * Timer.h
+ *
+ * Created on: Apr 4, 2017
+ * Author: wfg
+ */
+
+#ifndef TIMER_H_
+#define TIMER_H_
+
+/// \cond EXCLUDE_FROM_DOXYGEN
+#include <chrono>
+#include <string>
+/// \endcond
+
+#include "core/Support.h"
+
+namespace adios
+{
+namespace profiling
+{
+
+class Timer
+{
+
+public:
+ const std::string m_Process;
+ unsigned long long int m_ProcessTime = 0;
+
+ /**
+ *
+ */
+ /**
+ *
+ * Timer object constructor using std::chrono class
+ * @param process name of process to be measured
+ * @param resolution time resolution (mus, ms, s, etc.) from Resolutions in
+ * core/Support.h
+ * @param debugMode true: additional checks and exceptions, false: no checks
+ */
+ Timer(const std::string process, const Support::Resolutions resolution,
+ const bool debugMode = false);
+
+ ~Timer();
+
+ void SetInitialTime(); ///< set time to start counting for a process
+ void SetTime(); ///< sets time to measure, it will add to overall process time
+ std::string GetUnits() const;
+
+private:
+ const Support::Resolutions m_Resolution;
+ const bool m_DebugMode = false;
+ std::chrono::time_point<std::chrono::high_resolution_clock> m_InitialTime;
+ std::chrono::time_point<std::chrono::high_resolution_clock> m_ElapsedTime;
+ bool m_InitialTimeSet = false;
+ long long int GetCurrentTime(); ///< called by SetTime
+};
+
+} // end namespace profiling
+} // end namespace adios
+
+#endif /* TIMER_H_ */
diff --git a/source/ADIOS.cpp b/source/ADIOS.cpp
index a517acd27..efd3e2eac 100644
--- a/source/ADIOS.cpp
+++ b/source/ADIOS.cpp
@@ -129,14 +129,12 @@ std::shared_ptr<Engine> ADIOS::Open(const std::string &name,
if (isDefaultWriter || type == "BPFileWriter" || type == "bpfilewriter")
{
return std::make_shared<BPFileWriter>(*this, name, accessMode, mpiComm,
- method, iomode, timeout_sec,
- m_DebugMode, method.m_nThreads);
+ method);
}
else if (isDefaultReader || type == "BPReader" || type == "bpreader")
{
return std::make_shared<BPFileReader>(*this, name, accessMode, mpiComm,
- method, iomode, timeout_sec,
- m_DebugMode, method.m_nThreads);
+ method);
}
else if (type == "SIRIUS" || type == "sirius" || type == "Sirius")
{
@@ -148,8 +146,7 @@ std::shared_ptr<Engine> ADIOS::Open(const std::string &name,
{
#ifdef HAVE_DATAMAN
return std::make_shared<DataManWriter>(*this, name, accessMode, mpiComm,
- method, iomode, timeout_sec,
- m_DebugMode, method.m_nThreads);
+ method);
#else
throw std::invalid_argument("ERROR: this version didn't compile with "
"Dataman library, can't Open DataManWriter\n");
@@ -159,8 +156,7 @@ std::shared_ptr<Engine> ADIOS::Open(const std::string &name,
{
#ifdef HAVE_DATAMAN
return std::make_shared<DataManReader>(*this, name, accessMode, mpiComm,
- method, iomode, timeout_sec,
- m_DebugMode, method.m_nThreads);
+ method);
#else
throw std::invalid_argument("ERROR: this version didn't compile with "
"Dataman library, can't Open DataManReader\n");
@@ -170,8 +166,7 @@ std::shared_ptr<Engine> ADIOS::Open(const std::string &name,
{
#ifdef HAVE_ADIOS1
return std::make_shared<ADIOS1Writer>(*this, name, accessMode, mpiComm,
- method, iomode, timeout_sec,
- m_DebugMode, method.m_nThreads);
+ method);
#else
throw std::invalid_argument("ERROR: this version didn't compile with ADIOS "
"1.x library, can't Open ADIOS1Writer\n");
diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt
index 1b47d0fa6..731a1153f 100644
--- a/source/CMakeLists.txt
+++ b/source/CMakeLists.txt
@@ -20,9 +20,11 @@ add_library(adios2
engine/bp/BPFileReader.cpp
engine/bp/BPFileWriter.cpp
- format/BP1.cpp
- format/BP1Aggregator.cpp
- format/BP1Writer.cpp
+ packages/format/bp1/BP1Base.cpp
+ packages/format/bp1/BP1Aggregator.cpp
+ packages/format/bp1/BP1Writer.cpp
+
+ packages/profiling/iochrono/Timer.cpp
functions/adiosFunctions.cpp
diff --git a/source/core/Engine.cpp b/source/core/Engine.cpp
index 4f2cb8a44..df35828eb 100644
--- a/source/core/Engine.cpp
+++ b/source/core/Engine.cpp
@@ -15,13 +15,13 @@
namespace adios
{
-Engine::Engine(ADIOS &adios, std::string engineType, std::string name,
- std::string accessMode, MPI_Comm mpiComm, const Method &method,
- bool debugMode, unsigned int nthreads, std::string endMessage)
+Engine::Engine(ADIOS &adios, const std::string engineType,
+ const std::string name, const std::string accessMode,
+ MPI_Comm mpiComm, const Method &method, std::string endMessage)
: m_MPIComm{mpiComm}, m_EngineType{std::move(engineType)},
m_Name{std::move(name)}, m_AccessMode{std::move(accessMode)},
- m_Method{method}, m_ADIOS(adios), m_DebugMode{debugMode},
- m_nThreads{nthreads}, m_EndMessage(std::move(endMessage))
+ m_Method{method}, m_ADIOS(adios), m_EndMessage(std::move(endMessage)),
+ m_DebugMode{m_Method.m_DebugMode}, m_nThreads{m_Method.m_nThreads}
{
if (m_DebugMode == true)
{
@@ -39,242 +39,158 @@ Engine::Engine(ADIOS &adios, std::string engineType, std::string name,
void Engine::SetCallBack(std::function<void(const void *, std::string,
std::string, std::string, Dims)>
- /*callback*/)
+ callback)
{
}
// should these functions throw an exception?
-void Engine::Write(Variable<char> & /*variable*/, const char * /*values*/) {}
-void Engine::Write(Variable<unsigned char> & /*variable*/,
- const unsigned char * /*values*/)
-{
-}
-void Engine::Write(Variable<short> & /*variable*/, const short * /*values*/) {}
-void Engine::Write(Variable<unsigned short> & /*variable*/,
- const unsigned short * /*values*/)
-{
-}
-void Engine::Write(Variable<int> & /*variable*/, const int * /*values*/) {}
-void Engine::Write(Variable<unsigned int> & /*variable*/,
- const unsigned int * /*values*/)
-{
-}
-void Engine::Write(Variable<long int> & /*variable*/,
- const long int * /*values*/)
-{
-}
-void Engine::Write(Variable<unsigned long int> & /*variable*/,
- const unsigned long int * /*values*/)
-{
-}
-void Engine::Write(Variable<long long int> & /*variable*/,
- const long long int * /*values*/)
-{
-}
-void Engine::Write(Variable<unsigned long long int> & /*variable*/,
- const unsigned long long int * /*values*/)
-{
-}
-void Engine::Write(Variable<float> & /*variable*/, const float * /*values*/) {}
-void Engine::Write(Variable<double> & /*variable*/, const double * /*values*/)
-{
-}
-void Engine::Write(Variable<long double> & /*variable*/,
- const long double * /*values*/)
-{
-}
-void Engine::Write(Variable<std::complex<float>> & /*variable*/,
- const std::complex<float> * /*values*/)
-{
-}
-void Engine::Write(Variable<std::complex<double>> & /*variable*/,
- const std::complex<double> * /*values*/)
-{
-}
-void Engine::Write(Variable<std::complex<long double>> & /*variable*/,
- const std::complex<long double> * /*values*/)
-{
-}
-void Engine::Write(VariableCompound & /*variable*/, const void * /*values*/) {}
-void Engine::Write(const std::string & /*variableName*/,
- const char * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const unsigned char * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const short * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const unsigned short * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/, const int * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const unsigned int * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const long int * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const unsigned long int * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const long long int * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const unsigned long long int * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const float * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const double * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const long double * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const std::complex<float> * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const std::complex<double> * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const std::complex<long double> * /*values*/)
-{
-}
-void Engine::Write(const std::string & /*variableName*/,
- const void * /*values*/)
-{
-}
+// clang-format off
+void Engine::Write(Variable<char> &variable, const char *values) {}
+void Engine::Write(Variable<unsigned char> &variable, const unsigned char *values){ }
+void Engine::Write(Variable<short> &variable, const short *values) {}
+void Engine::Write(Variable<unsigned short> &variable, const unsigned short *values) {}
+void Engine::Write(Variable<int> &variable, const int *values) {}
+void Engine::Write(Variable<unsigned int> &variable, const unsigned int *values) {}
+void Engine::Write(Variable<long int> &variable, const long int *values) {}
+void Engine::Write(Variable<unsigned long int> &variable, const unsigned long int *values) {}
+void Engine::Write(Variable<long long int> &variable, const long long int *values) {}
+void Engine::Write(Variable<unsigned long long int> &variable, const unsigned long long int *values) {}
+void Engine::Write(Variable<float> &variable, const float *values) {}
+void Engine::Write(Variable<double> &variable, const double *values) {}
+void Engine::Write(Variable<long double> &variable, const long double *values) {}
+void Engine::Write(Variable<std::complex<float>> &variable, const std::complex<float> *values) {}
+void Engine::Write(Variable<std::complex<double>> &variable, const std::complex<double> *values) {}
+void Engine::Write(Variable<std::complex<long double>> &variable, const std::complex<long double> *values) {}
+void Engine::Write(VariableCompound &variable, const void *values) {}
+
+// String version
+void Engine::Write(const std::string variableName, const char *values) {}
+void Engine::Write(const std::string variableName, const unsigned char *values) {}
+void Engine::Write(const std::string variableName, const short *values) {}
+void Engine::Write(const std::string variableName, const unsigned short *values) {}
+void Engine::Write(const std::string variableName, const int *values) {}
+void Engine::Write(const std::string variableName, const unsigned int *values) {}
+void Engine::Write(const std::string variableName, const long int *values) {}
+void Engine::Write(const std::string variableName, const unsigned long int *values) {}
+void Engine::Write(const std::string variableName, const long long int *values) {}
+void Engine::Write(const std::string variableName, const unsigned long long int *values) {}
+void Engine::Write(const std::string variableName, const float *values) {}
+void Engine::Write(const std::string variableName, const double *values) {}
+void Engine::Write(const std::string variableName, const long double *values) {}
+void Engine::Write(const std::string variableName, const std::complex<float> *values) {}
+void Engine::Write(const std::string variableName, const std::complex<double> *values) {}
+void Engine::Write(const std::string variableName, const std::complex<long double> *values) {}
+void Engine::Write(const std::string variableName, const void *values) {}
+// clang-format on
-void Engine::Advance(float /*timeout_sec*/) {}
-void Engine::Advance(AdvanceMode /*mode*/, float /*timeout_sec*/) {}
+void Engine::Advance(float timeout_sec) {}
+void Engine::Advance(AdvanceMode mode, float timeout_sec) {}
void Engine::AdvanceAsync(
- AdvanceMode /*mode*/,
- std::function<void(std::shared_ptr<adios::Engine>)> /*callback*/)
+ AdvanceMode mode,
+ std::function<void(std::shared_ptr<adios::Engine>)> callback)
{
}
-void Engine::Close(const int /*transportIndex*/) {}
-
// READ
-Variable<void> *Engine::InquireVariable(const std::string /*name*/,
- const bool /*readIn*/)
+Variable<void> *Engine::InquireVariable(const std::string name,
+ const bool readIn)
{
return nullptr;
}
-Variable<char> *Engine::InquireVariableChar(const std::string /*name*/,
- const bool /*readIn*/)
+
+Variable<char> *Engine::InquireVariableChar(const std::string name,
+ const bool readIn)
{
return nullptr;
}
-Variable<unsigned char> *
-Engine::InquireVariableUChar(const std::string /*name*/, const bool /*readIn*/)
+
+Variable<unsigned char> *Engine::InquireVariableUChar(const std::string name,
+ const bool readIn)
{
return nullptr;
}
-Variable<short> *Engine::InquireVariableShort(const std::string /*name*/,
- const bool /*readIn*/)
+
+Variable<short> *Engine::InquireVariableShort(const std::string name,
+ const bool readIn)
{
return nullptr;
}
-Variable<unsigned short> *
-Engine::InquireVariableUShort(const std::string /*name*/, const bool /*readIn*/)
+
+Variable<unsigned short> *Engine::InquireVariableUShort(const std::string name,
+ const bool readIn)
{
return nullptr;
}
-Variable<int> *Engine::InquireVariableInt(const std::string /*name*/,
- const bool /*readIn*/)
+Variable<int> *Engine::InquireVariableInt(const std::string name,
+ const bool readIn)
{
return nullptr;
}
-Variable<unsigned int> *Engine::InquireVariableUInt(const std::string /*name*/,
- const bool /*readIn*/)
+Variable<unsigned int> *Engine::InquireVariableUInt(const std::string name,
+ const bool readIn)
{
return nullptr;
}
-Variable<long int> *Engine::InquireVariableLInt(const std::string /*name*/,
- const bool /*readIn*/)
+Variable<long int> *Engine::InquireVariableLInt(const std::string name,
+ const bool readIn)
{
return nullptr;
}
Variable<unsigned long int> *
-Engine::InquireVariableULInt(const std::string /*name*/, const bool /*readIn*/)
+Engine::InquireVariableULInt(const std::string name, const bool readIn)
{
return nullptr;
}
-Variable<long long int> *
-Engine::InquireVariableLLInt(const std::string /*name*/, const bool /*readIn*/)
+Variable<long long int> *Engine::InquireVariableLLInt(const std::string name,
+ const bool readIn)
{
return nullptr;
}
Variable<unsigned long long int> *
-Engine::InquireVariableULLInt(const std::string /*name*/, const bool /*readIn*/)
+Engine::InquireVariableULLInt(const std::string name, const bool readIn)
{
return nullptr;
}
-Variable<float> *Engine::InquireVariableFloat(const std::string /*name*/,
- const bool /*readIn*/)
+Variable<float> *Engine::InquireVariableFloat(const std::string name,
+ const bool readIn)
{
return nullptr;
}
-Variable<double> *Engine::InquireVariableDouble(const std::string /*name*/,
- const bool /*readIn*/)
+Variable<double> *Engine::InquireVariableDouble(const std::string name,
+ const bool readIn)
{
return nullptr;
}
-Variable<long double> *
-Engine::InquireVariableLDouble(const std::string /*name*/,
- const bool /*readIn*/)
+
+Variable<long double> *Engine::InquireVariableLDouble(const std::string name,
+ const bool readIn)
{
return nullptr;
}
Variable<std::complex<float>> *
-Engine::InquireVariableCFloat(const std::string /*name*/, const bool /*readIn*/)
+Engine::InquireVariableCFloat(const std::string name, const bool readIn)
{
return nullptr;
}
Variable<std::complex<double>> *
-Engine::InquireVariableCDouble(const std::string /*name*/,
- const bool /*readIn*/)
+Engine::InquireVariableCDouble(const std::string name, const bool readIn)
{
return nullptr;
}
Variable<std::complex<long double>> *
-Engine::InquireVariableCLDouble(const std::string /*name*/,
- const bool /*readIn*/)
+Engine::InquireVariableCLDouble(const std::string name, const bool readIn)
{
return nullptr;
}
-VariableCompound *Engine::InquireVariableCompound(const std::string /*name*/,
- const bool /*readIn*/)
+VariableCompound *Engine::InquireVariableCompound(const std::string name,
+ const bool readIn)
{
return nullptr;
}
-void Engine::Read(Variable<double> & /*variable*/, const double * /*values*/) {}
-void Engine::ScheduleRead(Variable<double> & /*variable*/,
- const double * /*values*/)
-{
-}
+void Engine::Read(Variable<double> &variable, const double *values) {}
+void Engine::ScheduleRead(Variable<double> &variable, const double *values) {}
void Engine::Release() {}
// PROTECTED
diff --git a/source/core/Method.cpp b/source/core/Method.cpp
index a97f101fc..ee22d5bdc 100644
--- a/source/core/Method.cpp
+++ b/source/core/Method.cpp
@@ -30,7 +30,7 @@ bool Method::isUserDefined()
void Method::SetEngine(const std::string type) { m_Type = type; }
-void Method::AllowThreads(const int nThreads)
+void Method::AllowThreads(const unsigned int nThreads)
{
if (nThreads > 1)
m_nThreads = nThreads;
diff --git a/source/core/Transport.cpp b/source/core/Transport.cpp
index 3f3fd7728..42fcb79ee 100644
--- a/source/core/Transport.cpp
+++ b/source/core/Transport.cpp
@@ -32,21 +32,21 @@ void Transport::Close() {}
void Transport::InitProfiler(const std::string accessMode,
const Support::Resolutions resolution)
{
- m_Profiler.m_Timers.emplace_back("open", Support::Resolutions::mus);
+ m_Profiler.Timers.emplace_back("open", Support::Resolutions::mus);
if (accessMode == "w" || accessMode == "write")
- m_Profiler.m_Timers.emplace_back("write", resolution);
+ m_Profiler.Timers.emplace_back("write", resolution);
else if (accessMode == "a" || accessMode == "append")
- m_Profiler.m_Timers.emplace_back("append", resolution);
+ m_Profiler.Timers.emplace_back("append", resolution);
else if (accessMode == "r" || accessMode == "read")
- m_Profiler.m_Timers.emplace_back("read", resolution);
+ m_Profiler.Timers.emplace_back("read", resolution);
- m_Profiler.m_Timers.emplace_back("close", Support::Resolutions::mus);
+ m_Profiler.Timers.emplace_back("close", Support::Resolutions::mus);
- m_Profiler.m_TotalBytes.push_back(0);
- m_Profiler.m_IsActive = true;
+ m_Profiler.TotalBytes.push_back(0);
+ m_Profiler.IsActive = true;
}
} // end namespace
diff --git a/source/engine/bp/BPFileReader.cpp b/source/engine/bp/BPFileReader.cpp
index 406c752d3..4a542c4a9 100644
--- a/source/engine/bp/BPFileReader.cpp
+++ b/source/engine/bp/BPFileReader.cpp
@@ -23,11 +23,9 @@ namespace adios
BPFileReader::BPFileReader(ADIOS &adios, const std::string name,
const std::string accessMode, MPI_Comm mpiComm,
- const Method &method, const IOMode iomode,
- const float timeout_sec, const bool debugMode,
- const unsigned int nthreads)
-: Engine(adios, "BPFileReader", name, accessMode, mpiComm, method, debugMode,
- nthreads, " BPFileReader constructor (or call to ADIOS Open).\n"),
+ const Method &method)
+: Engine(adios, "BPFileReader", name, accessMode, mpiComm, method,
+ " BPFileReader constructor (or call to ADIOS Open).\n"),
m_Buffer(accessMode, m_RankMPI, m_DebugMode)
{
Init();
diff --git a/source/engine/bp/BPFileWriter.cpp b/source/engine/bp/BPFileWriter.cpp
index 2785781f0..09d6a90c8 100644
--- a/source/engine/bp/BPFileWriter.cpp
+++ b/source/engine/bp/BPFileWriter.cpp
@@ -21,13 +21,11 @@ namespace adios
BPFileWriter::BPFileWriter(ADIOS &adios, const std::string name,
const std::string accessMode, MPI_Comm mpiComm,
- const Method &method, const IOMode iomode,
- const float timeout_sec, const bool debugMode,
- const unsigned int nthreads)
-: Engine(adios, "BPFileWriter", name, accessMode, mpiComm, method, debugMode,
- nthreads, " BPFileWriter constructor (or call to ADIOS Open).\n"),
+ const Method &method)
+: Engine(adios, "BPFileWriter", name, accessMode, mpiComm, method,
+ " BPFileWriter constructor (or call to ADIOS Open).\n"),
m_Buffer{capsule::STLVector(accessMode, m_RankMPI, m_DebugMode)},
- m_BP1Aggregator{format::BP1Aggregator(m_MPIComm, debugMode)},
+ m_BP1Aggregator{format::BP1Aggregator(m_MPIComm, m_DebugMode)},
m_MaxBufferSize{m_Buffer.m_Data.max_size()}
{
m_MetadataSet.TimeStep = 1; // starting at one to be compatible with ADIOS1.x
@@ -259,7 +257,7 @@ void BPFileWriter::Close(const int transportIndex)
false); // false: not using aggregation for now
}
- if (m_MetadataSet.Log.m_IsActive == true)
+ if (m_MetadataSet.Log.IsActive == true)
{
bool allClose = true;
for (auto &transport : m_Transports)
@@ -336,19 +334,19 @@ void BPFileWriter::InitParameters()
auto &profiler = m_MetadataSet.Log;
if (itProfile->second == "mus" || itProfile->second == "microseconds")
- profiler.m_Timers.emplace_back("buffering", Support::Resolutions::mus);
+ profiler.Timers.emplace_back("buffering", Support::Resolutions::mus);
else if (itProfile->second == "ms" || itProfile->second == "milliseconds")
- profiler.m_Timers.emplace_back("buffering", Support::Resolutions::ms);
+ profiler.Timers.emplace_back("buffering", Support::Resolutions::ms);
else if (itProfile->second == "s" || itProfile->second == "seconds")
- profiler.m_Timers.emplace_back("buffering", Support::Resolutions::s);
+ profiler.Timers.emplace_back("buffering", Support::Resolutions::s);
else if (itProfile->second == "min" || itProfile->second == "minutes")
- profiler.m_Timers.emplace_back("buffering", Support::Resolutions::m);
+ profiler.Timers.emplace_back("buffering", Support::Resolutions::m);
else if (itProfile->second == "h" || itProfile->second == "hours")
- profiler.m_Timers.emplace_back("buffering", Support::Resolutions::h);
+ profiler.Timers.emplace_back("buffering", Support::Resolutions::h);
else
{
if (m_DebugMode == true)
@@ -357,7 +355,7 @@ void BPFileWriter::InitParameters()
"call to Open or Engine constructor\n");
}
- profiler.m_IsActive = true;
+ profiler.IsActive = true;
}
}
@@ -468,8 +466,8 @@ void BPFileWriter::InitTransports()
void BPFileWriter::InitProcessGroup()
{
- if (m_MetadataSet.Log.m_IsActive == true)
- m_MetadataSet.Log.m_Timers[0].SetInitialTime();
+ if (m_MetadataSet.Log.IsActive == true)
+ m_MetadataSet.Log.Timers[0].SetInitialTime();
if (m_AccessMode == "a")
{
@@ -479,8 +477,8 @@ void BPFileWriter::InitProcessGroup()
WriteProcessGroupIndex();
- if (m_MetadataSet.Log.m_IsActive == true)
- m_MetadataSet.Log.m_Timers[0].SetTime();
+ if (m_MetadataSet.Log.IsActive == true)
+ m_MetadataSet.Log.Timers[0].SetTime();
}
void BPFileWriter::WriteProcessGroupIndex()
diff --git a/source/engine/dataman/DataManReader.cpp b/source/engine/dataman/DataManReader.cpp
index 941fe8f60..6d749319a 100644
--- a/source/engine/dataman/DataManReader.cpp
+++ b/source/engine/dataman/DataManReader.cpp
@@ -14,11 +14,11 @@
#include "functions/adiosFunctions.h" //CSVToVector
// supported transports
-#include "transport/file/FStream.h" // uses C++ fstream
-#include "transport/wan/MdtmMan.h" //uses Mdtm library
+#include "transport/file/FStream.h" // uses C++ fstream
+#include "transport/file/FileDescriptor.h" // uses POSIX
+#include "transport/file/FilePointer.h" // uses C FILE*
+#include "transport/wan/MdtmMan.h" //uses Mdtm library
-#include "../../../include/transport/file/FileDescriptor.h" // uses POSIX
-#include "../../../include/transport/file/FilePointer.h" // uses C FILE*
#include "DataMan.h" //here comes your DataMan header from external dataman library
namespace adios
@@ -26,12 +26,10 @@ namespace adios
DataManReader::DataManReader(ADIOS &adios, const std::string name,
const std::string accessMode, MPI_Comm mpiComm,
- const Method &method, const IOMode iomode,
- const float timeout_sec, const bool debugMode,
- const unsigned int nthreads)
-: Engine(adios, "DataManReader", name, accessMode, mpiComm, method, debugMode,
- nthreads, " DataManReader constructor (or call to ADIOS Open).\n"),
- m_Buffer(accessMode, m_RankMPI, m_DebugMode)
+ const Method &method)
+: Engine(adios, "DataManReader", name, accessMode, mpiComm, method,
+ " DataManReader constructor (or call to ADIOS Open).\n")
+// m_Buffer(accessMode, m_RankMPI, m_DebugMode)
{
Init();
}
diff --git a/source/engine/dataman/DataManWriter.cpp b/source/engine/dataman/DataManWriter.cpp
index edb527579..8e5383fff 100644
--- a/source/engine/dataman/DataManWriter.cpp
+++ b/source/engine/dataman/DataManWriter.cpp
@@ -27,8 +27,8 @@ DataManWriter::DataManWriter(ADIOS &adios, const std::string name,
const Method &method, const IOMode iomode,
const float timeout_sec, const bool debugMode,
const unsigned int nthreads)
-: Engine(adios, "DataManWriter", name, accessMode, mpiComm, method, debugMode,
- nthreads, " DataManWriter constructor (or call to ADIOS Open).\n"),
+: Engine(adios, "DataManWriter", name, accessMode, mpiComm, method,
+ " DataManWriter constructor (or call to ADIOS Open).\n"),
m_Buffer(accessMode, m_RankMPI, m_DebugMode)
{
Init();
diff --git a/source/format/BP1Aggregator.cpp b/source/packages/format/bp1/BP1Aggregator.cpp
similarity index 91%
rename from source/format/BP1Aggregator.cpp
rename to source/packages/format/bp1/BP1Aggregator.cpp
index c81a72bbf..aa6a5d2f6 100644
--- a/source/format/BP1Aggregator.cpp
+++ b/source/packages/format/bp1/BP1Aggregator.cpp
@@ -14,7 +14,7 @@
#include <vector>
/// \endcond
-#include "format/BP1Aggregator.h"
+#include "packages/format/bp1/BP1Aggregator.h"
namespace adios
{
@@ -36,9 +36,8 @@ void BP1Aggregator::WriteProfilingLog(const std::string fileName,
if (m_RankMPI == 0)
{
unsigned int sizeMPI = static_cast<unsigned int>(m_SizeMPI);
- std::vector<std::vector<char>> rankLogs(sizeMPI -
- 1); // rankLogs from other processes
- std::vector<int> rankLogsSizes(sizeMPI - 1, -1); // init with -1
+ std::vector<std::vector<char>> rankLogs(sizeMPI - 1); // other ranks
+ std::vector<int> rankLogsSizes(sizeMPI - 1, -1); // init with -1
std::vector<MPI_Request> requests(sizeMPI);
std::vector<MPI_Status> statuses(sizeMPI);
diff --git a/source/format/BP1.cpp b/source/packages/format/bp1/BP1Base.cpp
similarity index 83%
rename from source/format/BP1.cpp
rename to source/packages/format/bp1/BP1Base.cpp
index c81e7b6fe..ddb0b042e 100644
--- a/source/format/BP1.cpp
+++ b/source/packages/format/bp1/BP1Base.cpp
@@ -8,7 +8,7 @@
* Author: wfg
*/
-#include "format/BP1.h"
+#include "packages/format/bp1/BP1Base.h"
#include "functions/adiosFunctions.h"
namespace adios
@@ -16,7 +16,7 @@ namespace adios
namespace format
{
-std::string BP1::GetDirectoryName(const std::string name) const noexcept
+std::string BP1Base::GetDirectoryName(const std::string name) const noexcept
{
std::string directory;
@@ -28,8 +28,9 @@ std::string BP1::GetDirectoryName(const std::string name) const noexcept
return directory;
}
-void BP1::OpenRankFiles(const std::string name, const std::string accessMode,
- Transport &file) const
+// this should go outside
+void BP1Base::OpenRankFiles(const std::string name,
+ const std::string accessMode, Transport &file) const
{
const std::string directory = GetDirectoryName(name);
CreateDirectory(
@@ -42,7 +43,7 @@ void BP1::OpenRankFiles(const std::string name, const std::string accessMode,
// location fro writing
}
-std::vector<std::uint8_t> BP1::GetMethodIDs(
+std::vector<std::uint8_t> BP1Base::GetMethodIDs(
const std::vector<std::shared_ptr<Transport>> &transports) const noexcept
{
auto lf_GetMethodID = [](const std::string method) -> std::uint8_t {
diff --git a/source/format/BP1Writer.cpp b/source/packages/format/bp1/BP1Writer.cpp
similarity index 94%
rename from source/format/BP1Writer.cpp
rename to source/packages/format/bp1/BP1Writer.cpp
index b5aed5404..4f908f293 100644
--- a/source/format/BP1Writer.cpp
+++ b/source/packages/format/bp1/BP1Writer.cpp
@@ -13,8 +13,7 @@
#include <vector>
/// \endcond
-#include "core/Profiler.h"
-#include "format/BP1Writer.h"
+#include "packages/format/bp1/BP1Writer.h"
namespace adios
{
@@ -120,8 +119,8 @@ void BP1Writer::Close(BP1MetadataSet &metadataSet, capsule::STLVector &heap,
Transport &transport, bool &isFirstClose,
const bool doAggregation) const noexcept
{
- if (metadataSet.Log.m_IsActive == true)
- metadataSet.Log.m_Timers[0].SetInitialTime();
+ if (metadataSet.Log.IsActive == true)
+ metadataSet.Log.Timers[0].SetInitialTime();
if (isFirstClose == true)
{
@@ -130,8 +129,8 @@ void BP1Writer::Close(BP1MetadataSet &metadataSet, capsule::STLVector &heap,
FlattenMetadata(metadataSet, heap);
- if (metadataSet.Log.m_IsActive == true)
- metadataSet.Log.m_Timers[0].SetInitialTime();
+ if (metadataSet.Log.IsActive == true)
+ metadataSet.Log.Timers[0].SetInitialTime();
if (doAggregation == true) // N-to-M where 1 <= M <= N-1, might need a new
// Log metadataSet.Log.m_Timers just for
@@ -157,21 +156,22 @@ std::string BP1Writer::GetRankProfilingLog(
const int rank, const BP1MetadataSet &metadataSet,
const std::vector<std::shared_ptr<Transport>> &transports) const noexcept
{
- auto lf_WriterTimer = [](std::string &rankLog, const Timer &timer) {
- rankLog += "'" + timer.Process + "_" + timer.GetUnits() + "': " +
- std::to_string(timer.ProcessTime) + ", ";
+ auto lf_WriterTimer = [](std::string &rankLog,
+ const profiling::Timer &timer) {
+ rankLog += "'" + timer.m_Process + "_" + timer.GetUnits() + "': " +
+ std::to_string(timer.m_ProcessTime) + ", ";
};
// prepare string dictionary per rank
std::string rankLog("'rank_" + std::to_string(rank) + "': { ");
auto &profiler = metadataSet.Log;
- rankLog += "'bytes': " + std::to_string(profiler.m_TotalBytes[0]) + ", ";
- lf_WriterTimer(rankLog, profiler.m_Timers[0]);
+ rankLog += "'bytes': " + std::to_string(profiler.TotalBytes[0]) + ", ";
+ lf_WriterTimer(rankLog, profiler.Timers[0]);
for (unsigned int t = 0; t < transports.size(); ++t)
{
- auto &timers = transports[t]->m_Profiler.m_Timers;
+ auto &timers = transports[t]->m_Profiler.Timers;
rankLog += "'transport_" + std::to_string(t) + "': { ";
rankLog += "'lib': " + transports[t]->m_Type + ", ";
@@ -307,6 +307,7 @@ void BP1Writer::FlattenMetadata(BP1MetadataSet &metadataSet,
auto lf_IndexCountLength =
[](std::unordered_map<std::string, BP1Index> &indices,
std::uint32_t &count, std::uint64_t &length) {
+
count = indices.size();
length = 0;
for (auto &indexPair : indices) // set each index length
@@ -390,8 +391,8 @@ void BP1Writer::FlattenMetadata(BP1MetadataSet &metadataSet,
heap.m_DataAbsolutePosition += footerSize;
- if (metadataSet.Log.m_IsActive == true)
- metadataSet.Log.m_TotalBytes.push_back(heap.m_DataAbsolutePosition);
+ if (metadataSet.Log.IsActive == true)
+ metadataSet.Log.TotalBytes.push_back(heap.m_DataAbsolutePosition);
}
} // end namespace format
diff --git a/source/packages/profiling/iochrono/Timer.cpp b/source/packages/profiling/iochrono/Timer.cpp
new file mode 100644
index 000000000..9f6356f4a
--- /dev/null
+++ b/source/packages/profiling/iochrono/Timer.cpp
@@ -0,0 +1,100 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * Timer.cpp
+ *
+ * Created on: Apr 4, 2017
+ * Author: wfg
+ */
+
+#include "packages/profiling/iochrono/Timer.h"
+
+namespace adios
+{
+namespace profiling
+{
+
+Timer::Timer(const std::string process, const Support::Resolutions resolution,
+ const bool debug)
+: m_Process{process}, m_Resolution{resolution}, m_DebugMode{debug}
+{
+}
+
+Timer::~Timer() {}
+
+void Timer::SetInitialTime()
+{
+ m_InitialTime = std::chrono::high_resolution_clock::now();
+ m_InitialTimeSet = true;
+}
+
+void Timer::SetTime()
+{
+ m_ElapsedTime = std::chrono::high_resolution_clock::now();
+ m_ProcessTime += GetCurrentTime();
+}
+
+std::string Timer::GetUnits() const
+{
+ std::string units;
+ if (m_Resolution == Support::Resolutions::mus)
+ units = "mus";
+ else if (m_Resolution == Support::Resolutions::ms)
+ units = "ms";
+ else if (m_Resolution == Support::Resolutions::s)
+ units = "s";
+ else if (m_Resolution == Support::Resolutions::m)
+ units = "m";
+ else if (m_Resolution == Support::Resolutions::h)
+ units = "h";
+ return units;
+}
+
+// PRIVATE
+
+long long int Timer::GetCurrentTime()
+{
+ if (m_DebugMode == true)
+ {
+ if (m_InitialTimeSet == false)
+ throw std::invalid_argument("ERROR: SetInitialTime() in process " +
+ m_Process + " not called\n");
+ }
+
+ if (m_Resolution == Support::Resolutions::mus)
+ {
+ return std::chrono::duration_cast<std::chrono::microseconds>(m_ElapsedTime -
+ m_InitialTime)
+ .count();
+ }
+ else if (m_Resolution == Support::Resolutions::ms)
+ {
+ return std::chrono::duration_cast<std::chrono::milliseconds>(m_ElapsedTime -
+ m_InitialTime)
+ .count();
+ }
+ else if (m_Resolution == Support::Resolutions::s)
+ {
+ return std::chrono::duration_cast<std::chrono::seconds>(m_ElapsedTime -
+ m_InitialTime)
+ .count();
+ }
+ else if (m_Resolution == Support::Resolutions::m)
+ {
+ return std::chrono::duration_cast<std::chrono::minutes>(m_ElapsedTime -
+ m_InitialTime)
+ .count();
+ }
+ else if (m_Resolution == Support::Resolutions::h)
+ {
+ return std::chrono::duration_cast<std::chrono::hours>(m_ElapsedTime -
+ m_InitialTime)
+ .count();
+ }
+
+ return -1; // failure
+}
+
+} // end namespace
+} // end namespace
diff --git a/source/transport/file/FileDescriptor.cpp b/source/transport/file/FileDescriptor.cpp
index d3433d2e5..e64446e49 100644
--- a/source/transport/file/FileDescriptor.cpp
+++ b/source/transport/file/FileDescriptor.cpp
@@ -44,34 +44,34 @@ void FileDescriptor::Open(const std::string name, const std::string accessMode)
if (accessMode == "w" || accessMode == "write")
{
- if (m_Profiler.m_IsActive == true)
- m_Profiler.m_Timers[0].SetInitialTime();
+ if (m_Profiler.IsActive == true)
+ m_Profiler.Timers[0].SetInitialTime();
m_FileDescriptor = open(m_Name.c_str(), O_WRONLY | O_CREAT, 0777);
- if (m_Profiler.m_IsActive == true)
- m_Profiler.m_Timers[0].SetTime();
+ if (m_Profiler.IsActive == true)
+ m_Profiler.Timers[0].SetTime();
}
else if (accessMode == "a" || accessMode == "append")
{
- if (m_Profiler.m_IsActive == true)
- m_Profiler.m_Timers[0].SetInitialTime();
+ if (m_Profiler.IsActive == true)
+ m_Profiler.Timers[0].SetInitialTime();
m_FileDescriptor =
open(m_Name.c_str(), O_WRONLY | O_APPEND); // we need to change this
- if (m_Profiler.m_IsActive == true)
- m_Profiler.m_Timers[0].SetTime();
+ if (m_Profiler.IsActive == true)
+ m_Profiler.Timers[0].SetTime();
}
else if (accessMode == "r" || accessMode == "read")
{
- if (m_Profiler.m_IsActive == true)
- m_Profiler.m_Timers[0].SetInitialTime();
+ if (m_Profiler.IsActive == true)
+ m_Profiler.Timers[0].SetInitialTime();
m_FileDescriptor = open(m_Name.c_str(), O_RDONLY);
- if (m_Profiler.m_IsActive == true)
- m_Profiler.m_Timers[0].SetTime();
+ if (m_Profiler.IsActive == true)
+ m_Profiler.Timers[0].SetTime();
}
if (m_DebugMode == true)
@@ -85,13 +85,13 @@ void FileDescriptor::Open(const std::string name, const std::string accessMode)
void FileDescriptor::Write(const char *buffer, std::size_t size)
{
- if (m_Profiler.m_IsActive == true)
- m_Profiler.m_Timers[1].SetInitialTime();
+ if (m_Profiler.IsActive == true)
+ m_Profiler.Timers[1].SetInitialTime();
auto writtenSize = write(m_FileDescriptor, buffer, size);
- if (m_Profiler.m_IsActive == true)
- m_Profiler.m_Timers[1].SetTime();
+ if (m_Profiler.IsActive == true)
+ m_Profiler.Timers[1].SetTime();
if (m_DebugMode == true)
{
@@ -109,13 +109,13 @@ void FileDescriptor::Write(const char *buffer, std::size_t size)
void FileDescriptor::Close()
{
- if (m_Profiler.m_IsActive == true)
- m_Profiler.m_Timers[2].SetInitialTime();
+ if (m_Profiler.IsActive == true)
+ m_Profiler.Timers[2].SetInitialTime();
int status = close(m_FileDescriptor);
- if (m_Profiler.m_IsActive == true)
- m_Profiler.m_Timers[2].SetTime();
+ if (m_Profiler.IsActive == true)
+ m_Profiler.Timers[2].SetTime();
if (m_DebugMode == true)
{
--
GitLab