From cb0d2c11ad46064665965ba355e84e704d683906 Mon Sep 17 00:00:00 2001
From: wfg <wfg@pc0098504.ornl.gov>
Date: Tue, 24 Jan 2017 18:13:10 -0500
Subject: [PATCH] Working on format/BP1Writer.h
Template functions for Writing in BP Format
Modified Makefile to take default g++ and mpic++ in PATH
Modified DataMan.cpp and
Removed functions/bp1Write.h, in favor of class format/BP1Writer.h
To do:
Buffer management (maybe capsule needs data offset variables?)
Work on BP format under BP1Writer.h
---
Makefile | 8 +-
include/core/Engine.h | 2 +-
include/engine/writer/Writer.h | 5 +-
include/engine/writer/WriterTemplates.h | 33 +-
include/format/BP1Writer.h | 413 ++++++++++++++++++++++++
include/functions/bp1Write.h | 311 ------------------
include/functions/capsuleTemplates.h | 4 +-
src/engine/dataman/DataMan.cpp | 68 +---
src/engine/writer/Writer.cpp | 113 +++++--
9 files changed, 546 insertions(+), 411 deletions(-)
create mode 100644 include/format/BP1Writer.h
delete mode 100644 include/functions/bp1Write.h
diff --git a/Makefile b/Makefile
index 00e1f0d1d..898f40eee 100644
--- a/Makefile
+++ b/Makefile
@@ -7,10 +7,10 @@ SYS_BIN:=/usr/bin
SYS_LIB:=/usr/lib/x86_64-linux-gnu
LOCAL_LIB:=/usr/local/lib
-#COMPILERS AND LIBRARY LOCATIONS
-CC:=$(SYS_BIN)/g++
-AR:=$(SYS_BIN)/ar
-MPICC:=$(SYS_BIN)/mpic++
+#COMPILERS IN PATH AND LIBRARY LOCATIONS
+CC:=g++
+AR:=ar
+MPICC:=mpic++
LIBS:= -L$(SYS_LIB) -L$(LOCAL_LIB)
CFLAGS:=-c -Wall -Wpedantic -Woverloaded-virtual -std=c++11 -O0 -g
diff --git a/include/core/Engine.h b/include/core/Engine.h
index 06eef756f..a28b7500c 100644
--- a/include/core/Engine.h
+++ b/include/core/Engine.h
@@ -118,7 +118,7 @@ public:
virtual void Write( const std::string variableName, const double* values ) = 0;
virtual void Write( const std::string variableName, const long double* values ) = 0;
- virtual void Close( int transportIndex = -1 ); ///< Closes a particular transport
+ virtual void Close( const int transportIndex = -1 ); ///< Closes a particular transport
protected:
diff --git a/include/engine/writer/Writer.h b/include/engine/writer/Writer.h
index d995aa4e5..676faa488 100644
--- a/include/engine/writer/Writer.h
+++ b/include/engine/writer/Writer.h
@@ -9,6 +9,7 @@
#define WRITER_H_
#include "core/Engine.h"
+#include "format/BP1Writer.h"
namespace adios
@@ -61,10 +62,12 @@ public:
void Write( const std::string variableName, const double* values );
void Write( const std::string variableName, const long double* values );
- void Close( int transportIndex = -1 );
+ void Close( const int transportIndex = -1 );
private:
+ format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Capsules and m_Transports
+
void Init( );
void InitCapsules( );
void InitTransports( );
diff --git a/include/engine/writer/WriterTemplates.h b/include/engine/writer/WriterTemplates.h
index 314d6c3a4..a572c2300 100644
--- a/include/engine/writer/WriterTemplates.h
+++ b/include/engine/writer/WriterTemplates.h
@@ -9,26 +9,43 @@
#define WRITERTEMPLATES_H_
#include <string>
+#include <vector>
+#include <iostream>
+#include <memory>
#include "core/Group.h"
#include "core/Variable.h"
#include "capsule/Heap.h"
-#include "functions/adiosFunctions.h"
-#include "functions/capsuleTemplates.h"
-
+#include "format/BP1Writer.h"
namespace adios
{
+/**
+ * Unique template function that replaces macros to write any variable type to a single heap capsule
+ * @param group variable owner
+ * @param variableName
+ * @param variable
+ * @param buffers single heap capsule containing data and metadata buffers
+ * @param transports all transports from Writer Engine, info is flushed in case of buffer overflow
+ * @param bp1Writer from Writer Engine
+ */
template <class T>
-void WriterWriteVariable( Group& group, const std::string variableName, Variable<T>& variable, Heap& capsule )
+void WriterWriteVariable( const Group& group, const std::string variableName, const Variable<T>& variable,
+ Capsule& buffers, std::vector< std::shared_ptr<Transport> >& transports,
+ format::BP1Writer& bp1Writer )
{
- const auto localDimensions = group.GetDimensions( variable.DimensionsCSV );
- const auto size = GetTotalSize( localDimensions );
- T min, max;
- GetMinMax( variable.Values, size, min, max );
+
+ std::cout << "Hello from writing variable " << variableName << "of type " << typeid(T).name() << "\n";
+ //here deal with buffers allocation
+
+
+// const auto localDimensions = group.GetDimensions( variable.DimensionsCSV );
+// const auto size = GetTotalSize( localDimensions );
+// T min, max;
+// GetMinMax( variable.Values, size, min, max );
//
// const std::size_t bytesToWrite = localSize * sizeof( double ); //size of values + min + max in bytes
diff --git a/include/format/BP1Writer.h b/include/format/BP1Writer.h
new file mode 100644
index 000000000..5c26215b4
--- /dev/null
+++ b/include/format/BP1Writer.h
@@ -0,0 +1,413 @@
+/*
+ * BP1.h
+ *
+ * Created on: Jan 24, 2017
+ * Author: wfg
+ */
+
+#ifndef BP1WRITER_H_
+#define BP1WRITER_H_
+
+
+#include <vector>
+#include <cstdint>
+#include <algorithm> //std::count
+#include <cstring> //std::memcpy
+
+#include "core/Variable.h"
+#include "core/Group.h"
+
+
+
+namespace adios
+{
+namespace format
+{
+
+
+class BP1Writer
+{
+
+public:
+
+ std::vector<char> m_PGIndex; ///< process group index
+ std::vector<char> m_VariableIndex; ///< metadata variable index
+ std::vector<char> m_AttributeIndex; ///< metadata attribute index
+
+ const unsigned int m_Cores = 1;
+
+ /**
+ * DataTypes mapping in BP Format
+ */
+ enum DataTypes
+ {
+ type_unknown = -1, //!< type_unknown
+ type_byte = 0, //!< type_byte
+ type_short = 1, //!< type_short
+ type_integer = 2, //!< type_integer
+ type_long = 4, //!< type_long
+
+ type_unsigned_byte = 50, //!< type_unsigned_byte
+ type_unsigned_short = 51, //!< type_unsigned_short
+ type_unsigned_integer = 52,//!< type_unsigned_integer
+ type_unsigned_long = 54, //!< type_unsigned_long
+
+ type_real = 5, //!< type_real or float
+ type_double = 6, //!< type_double
+ type_long_double = 7, //!< type_long_double
+
+ type_string = 9, //!< type_string
+ type_complex = 10, //!< type_complex
+ type_double_complex = 11, //!< type_double_complex
+ type_string_array = 12 //!< type_string_array
+ };
+
+ /**
+ * Characteristic ID in variable metadata
+ */
+ enum VariableCharacteristicID
+ {
+ characteristic_value = 0, //!< characteristic_value
+ characteristic_min = 1, //!< This is no longer used. Used to read in older bp file format
+ characteristic_max = 2, //!< This is no longer used. Used to read in older bp file format
+ characteristic_offset = 3, //!< characteristic_offset
+ characteristic_dimensions = 4, //!< characteristic_dimensions
+ characteristic_var_id = 5, //!< characteristic_var_id
+ characteristic_payload_offset = 6, //!< characteristic_payload_offset
+ characteristic_file_index = 7, //!< characteristic_file_index
+ characteristic_time_index = 8, //!< characteristic_time_index
+ characteristic_bitmap = 9, //!< characteristic_bitmap
+ characteristic_stat = 10,//!< characteristic_stat
+ characteristic_transform_type = 11 //!< characteristic_transform_type
+ };
+
+
+ /** Define statistics type for characteristic ID = 10 in bp1 format */
+ enum VariableStatistics
+ {
+ statistic_min = 0,
+ statistic_max = 1,
+ statistic_cnt = 2,
+ statistic_sum = 3,
+ statistic_sum_square = 4,
+ statistic_hist = 5,
+ statistic_finite = 6
+ };
+
+
+ /**
+ * Returns data type index from enum Datatypes
+ * @param variable input variable
+ * @return data type
+ */
+ template< class T >
+ std::int8_t GetDataType( const Variable<T>& variable )
+ {
+ std::int8_t dataType = -1;
+ if( std::is_same<T,char>::value )
+ dataType = type_byte;
+
+ else if( std::is_same<T,short>::value )
+ dataType = type_short;
+
+ else if( std::is_same<T,int>::value )
+ dataType = type_integer;
+
+ else if( std::is_same<T,long int>::value )
+ dataType = type_long;
+
+ else if( std::is_same<T,unsigned char>::value )
+ dataType = type_unsigned_byte;
+
+ else if( std::is_same<T,unsigned short>::value )
+ dataType = type_unsigned_short;
+
+ else if( std::is_same<T,unsigned int>::value )
+ dataType = type_unsigned_integer;
+
+ else if( std::is_same<T,unsigned long int>::value )
+ dataType = type_unsigned_long;
+
+ else if( std::is_same<T,float>::value )
+ dataType = type_real;
+
+ else if( std::is_same<T,double>::value )
+ dataType = type_double;
+
+ else if( std::is_same<T,long double>::value )
+ dataType = type_long_double;
+ //need to implement complex and string
+ return dataType;
+ }
+
+
+ /**
+ * Returns the estimated variable index size
+ * @param group
+ * @param variableName
+ * @param variable
+ * @param verbosity
+ * @return variable index size
+ */
+ template< class T >
+ size_t GetVariableIndexSize( const Group& group, const std::string variableName,
+ const Variable<T> variable, const unsigned int verbosity )
+ {
+ //size_t indexSize = varEntryLength + memberID + lengthGroupName + groupName + lengthVariableName + lengthOfPath + path + datatype
+ size_t indexSize = 23; //without characteristics
+ indexSize += group.m_Name.size();
+ indexSize += variableName.size();
+
+ // characteristics 3 and 4, check variable number of dimensions
+ std::size_t dimensions = std::count( variable.DimensionsCSV.begin(), variable.DimensionsCSV.end(), ',' ) + 1; //number of commas in CSV + 1
+ indexSize += 28 * dimensions; //28 bytes per dimension
+ indexSize += 1; //id
+
+ //characteristics 3, variable offset in data
+ indexSize += 8;
+ indexSize += 1; //id
+ //characteristics 6, variable payload offset in data
+ indexSize += 8;
+ indexSize += 1; //id
+
+ //characteristic 0, if scalar add value, for now only allowing string
+ if( dimensions == 1 )
+ {
+ indexSize += sizeof(T);
+ indexSize += 1; //id
+ //must have an if here
+ indexSize += 2 + variableName.size();
+ indexSize += 1; //id
+ }
+
+ //characteristic statistics
+ if( verbosity == 0 ) //default, only min and max
+ {
+ indexSize += 2 * ( sizeof(T) + 1 );
+ indexSize += 1 + 1; //id
+ }
+
+ return indexSize;
+ //need to add transform characteristics
+ }
+
+ /**
+ * Write to many buffers and updates offset
+ * @param buffers
+ * @param source
+ * @param size
+ * @param offset
+ */
+ template< class T >
+ void WriteToBuffers( std::vector<char*>& buffers, const T* source, std::size_t size, std::size_t& offset )
+ {
+ for( auto& buffer : buffers )
+ {
+ std::memcpy( &buffer[offset], source, size );
+ }
+ offset += size;
+ }
+
+ /**
+ *
+ * @param group variable owner
+ * @param variableName name of variable to be written
+ * @param variable object carrying variable information
+ * @param dataBuffers buffers to which variable metadata and payload (values) will be written. Metadata is added in case of system corruption to allow regeneration.
+ * @param metadataBuffers buffers to which only variable metadata will be written
+ * @param characteristicsVerbosity default = 0 min and max only,
+ */
+ template< class T >
+ void WriteVariable( const Group& group, const std::string variableName,
+ const Variable<T>& variable,
+ std::vector<char*>& dataBuffers, const std::size_t dataPosition,
+ std::vector<char*>& metadataBuffers, const std::size_t metadataPosition,
+ const unsigned int memberID, const bool writeDimensionsInData,
+ const unsigned int verbose )
+ {
+ std::size_t metadataOffset = metadataPosition + 8; //length of var, will come at the end from this offset
+ std::size_t dataOffset = dataPosition + 8; //length of var, will come at the end from this offset
+
+ //memberID
+ WriteToBuffers( metadataBuffers, &memberID, 4, metadataOffset );
+ //group
+ const std::uint16_t lengthGroupName = group.m_Name.length();
+ WriteToBuffers( metadataBuffers, &lengthGroupName, 2, metadataOffset ); //2 bytes
+ //const char* groupName = ;
+ WriteToBuffers( metadataBuffers, group.m_Name.c_str(), lengthGroupName, metadataOffset );
+ //variable name to metadata and data
+ const std::uint16_t lengthVariableName = variableName.length();
+ WriteToBuffers( metadataBuffers, &lengthVariableName, 2, metadataOffset );
+ WriteToBuffers( metadataBuffers, variableName.c_str(), lengthVariableName, metadataOffset );
+ WriteToBuffers( dataBuffers, &lengthVariableName, 2, dataOffset );
+ WriteToBuffers( dataBuffers, variableName.c_str(), lengthVariableName, dataOffset );
+ //skip path (jump 2 bytes)
+ metadataOffset += 2;
+ dataOffset += 2;
+ //dataType
+ const std::uint8_t dataType = GetBPDataType( variable );
+ WriteToBuffers( metadataBuffers, &dataType, 1, metadataOffset );
+ WriteToBuffers( dataBuffers, &dataType, 1, dataOffset );
+
+ //Characteristics Sets in Metadata and Data
+ const std::size_t metadataCharacteristicsSetsCountPosition = metadataOffset; //very important piece
+ std::size_t dataCharacteristicsCountPosition = dataOffset;
+ const std::uint64_t sets = 1; //write one for now
+ WriteToBuffers( metadataBuffers, &sets, 8, metadataOffset );
+
+ unsigned int characteristicsCounter = 0; //used for characteristics count, characteristics length will be calculated at the end
+
+ //DIMENSIONS CHARACTERISTIC
+ const std::vector<unsigned long long int> localDimensions = group.GetDimensions( variable.DimensionsCSV );
+
+ //write to metadata characteristic
+ //characteristic: dimension
+ std::uint8_t characteristicID = characteristic_dimensions;
+ WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset );
+ const std::uint8_t dimensions = localDimensions.size();
+ WriteToBuffers( metadataBuffers, &dimensions, 1, metadataOffset );
+ const std::uint16_t dimensionsLength = dimensions * 24; //24 is from 8 bytes for each: local dimension, global dimension, global offset
+ WriteToBuffers( metadataBuffers, &dimensionsLength, 2, metadataOffset );
+
+ //dimensions in data buffer, header
+ if( writeDimensionsInData == true )
+ {
+ const char yes = 'y'; // dimension y or n
+ WriteToBuffers( dataBuffers, &yes, 1, dataOffset );
+ //for now only support unsigned long integer value (8 bytes), as in metadata
+ WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
+ const std::uint16_t dimensionsLengthInData = dimensions * 27; //27 is from 9 bytes for each: var y/n + local, var y/n + global dimension, var y/n + global offset
+ WriteToBuffers( dataBuffers, &dimensionsLengthInData, 2, dataOffset );
+ }
+ else
+ {
+ const char no = 'n'; // dimension y or n
+ WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+ dataCharacteristicsCountPosition += 1;
+ }
+
+ if( variable.GlobalBoundsIndex == -1 ) //local variable
+ {
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ //metadata characteristics
+ WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
+ metadataOffset += 16; //skipping global dimension(8), global offset (8)
+ }
+
+ if( writeDimensionsInData == true )
+ {
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ const char no = 'n'; //dimension format unsigned int value
+ WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+ dataOffset += 18; //skipping var no + global dimension(8), var no + global offset (8)
+ }
+ }
+ //dimensions in data characteristic entry (might not be required)
+ dataCharacteristicsCountPosition = dataOffset;
+ dataOffset += 5; //skip characteristics count(1) + length (4)
+ WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset );
+
+ std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
+ WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset );
+
+ WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset );
+
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ //metadata characteristics
+ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+ metadataOffset += 16; //skipping global dimension(8), global offset (8)
+ }
+ }
+ else //global variable
+ {
+ std::vector<unsigned long long int> globalDimensions = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].first );
+ std::vector<unsigned long long int> globalOffsets = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].second );
+
+ //metadata, writing in characteristics
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
+ WriteToBuffers( metadataBuffers, &globalDimensions[d], 8, metadataOffset );
+ WriteToBuffers( metadataBuffers, &globalOffsets[d], 8, metadataOffset );
+ }
+
+ //data dimensions entry
+ if( writeDimensionsInData == true )
+ {
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ const char no = 'n'; //dimension format unsigned int value
+ WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+ WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+ }
+ }
+ //dimensions in data characteristic entry (might not be required)
+ dataCharacteristicsCountPosition = dataOffset;
+ dataOffset += 5; //skip characteristics count(1) + length (4)
+ WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //id
+
+ std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
+ WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset );
+
+ WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset );
+
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+ WriteToBuffers( dataBuffers, &globalDimensions[d], 8, dataOffset );
+ WriteToBuffers( dataBuffers, &globalOffsets[d], 8, dataOffset );
+ }
+ }
+
+ ++characteristicsCounter;
+
+ //Stat -> Min, Max
+ characteristicID = characteristic_stat;
+ if( verbose == 0 ) //default verbose
+ {
+ const std::size_t valuesSize = GetTotalSize( localDimensions );
+ //here we can make decisions for threads based on valuesSize
+ T min, max;
+
+ if( valuesSize >= 10000000 ) //ten million?
+ GetMinMax( variable.Values, valuesSize, min, max, m_Cores ); //here we can add cores from constructor
+ else
+ GetMinMax( variable.Values, valuesSize, min, max );
+
+ const std::int8_t statisticMinID = statistic_min;
+ const std::int8_t statisticMaxID = statistic_max;
+ //write statistics here
+
+ }
+ //Offsets should be last
+
+
+
+ } //end of function
+
+
+
+
+private:
+
+ bool m_DebugMode = false;
+
+};
+
+
+
+
+
+
+} //end namespace format
+} //end namespace adios
+
+#endif /* BP1WRITER_H_ */
diff --git a/include/functions/bp1Write.h b/include/functions/bp1Write.h
deleted file mode 100644
index 9c490de33..000000000
--- a/include/functions/bp1Write.h
+++ /dev/null
@@ -1,311 +0,0 @@
-/*
- * bp1Format.h
- *
- * Created on: Jan 12, 2017
- * Author: wfg
- */
-
-#ifndef BP1WRITE_H_
-#define BP1WRITE_H_
-
-#include <algorithm> //std::count
-#include <cstring> //std::memcpy
-
-
-#include "core/Group.h"
-#include "core/Variable.h"
-#include "core/Capsule.h"
-
-
-namespace adios
-{
-
-
-enum DataTypes {
- type_unknown = -1,
- type_byte = 0, // char type
- type_short = 1,
- type_integer = 2,
- type_long = 4,
-
- type_unsigned_byte = 50,
- type_unsigned_short = 51,
- type_unsigned_integer = 52,
- type_unsigned_long = 54,
-
- type_real = 5, // float
- type_double = 6,
- type_long_double = 7,
-
- type_string = 9,
- type_complex = 10,
- type_double_complex = 11,
- type_string_array = 12
-};
-
-
-
-/** Define variable characteristic IDs in bp1 format */
-enum VariableCharacteristicID
-{
- characteristic_value = 0, //!< characteristic_value
- characteristic_min = 1, //!< This is no longer used. Used to read in older bp file format
- characteristic_max = 2, //!< This is no longer used. Used to read in older bp file format
- characteristic_offset = 3, //!< characteristic_offset
- characteristic_dimensions = 4, //!< characteristic_dimensions
- characteristic_var_id = 5, //!< characteristic_var_id
- characteristic_payload_offset = 6, //!< characteristic_payload_offset
- characteristic_file_index = 7, //!< characteristic_file_index
- characteristic_time_index = 8, //!< characteristic_time_index
- characteristic_bitmap = 9, //!< characteristic_bitmap
- characteristic_stat = 10,//!< characteristic_stat
- characteristic_transform_type = 11 //!< characteristic_transform_type
-};
-
-/** Define statistics type for characteristic ID = 10 in bp1 format */
-enum VariableStatistics
-{
- statistic_min = 0,
- statistic_max = 1,
- statistic_cnt = 2,
- statistic_sum = 3,
- statistic_sum_square = 4,
- statistic_hist = 5,
- statistic_finite = 6
-};
-
-/**
- * Write a piece of
- * @param buffers
- * @param source
- * @param size
- * @param offset
- */
-template< class T >
-void WriteToBuffers( std::vector<char*>& buffers, const T* source, std::size_t size, std::size_t& offset )
-{
- for( auto& buffer : buffers )
- {
- std::memcpy( &buffer[offset], source, size );
- }
- offset += size;
-}
-
-
-template< class T>
-std::int8_t GetBPDataType( const Variable<T>& variable )
-{
- int dataType = -1;
- if( std::is_same<T,char> )
- dataType = type_byte;
-
- else if( std::is_same<T,short> )
- dataType = type_short;
-
- else if( std::is_same<T,int> )
- dataType = type_integer;
-
- else if( std::is_same<T,long int> )
- dataType = type_long;
-
- else if( std::is_same<T,unsigned char> )
- dataType = type_unsigned_byte;
-
- else if( std::is_same<T,unsigned short> )
- dataType = type_unsigned_short;
-
- else if( std::is_same<T,unsigned int> )
- dataType = type_unsigned_integer;
-
- else if( std::is_same<T,unsigned long int> )
- dataType = type_unsigned_long;
-
- else if( std::is_same<T,float> )
- dataType = type_real;
-
- else if( std::is_same<T,double> )
- dataType = type_double;
-
- else if( std::is_same<T,long double> )
- dataType = type_long_double;
-
- //need to implement complex and string
-}
-
-
-/**
- *
- * @param group variable owner
- * @param variableName name of variable to be written
- * @param variable object carrying variable information
- * @param dataBuffers buffers to which variable metadata and payload (values) will be written. Metadata is added in case of system corruption to allow regeneration.
- * @param metadataBuffers buffers to which only variable metadata will be written
- * @param characteristicsVerbosity default = 0 min and max only,
- */
-template< class T >
-void WriteVariableToBuffers( const Group& group, const std::string variableName,
- const Variable<T>& variable, std::vector<char*>& dataBuffers,
- const std::size_t dataBuffersPosition,
- std::vector<char*>& metadataBuffers,
- const std::size_t metadataBuffersPosition,
- const unsigned int memberID, const bool writeDimensionsInData,
- const unsigned int characteristicsVerbosity )
-{
- std::size_t metadataOffset = metadataBuffersPosition + 8; //length of var, will come at the end from this offset
- std::size_t dataOffset = dataBuffersPosition + 8; //length of var, will come at the end from this offset
-
- //memberID
- WriteToBuffers( metadataBuffers, &memberID, 4, metadataOffset );
- //group
- const std::uint16_t lengthGroupName = group.m_Name.length();
- WriteToBuffers( metadataBuffers, &lengthGroupName, 2, metadataOffset ); //2 bytes
- WriteToBuffers( metadataBuffers, &group.m_Name.c_str(), lengthGroupName, metadataOffset );
- //variable name to metadata and data
- const std::uint16_t lengthVariableName = variableName.length();
- WriteToBuffers( metadataBuffers, &lengthVariableName, 2, metadataOffset );
- WriteToBuffers( metadataBuffers, variableName.c_str(), lengthVariableName, metadataOffset );
- WriteToBuffers( dataBuffers, &lengthVariableName, 2, dataOffset );
- WriteToBuffers( dataBuffers, variableName.c_str(), lengthVariableName, dataOffset );
- //skip path (jump 2 bytes)
- metadataOffset += 2;
- dataOffset += 2;
- //dataType
- const std::uint8_t dataType = GetBPDataType( variable );
- WriteToBuffers( metadataBuffers, &dataType, 1, metadataOffset );
- WriteToBuffers( dataBuffers, &dataType, 1, dataOffset );
-
- //Characteristics Sets in Metadata
- const std::size_t metadataCharacteristicPosition = metadataOffset;
- const std::uint64_t sets = 1; //write one for now
- WriteToBuffers( metadataBuffers, &sets, 8, metadataOffset );
-
- unsigned int characteristicsCounter = 0; //used for characteristics count, characteristics length will be calculated at the end
-
- //Get dimensions
- const std::vector<unsigned long long int> localDimensions = group.GetDimensions( variable.DimensionsCSV );
-
- //write to metadata characteristic
- //characteristic: dimension
- const std::uint8_t characteristicID = characteristic_dimensions;
- WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset );
- const std::uint8_t dimensions = localDimensions.size();
- WriteToBuffers( metadataBuffers, &dimensions, 1, metadataOffset );
- const std::uint16_t dimensionsLength = dimensions * 24; //24 is from 8 bytes for each: local, global dimension, global offset
- WriteToBuffers( metadataBuffers, &dimensions, 2, metadataOffset );
-
- //dimensions in data buffer
- if( writeDimensionsInData == true )
- {
- const char yes = 'y';
- WriteToBuffers( dataBuffers, &yes, 1, dataOffset );
- //for now only support unsigned long integer value (8 bytes), as in metadata
- WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
- const std::uint16_t dimensionsLengthInData = dimensions * 27; //27 is from 9 bytes for each: var y/n + local, var y/n + global dimension, var y/n + global offset
- WriteToBuffers( dataBuffers, &dimensionsLengthInData, 2, dataOffset );
- }
- else
- {
- const char no = 'n';
- WriteToBuffers( dataBuffers, &no, 1, dataOffset );
- }
-
-
- if( variable.GlobalBoundsIndex == -1 ) //local variable
- {
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- //metadata
- WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
- metadataOffset += 16; //skipping global dimension(8), global offset (8)
-
- //data
- if( writeDimensionsInData == true )
- {
- const char no = 'n';
- WriteToBuffers( dataBuffers, &no, 1, dataOffset );
- WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
- dataOffset += 18; //skipping var no + global dimension(8), var no + global offset (8)
- }
- }
- }
- else //global variable
- {
- std::vector<unsigned long long int> globalDimensions = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].first );
- std::vector<unsigned long long int> globalOffsets = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].second );
-
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- //metadata
- WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
- WriteToBuffers( metadataBuffers, &globalDimensions[d], 8, metadataOffset );
- WriteToBuffers( metadataBuffers, &globalOffsets[d], 8, metadataOffset );
-
- //data
- if( writeDimensionsInData == true )
- {
- const char no = 'n';
- WriteToBuffers( dataBuffers, &no, 1, dataOffset );
- WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
- WriteToBuffers( dataBuffers, &no, 1, dataOffset );
- WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
-
- }
- }
- }
-
-
-}
-
-
-
-
-template< class T >
-size_t GetVariableIndexSize( const Group& group, const std::string variableName,
- const Variable<T> variable, const unsigned int characteristicsVerbosity )
-{
- //size_t indexSize = varEntryLength + memberID + lengthGroupName + groupName + lengthVariableName + lengthOfPath + path + datatype
- size_t indexSize = 23; //without characteristics
- indexSize += group.m_Name.size();
- indexSize += variableName.size();
-
- // characteristics 3 and 4, check variable number of dimensions
- std::size_t dimensions = std::count( variable.DimensionsCSV.begin(), variable.DimensionsCSV.end(), ',' ) + 1; //number of commas in CSV + 1
- indexSize += 28 * dimensions; //28 bytes per dimension
- indexSize += 1; //id
-
- //characteristics 3, variable offset in data
- indexSize += 8;
- indexSize += 1; //id
- //characteristics 6, variable payload offset in data
- indexSize += 8;
- indexSize += 1; //id
-
- //characteristic 0, if scalar add value, for now only allowing string
- if( dimensions == 1 )
- {
- indexSize += sizeof(T);
- indexSize += 1; //id
- //must have an if here
- indexSize += 2 + variableName.size();
- indexSize += 1; //id
- }
-
- //characteristic statistics
- if( characteristicsVerbosity == 0 ) //default, only min and max
- {
- indexSize += 2 * ( sizeof(T) + 1 );
- indexSize += 1 + 1; //id
- }
-
- //need to add transform
-}
-
-
-
-
-
-} //end namespace
-
-
-
-#endif /* BP1WRITE_H_ */
diff --git a/include/functions/capsuleTemplates.h b/include/functions/capsuleTemplates.h
index 802c12244..d8f234d6e 100644
--- a/include/functions/capsuleTemplates.h
+++ b/include/functions/capsuleTemplates.h
@@ -30,7 +30,7 @@ namespace adios
* @param max
*/
template<class T>
-void GetMinMax( const T* values, const size_t size, T& min, T& max, const unsigned int cores = 1 )
+void GetMinMax( const T* values, const size_t size, T& min, T& max, const unsigned int cores = 1 ) noexcept
{
min = values[0];
max = values[0];
@@ -78,7 +78,7 @@ void MemcpyThreads( T* destination, const U* source, std::size_t count, const un
else
memcpyThreads.push_back( std::thread( std::memcpy, &destination[initialDestination], &source[initialSource], stride ) );
}
- //Now join the threads
+ //Now join the threads (is this really needed?)
for( auto& thread : memcpyThreads )
thread.join( );
}
diff --git a/src/engine/dataman/DataMan.cpp b/src/engine/dataman/DataMan.cpp
index 56dbf1af0..8c784b970 100644
--- a/src/engine/dataman/DataMan.cpp
+++ b/src/engine/dataman/DataMan.cpp
@@ -162,108 +162,67 @@ void DataMan::Write( Group& group, const std::string variableName, const long do
//USING Preset Group
void DataMan::Write( const std::string variableName, const char* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("char"), " from call to Write char*" );
- Variable<char>& variable = m_Group->m_Char[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const unsigned char* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned char"), " from call to Write unsigned char*" );
- Variable<unsigned char>& variable = m_Group->m_UChar[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const short* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("short"), " from call to Write short*" );
- Variable<short>& variable = m_Group->m_Short[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const unsigned short* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned short"), " from call to Write unsigned short*" );
- Variable<unsigned short>& variable = m_Group->m_UShort[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const int* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("int"), " from call to Write int*" );
- Variable<int>& variable = m_Group->m_Int[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
-
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const unsigned int* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned int"), " from call to Write unsigned int*" );
- Variable<unsigned int>& variable = m_Group->m_UInt[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const long int* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("long int"), " from call to Write long int*" );
- Variable<long int>& variable = m_Group->m_LInt[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const unsigned long int* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned long int"), " from call to Write unsigned long int*" );
- Variable<unsigned long int>& variable = m_Group->m_ULInt[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const long long int* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("long long int"), " from call to Write long long int*" );
- Variable<long long int>& variable = m_Group->m_LLInt[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const unsigned long long int* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned long long int"), " from call to Write unsigned long long int*" );
- Variable<unsigned long long int>& variable = m_Group->m_ULLInt[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const float* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("float"), " from call to Write float*" );
- Variable<float>& variable = m_Group->m_Float[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
void DataMan::Write( const std::string variableName, const double* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("double"), " from call to Write double*" );
- Variable<double>& variable = m_Group->m_Double[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
-
void DataMan::Write( const std::string variableName, const long double* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("long double"), " from call to Write long double*" );
- Variable<long double>& variable = m_Group->m_LDouble[index]; //must be a reference
- variable.Values = values;
- DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+ Write( *m_Group, variableName, values );
}
@@ -274,7 +233,6 @@ void DataMan::InitCapsules( )
}
-
void DataMan::InitTransports( )
{
std::set< std::string > transportStreamNames; //used to check for name conflict between transports
diff --git a/src/engine/writer/Writer.cpp b/src/engine/writer/Writer.cpp
index 5819e3634..73d3730b9 100644
--- a/src/engine/writer/Writer.cpp
+++ b/src/engine/writer/Writer.cpp
@@ -46,139 +46,186 @@ void Writer::Init( )
void Writer::Write( Group& group, const std::string variableName, const char* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("char"), " from call to Write char*" );
+ Variable<char>& variable = group.m_Char[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const unsigned char* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned char"), " from call to Write unsigned char*" );
+ Variable<unsigned char>& variable = group.m_UChar[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const short* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("short"), " from call to Write short*" );
+ Variable<short>& variable = group.m_Short[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const unsigned short* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned short"), " from call to Write unsigned short*" );
+ Variable<unsigned short>& variable = group.m_UShort[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const int* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("int"), " from call to Write int*" );
+ Variable<int>& variable = group.m_Int[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const unsigned int* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned int"), " from call to Write unsigned int*" );
+ Variable<unsigned int>& variable = group.m_UInt[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const long int* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("long int"), " from call to Write long int*" );
+ Variable<long int>& variable = group.m_LInt[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const unsigned long int* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned long int"), " from call to Write unsigned long int*" );
+ Variable<unsigned long int>& variable = group.m_ULInt[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const long long int* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("long long int"), " from call to Write long long int*" );
+ Variable<long long int>& variable = group.m_LLInt[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const unsigned long long int* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned long long int"), " from call to Write unsigned long long int*" );
+ Variable<unsigned long long int>& variable = group.m_ULLInt[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const float* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("float"), " from call to Write float*" );
+ Variable<float>& variable = group.m_Float[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const double* values )
{
- //auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("double"), " from call to Write double*" );
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("double"), " from call to Write double*" );
+ Variable<double>& variable = group.m_Double[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( Group& group, const std::string variableName, const long double* values )
{
-
+ auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("long double"), " from call to Write long double*" );
+ Variable<long double>& variable = group.m_LDouble[index]; //must be a reference
+ variable.Values = values;
+ WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
}
void Writer::Write( const std::string variableName, const char* values )
{
-
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const unsigned char* values )
{
-
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const short* values )
{
-
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const unsigned short* values )
{
-
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const int* values )
{
- const unsigned int index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("int"), " from call to Write int*" );
- m_Group->m_Int[index].Values = values;
- //here call the Template function WriteHelper( m_Group, variable, )
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const unsigned int* values )
{
-
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const long int* values )
{
-
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const unsigned long int* values )
{
-
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const long long int* values )
{
-
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const unsigned long long int* values )
{
-
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const float* values )
{
-
+ Write( *m_Group, variableName, values );
}
+
void Writer::Write( const std::string variableName, const double* values )
{
- auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("double"), " from call to Write double*" );
- std::cout << "Hello from SingleBP Write double with index " << index << "\n";
+ Write( *m_Group, variableName, values );
}
void Writer::Write( const std::string variableName, const long double* values )
{
-
+ Write( *m_Group, variableName, values );
}
@@ -192,6 +239,7 @@ void Writer::InitCapsules( )
void Writer::InitTransports( )
{
+ //Let BPFormat handle this??
std::set< std::string > transportStreamNames; //used to check for name conflict between transports
const unsigned int transportsSize = m_Method.m_TransportParameters.size();
@@ -253,6 +301,13 @@ void Writer::InitTransports( )
}
+void Writer::Close( const int transportIndex )
+{
+
+}
+
+
+
} //end namespace adios
--
GitLab