Working on format/BP1Writer.h

Template functions for Writing in BP Format

Modified Makefile to take default g++ and mpic++ in PATH

Modified DataMan.cpp and 

Removed functions/bp1Write.h, in favor of class format/BP1Writer.h

To do:
Buffer management (maybe capsule needs data offset variables?)
Work on BP format under BP1Writer.h

Makefile

@@ -7,10 +7,10 @@ SYS_BIN:=/usr/bin
 SYS_LIB:=/usr/lib/x86_64-linux-gnu
 LOCAL_LIB:=/usr/local/lib
 
-#COMPILERS AND LIBRARY LOCATIONS
-CC:=$(SYS_BIN)/g++
-AR:=$(SYS_BIN)/ar
-MPICC:=$(SYS_BIN)/mpic++
+#COMPILERS IN PATH AND LIBRARY LOCATIONS
+CC:=g++
+AR:=ar
+MPICC:=mpic++
 LIBS:= -L$(SYS_LIB) -L$(LOCAL_LIB)
 
 CFLAGS:=-c -Wall -Wpedantic -Woverloaded-virtual -std=c++11 -O0 -g

include/core/Engine.h

@@ -118,7 +118,7 @@ public:
     virtual void Write( const std::string variableName, const double* values ) = 0;
     virtual void Write( const std::string variableName, const long double* values ) = 0;
 
-    virtual void Close( int transportIndex = -1  ); ///< Closes a particular transport
+    virtual void Close( const int transportIndex = -1  ); ///< Closes a particular transport
 
 
 protected:

include/engine/writer/Writer.h

@@ -9,6 +9,7 @@
 #define WRITER_H_
 
 #include "core/Engine.h"
+#include "format/BP1Writer.h"
 
 
 namespace adios
@@ -61,10 +62,12 @@ public:
     void Write( const std::string variableName, const double* values );
     void Write( const std::string variableName, const long double* values );
 
-    void Close( int transportIndex = -1 );
+    void Close( const int transportIndex = -1 );
 
 private:
 
+    format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Capsules and m_Transports
+
     void Init( );
     void InitCapsules( );
     void InitTransports( );

include/engine/writer/WriterTemplates.h

@@ -9,26 +9,43 @@
 #define WRITERTEMPLATES_H_
 
 #include <string>
+#include <vector>
+#include <iostream>
+#include <memory>
 
 
 #include "core/Group.h"
 #include "core/Variable.h"
 #include "capsule/Heap.h"
-#include "functions/adiosFunctions.h"
-#include "functions/capsuleTemplates.h"
-
+#include "format/BP1Writer.h"
 
 
 namespace adios
 {
 
+/**
+ * Unique template function that replaces macros to write any variable type to a single heap capsule
+ * @param group variable owner
+ * @param variableName
+ * @param variable
+ * @param buffers single heap capsule containing data and metadata buffers
+ * @param transports all transports from Writer Engine, info is flushed in case of buffer overflow
+ * @param bp1Writer from Writer Engine
+ */
 template <class T>
-void WriterWriteVariable( Group& group, const std::string variableName, Variable<T>& variable, Heap& capsule )
+void WriterWriteVariable( const Group& group, const std::string variableName, const Variable<T>& variable,
+                          Capsule& buffers, std::vector< std::shared_ptr<Transport> >& transports,
+                          format::BP1Writer& bp1Writer )
 {
-    const auto localDimensions = group.GetDimensions( variable.DimensionsCSV );
-    const auto size = GetTotalSize( localDimensions );
-    T min, max;
-    GetMinMax( variable.Values, size, min, max );
+
+    std::cout << "Hello from writing variable " << variableName << "of type " << typeid(T).name() << "\n";
+    //here deal with buffers allocation
+
+
+//    const auto localDimensions = group.GetDimensions( variable.DimensionsCSV );
+//    const auto size = GetTotalSize( localDimensions );
+//    T min, max;
+//    GetMinMax( variable.Values, size, min, max );
 
 //
 //    const std::size_t bytesToWrite = localSize * sizeof( double ); //size of values + min + max in bytes

include/format/BP1Writer.h

+/*
+ * BP1.h
+ *
+ *  Created on: Jan 24, 2017
+ *      Author: wfg
+ */
+
+#ifndef BP1WRITER_H_
+#define BP1WRITER_H_
+
+
+#include <vector>
+#include <cstdint>
+#include <algorithm> //std::count
+#include <cstring> //std::memcpy
+
+#include "core/Variable.h"
+#include "core/Group.h"
+
+
+
+namespace adios
+{
+namespace format
+{
+
+
+class BP1Writer
+{
+
+public:
+
+    std::vector<char> m_PGIndex; ///< process group index
+    std::vector<char> m_VariableIndex; ///< metadata variable index
+    std::vector<char> m_AttributeIndex; ///< metadata attribute index
+
+    const unsigned int m_Cores = 1;
+
+    /**
+     * DataTypes mapping in BP Format
+     */
+    enum DataTypes
+    {
+        type_unknown = -1,         //!< type_unknown
+        type_byte = 0,             //!< type_byte
+        type_short = 1,            //!< type_short
+        type_integer = 2,          //!< type_integer
+        type_long = 4,             //!< type_long
+
+        type_unsigned_byte = 50,   //!< type_unsigned_byte
+        type_unsigned_short = 51,  //!< type_unsigned_short
+        type_unsigned_integer = 52,//!< type_unsigned_integer
+        type_unsigned_long = 54,   //!< type_unsigned_long
+
+        type_real = 5,             //!< type_real or float
+        type_double = 6,           //!< type_double
+        type_long_double = 7,      //!< type_long_double
+
+        type_string = 9,           //!< type_string
+        type_complex = 10,         //!< type_complex
+        type_double_complex = 11,  //!< type_double_complex
+        type_string_array = 12     //!< type_string_array
+    };
+
+    /**
+     * Characteristic ID in variable metadata
+     */
+    enum VariableCharacteristicID
+    {
+        characteristic_value          = 0, //!< characteristic_value
+        characteristic_min            = 1, //!< This is no longer used. Used to read in older bp file format
+        characteristic_max            = 2, //!< This is no longer used. Used to read in older bp file format
+        characteristic_offset         = 3, //!< characteristic_offset
+        characteristic_dimensions     = 4, //!< characteristic_dimensions
+        characteristic_var_id         = 5, //!< characteristic_var_id
+        characteristic_payload_offset = 6, //!< characteristic_payload_offset
+        characteristic_file_index     = 7, //!< characteristic_file_index
+        characteristic_time_index     = 8, //!< characteristic_time_index
+        characteristic_bitmap         = 9, //!< characteristic_bitmap
+        characteristic_stat           = 10,//!< characteristic_stat
+        characteristic_transform_type = 11 //!< characteristic_transform_type
+    };
+
+
+    /** Define statistics type for characteristic ID = 10 in bp1 format */
+    enum VariableStatistics
+    {
+        statistic_min             = 0,
+        statistic_max             = 1,
+        statistic_cnt             = 2,
+        statistic_sum             = 3,
+        statistic_sum_square      = 4,
+        statistic_hist            = 5,
+        statistic_finite          = 6
+    };
+
+
+    /**
+     * Returns data type index from enum Datatypes
+     * @param variable input variable
+     * @return data type
+     */
+    template< class T >
+    std::int8_t GetDataType( const Variable<T>& variable )
+    {
+        std::int8_t dataType = -1;
+        if( std::is_same<T,char>::value )
+            dataType = type_byte;
+
+        else if( std::is_same<T,short>::value )
+            dataType = type_short;
+
+        else if( std::is_same<T,int>::value )
+            dataType = type_integer;
+
+        else if( std::is_same<T,long int>::value )
+            dataType = type_long;
+
+        else if( std::is_same<T,unsigned char>::value )
+            dataType = type_unsigned_byte;
+
+        else if( std::is_same<T,unsigned short>::value )
+            dataType = type_unsigned_short;
+
+        else if( std::is_same<T,unsigned int>::value )
+            dataType = type_unsigned_integer;
+
+        else if( std::is_same<T,unsigned long int>::value )
+            dataType = type_unsigned_long;
+
+        else if( std::is_same<T,float>::value )
+            dataType = type_real;
+
+        else if( std::is_same<T,double>::value )
+            dataType = type_double;
+
+        else if( std::is_same<T,long double>::value )
+            dataType = type_long_double;
+        //need to implement complex and string
+        return dataType;
+    }
+
+
+    /**
+     * Returns the estimated variable index size
+     * @param group
+     * @param variableName
+     * @param variable
+     * @param verbosity
+     * @return variable index size
+     */
+    template< class T >
+    size_t GetVariableIndexSize( const Group& group, const std::string variableName,
+                                 const Variable<T> variable, const unsigned int verbosity )
+    {
+        //size_t indexSize = varEntryLength + memberID + lengthGroupName + groupName + lengthVariableName + lengthOfPath + path + datatype
+        size_t indexSize = 23; //without characteristics
+        indexSize += group.m_Name.size();
+        indexSize += variableName.size();
+
+        // characteristics 3 and 4, check variable number of dimensions
+        std::size_t dimensions = std::count( variable.DimensionsCSV.begin(), variable.DimensionsCSV.end(), ',' ) + 1; //number of commas in CSV + 1
+        indexSize += 28 * dimensions; //28 bytes per dimension
+        indexSize += 1; //id
+
+        //characteristics 3, variable offset in data
+        indexSize += 8;
+        indexSize += 1; //id
+        //characteristics 6, variable payload offset in data
+        indexSize += 8;
+        indexSize += 1; //id
+
+        //characteristic 0, if scalar add value, for now only allowing string
+        if( dimensions == 1 )
+        {
+            indexSize += sizeof(T);
+            indexSize += 1; //id
+            //must have an if here
+            indexSize += 2 + variableName.size();
+            indexSize += 1; //id
+        }
+
+        //characteristic statistics
+        if( verbosity == 0 ) //default, only min and max
+        {
+            indexSize += 2 * ( sizeof(T) + 1 );
+            indexSize += 1 + 1; //id
+        }
+
+        return indexSize;
+        //need to add transform characteristics
+    }
+
+    /**
+     * Write to many buffers and updates offset
+     * @param buffers
+     * @param source
+     * @param size
+     * @param offset
+     */
+    template< class T >
+    void WriteToBuffers( std::vector<char*>& buffers, const T* source, std::size_t size, std::size_t& offset )
+    {
+        for( auto& buffer : buffers )
+        {
+            std::memcpy( &buffer[offset], source, size );
+        }
+        offset += size;
+    }
+
+    /**
+     *
+     * @param group variable owner
+     * @param variableName name of variable to be written
+     * @param variable object carrying variable information
+     * @param dataBuffers buffers to which variable metadata and payload (values) will be written. Metadata is added in case of system corruption to allow regeneration.
+     * @param metadataBuffers buffers to which only variable metadata will be written
+     * @param characteristicsVerbosity default = 0 min and max only,
+     */
+    template< class T >
+    void WriteVariable( const Group& group, const std::string variableName,
+                        const Variable<T>& variable,
+                        std::vector<char*>& dataBuffers, const std::size_t dataPosition,
+                        std::vector<char*>& metadataBuffers, const std::size_t metadataPosition,
+                        const unsigned int memberID, const bool writeDimensionsInData,
+                        const unsigned int verbose )
+    {
+       std::size_t metadataOffset = metadataPosition + 8; //length of var, will come at the end from this offset
+       std::size_t dataOffset = dataPosition + 8; //length of var, will come at the end from this offset
+
+       //memberID
+       WriteToBuffers( metadataBuffers, &memberID, 4, metadataOffset );
+       //group
+       const std::uint16_t lengthGroupName = group.m_Name.length();
+       WriteToBuffers( metadataBuffers, &lengthGroupName, 2, metadataOffset ); //2 bytes
+       //const char* groupName = ;
+       WriteToBuffers( metadataBuffers, group.m_Name.c_str(), lengthGroupName, metadataOffset );
+       //variable name to metadata and data
+       const std::uint16_t lengthVariableName = variableName.length();
+       WriteToBuffers( metadataBuffers, &lengthVariableName, 2, metadataOffset );
+       WriteToBuffers( metadataBuffers, variableName.c_str(), lengthVariableName, metadataOffset );
+       WriteToBuffers( dataBuffers, &lengthVariableName, 2, dataOffset );
+       WriteToBuffers( dataBuffers, variableName.c_str(), lengthVariableName, dataOffset );
+       //skip path (jump 2 bytes)
+       metadataOffset += 2;
+       dataOffset += 2;
+       //dataType
+       const std::uint8_t dataType = GetBPDataType( variable );
+       WriteToBuffers( metadataBuffers, &dataType, 1, metadataOffset );
+       WriteToBuffers( dataBuffers, &dataType, 1, dataOffset );
+
+       //Characteristics Sets in Metadata and Data
+       const std::size_t metadataCharacteristicsSetsCountPosition = metadataOffset; //very important piece
+       std::size_t dataCharacteristicsCountPosition = dataOffset;
+       const std::uint64_t sets = 1; //write one for now
+       WriteToBuffers( metadataBuffers, &sets, 8, metadataOffset );
+
+       unsigned int characteristicsCounter = 0; //used for characteristics count, characteristics length will be calculated at the end
+
+       //DIMENSIONS CHARACTERISTIC
+       const std::vector<unsigned long long int> localDimensions = group.GetDimensions( variable.DimensionsCSV );
+
+       //write to metadata characteristic
+       //characteristic: dimension
+       std::uint8_t characteristicID = characteristic_dimensions;
+       WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset );
+       const std::uint8_t dimensions = localDimensions.size();
+       WriteToBuffers( metadataBuffers, &dimensions, 1, metadataOffset );
+       const std::uint16_t dimensionsLength = dimensions * 24; //24 is from 8 bytes for each: local dimension, global dimension, global offset
+       WriteToBuffers( metadataBuffers, &dimensionsLength, 2, metadataOffset );
+
+       //dimensions in data buffer, header
+       if( writeDimensionsInData == true )
+       {
+           const char yes = 'y'; // dimension y or n
+           WriteToBuffers( dataBuffers, &yes, 1, dataOffset );
+           //for now only support unsigned long integer value (8 bytes), as in metadata
+           WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
+           const std::uint16_t dimensionsLengthInData = dimensions * 27; //27 is from 9 bytes for each: var y/n + local, var y/n + global dimension, var y/n + global offset
+           WriteToBuffers( dataBuffers, &dimensionsLengthInData, 2, dataOffset );
+       }
+       else
+       {
+           const char no = 'n'; // dimension y or n
+           WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+           dataCharacteristicsCountPosition += 1;
+       }
+
+       if( variable.GlobalBoundsIndex == -1 ) //local variable
+       {
+           for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+           {
+               //metadata characteristics
+               WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
+               metadataOffset += 16; //skipping global dimension(8), global offset (8)
+           }
+
+           if( writeDimensionsInData == true )
+           {
+               for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+               {
+                   const char no = 'n'; //dimension format unsigned int value
+                   WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+                   WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+                   dataOffset += 18; //skipping var no + global dimension(8), var no + global offset (8)
+               }
+           }
+           //dimensions in data characteristic entry (might not be required)
+           dataCharacteristicsCountPosition = dataOffset;
+           dataOffset += 5; //skip characteristics count(1) + length (4)
+           WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset );
+
+           std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
+           WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset );
+
+           WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
+           WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset );
+
+           for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+           {
+               //metadata characteristics
+               WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+               metadataOffset += 16; //skipping global dimension(8), global offset (8)
+           }
+       }
+       else //global variable
+       {
+           std::vector<unsigned long long int> globalDimensions = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].first );
+           std::vector<unsigned long long int> globalOffsets = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].second );
+
+           //metadata, writing in characteristics
+           for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+           {
+               WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
+               WriteToBuffers( metadataBuffers, &globalDimensions[d], 8, metadataOffset );
+               WriteToBuffers( metadataBuffers, &globalOffsets[d], 8, metadataOffset );
+           }
+
+           //data dimensions entry
+           if( writeDimensionsInData == true )
+           {
+               for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+               {
+                   const char no = 'n'; //dimension format unsigned int value
+                   WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+                   WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+                   WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+                   WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+               }
+           }
+           //dimensions in data characteristic entry (might not be required)
+           dataCharacteristicsCountPosition = dataOffset;
+           dataOffset += 5; //skip characteristics count(1) + length (4)
+           WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //id
+
+           std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
+           WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset );
+
+           WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
+           WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset );
+
+           for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+           {
+               WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+               WriteToBuffers( dataBuffers, &globalDimensions[d], 8, dataOffset );
+               WriteToBuffers( dataBuffers, &globalOffsets[d], 8, dataOffset );
+           }
+       }
+
+       ++characteristicsCounter;
+
+       //Stat -> Min, Max
+       characteristicID = characteristic_stat;
+       if( verbose == 0 ) //default verbose
+       {
+           const std::size_t valuesSize = GetTotalSize( localDimensions );
+           //here we can make decisions for threads based on valuesSize
+           T min, max;
+
+           if( valuesSize >= 10000000 ) //ten million?
+               GetMinMax( variable.Values, valuesSize, min, max, m_Cores ); //here we can add cores from constructor
+           else
+               GetMinMax( variable.Values, valuesSize, min, max );
+
+           const std::int8_t statisticMinID = statistic_min;
+           const std::int8_t statisticMaxID = statistic_max;
+           //write statistics here
+
+       }
+       //Offsets should be last
+
+
+
+    } //end of function
+
+
+
+
+private:
+
+    bool m_DebugMode = false;
+
+};
+
+
+
+
+
+
+} //end namespace format
+} //end namespace adios
+
+#endif /* BP1WRITER_H_ */

include/functions/bp1Write.h

-/*
- * bp1Format.h
- *
- *  Created on: Jan 12, 2017
- *      Author: wfg
- */
-
-#ifndef BP1WRITE_H_
-#define BP1WRITE_H_
-
-#include <algorithm> //std::count
-#include <cstring> //std::memcpy
-
-
-#include "core/Group.h"
-#include "core/Variable.h"
-#include "core/Capsule.h"
-
-
-namespace adios
-{
-
-
-enum DataTypes {
-    type_unknown = -1,
-    type_byte = 0, // char type
-    type_short = 1,
-    type_integer = 2,
-    type_long = 4,
-
-    type_unsigned_byte = 50,
-    type_unsigned_short = 51,
-    type_unsigned_integer = 52,
-    type_unsigned_long = 54,
-
-    type_real = 5, // float
-    type_double = 6,
-    type_long_double = 7,
-
-    type_string = 9,
-    type_complex = 10,
-    type_double_complex = 11,
-    type_string_array = 12
-};
-
-
-
-/** Define variable characteristic IDs in bp1 format */
-enum VariableCharacteristicID
-{
-    characteristic_value          = 0, //!< characteristic_value
-    characteristic_min            = 1, //!< This is no longer used. Used to read in older bp file format
-    characteristic_max            = 2, //!< This is no longer used. Used to read in older bp file format
-    characteristic_offset         = 3, //!< characteristic_offset
-    characteristic_dimensions     = 4, //!< characteristic_dimensions
-    characteristic_var_id         = 5, //!< characteristic_var_id
-    characteristic_payload_offset = 6, //!< characteristic_payload_offset
-    characteristic_file_index     = 7, //!< characteristic_file_index
-    characteristic_time_index     = 8, //!< characteristic_time_index
-    characteristic_bitmap         = 9, //!< characteristic_bitmap
-    characteristic_stat           = 10,//!< characteristic_stat
-    characteristic_transform_type = 11 //!< characteristic_transform_type
-};
-
-/** Define statistics type for characteristic ID = 10 in bp1 format */
-enum VariableStatistics
-{
-    statistic_min             = 0,
-    statistic_max             = 1,
-    statistic_cnt             = 2,
-    statistic_sum             = 3,
-    statistic_sum_square      = 4,
-    statistic_hist            = 5,
-    statistic_finite          = 6
-};
-
-/**
- * Write a piece of
- * @param buffers
- * @param source
- * @param size
- * @param offset
- */
-template< class T >
-void WriteToBuffers( std::vector<char*>& buffers, const T* source, std::size_t size, std::size_t& offset )
-{
-    for( auto& buffer : buffers )
-    {
-        std::memcpy( &buffer[offset], source, size );
-    }
-    offset += size;
-}
-
-
-template< class T>
-std::int8_t GetBPDataType( const Variable<T>& variable )
-{
-    int dataType = -1;
-    if( std::is_same<T,char> )
-        dataType = type_byte;
-
-    else if( std::is_same<T,short> )
-        dataType = type_short;
-
-    else if( std::is_same<T,int> )
-        dataType = type_integer;
-
-    else if( std::is_same<T,long int> )
-        dataType = type_long;
-
-    else if( std::is_same<T,unsigned char> )
-        dataType = type_unsigned_byte;
-
-    else if( std::is_same<T,unsigned short> )
-        dataType = type_unsigned_short;
-
-    else if( std::is_same<T,unsigned int> )
-        dataType = type_unsigned_integer;
-
-    else if( std::is_same<T,unsigned long int> )
-        dataType = type_unsigned_long;
-
-    else if( std::is_same<T,float> )
-        dataType = type_real;
-
-    else if( std::is_same<T,double> )
-        dataType = type_double;
-
-    else if( std::is_same<T,long double> )
-        dataType = type_long_double;
-
-    //need to implement complex and string
-}
-
-
-/**
- *
- * @param group variable owner
- * @param variableName name of variable to be written
- * @param variable object carrying variable information
- * @param dataBuffers buffers to which variable metadata and payload (values) will be written. Metadata is added in case of system corruption to allow regeneration.
- * @param metadataBuffers buffers to which only variable metadata will be written
- * @param characteristicsVerbosity default = 0 min and max only,
- */
-template< class T >
-void WriteVariableToBuffers( const Group& group, const std::string variableName,
-                             const Variable<T>& variable, std::vector<char*>& dataBuffers,
-                             const std::size_t dataBuffersPosition,
-                             std::vector<char*>& metadataBuffers,
-                             const std::size_t metadataBuffersPosition,
-                             const unsigned int memberID, const bool writeDimensionsInData,
-                             const unsigned int characteristicsVerbosity )
-{
-   std::size_t metadataOffset = metadataBuffersPosition + 8; //length of var, will come at the end from this offset
-   std::size_t dataOffset = dataBuffersPosition + 8; //length of var, will come at the end from this offset
-
-   //memberID
-   WriteToBuffers( metadataBuffers, &memberID, 4, metadataOffset );
-   //group
-   const std::uint16_t lengthGroupName = group.m_Name.length();
-   WriteToBuffers( metadataBuffers, &lengthGroupName, 2, metadataOffset ); //2 bytes
-   WriteToBuffers( metadataBuffers, &group.m_Name.c_str(), lengthGroupName, metadataOffset );
-   //variable name to metadata and data
-   const std::uint16_t lengthVariableName = variableName.length();
-   WriteToBuffers( metadataBuffers, &lengthVariableName, 2, metadataOffset );
-   WriteToBuffers( metadataBuffers, variableName.c_str(), lengthVariableName, metadataOffset );
-   WriteToBuffers( dataBuffers, &lengthVariableName, 2, dataOffset );
-   WriteToBuffers( dataBuffers, variableName.c_str(), lengthVariableName, dataOffset );
-   //skip path (jump 2 bytes)
-   metadataOffset += 2;
-   dataOffset += 2;
-   //dataType
-   const std::uint8_t dataType = GetBPDataType( variable );
-   WriteToBuffers( metadataBuffers, &dataType, 1, metadataOffset );
-   WriteToBuffers( dataBuffers, &dataType, 1, dataOffset );
-
-   //Characteristics Sets in Metadata
-   const std::size_t metadataCharacteristicPosition = metadataOffset;
-   const std::uint64_t sets = 1; //write one for now
-   WriteToBuffers( metadataBuffers, &sets, 8, metadataOffset );
-
-   unsigned int characteristicsCounter = 0; //used for characteristics count, characteristics length will be calculated at the end
-
-   //Get dimensions
-   const std::vector<unsigned long long int> localDimensions = group.GetDimensions( variable.DimensionsCSV );
-
-   //write to metadata characteristic
-   //characteristic: dimension
-   const std::uint8_t characteristicID = characteristic_dimensions;
-   WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset );
-   const std::uint8_t dimensions = localDimensions.size();
-   WriteToBuffers( metadataBuffers, &dimensions, 1, metadataOffset );
-   const std::uint16_t dimensionsLength = dimensions * 24; //24 is from 8 bytes for each: local, global dimension, global offset
-   WriteToBuffers( metadataBuffers, &dimensions, 2, metadataOffset );
-
-   //dimensions in data buffer
-   if( writeDimensionsInData == true )
-   {
-       const char yes = 'y';
-       WriteToBuffers( dataBuffers, &yes, 1, dataOffset );
-       //for now only support unsigned long integer value (8 bytes), as in metadata
-       WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
-       const std::uint16_t dimensionsLengthInData = dimensions * 27; //27 is from 9 bytes for each: var y/n + local, var y/n + global dimension, var y/n + global offset
-       WriteToBuffers( dataBuffers, &dimensionsLengthInData, 2, dataOffset );
-   }
-   else
-   {
-       const char no = 'n';
-       WriteToBuffers( dataBuffers, &no, 1, dataOffset );
-   }
-
-
-   if( variable.GlobalBoundsIndex == -1 ) //local variable
-   {
-       for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
-       {
-           //metadata
-           WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
-           metadataOffset += 16; //skipping global dimension(8), global offset (8)
-
-           //data
-           if( writeDimensionsInData == true )
-           {
-               const char no = 'n';
-               WriteToBuffers( dataBuffers, &no, 1, dataOffset );
-               WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
-               dataOffset += 18; //skipping var no + global dimension(8), var no + global offset (8)
-           }
-       }
-   }
-   else //global variable
-   {
-       std::vector<unsigned long long int> globalDimensions = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].first );
-       std::vector<unsigned long long int> globalOffsets = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].second );
-
-       for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
-       {
-           //metadata
-           WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
-           WriteToBuffers( metadataBuffers, &globalDimensions[d], 8, metadataOffset );
-           WriteToBuffers( metadataBuffers, &globalOffsets[d], 8, metadataOffset );
-
-           //data
-           if( writeDimensionsInData == true )
-           {
-               const char no = 'n';
-               WriteToBuffers( dataBuffers, &no, 1, dataOffset );
-               WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
-               WriteToBuffers( dataBuffers, &no, 1, dataOffset );
-               WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
-
-           }
-       }
-   }
-
-
-}
-
-
-
-
-template< class T >
-size_t GetVariableIndexSize( const Group& group, const std::string variableName,
-                             const Variable<T> variable, const unsigned int characteristicsVerbosity )
-{
-    //size_t indexSize = varEntryLength + memberID + lengthGroupName + groupName + lengthVariableName + lengthOfPath + path + datatype
-    size_t indexSize = 23; //without characteristics
-    indexSize += group.m_Name.size();
-    indexSize += variableName.size();
-
-    // characteristics 3 and 4, check variable number of dimensions
-    std::size_t dimensions = std::count( variable.DimensionsCSV.begin(), variable.DimensionsCSV.end(), ',' ) + 1; //number of commas in CSV + 1
-    indexSize += 28 * dimensions; //28 bytes per dimension
-    indexSize += 1; //id
-
-    //characteristics 3, variable offset in data
-    indexSize += 8;
-    indexSize += 1; //id
-    //characteristics 6, variable payload offset in data
-    indexSize += 8;
-    indexSize += 1; //id
-
-    //characteristic 0, if scalar add value, for now only allowing string
-    if( dimensions == 1 )
-    {
-        indexSize += sizeof(T);
-        indexSize += 1; //id
-        //must have an if here
-        indexSize += 2 + variableName.size();
-        indexSize += 1; //id
-    }
-
-    //characteristic statistics
-    if( characteristicsVerbosity == 0 ) //default, only min and max
-    {
-        indexSize += 2 * ( sizeof(T) + 1 );
-        indexSize += 1 + 1; //id
-    }
-
-    //need to add transform
-}
-
-
-
-
-
-} //end namespace
-
-
-
-#endif /* BP1WRITE_H_ */

include/functions/capsuleTemplates.h

@@ -30,7 +30,7 @@ namespace adios
  * @param max
  */
 template<class T>
-void GetMinMax( const T* values, const size_t size, T& min, T& max, const unsigned int cores = 1 )
+void GetMinMax( const T* values, const size_t size, T& min, T& max, const unsigned int cores = 1 ) noexcept
 {
     min = values[0];
     max = values[0];
@@ -78,7 +78,7 @@ void MemcpyThreads( T* destination, const U* source, std::size_t count, const un
         else
             memcpyThreads.push_back( std::thread( std::memcpy, &destination[initialDestination], &source[initialSource], stride ) );
     }
-    //Now join the threads
+    //Now join the threads (is this really needed?)
     for( auto& thread : memcpyThreads )
         thread.join( );
 }

src/engine/dataman/DataMan.cpp

@@ -162,108 +162,67 @@ void DataMan::Write( Group& group, const std::string variableName, const long do
 //USING Preset Group
 void DataMan::Write( const std::string variableName, const char* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("char"), " from call to Write char*" );
-	Variable<char>& variable = m_Group->m_Char[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+	Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const unsigned char* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned char"), " from call to Write unsigned char*" );
-	Variable<unsigned char>& variable = m_Group->m_UChar[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const short* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("short"), " from call to Write short*" );
-	Variable<short>& variable = m_Group->m_Short[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const unsigned short* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned short"), " from call to Write unsigned short*" );
-	Variable<unsigned short>& variable = m_Group->m_UShort[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const int* values )
 {
-    auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("int"), " from call to Write int*" );
-	Variable<int>& variable = m_Group->m_Int[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
-
+    Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const unsigned int* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned int"), " from call to Write unsigned int*" );
-	Variable<unsigned int>& variable = m_Group->m_UInt[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const long int* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("long int"), " from call to Write long int*" );
-	Variable<long int>& variable = m_Group->m_LInt[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const unsigned long int* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned long int"), " from call to Write unsigned long int*" );
-	Variable<unsigned long int>& variable = m_Group->m_ULInt[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const long long int* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("long long int"), " from call to Write long long int*" );
-	Variable<long long int>& variable = m_Group->m_LLInt[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const unsigned long long int* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned long long int"), " from call to Write unsigned long long int*" );
-	Variable<unsigned long long int>& variable = m_Group->m_ULLInt[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const float* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("float"), " from call to Write float*" );
-	Variable<float>& variable = m_Group->m_Float[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
 void DataMan::Write( const std::string variableName, const double* values )
 {
-    auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("double"), " from call to Write double*" );
-	Variable<double>& variable = m_Group->m_Double[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
-
 void DataMan::Write( const std::string variableName, const long double* values )
 {
-	auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("long double"), " from call to Write long double*" );
-	Variable<long double>& variable = m_Group->m_LDouble[index]; //must be a reference
-	variable.Values = values;
-	DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports );
+    Write( *m_Group, variableName, values );
 }
 
 
@@ -274,7 +233,6 @@ void DataMan::InitCapsules( )
 }
 
 
-
 void DataMan::InitTransports( )
 {
     std::set< std::string > transportStreamNames; //used to check for name conflict between transports

src/engine/writer/Writer.cpp

@@ -46,139 +46,186 @@ void Writer::Init( )
 
 void Writer::Write( Group& group, const std::string variableName, const char* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("char"), " from call to Write char*" );
+    Variable<char>& variable = group.m_Char[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 void Writer::Write( Group& group, const std::string variableName, const unsigned char* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned char"), " from call to Write unsigned char*" );
+    Variable<unsigned char>& variable = group.m_UChar[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 void Writer::Write( Group& group, const std::string variableName, const short* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("short"), " from call to Write short*" );
+    Variable<short>& variable = group.m_Short[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 void Writer::Write( Group& group, const std::string variableName, const unsigned short* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned short"), " from call to Write unsigned short*" );
+    Variable<unsigned short>& variable = group.m_UShort[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 void Writer::Write( Group& group, const std::string variableName, const int* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("int"), " from call to Write int*" );
+    Variable<int>& variable = group.m_Int[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 void Writer::Write( Group& group, const std::string variableName, const unsigned int* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned int"), " from call to Write unsigned int*" );
+    Variable<unsigned int>& variable = group.m_UInt[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 void Writer::Write( Group& group, const std::string variableName, const long int* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("long int"), " from call to Write long int*" );
+    Variable<long int>& variable = group.m_LInt[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 void Writer::Write( Group& group, const std::string variableName, const unsigned long int* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned long int"), " from call to Write unsigned long int*" );
+    Variable<unsigned long int>& variable = group.m_ULInt[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 void Writer::Write( Group& group, const std::string variableName, const long long int* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("long long int"), " from call to Write long long int*" );
+    Variable<long long int>& variable = group.m_LLInt[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 void Writer::Write( Group& group, const std::string variableName, const unsigned long long int* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned long long int"), " from call to Write unsigned long long int*" );
+    Variable<unsigned long long int>& variable = group.m_ULLInt[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 void Writer::Write( Group& group, const std::string variableName, const float* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("float"), " from call to Write float*" );
+    Variable<float>& variable = group.m_Float[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 
 void Writer::Write( Group& group, const std::string variableName, const double* values )
 {
-    //auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("double"), " from call to Write double*" );
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("double"), " from call to Write double*" );
+    Variable<double>& variable = group.m_Double[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 
 void Writer::Write( Group& group, const std::string variableName, const long double* values )
 {
-
+    auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("long double"), " from call to Write long double*" );
+    Variable<long double>& variable = group.m_LDouble[index]; //must be a reference
+    variable.Values = values;
+    WriterWriteVariable( group, variableName, variable, *m_Capsules[0], m_Transports, m_BP1Writer );
 }
 
 
 void Writer::Write( const std::string variableName, const char* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const unsigned char* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const short* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const unsigned short* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const int* values )
 {
-    const unsigned int index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("int"), " from call to Write int*" );
-    m_Group->m_Int[index].Values = values;
-    //here call the Template function WriteHelper( m_Group, variable, )
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const unsigned int* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const long int* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const unsigned long int* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const long long int* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const unsigned long long int* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const float* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
+
 void Writer::Write( const std::string variableName, const double* values )
 {
-    auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("double"), " from call to Write double*" );
-    std::cout << "Hello from SingleBP Write double with index " << index << "\n";
+    Write( *m_Group, variableName, values );
 }
 
 
 void Writer::Write( const std::string variableName, const long double* values )
 {
-
+    Write( *m_Group, variableName, values );
 }
 
 
@@ -192,6 +239,7 @@ void Writer::InitCapsules( )
 
 void Writer::InitTransports( )
 {
+    //Let BPFormat handle this??
     std::set< std::string > transportStreamNames; //used to check for name conflict between transports
 
     const unsigned int transportsSize = m_Method.m_TransportParameters.size();
@@ -253,6 +301,13 @@ void Writer::InitTransports( )
 }
 
 
+void Writer::Close( const int transportIndex )
+{
+
+}
+
+
+
 
 } //end namespace adios