diff --git a/examples/hello/writer/helloWriter_OOP.cpp b/examples/hello/writer/helloWriter_OOP.cpp
index 3996dca86a0aa99da7d693209656cf2d6aa1a9ea..ab19925677f177e4f22e7f8167aafd87902b644a 100644
--- a/examples/hello/writer/helloWriter_OOP.cpp
+++ b/examples/hello/writer/helloWriter_OOP.cpp
@@ -39,8 +39,10 @@ int main( int argc, char* argv [] )
//Define group and variables with transforms, variables don't have functions, only group can access variables
adios::Group& ioGroup = adios.DeclareGroup( "ioGroup" );
adios::Var ioNx = ioGroup.DefineVariable<unsigned int>( "Nx" );
- adios::Var ioMyDoubles = ioGroup.DefineVariable<double>( "myDoubles", "Nx" );
- adios::Var ioMyFloats = ioGroup.DefineVariable<float>( "myFloats", "Nx" );
+
+ adios::Dims dimNx = ioGroup.SetDimensions( {ioNx} );
+ adios::Var ioMyDoubles = ioGroup.DefineVariable<double>( "myDoubles", dimNx );
+ adios::Var ioMyFloats = ioGroup.DefineVariable<float>( "myFloats", dimNx );
//add transform to variable in group...not executed (just testing API)
adios::Transform bzip2 = adios::transform::BZIP2( );
diff --git a/include/capsule/Heap.h b/include/capsule/Heap.h
index d6d95fc5a336320588fffd938b5bb80cc118e6f1..9138e4c7d3fbfd16f206b8ad74068a020380f92e 100644
--- a/include/capsule/Heap.h
+++ b/include/capsule/Heap.h
@@ -23,6 +23,9 @@ class Heap : public Capsule
public:
+ std::vector<char> m_Data; ///< data buffer allocated using the STL in heap memory, default size = 16 Mb
+ std::vector<char> m_Metadata; ///< metadata buffer allocated using the STL in heap memory, default size = 100 Kb
+
/**
* Unique constructor
* @param accessMode read, write or append
@@ -44,12 +47,6 @@ public:
void ResizeData( const std::size_t size );
void ResizeMetadata( const std::size_t size );
-
-private:
-
- std::vector<char> m_Data; ///< data buffer allocated using the STL in heap memory, default size = 16 Mb
- std::vector<char> m_Metadata; ///< metadata buffer allocated using the STL in heap memory, default size = 100 Kb
-
};
diff --git a/include/core/Capsule.h b/include/core/Capsule.h
index ebc36d30e9ffb2515ca825de74de74303d3090f7..372ffdfaffdff6169b428b9687235606586c18ab 100644
--- a/include/core/Capsule.h
+++ b/include/core/Capsule.h
@@ -27,6 +27,9 @@ public:
const std::string m_Type; ///< buffer type
const std::string m_AccessMode; ///< 'w': write, 'r': read, 'a': append
+ std::size_t m_AbsoluteDataPosition; ///< includes the data flushed to transports
+ std::size_t m_AbsoluteMetadataPosition; ///< includes the metadata flushed to transports
+
/**
* Base class constructor providing type from derived class and accessMode
* @param type derived class type
diff --git a/include/core/Group.h b/include/core/Group.h
index 60b983064796a4b63086b00cd6174fc8ff052c20..879d859035ac510201d93fbef9819f48e8aed1a4 100644
--- a/include/core/Group.h
+++ b/include/core/Group.h
@@ -15,6 +15,7 @@
#include <vector>
#include <ostream>
#include <set>
+#include <initializer_list> //SetDimensions
/// \endcond
#ifdef HAVE_MPI
@@ -35,6 +36,8 @@ namespace adios
{
using Var = std::string; ///< used for returning variables from DefineVariable
+using Dims = std::string;
+
/**
* Class that defines each ADIOS Group composed of Variables, Attributes and GlobalBounds (if global variables exist)
*/
@@ -84,6 +87,9 @@ public:
~Group( ); ///< Using STL containers, no deallocation
+
+ Dims SetDimensions( std::initializer_list<Var> variables ); ///< returns adios::Dims object from a list of existing adios::Var objects
+
/**
* Define a new variable in the group object
* @param name variable name, must be unique in the group. If name exists it removes the current variable. In debug mode program will exit.
@@ -95,16 +101,16 @@ public:
* @param parameters corresponding parameter used by a Transform object in transforms (index should match), default is empty
*/
Var DefineVariable( const std::string variableName, const std::string type,
- const std::string dimensionsCSV = "",
- const std::string globalDimensionsCSV = "", const std::string globalOffsetsCSV = "",
+ const Dims dimensionsCSV = "1",
+ const Dims globalDimensionsCSV = "", const Dims globalOffsetsCSV = "",
const std::vector<Transform*> transforms = std::vector<Transform*>(),
const std::vector<int> parameters = std::vector<int>() );
template< class T >
Var DefineVariable( const std::string variableName,
- const std::string dimensionsCSV = "",
- const std::string globalDimensionsCSV = "", const std::string globalOffsetsCSV = "",
+ const Dims dimensionsCSV = "",
+ const Dims globalDimensionsCSV = "", const Dims globalOffsetsCSV = "",
const std::vector<Transform*> transforms = std::vector<Transform*>(),
const std::vector<int> parameters = std::vector<int>() )
{
@@ -117,7 +123,7 @@ public:
* @param transform corresponding transform object, non-const as a pointer is created and pushed to a vector
* @param parameter optional parameter interpreted by the corresponding Transform, default = -1
*/
- void AddTransform( const std::string variableName, Transform& transform, const int parameter = -1 );
+ void AddTransform( const Var variableName, Transform& transform, const int parameter = -1 );
/**
* Define a new attribute
@@ -139,7 +145,7 @@ public:
* @param dimensionsCSV comma separated dimensions "Nx,Ny,Nz"
* @return actual vector values = { Nx, Ny, Nz }
*/
- std::vector<unsigned long long int> GetDimensions( const std::string dimensionsCSV ) const;
+ std::vector<unsigned long long int> GetDimensions( const Dims dimensionsCSV ) const;
private:
diff --git a/include/format/BP1Writer.h b/include/format/BP1Writer.h
index 3ec8674d39fade92cdc4ce212b78a0236d0a8ccf..0aef1cf1b8b6f78a40c92cd836a1829face11702 100644
--- a/include/format/BP1Writer.h
+++ b/include/format/BP1Writer.h
@@ -11,7 +11,7 @@
#include <vector>
#include <cstdint>
-#include <algorithm> //std::count
+#include <algorithm> //std::count, std::copy
#include <cstring> //std::memcpy
#include "core/Variable.h"
@@ -30,11 +30,20 @@ class BP1Writer
public:
- std::vector<char> m_PGIndex; ///< process group index
+ std::uint64_t m_ProcessGroupsCount = 0; ///< number of process groups
+ std::uint64_t m_ProcessGroupsLength = 0; ///< length in bytes of process groups
+ std::vector<char> m_ProcessGroupIndex; ///< process group index metadata
+
+ std::uint32_t m_VariablesCount = 0; ///< number of written Variables
+ std::uint64_t m_VariablesLength = 0; ///< length in bytes of written Variables
std::vector<char> m_VariableIndex; ///< metadata variable index
+
+ std::uint32_t m_AttributesCount = 0; ///< number of Attributes
+ std::uint64_t m_AttributesLength = 0; ///< length in bytes of Attributes
std::vector<char> m_AttributeIndex; ///< metadata attribute index
const unsigned int m_Cores = 1;
+ const unsigned int m_Verbosity = 0;
/**
* DataTypes mapping in BP Format
@@ -151,7 +160,7 @@ public:
*/
template< class T >
size_t GetVariableIndexSize( const Group& group, const std::string variableName,
- const Variable<T> variable, const unsigned int verbosity ) noexcept
+ const Variable<T> variable ) noexcept
{
//size_t indexSize = varEntryLength + memberID + lengthGroupName + groupName + lengthVariableName + lengthOfPath + path + datatype
size_t indexSize = 23; //without characteristics
@@ -181,7 +190,7 @@ public:
}
//characteristic statistics
- if( verbosity == 0 ) //default, only min and max
+ if( m_Verbosity == 0 ) //default, only min and max
{
indexSize += 2 * ( sizeof(T) + 1 );
indexSize += 1 + 1; //id
@@ -208,218 +217,244 @@ public:
offset += size;
}
- template< class T >
- void CopyToBuffers( std::vector<char*>& buffers, const T* source, std::size_t size, std::size_t& offset ) noexcept
+ template< class T, class U >
+ void CopyToBuffers( std::vector<char*>& buffers, const T* source, U size, std::size_t& offset ) noexcept
{
for( auto& buffer : buffers )
{
std::copy( source, source+size, &buffer[offset] );
- //std::memcpy( &buffer[offset], source, size );
}
offset += size;
}
-
-
/**
- *
+ * Writes a variable to BP format in data and metadata (index) buffers
* @param group variable owner
* @param variableName name of variable to be written
* @param variable object carrying variable information
* @param dataBuffers buffers to which variable metadata and payload (values) will be written. Metadata is added in case of system corruption to allow regeneration.
+ * @param dataPosition initial data position
* @param metadataBuffers buffers to which only variable metadata will be written
- * @param characteristicsVerbosity default = 0 min and max only,
+ * @param metadataPosition
+ * @param memberID
+ * @return number of bytes written, needed for variable entry length
*/
template< class T >
- void WriteVariable( const Group& group, const std::string variableName,
+ void WriteVariable( const Group& group, const Var variableName,
const Variable<T>& variable,
std::vector<char*>& dataBuffers, const std::size_t dataPosition,
- std::vector<char*>& metadataBuffers, const std::size_t metadataPosition,
- const unsigned int memberID, const bool writeDimensionsInData,
- const unsigned int verbose ) noexcept
+ std::vector<char*>& metadataBuffers, const std::size_t metadataPosition ) noexcept
{
- std::size_t metadataOffset = metadataPosition + 8; //length of var, will come at the end from this offset
- std::size_t dataOffset = dataPosition + 8; //length of var, will come at the end from this offset
-
- //memberID
- WriteToBuffers( metadataBuffers, &memberID, 4, metadataOffset );
- //group
- const std::uint16_t lengthGroupName = group.m_Name.length();
- WriteToBuffers( metadataBuffers, &lengthGroupName, 2, metadataOffset ); //2 bytes
- //const char* groupName = ;
- WriteToBuffers( metadataBuffers, group.m_Name.c_str(), lengthGroupName, metadataOffset );
- //variable name to metadata and data
- const std::uint16_t lengthVariableName = variableName.length();
- WriteToBuffers( metadataBuffers, &lengthVariableName, 2, metadataOffset );
- WriteToBuffers( metadataBuffers, variableName.c_str(), lengthVariableName, metadataOffset );
- WriteToBuffers( dataBuffers, &lengthVariableName, 2, dataOffset );
- WriteToBuffers( dataBuffers, variableName.c_str(), lengthVariableName, dataOffset );
- //skip path (jump 2 bytes)
- metadataOffset += 2;
- dataOffset += 2;
- //dataType
- const std::uint8_t dataType = GetBPDataType( variable );
- WriteToBuffers( metadataBuffers, &dataType, 1, metadataOffset );
- WriteToBuffers( dataBuffers, &dataType, 1, dataOffset );
-
- //Characteristics Sets in Metadata and Data
- const std::size_t metadataCharacteristicsSetsCountPosition = metadataOffset; //very important piece
- std::size_t dataCharacteristicsCountPosition = dataOffset;
- const std::uint64_t sets = 1; //write one for now
- WriteToBuffers( metadataBuffers, &sets, 8, metadataOffset );
-
- unsigned int characteristicsCounter = 0; //used for characteristics count, characteristics length will be calculated at the end
-
- //DIMENSIONS CHARACTERISTIC
- const std::vector<unsigned long long int> localDimensions = group.GetDimensions( variable.DimensionsCSV );
-
- //write to metadata characteristic
- //characteristic: dimension
- std::uint8_t characteristicID = characteristic_dimensions;
- WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset );
- const std::uint8_t dimensions = localDimensions.size();
- WriteToBuffers( metadataBuffers, &dimensions, 1, metadataOffset );
- const std::uint16_t dimensionsLength = dimensions * 24; //24 is from 8 bytes for each: local dimension, global dimension, global offset
- WriteToBuffers( metadataBuffers, &dimensionsLength, 2, metadataOffset );
-
- //dimensions in data buffer, header
- if( writeDimensionsInData == true )
- {
- const char yes = 'y'; // dimension y or n
- WriteToBuffers( dataBuffers, &yes, 1, dataOffset );
- //for now only support unsigned long integer value (8 bytes), as in metadata
- WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
- const std::uint16_t dimensionsLengthInData = dimensions * 27; //27 is from 9 bytes for each: var y/n + local, var y/n + global dimension, var y/n + global offset
- WriteToBuffers( dataBuffers, &dimensionsLengthInData, 2, dataOffset );
- }
- else
- {
- const char no = 'n'; // dimension y or n
- WriteToBuffers( dataBuffers, &no, 1, dataOffset );
- dataCharacteristicsCountPosition += 1;
- }
-
- if( variable.GlobalBoundsIndex == -1 ) //local variable
- {
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- //metadata characteristics
- WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
- metadataOffset += 16; //skipping global dimension(8), global offset (8)
- }
-
- if( writeDimensionsInData == true )
- {
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- const char no = 'n'; //dimension format unsigned int value
- WriteToBuffers( dataBuffers, &no, 1, dataOffset );
- WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
- dataOffset += 18; //skipping var no + global dimension(8), var no + global offset (8)
- }
- }
- //dimensions in data characteristic entry (might not be required)
- dataCharacteristicsCountPosition = dataOffset;
- dataOffset += 5; //skip characteristics count(1) + length (4)
- WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset );
-
- std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
- WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset );
-
- WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
- WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset );
-
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- //metadata characteristics
- WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
- metadataOffset += 16; //skipping global dimension(8), global offset (8)
- }
- }
- else //global variable
- {
- std::vector<unsigned long long int> globalDimensions = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].first );
- std::vector<unsigned long long int> globalOffsets = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].second );
-
- //metadata, writing in characteristics
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
- WriteToBuffers( metadataBuffers, &globalDimensions[d], 8, metadataOffset );
- WriteToBuffers( metadataBuffers, &globalOffsets[d], 8, metadataOffset );
- }
-
- //data dimensions entry
- if( writeDimensionsInData == true )
- {
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- const char no = 'n'; //dimension format unsigned int value
- WriteToBuffers( dataBuffers, &no, 1, dataOffset );
- WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
- WriteToBuffers( dataBuffers, &no, 1, dataOffset );
- WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
- }
- }
- //dimensions in data characteristic entry (might not be required)
- dataCharacteristicsCountPosition = dataOffset;
- dataOffset += 5; //skip characteristics count(1) + length (4)
- WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //id
-
- std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
- WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset );
-
- WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
- WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset );
-
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
- WriteToBuffers( dataBuffers, &globalDimensions[d], 8, dataOffset );
- WriteToBuffers( dataBuffers, &globalOffsets[d], 8, dataOffset );
- }
- }
-
- ++characteristicsCounter;
- //Stat -> Min, Max
- characteristicID = characteristic_stat;
- if( verbose == 0 ) //default verbose
- {
- const std::size_t valuesSize = GetTotalSize( localDimensions );
- //here we can make decisions for threads based on valuesSize
- T min, max;
-
- if( valuesSize >= 10000000 ) //ten million? this needs actual results
- GetMinMax( variable.Values, valuesSize, min, max, m_Cores ); //here we can add cores from constructor
- else
- GetMinMax( variable.Values, valuesSize, min, max );
-
- const std::int8_t statisticMinID = statistic_min;
- const std::int8_t statisticMaxID = statistic_max;
-
- //write min and max to metadata
- WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //min
- WriteToBuffers( metadataBuffers, &statisticMinID, 1, metadataOffset );
- WriteToBuffers( metadataBuffers, &min, sizeof(T), metadataOffset );
-
- WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //max
- WriteToBuffers( metadataBuffers, &statisticMaxID, 1, metadataOffset );
- WriteToBuffers( metadataBuffers, &max, sizeof(T), metadataOffset );
-
- //write min and max to data
- WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //min
- const std::int16_t lengthCharacteristic = 1 + sizeof( T );
- WriteToBuffers( dataBuffers, &lengthCharacteristic, 2, dataOffset );
- WriteToBuffers( dataBuffers, &statisticMinID, 1, dataOffset );
- WriteToBuffers( dataBuffers, &min, sizeof(T), dataOffset );
-
- WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //max
- WriteToBuffers( dataBuffers, &lengthCharacteristic, 2, dataOffset );
- WriteToBuffers( dataBuffers, &statisticMaxID, 1, dataOffset );
- WriteToBuffers( dataBuffers, &max, sizeof(T), dataOffset );
- }
- //Offsets should be last and only written to metadata
+ std::size_t metadataOffset = metadataPosition + 4; //length of var, will come at the end from this offset
+ std::size_t dataOffset = dataPosition + 8; //length of var, will come at the end from this offset
+
+ //memberID
+ WriteToBuffers( metadataBuffers, &m_VariablesCount, 4, metadataOffset );
+ //group, only in metadata
+ const std::uint16_t lengthGroupName = group.m_Name.length();
+ WriteToBuffers( metadataBuffers, &lengthGroupName, 2, metadataOffset ); //2 bytes
+ WriteToBuffers( metadataBuffers, group.m_Name.c_str(), lengthGroupName, metadataOffset );
+ //variable name to metadata and data
+ const std::uint16_t lengthVariableName = variableName.length();
+ WriteToBuffers( metadataBuffers, &lengthVariableName, 2, metadataOffset );
+ WriteToBuffers( metadataBuffers, variableName.c_str(), lengthVariableName, metadataOffset );
+ WriteToBuffers( dataBuffers, &lengthVariableName, 2, dataOffset );
+ WriteToBuffers( dataBuffers, variableName.c_str(), lengthVariableName, dataOffset );
+ //skip path (jump 2 bytes, already set to zero)
+ metadataOffset += 2;
+ dataOffset += 2;
+ //dataType
+ const std::uint8_t dataType = GetDataType( variable );
+ WriteToBuffers( metadataBuffers, &dataType, 1, metadataOffset );
+ WriteToBuffers( dataBuffers, &dataType, 1, dataOffset );
+
+ //Characteristics Sets in Metadata and Data
+ const std::size_t metadataCharacteristicsSetsCountPosition = metadataOffset; //very important piece
+ std::size_t dataCharacteristicsCountPosition = dataOffset;
+ const std::uint64_t sets = 1; //write one for now
+ WriteToBuffers( metadataBuffers, &sets, 8, metadataOffset );
+
+ std::uint8_t characteristicsCounter = 0; //used for characteristics count, characteristics length will be calculated at the end
+
+ //DIMENSIONS CHARACTERISTIC
+ const std::vector<unsigned long long int> localDimensions = group.GetDimensions( variable.DimensionsCSV );
+
+ //write to metadata characteristic
+ //characteristic: dimension
+ std::uint8_t characteristicID = characteristic_dimensions;
+ WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset );
+ const std::uint8_t dimensions = localDimensions.size();
+ WriteToBuffers( metadataBuffers, &dimensions, 1, metadataOffset );
+ const std::uint16_t dimensionsLength = dimensions * 24; //24 is from 8 bytes for each: local dimension, global dimension, global offset
+ WriteToBuffers( metadataBuffers, &dimensionsLength, 2, metadataOffset );
+
+ //write in data if it's a dimension variable (scalar) y or n
+ const char dimensionYorN = ( variable.IsDimension ) ? 'y' : 'n';
+ WriteToBuffers( dataBuffers, &dimensionYorN, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
+ const std::uint16_t dimensionsLengthInData = dimensions * 27; //27 is from 9 bytes for each: var y/n + local, var y/n + global dimension, var y/n + global offset
+ WriteToBuffers( dataBuffers, &dimensionsLengthInData, 2, dataOffset );
+
+ if( variable.GlobalBoundsIndex == -1 ) //local variable
+ {
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ //metadata characteristics
+ WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
+ metadataOffset += 16; //skipping global dimension(8), global offset (8)
+ }
+
+ const char no = 'n'; //dimension format unsigned int value (not using memberID for now)
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ //data dimensions
+ WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+ dataOffset += 18; //skipping var no + global dimension(8), var no + global offset (8)
+ }
+
+ //dimensions in data characteristic entry
+ dataCharacteristicsCountPosition = dataOffset;
+ dataOffset += 5; //skip characteristics count(1) + length (4)
+ WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset );
+
+ std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
+ WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset );
+
+ WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset );
+
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ //data characteristics
+ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+ metadataOffset += 16; //skipping global dimension(8), global offset (8)
+ }
+ }
+ else //global variable
+ {
+ std::vector<unsigned long long int> globalDimensions = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].first );
+ std::vector<unsigned long long int> globalOffsets = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].second );
+
+ //metadata, writing in characteristics
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
+ WriteToBuffers( metadataBuffers, &globalDimensions[d], 8, metadataOffset );
+ WriteToBuffers( metadataBuffers, &globalOffsets[d], 8, metadataOffset );
+ }
+
+ //data dimensions entry
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ const char no = 'n'; //dimension format unsigned int value
+ WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+ WriteToBuffers( dataBuffers, &no, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+ }
+
+ //dimensions in data characteristic entry (might not be required)
+ dataCharacteristicsCountPosition = dataOffset;
+ dataOffset += 5; //skip characteristics count(1) + length (4)
+ WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //id
+
+ std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
+ WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset );
+
+ WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset );
+
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
+ WriteToBuffers( dataBuffers, &globalDimensions[d], 8, dataOffset );
+ WriteToBuffers( dataBuffers, &globalOffsets[d], 8, dataOffset );
+ }
+ }
+ ++characteristicsCounter;
+
+ //VALUE for SCALAR or STAT min, max for ARRAY
+ //Value for scalar
+ if( variable.DimensionsCSV == "1" ) //scalar //just doing string scalars for now (by name), needs to be modified when user passes value
+ {
+ characteristicID = characteristic_value;
+ std::int16_t lenghtOfName = variableName.length();
+ //metadata
+ WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset );
+ WriteToBuffers( metadataBuffers, &lenghtOfName, 2, metadataOffset );
+ WriteToBuffers( metadataBuffers, variableName.c_str(), lenghtOfName, metadataOffset );
+
+ //data
+ WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset );
+ std::int16_t lengthOfCharacteristic = 2 + lenghtOfName;
+ WriteToBuffers( dataBuffers, &lengthOfCharacteristic, 2, dataOffset );
+ WriteToBuffers( dataBuffers, &lenghtOfName, 2, dataOffset );
+ WriteToBuffers( dataBuffers, variableName.c_str(), lenghtOfName, dataOffset );
+ }
+ else // Stat -> Min, Max for arrays,
+ {
+ if( m_Verbosity == 0 ) //default verbose
+ {
+ //Get min and max
+ const std::size_t valuesSize = GetTotalSize( localDimensions );
+ //here we can make decisions for threads based on valuesSize
+ T min, max;
+
+ if( valuesSize >= 10000000 ) //ten million? this needs actual results
+ GetMinMax( variable.Values, valuesSize, min, max, m_Cores ); //here we can add cores from constructor
+ else
+ GetMinMax( variable.Values, valuesSize, min, max );
+
+ //set characteristic ids for min and max
+ characteristicID = characteristic_stat;
+ const std::int8_t statisticMinID = statistic_min;
+ const std::int8_t statisticMaxID = statistic_max;
+
+ //write min and max to metadata
+ WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //min
+ WriteToBuffers( metadataBuffers, &statisticMinID, 1, metadataOffset );
+ WriteToBuffers( metadataBuffers, &min, sizeof(T), metadataOffset );
+
+ WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //max
+ WriteToBuffers( metadataBuffers, &statisticMaxID, 1, metadataOffset );
+ WriteToBuffers( metadataBuffers, &max, sizeof(T), metadataOffset );
+
+ //write min and max to data
+ WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //min
+ const std::int16_t lengthCharacteristic = 1 + sizeof( T );
+ WriteToBuffers( dataBuffers, &lengthCharacteristic, 2, dataOffset );
+ WriteToBuffers( dataBuffers, &statisticMinID, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &min, sizeof(T), dataOffset );
+
+ WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //max
+ WriteToBuffers( dataBuffers, &lengthCharacteristic, 2, dataOffset );
+ WriteToBuffers( dataBuffers, &statisticMaxID, 1, dataOffset );
+ WriteToBuffers( dataBuffers, &max, sizeof(T), dataOffset );
+ }
+ }
+ ++characteristicsCounter;
+ //here write characteristics count and length to data
+ std::uint32_t characteristicsLengthInData = dataOffset - dataCharacteristicsCountPosition;
+ WriteToBuffers( dataBuffers, &characteristicsCounter, 1, dataCharacteristicsCountPosition );
+ WriteToBuffers( dataBuffers, &characteristicsLengthInData, 4, dataCharacteristicsCountPosition );
+ dataCharacteristicsCountPosition -= 5;
+
+ //Offsets should be last and only written to metadata
+ characteristicID = characteristic_offset;
+ WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //variable offset id
+ WriteToBuffers( metadataBuffers, &dataPosition, 8, metadataOffset ); //variable offset
+ ++characteristicsCounter;
+
+ characteristicID = characteristic_payload_offset;
+ WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //variable payload offset id
+ WriteToBuffers( metadataBuffers, &dataOffset, 8, metadataOffset ); //variable offset
+ ++characteristicsCounter;
+
+ //here write var entry length in metadata
+
+ ++m_VariablesCount;
} //end of function
diff --git a/src/core/Group.cpp b/src/core/Group.cpp
index c2013331c6cd4b23c88da248f14960eb6a4e6edf..a0679e813853ef05f8c47e39fd40cbdbfa498d03 100644
--- a/src/core/Group.cpp
+++ b/src/core/Group.cpp
@@ -29,7 +29,8 @@ Group::Group( const std::string name, const bool debugMode ):
{ }
-Group::Group( const std::string name, const std::string& xmlGroup, std::vector< std::shared_ptr<Transform> >& transforms, const bool debugMode ):
+Group::Group( const std::string name, const std::string& xmlGroup, std::vector< std::shared_ptr<Transform> >& transforms,
+ const bool debugMode ):
m_Name{ name },
m_DebugMode{ debugMode }
{
@@ -41,9 +42,32 @@ Group::~Group( )
{ }
+Dims Group::SetDimensions( std::initializer_list<Var> variableList )
+{
+ if( m_DebugMode == true )
+ {
+ if( variableList.size() == 0 )
+ throw std::invalid_argument( "ERROR: variableList is empty, in call to SetDimensions\n" );
+ }
+
+ Dims dimensionsCSV;
+ for( const auto variable : variableList )
+ {
+ if( m_DebugMode == true )
+ {
+ if( variable.find(",") != variable.npos )
+ throw std::invalid_argument( "ERROR: variable can't contain a comma character, in call to SetDimensions\n" );
+ }
+ dimensionsCSV += variable + ",";
+ }
+ dimensionsCSV.pop_back();
+ return dimensionsCSV;
+}
+
+
Var Group::DefineVariable( const std::string variableName, const std::string type,
- const std::string dimensionsCSV,
- const std::string globalDimensionsCSV, const std::string globalOffsetsCSV,
+ const Dims dimensionsCSV,
+ const Dims globalDimensionsCSV, const Dims globalOffsetsCSV,
std::vector<Transform*> transforms, std::vector<int> parameters )
{
auto lf_CheckDimensionVariables = [&]( const std::string csv, const std::string dimensionType, const std::string variableName )
@@ -305,6 +329,11 @@ unsigned long long int Group::GetIntVariableValue( const std::string variableNam
std::vector<unsigned long long int> Group::GetDimensions( const std::string dimensionsCSV ) const
{
+ if( dimensionsCSV == "1" ) //scalar
+ {
+ return std::vector<unsigned long long int>{ 1 };
+ }
+
std::vector<unsigned long long int> dimensions;
if( dimensionsCSV.find(',') == dimensionsCSV.npos ) //check if 1D