Skip to content
Snippets Groups Projects
Commit 67ed835d authored by wfg's avatar wfg
Browse files

Working on BP1Writer.h and Group class 

BP1Writer: Continue working on functions to write data and metadata

Group: added Dims alias and SetDimensions function, so dimensions can be built from a Var List put in an std::initializer_list<Var>
parent 125ad551
No related branches found
No related tags found
1 merge request!8Integrate groupless
Hide whitespace changes
Inline Side-by-side
examples/hello/writer/helloWriter_OOP.cpp
+ 4
− 2
View file @ 67ed835d
...@@ -39,8 +39,10 @@ int main( int argc, char* argv [] ) ...@@ -39,8 +39,10 @@ int main( int argc, char* argv [] )
//Define group and variables with transforms, variables don't have functions, only group can access variables //Define group and variables with transforms, variables don't have functions, only group can access variables
adios::Group& ioGroup = adios.DeclareGroup( "ioGroup" ); adios::Group& ioGroup = adios.DeclareGroup( "ioGroup" );
adios::Var ioNx = ioGroup.DefineVariable<unsigned int>( "Nx" ); adios::Var ioNx = ioGroup.DefineVariable<unsigned int>( "Nx" );
adios::Var ioMyDoubles = ioGroup.DefineVariable<double>( "myDoubles", "Nx" );
adios::Var ioMyFloats = ioGroup.DefineVariable<float>( "myFloats", "Nx" ); adios::Dims dimNx = ioGroup.SetDimensions( {ioNx} );
adios::Var ioMyDoubles = ioGroup.DefineVariable<double>( "myDoubles", dimNx );
adios::Var ioMyFloats = ioGroup.DefineVariable<float>( "myFloats", dimNx );
//add transform to variable in group...not executed (just testing API) //add transform to variable in group...not executed (just testing API)
adios::Transform bzip2 = adios::transform::BZIP2( ); adios::Transform bzip2 = adios::transform::BZIP2( );
......
include/capsule/Heap.h
+ 3
− 6
View file @ 67ed835d
...@@ -23,6 +23,9 @@ class Heap : public Capsule ...@@ -23,6 +23,9 @@ class Heap : public Capsule
public: public:
std::vector<char> m_Data; ///< data buffer allocated using the STL in heap memory, default size = 16 Mb
std::vector<char> m_Metadata; ///< metadata buffer allocated using the STL in heap memory, default size = 100 Kb
/** /**
* Unique constructor * Unique constructor
* @param accessMode read, write or append * @param accessMode read, write or append
...@@ -44,12 +47,6 @@ public: ...@@ -44,12 +47,6 @@ public:
void ResizeData( const std::size_t size ); void ResizeData( const std::size_t size );
void ResizeMetadata( const std::size_t size ); void ResizeMetadata( const std::size_t size );
private:
std::vector<char> m_Data; ///< data buffer allocated using the STL in heap memory, default size = 16 Mb
std::vector<char> m_Metadata; ///< metadata buffer allocated using the STL in heap memory, default size = 100 Kb
}; };
......
include/core/Capsule.h
+ 3
− 0
View file @ 67ed835d
...@@ -27,6 +27,9 @@ public: ...@@ -27,6 +27,9 @@ public:
const std::string m_Type; ///< buffer type const std::string m_Type; ///< buffer type
const std::string m_AccessMode; ///< 'w': write, 'r': read, 'a': append const std::string m_AccessMode; ///< 'w': write, 'r': read, 'a': append
std::size_t m_AbsoluteDataPosition; ///< includes the data flushed to transports
std::size_t m_AbsoluteMetadataPosition; ///< includes the metadata flushed to transports
/** /**
* Base class constructor providing type from derived class and accessMode * Base class constructor providing type from derived class and accessMode
* @param type derived class type * @param type derived class type
......
include/core/Group.h
+ 12
− 6
View file @ 67ed835d
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
#include <vector> #include <vector>
#include <ostream> #include <ostream>
#include <set> #include <set>
#include <initializer_list> //SetDimensions
/// \endcond /// \endcond
#ifdef HAVE_MPI #ifdef HAVE_MPI
...@@ -35,6 +36,8 @@ namespace adios ...@@ -35,6 +36,8 @@ namespace adios
{ {
using Var = std::string; ///< used for returning variables from DefineVariable using Var = std::string; ///< used for returning variables from DefineVariable
using Dims = std::string;
/** /**
* Class that defines each ADIOS Group composed of Variables, Attributes and GlobalBounds (if global variables exist) * Class that defines each ADIOS Group composed of Variables, Attributes and GlobalBounds (if global variables exist)
*/ */
...@@ -84,6 +87,9 @@ public: ...@@ -84,6 +87,9 @@ public:
~Group( ); ///< Using STL containers, no deallocation ~Group( ); ///< Using STL containers, no deallocation
Dims SetDimensions( std::initializer_list<Var> variables ); ///< returns adios::Dims object from a list of existing adios::Var objects
/** /**
* Define a new variable in the group object * Define a new variable in the group object
* @param name variable name, must be unique in the group. If name exists it removes the current variable. In debug mode program will exit. * @param name variable name, must be unique in the group. If name exists it removes the current variable. In debug mode program will exit.
...@@ -95,16 +101,16 @@ public: ...@@ -95,16 +101,16 @@ public:
* @param parameters corresponding parameter used by a Transform object in transforms (index should match), default is empty * @param parameters corresponding parameter used by a Transform object in transforms (index should match), default is empty
*/ */
Var DefineVariable( const std::string variableName, const std::string type, Var DefineVariable( const std::string variableName, const std::string type,
const std::string dimensionsCSV = "", const Dims dimensionsCSV = "1",
const std::string globalDimensionsCSV = "", const std::string globalOffsetsCSV = "", const Dims globalDimensionsCSV = "", const Dims globalOffsetsCSV = "",
const std::vector<Transform*> transforms = std::vector<Transform*>(), const std::vector<Transform*> transforms = std::vector<Transform*>(),
const std::vector<int> parameters = std::vector<int>() ); const std::vector<int> parameters = std::vector<int>() );
template< class T > template< class T >
Var DefineVariable( const std::string variableName, Var DefineVariable( const std::string variableName,
const std::string dimensionsCSV = "", const Dims dimensionsCSV = "",
const std::string globalDimensionsCSV = "", const std::string globalOffsetsCSV = "", const Dims globalDimensionsCSV = "", const Dims globalOffsetsCSV = "",
const std::vector<Transform*> transforms = std::vector<Transform*>(), const std::vector<Transform*> transforms = std::vector<Transform*>(),
const std::vector<int> parameters = std::vector<int>() ) const std::vector<int> parameters = std::vector<int>() )
{ {
...@@ -117,7 +123,7 @@ public: ...@@ -117,7 +123,7 @@ public:
* @param transform corresponding transform object, non-const as a pointer is created and pushed to a vector * @param transform corresponding transform object, non-const as a pointer is created and pushed to a vector
* @param parameter optional parameter interpreted by the corresponding Transform, default = -1 * @param parameter optional parameter interpreted by the corresponding Transform, default = -1
*/ */
void AddTransform( const std::string variableName, Transform& transform, const int parameter = -1 ); void AddTransform( const Var variableName, Transform& transform, const int parameter = -1 );
/** /**
* Define a new attribute * Define a new attribute
...@@ -139,7 +145,7 @@ public: ...@@ -139,7 +145,7 @@ public:
* @param dimensionsCSV comma separated dimensions "Nx,Ny,Nz" * @param dimensionsCSV comma separated dimensions "Nx,Ny,Nz"
* @return actual vector values = { Nx, Ny, Nz } * @return actual vector values = { Nx, Ny, Nz }
*/ */
std::vector<unsigned long long int> GetDimensions( const std::string dimensionsCSV ) const; std::vector<unsigned long long int> GetDimensions( const Dims dimensionsCSV ) const;
private: private:
......
include/format/BP1Writer.h
+ 231
− 196
View file @ 67ed835d
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
#include <vector> #include <vector>
#include <cstdint> #include <cstdint>
#include <algorithm> //std::count #include <algorithm> //std::count, std::copy
#include <cstring> //std::memcpy #include <cstring> //std::memcpy
#include "core/Variable.h" #include "core/Variable.h"
...@@ -30,11 +30,20 @@ class BP1Writer ...@@ -30,11 +30,20 @@ class BP1Writer
public: public:
std::vector<char> m_PGIndex; ///< process group index std::uint64_t m_ProcessGroupsCount = 0; ///< number of process groups
std::uint64_t m_ProcessGroupsLength = 0; ///< length in bytes of process groups
std::vector<char> m_ProcessGroupIndex; ///< process group index metadata
std::uint32_t m_VariablesCount = 0; ///< number of written Variables
std::uint64_t m_VariablesLength = 0; ///< length in bytes of written Variables
std::vector<char> m_VariableIndex; ///< metadata variable index std::vector<char> m_VariableIndex; ///< metadata variable index
std::uint32_t m_AttributesCount = 0; ///< number of Attributes
std::uint64_t m_AttributesLength = 0; ///< length in bytes of Attributes
std::vector<char> m_AttributeIndex; ///< metadata attribute index std::vector<char> m_AttributeIndex; ///< metadata attribute index
const unsigned int m_Cores = 1; const unsigned int m_Cores = 1;
const unsigned int m_Verbosity = 0;
/** /**
* DataTypes mapping in BP Format * DataTypes mapping in BP Format
...@@ -151,7 +160,7 @@ public: ...@@ -151,7 +160,7 @@ public:
*/ */
template< class T > template< class T >
size_t GetVariableIndexSize( const Group& group, const std::string variableName, size_t GetVariableIndexSize( const Group& group, const std::string variableName,
const Variable<T> variable, const unsigned int verbosity ) noexcept const Variable<T> variable ) noexcept
{ {
//size_t indexSize = varEntryLength + memberID + lengthGroupName + groupName + lengthVariableName + lengthOfPath + path + datatype //size_t indexSize = varEntryLength + memberID + lengthGroupName + groupName + lengthVariableName + lengthOfPath + path + datatype
size_t indexSize = 23; //without characteristics size_t indexSize = 23; //without characteristics
...@@ -181,7 +190,7 @@ public: ...@@ -181,7 +190,7 @@ public:
} }
//characteristic statistics //characteristic statistics
if( verbosity == 0 ) //default, only min and max if( m_Verbosity == 0 ) //default, only min and max
{ {
indexSize += 2 * ( sizeof(T) + 1 ); indexSize += 2 * ( sizeof(T) + 1 );
indexSize += 1 + 1; //id indexSize += 1 + 1; //id
...@@ -208,218 +217,244 @@ public: ...@@ -208,218 +217,244 @@ public:
offset += size; offset += size;
} }
template< class T > template< class T, class U >
void CopyToBuffers( std::vector<char*>& buffers, const T* source, std::size_t size, std::size_t& offset ) noexcept void CopyToBuffers( std::vector<char*>& buffers, const T* source, U size, std::size_t& offset ) noexcept
{ {
for( auto& buffer : buffers ) for( auto& buffer : buffers )
{ {
std::copy( source, source+size, &buffer[offset] ); std::copy( source, source+size, &buffer[offset] );
//std::memcpy( &buffer[offset], source, size );
} }
offset += size; offset += size;
} }
/** /**
* * Writes a variable to BP format in data and metadata (index) buffers
* @param group variable owner * @param group variable owner
* @param variableName name of variable to be written * @param variableName name of variable to be written
* @param variable object carrying variable information * @param variable object carrying variable information
* @param dataBuffers buffers to which variable metadata and payload (values) will be written. Metadata is added in case of system corruption to allow regeneration. * @param dataBuffers buffers to which variable metadata and payload (values) will be written. Metadata is added in case of system corruption to allow regeneration.
* @param dataPosition initial data position
* @param metadataBuffers buffers to which only variable metadata will be written * @param metadataBuffers buffers to which only variable metadata will be written
* @param characteristicsVerbosity default = 0 min and max only, * @param metadataPosition
* @param memberID
* @return number of bytes written, needed for variable entry length
*/ */
template< class T > template< class T >
void WriteVariable( const Group& group, const std::string variableName, void WriteVariable( const Group& group, const Var variableName,
const Variable<T>& variable, const Variable<T>& variable,
std::vector<char*>& dataBuffers, const std::size_t dataPosition, std::vector<char*>& dataBuffers, const std::size_t dataPosition,
std::vector<char*>& metadataBuffers, const std::size_t metadataPosition, std::vector<char*>& metadataBuffers, const std::size_t metadataPosition ) noexcept
const unsigned int memberID, const bool writeDimensionsInData,
const unsigned int verbose ) noexcept
{ {
std::size_t metadataOffset = metadataPosition + 8; //length of var, will come at the end from this offset std::size_t metadataOffset = metadataPosition + 4; //length of var, will come at the end from this offset
std::size_t dataOffset = dataPosition + 8; //length of var, will come at the end from this offset std::size_t dataOffset = dataPosition + 8; //length of var, will come at the end from this offset
//memberID //memberID
WriteToBuffers( metadataBuffers, &memberID, 4, metadataOffset ); WriteToBuffers( metadataBuffers, &m_VariablesCount, 4, metadataOffset );
//group //group, only in metadata
const std::uint16_t lengthGroupName = group.m_Name.length(); const std::uint16_t lengthGroupName = group.m_Name.length();
WriteToBuffers( metadataBuffers, &lengthGroupName, 2, metadataOffset ); //2 bytes WriteToBuffers( metadataBuffers, &lengthGroupName, 2, metadataOffset ); //2 bytes
//const char* groupName = ; WriteToBuffers( metadataBuffers, group.m_Name.c_str(), lengthGroupName, metadataOffset );
WriteToBuffers( metadataBuffers, group.m_Name.c_str(), lengthGroupName, metadataOffset ); //variable name to metadata and data
//variable name to metadata and data const std::uint16_t lengthVariableName = variableName.length();
const std::uint16_t lengthVariableName = variableName.length(); WriteToBuffers( metadataBuffers, &lengthVariableName, 2, metadataOffset );
WriteToBuffers( metadataBuffers, &lengthVariableName, 2, metadataOffset ); WriteToBuffers( metadataBuffers, variableName.c_str(), lengthVariableName, metadataOffset );
WriteToBuffers( metadataBuffers, variableName.c_str(), lengthVariableName, metadataOffset ); WriteToBuffers( dataBuffers, &lengthVariableName, 2, dataOffset );
WriteToBuffers( dataBuffers, &lengthVariableName, 2, dataOffset ); WriteToBuffers( dataBuffers, variableName.c_str(), lengthVariableName, dataOffset );
WriteToBuffers( dataBuffers, variableName.c_str(), lengthVariableName, dataOffset ); //skip path (jump 2 bytes, already set to zero)
//skip path (jump 2 bytes) metadataOffset += 2;
metadataOffset += 2; dataOffset += 2;
dataOffset += 2; //dataType
//dataType const std::uint8_t dataType = GetDataType( variable );
const std::uint8_t dataType = GetBPDataType( variable ); WriteToBuffers( metadataBuffers, &dataType, 1, metadataOffset );
WriteToBuffers( metadataBuffers, &dataType, 1, metadataOffset ); WriteToBuffers( dataBuffers, &dataType, 1, dataOffset );
WriteToBuffers( dataBuffers, &dataType, 1, dataOffset );
//Characteristics Sets in Metadata and Data
//Characteristics Sets in Metadata and Data const std::size_t metadataCharacteristicsSetsCountPosition = metadataOffset; //very important piece
const std::size_t metadataCharacteristicsSetsCountPosition = metadataOffset; //very important piece std::size_t dataCharacteristicsCountPosition = dataOffset;
std::size_t dataCharacteristicsCountPosition = dataOffset; const std::uint64_t sets = 1; //write one for now
const std::uint64_t sets = 1; //write one for now WriteToBuffers( metadataBuffers, &sets, 8, metadataOffset );
WriteToBuffers( metadataBuffers, &sets, 8, metadataOffset );
std::uint8_t characteristicsCounter = 0; //used for characteristics count, characteristics length will be calculated at the end
unsigned int characteristicsCounter = 0; //used for characteristics count, characteristics length will be calculated at the end
//DIMENSIONS CHARACTERISTIC
//DIMENSIONS CHARACTERISTIC const std::vector<unsigned long long int> localDimensions = group.GetDimensions( variable.DimensionsCSV );
const std::vector<unsigned long long int> localDimensions = group.GetDimensions( variable.DimensionsCSV );
//write to metadata characteristic
//write to metadata characteristic //characteristic: dimension
//characteristic: dimension std::uint8_t characteristicID = characteristic_dimensions;
std::uint8_t characteristicID = characteristic_dimensions; WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset );
WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); const std::uint8_t dimensions = localDimensions.size();
const std::uint8_t dimensions = localDimensions.size(); WriteToBuffers( metadataBuffers, &dimensions, 1, metadataOffset );
WriteToBuffers( metadataBuffers, &dimensions, 1, metadataOffset ); const std::uint16_t dimensionsLength = dimensions * 24; //24 is from 8 bytes for each: local dimension, global dimension, global offset
const std::uint16_t dimensionsLength = dimensions * 24; //24 is from 8 bytes for each: local dimension, global dimension, global offset WriteToBuffers( metadataBuffers, &dimensionsLength, 2, metadataOffset );
WriteToBuffers( metadataBuffers, &dimensionsLength, 2, metadataOffset );
//write in data if it's a dimension variable (scalar) y or n
//dimensions in data buffer, header const char dimensionYorN = ( variable.IsDimension ) ? 'y' : 'n';
if( writeDimensionsInData == true ) WriteToBuffers( dataBuffers, &dimensionYorN, 1, dataOffset );
{ WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
const char yes = 'y'; // dimension y or n const std::uint16_t dimensionsLengthInData = dimensions * 27; //27 is from 9 bytes for each: var y/n + local, var y/n + global dimension, var y/n + global offset
WriteToBuffers( dataBuffers, &yes, 1, dataOffset ); WriteToBuffers( dataBuffers, &dimensionsLengthInData, 2, dataOffset );
//for now only support unsigned long integer value (8 bytes), as in metadata
WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset ); if( variable.GlobalBoundsIndex == -1 ) //local variable
const std::uint16_t dimensionsLengthInData = dimensions * 27; //27 is from 9 bytes for each: var y/n + local, var y/n + global dimension, var y/n + global offset {
WriteToBuffers( dataBuffers, &dimensionsLengthInData, 2, dataOffset ); for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
} {
else //metadata characteristics
{ WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
const char no = 'n'; // dimension y or n metadataOffset += 16; //skipping global dimension(8), global offset (8)
WriteToBuffers( dataBuffers, &no, 1, dataOffset ); }
dataCharacteristicsCountPosition += 1;
} const char no = 'n'; //dimension format unsigned int value (not using memberID for now)
for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
if( variable.GlobalBoundsIndex == -1 ) //local variable {
{ //data dimensions
for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d ) WriteToBuffers( dataBuffers, &no, 1, dataOffset );
{ WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
//metadata characteristics dataOffset += 18; //skipping var no + global dimension(8), var no + global offset (8)
WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset ); }
metadataOffset += 16; //skipping global dimension(8), global offset (8)
} //dimensions in data characteristic entry
dataCharacteristicsCountPosition = dataOffset;
if( writeDimensionsInData == true ) dataOffset += 5; //skip characteristics count(1) + length (4)
{ WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset );
for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
{ std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
const char no = 'n'; //dimension format unsigned int value WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset );
WriteToBuffers( dataBuffers, &no, 1, dataOffset );
WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset ); WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
dataOffset += 18; //skipping var no + global dimension(8), var no + global offset (8) WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset );
}
} for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
//dimensions in data characteristic entry (might not be required) {
dataCharacteristicsCountPosition = dataOffset; //data characteristics
dataOffset += 5; //skip characteristics count(1) + length (4) WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); metadataOffset += 16; //skipping global dimension(8), global offset (8)
}
std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each }
WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset ); else //global variable
{
WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset ); std::vector<unsigned long long int> globalDimensions = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].first );
WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset ); std::vector<unsigned long long int> globalOffsets = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].second );
for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d ) //metadata, writing in characteristics
{ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
//metadata characteristics {
WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset ); WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset );
metadataOffset += 16; //skipping global dimension(8), global offset (8) WriteToBuffers( metadataBuffers, &globalDimensions[d], 8, metadataOffset );
} WriteToBuffers( metadataBuffers, &globalOffsets[d], 8, metadataOffset );
} }
else //global variable
{ //data dimensions entry
std::vector<unsigned long long int> globalDimensions = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].first ); for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
std::vector<unsigned long long int> globalOffsets = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].second ); {
const char no = 'n'; //dimension format unsigned int value
//metadata, writing in characteristics WriteToBuffers( dataBuffers, &no, 1, dataOffset );
for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d ) WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
{ WriteToBuffers( dataBuffers, &no, 1, dataOffset );
WriteToBuffers( metadataBuffers, &localDimensions[d], 8, metadataOffset ); WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
WriteToBuffers( metadataBuffers, &globalDimensions[d], 8, metadataOffset ); }
WriteToBuffers( metadataBuffers, &globalOffsets[d], 8, metadataOffset );
} //dimensions in data characteristic entry (might not be required)
dataCharacteristicsCountPosition = dataOffset;
//data dimensions entry dataOffset += 5; //skip characteristics count(1) + length (4)
if( writeDimensionsInData == true ) WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //id
{
for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d ) std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
{ WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset );
const char no = 'n'; //dimension format unsigned int value
WriteToBuffers( dataBuffers, &no, 1, dataOffset ); WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset );
WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset ); WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset );
WriteToBuffers( dataBuffers, &no, 1, dataOffset );
WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset ); for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
} {
} WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset );
//dimensions in data characteristic entry (might not be required) WriteToBuffers( dataBuffers, &globalDimensions[d], 8, dataOffset );
dataCharacteristicsCountPosition = dataOffset; WriteToBuffers( dataBuffers, &globalOffsets[d], 8, dataOffset );
dataOffset += 5; //skip characteristics count(1) + length (4) }
WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //id }
++characteristicsCounter;
std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
WriteToBuffers( dataBuffers, &lengthOfDimensionsCharacteristic, 2, dataOffset ); //VALUE for SCALAR or STAT min, max for ARRAY
//Value for scalar
WriteToBuffers( dataBuffers, &dimensions, 1, dataOffset ); if( variable.DimensionsCSV == "1" ) //scalar //just doing string scalars for now (by name), needs to be modified when user passes value
WriteToBuffers( dataBuffers, &dimensionsLength, 2, dataOffset ); {
characteristicID = characteristic_value;
for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d ) std::int16_t lenghtOfName = variableName.length();
{ //metadata
WriteToBuffers( dataBuffers, &localDimensions[d], 8, dataOffset ); WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset );
WriteToBuffers( dataBuffers, &globalDimensions[d], 8, dataOffset ); WriteToBuffers( metadataBuffers, &lenghtOfName, 2, metadataOffset );
WriteToBuffers( dataBuffers, &globalOffsets[d], 8, dataOffset ); WriteToBuffers( metadataBuffers, variableName.c_str(), lenghtOfName, metadataOffset );
}
} //data
WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset );
++characteristicsCounter; std::int16_t lengthOfCharacteristic = 2 + lenghtOfName;
//Stat -> Min, Max WriteToBuffers( dataBuffers, &lengthOfCharacteristic, 2, dataOffset );
characteristicID = characteristic_stat; WriteToBuffers( dataBuffers, &lenghtOfName, 2, dataOffset );
if( verbose == 0 ) //default verbose WriteToBuffers( dataBuffers, variableName.c_str(), lenghtOfName, dataOffset );
{ }
const std::size_t valuesSize = GetTotalSize( localDimensions ); else // Stat -> Min, Max for arrays,
//here we can make decisions for threads based on valuesSize {
T min, max; if( m_Verbosity == 0 ) //default verbose
{
if( valuesSize >= 10000000 ) //ten million? this needs actual results //Get min and max
GetMinMax( variable.Values, valuesSize, min, max, m_Cores ); //here we can add cores from constructor const std::size_t valuesSize = GetTotalSize( localDimensions );
else //here we can make decisions for threads based on valuesSize
GetMinMax( variable.Values, valuesSize, min, max ); T min, max;
const std::int8_t statisticMinID = statistic_min; if( valuesSize >= 10000000 ) //ten million? this needs actual results
const std::int8_t statisticMaxID = statistic_max; GetMinMax( variable.Values, valuesSize, min, max, m_Cores ); //here we can add cores from constructor
else
//write min and max to metadata GetMinMax( variable.Values, valuesSize, min, max );
WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //min
WriteToBuffers( metadataBuffers, &statisticMinID, 1, metadataOffset ); //set characteristic ids for min and max
WriteToBuffers( metadataBuffers, &min, sizeof(T), metadataOffset ); characteristicID = characteristic_stat;
const std::int8_t statisticMinID = statistic_min;
WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //max const std::int8_t statisticMaxID = statistic_max;
WriteToBuffers( metadataBuffers, &statisticMaxID, 1, metadataOffset );
WriteToBuffers( metadataBuffers, &max, sizeof(T), metadataOffset ); //write min and max to metadata
WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //min
//write min and max to data WriteToBuffers( metadataBuffers, &statisticMinID, 1, metadataOffset );
WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //min WriteToBuffers( metadataBuffers, &min, sizeof(T), metadataOffset );
const std::int16_t lengthCharacteristic = 1 + sizeof( T );
WriteToBuffers( dataBuffers, &lengthCharacteristic, 2, dataOffset ); WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //max
WriteToBuffers( dataBuffers, &statisticMinID, 1, dataOffset ); WriteToBuffers( metadataBuffers, &statisticMaxID, 1, metadataOffset );
WriteToBuffers( dataBuffers, &min, sizeof(T), dataOffset ); WriteToBuffers( metadataBuffers, &max, sizeof(T), metadataOffset );
WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //max //write min and max to data
WriteToBuffers( dataBuffers, &lengthCharacteristic, 2, dataOffset ); WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //min
WriteToBuffers( dataBuffers, &statisticMaxID, 1, dataOffset ); const std::int16_t lengthCharacteristic = 1 + sizeof( T );
WriteToBuffers( dataBuffers, &max, sizeof(T), dataOffset ); WriteToBuffers( dataBuffers, &lengthCharacteristic, 2, dataOffset );
} WriteToBuffers( dataBuffers, &statisticMinID, 1, dataOffset );
//Offsets should be last and only written to metadata WriteToBuffers( dataBuffers, &min, sizeof(T), dataOffset );
WriteToBuffers( dataBuffers, &characteristicID, 1, dataOffset ); //max
WriteToBuffers( dataBuffers, &lengthCharacteristic, 2, dataOffset );
WriteToBuffers( dataBuffers, &statisticMaxID, 1, dataOffset );
WriteToBuffers( dataBuffers, &max, sizeof(T), dataOffset );
}
}
++characteristicsCounter;
//here write characteristics count and length to data
std::uint32_t characteristicsLengthInData = dataOffset - dataCharacteristicsCountPosition;
WriteToBuffers( dataBuffers, &characteristicsCounter, 1, dataCharacteristicsCountPosition );
WriteToBuffers( dataBuffers, &characteristicsLengthInData, 4, dataCharacteristicsCountPosition );
dataCharacteristicsCountPosition -= 5;
//Offsets should be last and only written to metadata
characteristicID = characteristic_offset;
WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //variable offset id
WriteToBuffers( metadataBuffers, &dataPosition, 8, metadataOffset ); //variable offset
++characteristicsCounter;
characteristicID = characteristic_payload_offset;
WriteToBuffers( metadataBuffers, &characteristicID, 1, metadataOffset ); //variable payload offset id
WriteToBuffers( metadataBuffers, &dataOffset, 8, metadataOffset ); //variable offset
++characteristicsCounter;
//here write var entry length in metadata
++m_VariablesCount;
} //end of function } //end of function
......
src/core/Group.cpp
+ 32
− 3
View file @ 67ed835d
...@@ -29,7 +29,8 @@ Group::Group( const std::string name, const bool debugMode ): ...@@ -29,7 +29,8 @@ Group::Group( const std::string name, const bool debugMode ):
{ } { }
Group::Group( const std::string name, const std::string& xmlGroup, std::vector< std::shared_ptr<Transform> >& transforms, const bool debugMode ): Group::Group( const std::string name, const std::string& xmlGroup, std::vector< std::shared_ptr<Transform> >& transforms,
const bool debugMode ):
m_Name{ name }, m_Name{ name },
m_DebugMode{ debugMode } m_DebugMode{ debugMode }
{ {
...@@ -41,9 +42,32 @@ Group::~Group( ) ...@@ -41,9 +42,32 @@ Group::~Group( )
{ } { }
Dims Group::SetDimensions( std::initializer_list<Var> variableList )
{
if( m_DebugMode == true )
{
if( variableList.size() == 0 )
throw std::invalid_argument( "ERROR: variableList is empty, in call to SetDimensions\n" );
}
Dims dimensionsCSV;
for( const auto variable : variableList )
{
if( m_DebugMode == true )
{
if( variable.find(",") != variable.npos )
throw std::invalid_argument( "ERROR: variable can't contain a comma character, in call to SetDimensions\n" );
}
dimensionsCSV += variable + ",";
}
dimensionsCSV.pop_back();
return dimensionsCSV;
}
Var Group::DefineVariable( const std::string variableName, const std::string type, Var Group::DefineVariable( const std::string variableName, const std::string type,
const std::string dimensionsCSV, const Dims dimensionsCSV,
const std::string globalDimensionsCSV, const std::string globalOffsetsCSV, const Dims globalDimensionsCSV, const Dims globalOffsetsCSV,
std::vector<Transform*> transforms, std::vector<int> parameters ) std::vector<Transform*> transforms, std::vector<int> parameters )
{ {
auto lf_CheckDimensionVariables = [&]( const std::string csv, const std::string dimensionType, const std::string variableName ) auto lf_CheckDimensionVariables = [&]( const std::string csv, const std::string dimensionType, const std::string variableName )
...@@ -305,6 +329,11 @@ unsigned long long int Group::GetIntVariableValue( const std::string variableNam ...@@ -305,6 +329,11 @@ unsigned long long int Group::GetIntVariableValue( const std::string variableNam
std::vector<unsigned long long int> Group::GetDimensions( const std::string dimensionsCSV ) const std::vector<unsigned long long int> Group::GetDimensions( const std::string dimensionsCSV ) const
{ {
if( dimensionsCSV == "1" ) //scalar
{
return std::vector<unsigned long long int>{ 1 };
}
std::vector<unsigned long long int> dimensions; std::vector<unsigned long long int> dimensions;
if( dimensionsCSV.find(',') == dimensionsCSV.npos ) //check if 1D if( dimensionsCSV.find(',') == dimensionsCSV.npos ) //check if 1D
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment