From 9b454fb0d0594eec860f033e044ae699c3e07e75 Mon Sep 17 00:00:00 2001
From: wfg <wfg@pc0098504.ornl.gov>
Date: Wed, 8 Mar 2017 18:05:31 -0500
Subject: [PATCH] First working version of BP1Writer
Tests with bpls indicate that variable payloads are recognized
Renamed BPWriter engine to BPFileWriter
Changed WriteVariableIndex function in BP1Writer to only include a
capsule::STLVector as buffers, other capsules will need to get it from
this heap buffer.
To do:
add profiling info (time)
---
.../hello/bpWriter/helloBPWriter_nompi.cpp | 14 +-
include/ADIOS_CPP.h | 2 +-
.../engine/bp/{BPReader.h => BPFileReader.h} | 12 +-
.../engine/bp/{BPWriter.h => BPFileWriter.h} | 21 +-
include/format/BP1Writer.h | 454 +++++++-----------
include/functions/adiosFunctions.h | 7 -
include/functions/adiosTemplates.h | 55 +--
src/ADIOS.cpp | 12 +-
.../bp/{BPReader.cpp => BPFileReader.cpp} | 58 +--
.../bp/{BPWriter.cpp => BPFileWriter.cpp} | 104 ++--
src/format/BP1Writer.cpp | 300 +++++-------
src/functions/adiosFunctions.cpp | 7 -
src/transport/file/FD.cpp | 2 +-
13 files changed, 406 insertions(+), 642 deletions(-)
rename include/engine/bp/{BPReader.h => BPFileReader.h} (93%)
rename include/engine/bp/{BPWriter.h => BPFileWriter.h} (94%)
rename src/engine/bp/{BPReader.cpp => BPFileReader.cpp} (65%)
rename src/engine/bp/{BPWriter.cpp => BPFileWriter.cpp} (66%)
diff --git a/examples/hello/bpWriter/helloBPWriter_nompi.cpp b/examples/hello/bpWriter/helloBPWriter_nompi.cpp
index 3cec9144d..a06b1b061 100644
--- a/examples/hello/bpWriter/helloBPWriter_nompi.cpp
+++ b/examples/hello/bpWriter/helloBPWriter_nompi.cpp
@@ -36,6 +36,7 @@ int main( int argc, char* argv [] )
adios::Variable<double>& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", adios::Dims{Nx} );
adios::Variable<float>& ioMyMatrix = adios.DefineVariable<float>( "myMatrix", adios::Dims{rows,columns} );
adios::Variable<float>& ioMyMatrix2 = adios.DefineVariable<float>( "myMatrix2", adios::Dims{rows,columns} );
+ adios::Variable<float>& ioMyMatrix3 = adios.DefineVariable<float>( "myMatrix3", adios::Dims{rows,columns} );
//Define method for engine creation, it is basically straight-forward parameters
adios::Method& bpWriterSettings = adios.DeclareMethod( "SinglePOSIXFile" ); //default method type is Writer
@@ -43,15 +44,16 @@ int main( int argc, char* argv [] )
//Create engine smart pointer due to polymorphism,
//Open returns a smart pointer to Engine containing the Derived class Writer
- auto bpWriter = adios.Open( "myDoubles_nompi.bp", "w", bpWriterSettings );
+ auto bpFileWriter = adios.Open( "myDoubles_nompi.bp", "w", bpWriterSettings );
- if( bpWriter == nullptr )
+ if( bpFileWriter == nullptr )
throw std::ios_base::failure( "ERROR: couldn't create bpWriter at Open\n" );
- bpWriter->Write<double>( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
- bpWriter->Write<float>( ioMyMatrix, myMatrix.data() ); //2d Example
- bpWriter->Write<float>( ioMyMatrix2, myMatrix2.data() ); //2d Example
- bpWriter->Close( );
+ bpFileWriter->Write<double>( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
+ bpFileWriter->Write<float>( ioMyMatrix, myMatrix.data() ); //2d Example
+ bpFileWriter->Write<float>( ioMyMatrix2, myMatrix2.data() ); //2d Example
+ bpFileWriter->Write<float>( ioMyMatrix3, myMatrix2.data() ); //2d Example
+ bpFileWriter->Close( );
//
}
catch( std::invalid_argument& e )
diff --git a/include/ADIOS_CPP.h b/include/ADIOS_CPP.h
index 1dd00e333..d0327b264 100644
--- a/include/ADIOS_CPP.h
+++ b/include/ADIOS_CPP.h
@@ -14,7 +14,7 @@
#include "core/Engine.h"
#include "core/Transform.h"
-#include "engine/bp/BPWriter.h"
+#include "engine/bp/BPFileWriter.h"
//Will allow to create engines directly (no polymorphism)
#ifdef HAVE_DATAMAN
diff --git a/include/engine/bp/BPReader.h b/include/engine/bp/BPFileReader.h
similarity index 93%
rename from include/engine/bp/BPReader.h
rename to include/engine/bp/BPFileReader.h
index 784d5703f..7693d57a7 100644
--- a/include/engine/bp/BPReader.h
+++ b/include/engine/bp/BPFileReader.h
@@ -5,8 +5,8 @@
* Author: wfg
*/
-#ifndef BPREADER_H_
-#define BPREADER_H_
+#ifndef BPFILEREADER_H_
+#define BPFILEREADER_H_
#include <iostream> //this must go away
@@ -20,7 +20,7 @@ namespace adios
{
-class BPReader : public Engine
+class BPFileReader : public Engine
{
public:
@@ -34,10 +34,10 @@ public:
* @param debugMode
* @param hostLanguage
*/
- BPReader( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
+ BPFileReader( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
const Method& method, const bool debugMode = false, const unsigned int cores = 1 );
- ~BPReader( );
+ ~BPFileReader( );
Variable<void>* InquireVariable( const std::string name, const bool readIn = true );
@@ -98,4 +98,4 @@ private:
} //end namespace adios
-#endif /* BPREADER_H_ */
+#endif /* BPFILEREADER_H_ */
diff --git a/include/engine/bp/BPWriter.h b/include/engine/bp/BPFileWriter.h
similarity index 94%
rename from include/engine/bp/BPWriter.h
rename to include/engine/bp/BPFileWriter.h
index 3c42b6c23..6b760ee69 100644
--- a/include/engine/bp/BPWriter.h
+++ b/include/engine/bp/BPFileWriter.h
@@ -5,8 +5,10 @@
* Author: wfg
*/
-#ifndef BPWRITER_H_
-#define BPWRITER_H_
+#ifndef BPFILEWRITER_H_
+#define BPFILEWRITER_H_
+
+#include <iostream> //will go away
#include "core/Engine.h"
#include "format/BP1Writer.h"
@@ -18,7 +20,7 @@
namespace adios
{
-class BPWriter : public Engine
+class BPFileWriter : public Engine
{
public:
@@ -31,10 +33,10 @@ public:
* @param method
* @param debugMode
*/
- BPWriter( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
- const Method& method, const bool debugMode = false, const unsigned int cores = 1 );
+ BPFileWriter( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
+ const Method& method, const bool debugMode = false, const unsigned int cores = 1 );
- ~BPWriter( );
+ ~BPFileWriter( );
void Advance( );
@@ -48,14 +50,13 @@ public:
private:
capsule::STLVector m_Buffer; ///< heap capsule using STL std::vector<char>
+ format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
+ format::BP1MetadataSet m_MetadataSet; ///< metadata set accompanying the heap buffer data in bp format. Needed by m_BP1Writer
bool m_IsFirstClose = true; ///< set to false after first Close is reached so metadata doesn't have to be accommodated for a subsequent Close
std::size_t m_MaxBufferSize; ///< maximum allowed memory to be allocated
float m_GrowthFactor = 1.5; ///< capsule memory growth factor, new_memory = m_GrowthFactor * current_memory
- format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
- format::BP1MetadataSet m_MetadataSet; ///< metadata set accompanying the heap buffer data in bp format. Needed by m_BP1Writer
-
bool m_TransportFlush = false; ///< true: transport flush happened, buffer must be reset
void Init( );
@@ -154,4 +155,4 @@ private:
} //end namespace adios
-#endif /* BPWRITER_H_ */
+#endif /* BPFILEWRITER_H_ */
diff --git a/include/format/BP1Writer.h b/include/format/BP1Writer.h
index 6f9daa0e3..d54d4212c 100644
--- a/include/format/BP1Writer.h
+++ b/include/format/BP1Writer.h
@@ -67,23 +67,6 @@ public:
const std::vector< std::shared_ptr<Transport> >& transports,
capsule::STLVector& buffer,
BP1MetadataSet& metadataSet ) const noexcept;
- /**
- * Writes a process group index PGIndex and list of methods (from transports), done at Open or aggregation of new time step
- * Version that operates on many capsules and metadatasets
- * @param isFortran
- * @param name
- * @param processID
- * @param timeStepName
- * @param timeStep
- * @param transports
- * @param capsules
- * @param metadataSets
- */
- void WriteProcessGroupIndex( const bool isFortran, const std::string name, const unsigned int processID,
- const std::string timeStepName, const unsigned int timeStep,
- const std::vector< std::shared_ptr<Transport> >& transports,
- std::vector< std::shared_ptr<Capsule> >& capsules,
- std::vector<BP1MetadataSet>& metadataSets ) const noexcept;
/**
* Returns the estimated variable index size
@@ -134,195 +117,76 @@ public:
}
/**
- * Version that takes directly a single Heap buffer class and a single BP1MetadataSet
- * Skip virtual if optimized to an object
- * @param group
- * @param variableName
- * @param variable
- * @param buffer
- * @param metadataSet
- */
- template< class T >
+ *
+ * @param variable
+ * @param dataBuffers
+ * @param dataPositions
+ * @param dataAbsolutePositions
+ * @param metadataBuffers
+ * @param metadataPositions
+ * @param variablesCount
+ */
+ template<class T>
void WriteVariableIndex( const Variable<T>& variable, capsule::STLVector& buffer, BP1MetadataSet& metadataSet ) const noexcept
{
- // adapt this part to local variables
- std::vector<char*> dataBuffers{ buffer.m_Data.data() };
- std::vector<size_t> dataPositions { buffer.m_DataPosition };
- std::vector<size_t> dataAbsolutePositions { buffer.m_DataAbsolutePosition };
-
- std::vector<char*> metadataBuffers{ metadataSet.VarsIndex.data() };
- std::vector<std::size_t> metadataPositions{ metadataSet.VarsIndexPosition };
- std::vector<unsigned int> variablesCount{ metadataSet.VarsCount };
-
- WriteVariableIndexCommon( variable, dataBuffers, dataPositions, dataAbsolutePositions,
- metadataBuffers, metadataPositions, variablesCount );
-
- //update positions and varsCount originally passed by value
- buffer.m_DataPosition = dataPositions[0];
- buffer.m_DataAbsolutePosition = dataAbsolutePositions[0];
- metadataSet.VarsIndexPosition = metadataPositions[0];
- metadataSet.VarsCount += 1;
- }
-
-
- /**
- * Version that writes to a vector of capsules
- * @param group variable owner
- * @param variableName name of variable to be written
- * @param variable object carrying variable information
- * @param capsules from Engine member m_Capsules
- * @param metadataSets
- */
- template< class T >
- void WriteVariableIndex( const Variable<T>& variable,
- std::vector< std::shared_ptr<Capsule> >& capsules,
- std::vector<BP1MetadataSet>& metadataSets ) const noexcept
- {
- // adapt this part to local variables
- std::vector<char*> metadataBuffers, dataBuffers;
- std::vector<std::size_t> metadataPositions, dataPositions, dataAbsolutePositions;
- std::vector<unsigned int> variablesCount;
-
- for( auto& metadataSet : metadataSets )
+ auto lf_String = []( const std::string name, const std::uint16_t length,
+ std::vector<char>& buffer, std::size_t& position )
{
- metadataBuffers.push_back( metadataSet.VarsIndex.data() );
- metadataPositions.push_back( metadataSet.VarsIndexPosition );
- variablesCount.push_back( metadataSet.VarsCount );
- }
+ MemcpyToBuffer( buffer, position, &length, 2 );
+ MemcpyToBuffer( buffer, position, name.c_str(), length );
+ };
- for( auto& capsule : capsules )
+ auto lf_MemberID = []( const std::uint32_t memberID, capsule::STLVector& buffer, BP1MetadataSet& metadataSet )
{
- dataBuffers.push_back( capsule->GetData( ) );
- dataPositions.push_back( capsule->m_DataPosition );
- dataAbsolutePositions.push_back( capsule->m_DataAbsolutePosition );
- }
-
- WriteVariableIndexCommon( variable, dataBuffers, dataPositions, dataAbsolutePositions,
- metadataBuffers, metadataPositions, variablesCount );
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &memberID, 4 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &memberID, 4 );
+ };
- //update positions and varsCount originally passed by value
- const unsigned int buffersSize = static_cast<unsigned int>( capsules.size() );
- for( unsigned int i = 0; i < buffersSize; ++i )
+ auto lf_VarName = [&]( const std::string name, capsule::STLVector& buffer, BP1MetadataSet& metadataSet )
{
- metadataSets[i].VarsIndexPosition = metadataPositions[i];
- metadataSets[i].VarsCount += 1;
+ const std::uint16_t length = name.length();
+ lf_String( name, length, buffer.m_Data, buffer.m_DataPosition );
+ lf_String( name, length, metadataSet.VarsIndex, metadataSet.VarsIndexPosition );
+ };
- capsules[i]->m_DataPosition = dataPositions[i];
- capsules[i]->m_DataAbsolutePosition = dataAbsolutePositions[i];
- }
- }
-
- /**
- * Expensive part this is only for heap buffers need to adapt to vector of capsules
- * @param variable
- * @param buffer
- */
- template< class T >
- void WriteVariablePayload( const Variable<T>& variable, capsule::STLVector& buffer, const unsigned int cores = 1 ) const noexcept
- {
- std::size_t payloadSize = variable.PayLoadSize();
- //serial for now, expensive part
- MemcpyThreads( &buffer.m_Data[buffer.m_DataPosition], variable.m_AppValues, payloadSize, cores ); //EXPENSIVE part, might want to use threads if large.
- //update indices
- buffer.m_DataPosition += payloadSize;
- buffer.m_DataAbsolutePosition += payloadSize;
- }
-
-
- /**
- * Function that sets metadata (if first close) and writes to a single transport
- * @param metadataSet current rank metadata set
- * @param capsule contains data and metadata buffers
- * @param transport does a write after data and metadata is setup
- * @param isFirstClose true: metadata has been set and aggregated
- * @param haveMetadata true: attach metadata buffer to each data buffer and do a transport write
- * @param haveTiming true: add timing.log file
- */
- void Close( BP1MetadataSet& metadataSet, Capsule& capsule, Transport& transport, bool& isFirstClose,
- const bool haveMetadata = true, const bool haveTiming = false ) const noexcept;
-
-
-private:
-
- /**
- * Common function that Writes a process group index PGIndex, done at Open or aggregation of new time step.
- * Called from public WriteProcessGroupIndex functions.
- * @param isFortran true: using Fortran, false: other language
- * @param name process group, usually the rank (maybe communicator?)
- * @param processID processID, usually the rank
- * @param timeStepName
- * @param timeStep
- * @param dataBuffers
- * @param dataPositions
- * @param dataAbsolutePositions
- * @param metadataBuffers
- * @param metadataPositions
- */
- void WriteProcessGroupIndexCommon( const bool isFortran, const std::string name, const unsigned int processID,
- const std::string timeStepName, const unsigned int timeStep,
- const std::vector<int>& methodIDs,
- std::vector<char*>& dataBuffers, std::vector<std::size_t>& dataPositions,
- std::vector<std::size_t>& dataAbsolutePositions,
- std::vector<char*>& metadataBuffers,
- std::vector<std::size_t>& metadataPositions ) const noexcept;
-
- /**
- *
- * @param variable
- * @param dataBuffers
- * @param dataPositions
- * @param dataAbsolutePositions
- * @param metadataBuffers
- * @param metadataPositions
- * @param variablesCount
- */
- template<class T>
- void WriteVariableIndexCommon( const Variable<T>& variable,
- std::vector<char*>& dataBuffers, std::vector<size_t>& dataPositions,
- std::vector<size_t>& dataAbsolutePositions,
- std::vector<char*>& metadataBuffers, std::vector<size_t>& metadataPositions,
- std::vector<unsigned int> variablesCount ) const noexcept
- {
- //capture initial positions
- std::vector<std::size_t> metadataLengthPositions( metadataPositions );
- std::vector<std::size_t> dataLengthPositions( dataPositions );
+ auto lf_DataType = []( const std::uint8_t dataType, capsule::STLVector& buffer, BP1MetadataSet& metadataSet )
+ {
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &dataType, 1 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &dataType, 1 );
+ };
- MovePositions( 4, metadataPositions ); //length of var, will come at the end from this offset
- MovePositions( 8, dataPositions ); //length of var, will come at the end from this offset
- //memberID in metadata and data
- MemcpyToBuffers( metadataBuffers, metadataPositions, variablesCount, 4 );
- MemcpyToBuffers( dataBuffers, dataPositions, variablesCount, 4 );
+ //BODY of function starts here
+ //capture initial positions storing the variable Length
+ const std::size_t metadataVarLengthPosition = metadataSet.VarsIndexPosition;
+ const std::size_t dataVarLengthPosition = buffer.m_DataPosition;
- //skipping 2 bytes for length of group name which is zero, only in metadata
- MovePositions( 2, metadataPositions ); //length of var, will come at the end from this offset
+ metadataSet.VarsIndexPosition += 4; //skip var length
+ buffer.m_DataPosition += 8; //skip var length
- //variable name to metadata and data
- const std::uint16_t lengthVariableName = variable.m_Name.length();
- WriteNameRecord( variable.m_Name, lengthVariableName, metadataBuffers, metadataPositions );
- WriteNameRecord( variable.m_Name, lengthVariableName, dataBuffers, dataPositions );
+ lf_MemberID( metadataSet.VarsCount, buffer, metadataSet ); //memberID in metadata and data
+ metadataSet.VarsIndexPosition += 2; //skipping 2 bytes for length of group name which is zero, only in metadata
+ lf_VarName( variable.m_Name, buffer, metadataSet ); //variable name to metadata and data
- //skip path (jump 2 bytes, already set to zero)
- MovePositions( 2, metadataPositions ); //length of var, will come at the end from this offset
- MovePositions( 2, dataPositions ); //length of var, will come at the end from this offset
+ metadataSet.VarsIndexPosition += 2; //skip path
+ buffer.m_DataPosition += 2; //skip path
//dataType
const std::uint8_t dataType = GetDataType<T>();
- MemcpyToBuffers( metadataBuffers, metadataPositions, &dataType, 1 );
- MemcpyToBuffers( dataBuffers, dataPositions, &dataType, 1 );
+ lf_DataType( dataType, buffer, metadataSet );
//write in data if it's a dimension variable (scalar) y or n
const char dimensionYorN = ( variable.m_IsDimension ) ? 'y' : 'n';
- MemcpyToBuffers( dataBuffers, dataPositions, &dimensionYorN, 1 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &dimensionYorN, 1 );
- //Characteristics Sets in Metadata and Data
+ //Characteristics Sets Count in Metadata
const std::uint64_t sets = 1; //write one for now
- MemcpyToBuffers( metadataBuffers, metadataPositions, &sets, 8 );
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &sets, 8 );
+ //Characteristics set
std::uint8_t characteristicsCounter = 0; //used for characteristics count, characteristics length will be calculated at the end
- std::vector<std::size_t> metadataCharacteristicsCountPositions( metadataPositions ); //very important, can't be const as it is updated by MemcpyToBuffer
- //here move positions 5 bytes in data and metadata for characteristics count + length
- MovePositions( 5, metadataPositions );
+ const std::size_t metadataCharacteristicsCountPosition = metadataSet.VarsIndexPosition;
+ metadataSet.VarsIndexPosition += 5; //here move positions 5 bytes in data and metadata for characteristics count + length
//DIMENSIONS CHARACTERISTIC
const std::vector<std::size_t>& localDimensions = variable.m_Dimensions;
@@ -330,53 +194,53 @@ private:
//write to metadata characteristic
//characteristic: dimension
std::uint8_t characteristicID = characteristic_dimensions;
- MemcpyToBuffers( metadataBuffers, metadataPositions, &characteristicID, 1 );
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &characteristicID, 1 );
const std::uint8_t dimensions = localDimensions.size();
- MemcpyToBuffers( metadataBuffers, metadataPositions, &dimensions, 1 );
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &dimensions, 1 );
const std::uint16_t dimensionsLength = dimensions * 24; //24 is from 8 bytes for each: local dimension, global dimension, global offset
- MemcpyToBuffers( metadataBuffers, metadataPositions, &dimensionsLength, 2 );
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &dimensionsLength, 2 );
//write dimensions count and length in data
- MemcpyToBuffers( dataBuffers, dataPositions, &dimensions, 1 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &dimensions, 1 );
const std::uint16_t dimensionsLengthInData = dimensions * 27; //27 is from 9 bytes for each: var y/n + local, var y/n + global dimension, var y/n + global offset
- MemcpyToBuffers( dataBuffers, dataPositions, &dimensionsLengthInData, 2 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &dimensionsLengthInData, 2 );
- std::vector<std::size_t> dataCharacteristicsCountPositions( dataPositions ); //very important piece, will redefine later
+ std::size_t dataCharacteristicsCountPosition = buffer.m_DataPosition; //will be modified
if( variable.m_GlobalDimensions.empty() ) //local variable
{
- WriteDimensionRecord( metadataBuffers, metadataPositions, localDimensions, 16 );
- WriteDimensionRecord( dataBuffers, dataPositions, localDimensions, 18, true ); //not using memberID for now
+ WriteDimensionRecord( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, localDimensions, 16 );
+ WriteDimensionRecord( buffer.m_Data, buffer.m_DataPosition, localDimensions, 18, true ); //not using memberID for now
- dataCharacteristicsCountPositions = dataPositions; //very important to track as writer is going back to this position
- MovePositions( 5, dataPositions ); //skip characteristics count(1) + length (4)
+ dataCharacteristicsCountPosition = buffer.m_DataPosition; //very important to track as writer is going back to this position
+ buffer.m_DataPosition += 5; //skip characteristics count(1) + length (4)
//dimensions in data characteristic entry
- MemcpyToBuffers( dataBuffers, dataPositions, &characteristicID, 1 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &characteristicID, 1 );
const std::int16_t lengthOfDimensionsCharacteristic = 24 * dimensions; // 24 = 3 local, global, global offset x 8 bytes/each
- MemcpyToBuffers( dataBuffers, dataPositions, &lengthOfDimensionsCharacteristic, 2 );
- MemcpyToBuffers( dataBuffers, dataPositions, &dimensions, 1 );
- MemcpyToBuffers( dataBuffers, dataPositions, &dimensionsLength, 2 );
- WriteDimensionRecord( dataBuffers, dataPositions, localDimensions, 16 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &lengthOfDimensionsCharacteristic, 2 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &dimensions, 1 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &dimensionsLength, 2 );
+ WriteDimensionRecord( buffer.m_Data, buffer.m_DataPosition, localDimensions, 16 );
}
else //global variable
{
const std::vector<std::size_t>& globalDimensions = variable.m_GlobalDimensions;
const std::vector<std::size_t>& globalOffsets = variable.m_GlobalOffsets;
- WriteDimensionRecord( metadataBuffers, metadataPositions, localDimensions, globalDimensions, globalOffsets );
- WriteDimensionRecord( dataBuffers, dataPositions, localDimensions, globalDimensions, globalOffsets, true );
+ WriteDimensionRecord( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, localDimensions, globalDimensions, globalOffsets );
+ WriteDimensionRecord( buffer.m_Data, buffer.m_DataPosition, localDimensions, globalDimensions, globalOffsets, true );
- dataCharacteristicsCountPositions = dataPositions; //very important, going back to these positions
- MovePositions( 5, dataPositions ); //skip characteristics count(1) + length (4)
+ dataCharacteristicsCountPosition = buffer.m_DataPosition; //very important, going back to these positions
+ buffer.m_DataPosition += 5; //skip characteristics count(1) + length (4)
//dimensions in data characteristic entry
- MemcpyToBuffers( dataBuffers, dataPositions, &characteristicID, 1 ); //id
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &characteristicID, 1 ); //id
const std::int16_t lengthOfDimensionsCharacteristic = 24 * dimensions; // 24 = 3 local, global, global offset x 8 bytes/each
- MemcpyToBuffers( dataBuffers, dataPositions, &lengthOfDimensionsCharacteristic, 2 );
- MemcpyToBuffers( dataBuffers, dataPositions, &dimensions, 1 );
- MemcpyToBuffers( dataBuffers, dataPositions, &dimensionsLength, 2 );
- WriteDimensionRecord( dataBuffers, dataPositions, localDimensions, globalDimensions, globalOffsets );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &lengthOfDimensionsCharacteristic, 2 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &dimensions, 1 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &dimensionsLength, 2 );
+ WriteDimensionRecord( buffer.m_Data, buffer.m_DataPosition, localDimensions, globalDimensions, globalOffsets );
}
++characteristicsCounter;
@@ -385,14 +249,14 @@ private:
if( variable.m_IsScalar ) //scalar //just doing string scalars for now (by name), needs to be modified when user passes value
{
characteristicID = characteristic_value;
- MemcpyToBuffers( metadataBuffers, metadataPositions, &characteristicID, 1 );
- MemcpyToBuffers( metadataBuffers, metadataPositions, variable.m_AppValues, sizeof(T) );
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &characteristicID, 1 );
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, variable.m_AppValues, sizeof(T) );
//data
- MemcpyToBuffers( dataBuffers, dataPositions, &characteristicID, 1 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &characteristicID, 1 );
const std::uint16_t lengthOfValue = sizeof( T );
- MemcpyToBuffers( dataBuffers, dataPositions, &lengthOfValue, 2 ); //add length of characteristic in data
- MemcpyToBuffers( dataBuffers, dataPositions, variable.m_AppValues, sizeof(T) );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &lengthOfValue, 2 ); //add length of characteristic in data
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, variable.m_AppValues, sizeof(T) );
++characteristicsCounter;
}
@@ -400,59 +264,86 @@ private:
{
if( m_Verbosity == 0 ) //default verbose
{
- WriteMinMax( variable, dataBuffers, dataPositions, metadataBuffers, metadataPositions );
+ WriteMinMax( variable, buffer, metadataSet );
characteristicsCounter += 2;
}
}
//Back to characteristics count and length in Data
- std::vector<std::uint32_t> dataCharacteristicsLengths( dataPositions.size() );
- for( unsigned int i = 0; i < dataPositions.size(); ++i )
- dataCharacteristicsLengths[i] = dataPositions[i] - dataCharacteristicsCountPositions[i] - 4 - 1; //remove its own length (4 bytes) + characteristic counter
-
- MemcpyToBuffers( dataBuffers, dataCharacteristicsCountPositions, &characteristicsCounter, 1 );
- MemcpyToBuffers( dataBuffers, dataCharacteristicsCountPositions, dataCharacteristicsLengths, 4 ); //vector to vector
+ //count
+ std::memcpy( &buffer.m_Data[dataCharacteristicsCountPosition], &characteristicsCounter, 1 );
+ //length
+ const std::uint32_t dataCharacteristicsLength = buffer.m_DataPosition - dataCharacteristicsCountPosition - 4 - 1; //remove its own length (4 bytes) + characteristic counter ( 1 byte )
+ std::memcpy( &buffer.m_Data[dataCharacteristicsCountPosition+1], &dataCharacteristicsLength, 4 );
//Metadata only: Offsets should be last, they come from data absolute positions
characteristicID = characteristic_offset;
- MemcpyToBuffers( metadataBuffers, metadataPositions, &characteristicID, 1 ); //variable offset id
- MemcpyToBuffers( metadataBuffers, metadataPositions, dataAbsolutePositions, 8 ); //variable offset
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &characteristicID, 1 ); //variable offset id
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &buffer.m_DataAbsolutePosition, 8 ); //variable offset
++characteristicsCounter;
- //update absolute positions with dataPositions, this is the payload offset
- std::vector<std::uint64_t> varLengths( dataPositions.size() );
- for( unsigned int i = 0; i < dataAbsolutePositions.size(); ++i )
- {
- varLengths[i] = dataPositions[i] - dataLengthPositions[i];
- dataAbsolutePositions[i] += varLengths[i];
- varLengths[i] += variable.PayLoadSize() - 8; //remove its own size
- }
+ //Back to length of var including payload size in data + update absolute position
+ const std::uint64_t varLength = buffer.m_DataPosition - dataVarLengthPosition + variable.PayLoadSize() - 8; //remove its own size
+ std::memcpy( &buffer.m_Data[dataVarLengthPosition], &varLength, 8 );
+ buffer.m_DataAbsolutePosition += buffer.m_DataPosition - dataVarLengthPosition; //payload offset
characteristicID = characteristic_payload_offset;
- MemcpyToBuffers( metadataBuffers, metadataPositions, &characteristicID, 1 ); //variable payload offset id
- MemcpyToBuffers( metadataBuffers, metadataPositions, dataAbsolutePositions, 8 ); //variable payload offset
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &characteristicID, 1 ); //variable payload offset id
+ MemcpyToBuffer( metadataSet.VarsIndex, metadataSet.VarsIndexPosition, &buffer.m_DataAbsolutePosition, 8 ); //variable payload offset
++characteristicsCounter;
//Back to writing characteristics count and length in Metadata
- std::vector<std::uint32_t> metadataCharacteristicsLengths( metadataPositions.size() );
- for( unsigned int i = 0; i < metadataPositions.size(); ++i )
- metadataCharacteristicsLengths[i] = metadataPositions[i] - metadataCharacteristicsCountPositions[i] - 4 - 1; //remove its own size and characteristic counter size
+ //count
+ std::memcpy( &metadataSet.VarsIndex[metadataCharacteristicsCountPosition], &characteristicsCounter, 1 );
+ //length
+ const std::uint32_t metadataCharacteristicsLength = metadataSet.VarsIndexPosition - metadataCharacteristicsCountPosition - 4 - 1; //remove its own size and characteristic counter size
+ std::memcpy( &metadataSet.VarsIndex[metadataCharacteristicsCountPosition+1], &metadataCharacteristicsLength, 4 );
+
+ //Back to writing var entry length in Metadata
+ const std::uint32_t metadataVarEntryLength = metadataSet.VarsIndexPosition - metadataVarLengthPosition - 4; //remove its own size
+ std::memcpy( &metadataSet.VarsIndex[metadataVarLengthPosition], &metadataVarEntryLength, 4 );
- MemcpyToBuffers( metadataBuffers, metadataCharacteristicsCountPositions, &characteristicsCounter, 1 );
- MemcpyToBuffers( metadataBuffers, metadataCharacteristicsCountPositions, metadataCharacteristicsLengths, 4 ); //vector to vector
+ ++metadataSet.VarsCount;
+ }
- //Back to var entry length
- std::vector<std::uint32_t> metadataVarEntryLengths( metadataPositions.size() );
- for( unsigned int i = 0; i < metadataPositions.size(); ++i )
- metadataVarEntryLengths[i] = metadataPositions[i] - metadataLengthPositions[i] - 4; //remove its own size
- MemcpyToBuffers( metadataBuffers, metadataLengthPositions, metadataVarEntryLengths, 4 ); //vector to vector
+ /**
+ * Expensive part this is only for heap buffers need to adapt to vector of capsules
+ * @param variable
+ * @param buffer
+ */
+ template< class T >
+ void WriteVariablePayload( const Variable<T>& variable, capsule::STLVector& buffer, const unsigned int cores = 1 ) const noexcept
+ {
+ std::size_t payloadSize = variable.PayLoadSize(); //not using const due to memcpy inside Memcpythreads
+// std::cout << variable.m_Name << "\n";
+// std::cout << "Payload position: " << buffer.m_DataPosition << "\n";
+// std::cout << "Payload size: " << payloadSize << "\n\n";
- //Need to add length of var including payload size
- MemcpyToBuffers( dataBuffers, dataLengthPositions, varLengths, 8 );
+
+ //EXPENSIVE part, might want to use threads if large, serial for now
+ MemcpyThreads( &buffer.m_Data[buffer.m_DataPosition], variable.m_AppValues, payloadSize, cores );
+ //update indices
+ buffer.m_DataPosition += payloadSize;
+ buffer.m_DataAbsolutePosition += payloadSize;
}
+ /**
+ * Function that sets metadata (if first close) and writes to a single transport
+ * @param metadataSet current rank metadata set
+ * @param buffer contains data
+ * @param transport does a write after data and metadata is setup
+ * @param isFirstClose true: metadata has been set and aggregated
+ * @param haveMetadata true: attach metadata buffer to each data buffer and do a transport write
+ * @param haveTiming true: add timing.log file
+ */
+ void Close( BP1MetadataSet& metadataSet, capsule::STLVector& buffer, Transport& transport, bool& isFirstClose,
+ const bool haveMetadata = true, const bool haveTiming = false ) const noexcept;
+
+
+private:
+
/**
* Writes name record using a
* @param name to be written
@@ -461,18 +352,18 @@ private:
* @param positions to be moved
*/
void WriteNameRecord( const std::string name, const std::uint16_t length,
- std::vector<char*>& buffers, std::vector<std::size_t>& positions ) const noexcept;
+ std::vector<char>& buffer, std::size_t& position ) const noexcept;
/**
* Write a dimension record for a global variable used by WriteVariableCommon
- * @param buffers
- * @param positions
+ * @param buffer
+ * @param position
* @param localDimensions
* @param globalDimensions
* @param globalOffsets
* @param addType true: for data buffers, false: for metadata buffer and data characteristic
*/
- void WriteDimensionRecord( std::vector<char*>& buffers, std::vector<std::size_t>& positions,
+ void WriteDimensionRecord( std::vector<char>& buffer, std::size_t& position,
const std::vector<std::size_t>& localDimensions,
const std::vector<std::size_t>& globalDimensions,
const std::vector<std::size_t>& globalOffsets,
@@ -480,13 +371,13 @@ private:
/**
* Write a dimension record for a local variable used by WriteVariableCommon
- * @param buffers
- * @param positions
+ * @param buffer
+ * @param position
* @param localDimensions
* @param skip
* @param addType true: for data buffers, false: for metadata buffer and data characteristic
*/
- void WriteDimensionRecord( std::vector<char*>& buffers, std::vector<std::size_t>& positions,
+ void WriteDimensionRecord( std::vector<char>& buffer, std::size_t& position,
const std::vector<std::size_t>& localDimensions,
const unsigned int skip,
const bool addType = false ) const noexcept;
@@ -501,9 +392,7 @@ private:
* @param metadataPositions
*/
template<class T> inline
- void WriteMinMax( const Variable<T>& variable,
- std::vector<char*>& dataBuffers, std::vector<size_t>& dataPositions,
- std::vector<char*>& metadataBuffers, std::vector<size_t>& metadataPositions ) const noexcept
+ void WriteMinMax( const Variable<T>& variable, capsule::STLVector& buffer, BP1MetadataSet& metadataSet ) const noexcept
{
T min, max;
const std::size_t valuesSize = variable.TotalSize();
@@ -512,7 +401,7 @@ private:
else
GetMinMax( variable.m_AppValues, valuesSize, min, max );
- WriteMinMaxValues( min, max, dataBuffers, dataPositions, metadataBuffers, metadataPositions );
+ WriteMinMaxValues( min, max, buffer, metadataSet );
}
@@ -520,17 +409,15 @@ private:
* Common part of WriteMinMax specialized templates. Writes to buffers after min and max are calculated.
*/
template<class T>
- void WriteMinMaxValues( const T min, const T max,
- std::vector<char*>& dataBuffers, std::vector<size_t>& dataPositions,
- std::vector<char*>& metadataBuffers, std::vector<size_t>& metadataPositions ) const noexcept
+ void WriteMinMaxValues( const T min, const T max, capsule::STLVector& buffer, BP1MetadataSet& metadataSet ) const noexcept
{
constexpr std::int8_t characteristicMinID = characteristic_min;
constexpr std::int8_t characteristicMaxID = characteristic_max;
- WriteValueRecord( characteristicMinID, min, metadataBuffers, metadataPositions );
- WriteValueRecord( characteristicMaxID, max, metadataBuffers, metadataPositions );
- WriteValueRecord( characteristicMinID, min, dataBuffers, dataPositions, true ); //addLength in between for data
- WriteValueRecord( characteristicMaxID, max, dataBuffers, dataPositions, true ); //addLength in between for data
+ WriteValueRecord( characteristicMinID, min, metadataSet.VarsIndex, metadataSet.VarsIndexPosition );
+ WriteValueRecord( characteristicMaxID, max, metadataSet.VarsIndex, metadataSet.VarsIndexPosition );
+ WriteValueRecord( characteristicMinID, min, buffer.m_Data, buffer.m_DataPosition, true ); //true: addLength in between for data
+ WriteValueRecord( characteristicMaxID, max, buffer.m_Data, buffer.m_DataPosition, true ); //true: addLength in between for data
}
@@ -544,41 +431,40 @@ private:
*/
template<class T>
void WriteValueRecord( const std::uint8_t& characteristicID, const T& value,
- std::vector<char*>& buffers, std::vector<std::size_t>& positions,
+ std::vector<char>& buffer, std::size_t& position,
const bool addLength = false ) const noexcept
{
- MemcpyToBuffers( buffers, positions, &characteristicID, 1 );
+ MemcpyToBuffer( buffer, position, &characteristicID, 1 );
if( addLength == true )
{
- const std::int16_t lengthCharacteristic = 1 + sizeof( T );
- MemcpyToBuffers( buffers, positions, &lengthCharacteristic, 2 );
+ const std::uint16_t lengthCharacteristic = 1 + sizeof( T ); //id
+ MemcpyToBuffer( buffer, position, &lengthCharacteristic, 2 );
}
- MemcpyToBuffers( buffers, positions, &value, sizeof(T) );
+ MemcpyToBuffer( buffer, position, &value, sizeof(T) );
}
/**
- * Flattens the metadata indices into a single metadata buffer in capsule
+ * Flattens the data and fills the pg length, vars count, vars length and attributes
* @param metadataSet
- * @param capsule
+ * @param buffer
*/
- void FlattenMetadata( BP1MetadataSet& metadataSet, Capsule& capsule ) const noexcept; ///< sets the metadata buffer in capsule with indices and minifooter
-
+ void FlattenData( BP1MetadataSet& metadataSet, capsule::STLVector& buffer ) const noexcept;
/**
- * Flattens the data and fills the pg length, vars count, vars length and attributes
+ * Flattens the metadata indices into a single metadata buffer in capsule
* @param metadataSet
- * @param capsule
+ * @param buffer
*/
- void FlattenData( BP1MetadataSet& metadataSet, Capsule& capsule ) const noexcept;
+ void FlattenMetadata( BP1MetadataSet& metadataSet, capsule::STLVector& buffer ) const noexcept; ///< sets the metadata buffer in capsule with indices and minifooter
};
/**
- * Specilized version of WriteMinMax for std::complex<float>
+ * Specialized version of WriteMinMax for std::complex<float>
* @param variable
* @param dataBuffers
* @param dataPositions
@@ -586,9 +472,8 @@ private:
* @param metadataPositions
*/
template<> inline
-void BP1Writer::WriteMinMax<std::complex<float>>( const Variable<std::complex<float>>& variable,
- std::vector<char*>& dataBuffers, std::vector<size_t>& dataPositions,
- std::vector<char*>& metadataBuffers, std::vector<size_t>& metadataPositions ) const noexcept
+void BP1Writer::WriteMinMax<std::complex<float>>( const Variable<std::complex<float>>& variable, capsule::STLVector& buffer,
+ BP1MetadataSet& metadataSet ) const noexcept
{
float min, max;
const std::size_t valuesSize = variable.TotalSize();
@@ -597,14 +482,13 @@ void BP1Writer::WriteMinMax<std::complex<float>>( const Variable<std::complex<fl
else
GetMinMax( variable.m_AppValues, valuesSize, min, max );
- WriteMinMaxValues( min, max, dataBuffers, dataPositions, metadataBuffers, metadataPositions );
+ WriteMinMaxValues( min, max, buffer, metadataSet );
}
template<> inline
-void BP1Writer::WriteMinMax<std::complex<double>>( const Variable<std::complex<double>>& variable,
- std::vector<char*>& dataBuffers, std::vector<size_t>& dataPositions,
- std::vector<char*>& metadataBuffers, std::vector<size_t>& metadataPositions ) const noexcept
+void BP1Writer::WriteMinMax<std::complex<double>>( const Variable<std::complex<double>>& variable, capsule::STLVector& buffer,
+ BP1MetadataSet& metadataSet ) const noexcept
{
double min, max;
const std::size_t valuesSize = variable.TotalSize();
@@ -613,14 +497,14 @@ void BP1Writer::WriteMinMax<std::complex<double>>( const Variable<std::complex<d
else
GetMinMax( variable.m_AppValues, valuesSize, min, max );
- WriteMinMaxValues( min, max, dataBuffers, dataPositions, metadataBuffers, metadataPositions );
+ WriteMinMaxValues( min, max, buffer, metadataSet );
}
template<> inline
void BP1Writer::WriteMinMax<std::complex<long double>>( const Variable<std::complex<long double>>& variable,
- std::vector<char*>& dataBuffers, std::vector<size_t>& dataPositions,
- std::vector<char*>& metadataBuffers, std::vector<size_t>& metadataPositions ) const noexcept
+ capsule::STLVector& buffer,
+ BP1MetadataSet& metadataSet ) const noexcept
{
long double min, max;
const std::size_t valuesSize = variable.TotalSize();
@@ -629,7 +513,7 @@ void BP1Writer::WriteMinMax<std::complex<long double>>( const Variable<std::comp
else
GetMinMax( variable.m_AppValues, valuesSize, min, max );
- WriteMinMaxValues( min, max, dataBuffers, dataPositions, metadataBuffers, metadataPositions );
+ WriteMinMaxValues( min, max, buffer, metadataSet );
}
diff --git a/include/functions/adiosFunctions.h b/include/functions/adiosFunctions.h
index 511e26765..d3e2b6bf5 100644
--- a/include/functions/adiosFunctions.h
+++ b/include/functions/adiosFunctions.h
@@ -170,13 +170,6 @@ std::vector<int> CSVToVectorInt( const std::string csv );
int GrowBuffer( const std::size_t incomingDataSize, const float growthFactor, const std::size_t currentPosition,
std::vector<char>& buffer );
-/**
- * Moves positions in a vector by a number of bytes
- * @param bytes input number of bytes
- * @param positions += bytes
- */
-void MovePositions( const int bytes, std::vector<std::size_t>& positions ) noexcept;
-
/**
* Check if system is little endian
diff --git a/include/functions/adiosTemplates.h b/include/functions/adiosTemplates.h
index 3af61fe0e..65998950a 100644
--- a/include/functions/adiosTemplates.h
+++ b/include/functions/adiosTemplates.h
@@ -15,6 +15,7 @@
#include <set>
#include <complex>
#include <cmath> //std::sqrt
+#include <iostream>
/// \endcond
@@ -140,7 +141,7 @@ void MemcpyThreads( T* destination, const U* source, std::size_t count, const un
{
if( cores == 1 )
{
- std::memcpy( destination, source, count ); //here is the bug!!!
+ std::memcpy( destination, source, count );
return;
}
@@ -167,60 +168,28 @@ void MemcpyThreads( T* destination, const U* source, std::size_t count, const un
}
-/**
- * Write to many buffers and updates positions a single piece of data source
- * @param buffers
- * @param positions each element is updated to += size
- * @param source
- * @param size
- */
template< class T >
-void MemcpyToBuffers( std::vector<char*>& buffers, std::vector<std::size_t>& positions, const T* source, std::size_t size ) noexcept
+void MemcpyToBuffer( std::vector<char>& raw, std::size_t& position, const T* source, std::size_t size ) noexcept
{
- const unsigned int length = buffers.size( );
-
- for( unsigned int i = 0; i < length; ++i )
- {
- char* buffer = buffers[i];
- std::memcpy( &buffer[ positions[i] ], source, size );
- //std::copy( source, source+size, &buffers[ positions[i] ] ); wrong version
- positions[i] += size;
- }
+ std::memcpy( &raw[position], source, size );
+ position += size;
}
-/**
- * Version that adds a source container for a 1 to 1 buffer memory copy
- * @param buffers
- * @param positions
- * @param source
- * @param size
- */
template< class T >
-void MemcpyToBuffers( std::vector<char*>& buffers, std::vector<std::size_t>& positions,
- const std::vector<T>& source, std::size_t size ) noexcept
+void PrintValues( const std::string name, const char* buffer, const std::size_t position, const std::size_t elements )
{
- const unsigned int length = buffers.size( );
-
- for( unsigned int i = 0; i < length; ++i )
- {
- char* buffer = buffers[i];
- std::memcpy( &buffer[ positions[i] ], &source[i], size );
- //std::copy( &source[i], &source[i]+size, &buffers[ positions[i] ] );
- positions[i] += size;
- }
-}
+ std::vector<T> values( elements );
+ std::memcpy( values.data(), &buffer[position], elements * sizeof(T) );
+ std::cout << "Read " << name << "\n";
+ for( const auto value : values )
+ std::cout << value << " ";
-template< class T >
-void MemcpyToBuffer( char* buffer, std::size_t& position, const T* source, std::size_t size ) noexcept
-{
- std::memcpy( &buffer[position], source, size );
- position += size;
+ std::cout << "\n";
}
-
} //end namespace
diff --git a/src/ADIOS.cpp b/src/ADIOS.cpp
index 90d60b81b..0d4eefce3 100644
--- a/src/ADIOS.cpp
+++ b/src/ADIOS.cpp
@@ -13,13 +13,11 @@
/// \endcond
#include "ADIOS.h"
-
-
#include "functions/adiosFunctions.h"
//Engines
-#include "engine/bp/BPWriter.h"
-#include "engine/bp/BPReader.h"
+#include "engine/bp/BPFileWriter.h"
+#include "engine/bp/BPFileReader.h"
#ifdef HAVE_DATAMAN //external dependencies
#include "engine/dataman/DataManWriter.h"
@@ -113,13 +111,13 @@ std::shared_ptr<Engine> ADIOS::Open( const std::string name, const std::string a
const bool isDefaultReader = ( accessMode == "r" || accessMode == "read" ) && type.empty() ? true : false;
- if( isDefaultWriter || type == "BPWriter" || type == "bpwriter" )
+ if( isDefaultWriter || type == "BPFileWriter" || type == "bpfilewriter" )
{
- return std::make_shared<BPWriter>( *this, name, accessMode, mpiComm, method, m_DebugMode, cores );
+ return std::make_shared<BPFileWriter>( *this, name, accessMode, mpiComm, method, m_DebugMode, cores );
}
else if( isDefaultReader || type == "BPReader" || type == "bpreader" )
{
- return std::make_shared<BPReader>( *this, name, accessMode, mpiComm, method, m_DebugMode, cores );
+ return std::make_shared<BPFileReader>( *this, name, accessMode, mpiComm, method, m_DebugMode, cores );
}
else if( type == "SIRIUS" || type == "sirius" || type == "Sirius" )
{
diff --git a/src/engine/bp/BPReader.cpp b/src/engine/bp/BPFileReader.cpp
similarity index 65%
rename from src/engine/bp/BPReader.cpp
rename to src/engine/bp/BPFileReader.cpp
index dd6481a2a..6ae61f522 100644
--- a/src/engine/bp/BPReader.cpp
+++ b/src/engine/bp/BPFileReader.cpp
@@ -1,5 +1,5 @@
/*
- * BPReader.cpp
+ * BPFileReader.cpp
*
* Created on: Feb 27, 2017
* Author: wfg
@@ -7,7 +7,7 @@
-#include "engine/bp/BPReader.h"
+#include "engine/bp/BPFileReader.h"
#include "core/Support.h"
#include "functions/adiosFunctions.h" //CSVToVector
@@ -21,87 +21,87 @@
namespace adios
{
-BPReader::BPReader( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
+BPFileReader::BPFileReader( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
const Method& method, const bool debugMode, const unsigned int cores ):
- Engine( adios, "BPReader", name, accessMode, mpiComm, method, debugMode, cores, " BPReader constructor (or call to ADIOS Open).\n" ),
+ Engine( adios, "BPFileReader", name, accessMode, mpiComm, method, debugMode, cores, " BPFileReader constructor (or call to ADIOS Open).\n" ),
m_Buffer( accessMode, m_RankMPI, m_DebugMode )
{
Init( );
}
-BPReader::~BPReader( )
+BPFileReader::~BPFileReader( )
{ }
-Variable<void>* BPReader::InquireVariable( const std::string name, const bool readIn ) //not yet implemented
+Variable<void>* BPFileReader::InquireVariable( const std::string name, const bool readIn ) //not yet implemented
{ return nullptr; }
-Variable<char>* BPReader::InquireVariableChar( const std::string name, const bool readIn )
+Variable<char>* BPFileReader::InquireVariableChar( const std::string name, const bool readIn )
{ return InquireVariableCommon<char>( name, readIn ); }
-Variable<unsigned char>* BPReader::InquireVariableUChar( const std::string name, const bool readIn )
+Variable<unsigned char>* BPFileReader::InquireVariableUChar( const std::string name, const bool readIn )
{ return InquireVariableCommon<unsigned char>( name, readIn ); }
-Variable<short>* BPReader::InquireVariableShort( const std::string name, const bool readIn )
+Variable<short>* BPFileReader::InquireVariableShort( const std::string name, const bool readIn )
{ return InquireVariableCommon<short>( name, readIn ); }
-Variable<unsigned short>* BPReader::InquireVariableUShort( const std::string name, const bool readIn )
+Variable<unsigned short>* BPFileReader::InquireVariableUShort( const std::string name, const bool readIn )
{ return InquireVariableCommon<unsigned short>( name, readIn ); }
-Variable<int>* BPReader::InquireVariableInt( const std::string name, const bool readIn )
+Variable<int>* BPFileReader::InquireVariableInt( const std::string name, const bool readIn )
{ return InquireVariableCommon<int>( name, readIn ); }
-Variable<unsigned int>* BPReader::InquireVariableUInt( const std::string name, const bool readIn )
+Variable<unsigned int>* BPFileReader::InquireVariableUInt( const std::string name, const bool readIn )
{ return InquireVariableCommon<unsigned int>( name, readIn ); }
-Variable<long int>* BPReader::InquireVariableLInt( const std::string name, const bool readIn )
+Variable<long int>* BPFileReader::InquireVariableLInt( const std::string name, const bool readIn )
{ return InquireVariableCommon<long int>( name, readIn ); }
-Variable<unsigned long int>* BPReader::InquireVariableULInt( const std::string name, const bool readIn )
+Variable<unsigned long int>* BPFileReader::InquireVariableULInt( const std::string name, const bool readIn )
{ return InquireVariableCommon<unsigned long int>( name, readIn ); }
-Variable<long long int>* BPReader::InquireVariableLLInt( const std::string name, const bool readIn )
+Variable<long long int>* BPFileReader::InquireVariableLLInt( const std::string name, const bool readIn )
{ return InquireVariableCommon<long long int>( name, readIn ); }
-Variable<unsigned long long int>* BPReader::InquireVariableULLInt( const std::string name, const bool readIn )
+Variable<unsigned long long int>* BPFileReader::InquireVariableULLInt( const std::string name, const bool readIn )
{ return InquireVariableCommon<unsigned long long int>( name, readIn ); }
-Variable<float>* BPReader::InquireVariableFloat( const std::string name, const bool readIn )
+Variable<float>* BPFileReader::InquireVariableFloat( const std::string name, const bool readIn )
{ return InquireVariableCommon<float>( name, readIn ); }
-Variable<double>* BPReader::InquireVariableDouble( const std::string name, const bool readIn )
+Variable<double>* BPFileReader::InquireVariableDouble( const std::string name, const bool readIn )
{ return InquireVariableCommon<double>( name, readIn ); }
-Variable<long double>* BPReader::InquireVariableLDouble( const std::string name, const bool readIn )
+Variable<long double>* BPFileReader::InquireVariableLDouble( const std::string name, const bool readIn )
{ return InquireVariableCommon<long double>( name, readIn ); }
-Variable<std::complex<float>>* BPReader::InquireVariableCFloat( const std::string name, const bool readIn )
+Variable<std::complex<float>>* BPFileReader::InquireVariableCFloat( const std::string name, const bool readIn )
{ return InquireVariableCommon<std::complex<float>>( name, readIn ); }
-Variable<std::complex<double>>* BPReader::InquireVariableCDouble( const std::string name, const bool readIn )
+Variable<std::complex<double>>* BPFileReader::InquireVariableCDouble( const std::string name, const bool readIn )
{ return InquireVariableCommon<std::complex<double>>( name, readIn ); }
-Variable<std::complex<long double>>* BPReader::InquireVariableCLDouble( const std::string name, const bool readIn )
+Variable<std::complex<long double>>* BPFileReader::InquireVariableCLDouble( const std::string name, const bool readIn )
{ return InquireVariableCommon<std::complex<long double>>( name, readIn ); }
-VariableCompound* BPReader::InquireVariableCompound( const std::string name, const bool readIn )
+VariableCompound* BPFileReader::InquireVariableCompound( const std::string name, const bool readIn )
{ return nullptr; }
-void BPReader::Close( const int transportIndex )
+void BPFileReader::Close( const int transportIndex )
{
}
//PRIVATE
-void BPReader::Init( )
+void BPFileReader::Init( )
{
if( m_DebugMode == true )
{
if( m_AccessMode != "r" && m_AccessMode != "read" )
- throw std::invalid_argument( "ERROR: BPReader doesn't support access mode " + m_AccessMode +
- ", in call to ADIOS Open or BPReader constructor\n" );
+ throw std::invalid_argument( "ERROR: BPFileReader doesn't support access mode " + m_AccessMode +
+ ", in call to ADIOS Open or BPFileReader constructor\n" );
}
InitCapsules( );
@@ -109,13 +109,13 @@ void BPReader::Init( )
}
-void BPReader::InitCapsules( )
+void BPFileReader::InitCapsules( )
{
//here init memory capsules
}
-void BPReader::InitTransports( ) //maybe move this?
+void BPFileReader::InitTransports( ) //maybe move this?
{
if( m_DebugMode == true )
{
diff --git a/src/engine/bp/BPWriter.cpp b/src/engine/bp/BPFileWriter.cpp
similarity index 66%
rename from src/engine/bp/BPWriter.cpp
rename to src/engine/bp/BPFileWriter.cpp
index 49c2dc0bb..2cbde1306 100644
--- a/src/engine/bp/BPWriter.cpp
+++ b/src/engine/bp/BPFileWriter.cpp
@@ -1,14 +1,13 @@
/*
- * BPWriter.cpp
+ * BPFileWriter.cpp
*
* Created on: Dec 19, 2016
* Author: wfg
*/
-#include "engine/bp/BPWriter.h"
+#include "engine/bp/BPFileWriter.h"
#include "ADIOS.h"
-
//supported transports
#include "transport/file/FD.h"
#include "transport/file/FP.h"
@@ -19,9 +18,9 @@ namespace adios
{
-BPWriter::BPWriter( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
- const Method& method, const bool debugMode, const unsigned int cores ):
- Engine( adios, "BPWriter", name, accessMode, mpiComm, method, debugMode, cores, " BPWriter constructor (or call to ADIOS Open).\n" ),
+BPFileWriter::BPFileWriter( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
+ const Method& method, const bool debugMode, const unsigned int cores ):
+ Engine( adios, "BPFileWriter", name, accessMode, mpiComm, method, debugMode, cores, " BPFileWriter constructor (or call to ADIOS Open).\n" ),
m_Buffer{ capsule::STLVector( accessMode, m_RankMPI, m_DebugMode ) },
m_MaxBufferSize{ m_Buffer.m_Data.max_size() }
{
@@ -29,11 +28,11 @@ BPWriter::BPWriter( ADIOS& adios, const std::string name, const std::string acce
}
-BPWriter::~BPWriter( )
+BPFileWriter::~BPFileWriter( )
{ }
-void BPWriter::Init( )
+void BPFileWriter::Init( )
{
auto itGrowthFactor = m_Method.m_Parameters.find( "buffer_growth" );
if( itGrowthFactor != m_Method.m_Parameters.end() )
@@ -55,117 +54,117 @@ void BPWriter::Init( )
}
-void BPWriter::Write( Variable<char>& variable, const char* values )
+void BPFileWriter::Write( Variable<char>& variable, const char* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<unsigned char>& variable, const unsigned char* values )
+void BPFileWriter::Write( Variable<unsigned char>& variable, const unsigned char* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<short>& variable, const short* values )
+void BPFileWriter::Write( Variable<short>& variable, const short* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<unsigned short>& variable, const unsigned short* values )
+void BPFileWriter::Write( Variable<unsigned short>& variable, const unsigned short* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<int>& variable, const int* values )
+void BPFileWriter::Write( Variable<int>& variable, const int* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<unsigned int>& variable, const unsigned int* values )
+void BPFileWriter::Write( Variable<unsigned int>& variable, const unsigned int* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<long int>& variable, const long int* values )
+void BPFileWriter::Write( Variable<long int>& variable, const long int* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<unsigned long int>& variable, const unsigned long int* values )
+void BPFileWriter::Write( Variable<unsigned long int>& variable, const unsigned long int* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<long long int>& variable, const long long int* values )
+void BPFileWriter::Write( Variable<long long int>& variable, const long long int* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<unsigned long long int>& variable, const unsigned long long int* values )
+void BPFileWriter::Write( Variable<unsigned long long int>& variable, const unsigned long long int* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<float>& variable, const float* values )
+void BPFileWriter::Write( Variable<float>& variable, const float* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<double>& variable, const double* values )
+void BPFileWriter::Write( Variable<double>& variable, const double* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<long double>& variable, const long double* values )
+void BPFileWriter::Write( Variable<long double>& variable, const long double* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<std::complex<float>>& variable, const std::complex<float>* values )
+void BPFileWriter::Write( Variable<std::complex<float>>& variable, const std::complex<float>* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<std::complex<double>>& variable, const std::complex<double>* values )
+void BPFileWriter::Write( Variable<std::complex<double>>& variable, const std::complex<double>* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( Variable<std::complex<long double>>& variable, const std::complex<long double>* values )
+void BPFileWriter::Write( Variable<std::complex<long double>>& variable, const std::complex<long double>* values )
{ WriteVariableCommon( variable, values ); }
-void BPWriter::Write( VariableCompound& variable, const void* values )
+void BPFileWriter::Write( VariableCompound& variable, const void* values )
{ }
//String version
-void BPWriter::Write( const std::string variableName, const char* values )
+void BPFileWriter::Write( const std::string variableName, const char* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<char>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const unsigned char* values )
+void BPFileWriter::Write( const std::string variableName, const unsigned char* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<unsigned char>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const short* values )
+void BPFileWriter::Write( const std::string variableName, const short* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<short>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const unsigned short* values )
+void BPFileWriter::Write( const std::string variableName, const unsigned short* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<unsigned short>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const int* values )
+void BPFileWriter::Write( const std::string variableName, const int* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<int>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const unsigned int* values )
+void BPFileWriter::Write( const std::string variableName, const unsigned int* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<unsigned int>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const long int* values )
+void BPFileWriter::Write( const std::string variableName, const long int* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<long int>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const unsigned long int* values )
+void BPFileWriter::Write( const std::string variableName, const unsigned long int* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<unsigned long int>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const long long int* values )
+void BPFileWriter::Write( const std::string variableName, const long long int* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<long long int>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const unsigned long long int* values )
+void BPFileWriter::Write( const std::string variableName, const unsigned long long int* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<unsigned long long int>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const float* values )
+void BPFileWriter::Write( const std::string variableName, const float* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<float>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const double* values )
+void BPFileWriter::Write( const std::string variableName, const double* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<double>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const long double* values )
+void BPFileWriter::Write( const std::string variableName, const long double* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<long double>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const std::complex<float>* values )
+void BPFileWriter::Write( const std::string variableName, const std::complex<float>* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<std::complex<float>>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const std::complex<double>* values )
+void BPFileWriter::Write( const std::string variableName, const std::complex<double>* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<std::complex<double>>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const std::complex<long double>* values )
+void BPFileWriter::Write( const std::string variableName, const std::complex<long double>* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<std::complex<long double>>( variableName ), values ); }
-void BPWriter::Write( const std::string variableName, const void* values )
+void BPFileWriter::Write( const std::string variableName, const void* values )
{ }
-void BPWriter::Advance( )
+void BPFileWriter::Advance( )
{
//first close current pg
}
-void BPWriter::Close( const int transportIndex )
+void BPFileWriter::Close( const int transportIndex )
{
CheckTransportIndex( transportIndex );
if( transportIndex == -1 )
@@ -181,7 +180,7 @@ void BPWriter::Close( const int transportIndex )
//PRIVATE FUNCTIONS
-void BPWriter::InitTransports( )
+void BPFileWriter::InitTransports( )
{
if( m_DebugMode == true )
{
@@ -238,7 +237,7 @@ void BPWriter::InitTransports( )
}
-void BPWriter::InitProcessGroup( )
+void BPFileWriter::InitProcessGroup( )
{
if( m_AccessMode == "a" )
{
@@ -249,7 +248,7 @@ void BPWriter::InitProcessGroup( )
-void BPWriter::WriteProcessGroupIndex( )
+void BPFileWriter::WriteProcessGroupIndex( )
{
//pg = process group
const std::string name( std::to_string( m_RankMPI ) ); //using rank as name
@@ -272,20 +271,17 @@ void BPWriter::WriteProcessGroupIndex( )
}
-
-
-
-bool BPWriter::CheckBuffersAllocation( const std::size_t indexSize, const std::size_t payloadSize )
+bool BPFileWriter::CheckBuffersAllocation( const std::size_t indexSize, const std::size_t payloadSize )
{
//Check if data in buffer needs to be reallocated
- const std::size_t neededSize = m_Buffer.m_DataPosition + payloadSize + indexSize + 60; //adding some bytes tolerance
+ const std::size_t neededSize = m_Buffer.m_DataPosition + payloadSize + indexSize + 500; //adding some bytes tolerance
// might need to write payload in batches
bool doTransportsFlush = ( neededSize > m_MaxBufferSize )? true : false;
- if( GrowBuffer( m_MaxBufferSize, m_GrowthFactor, m_Buffer.m_DataPosition, m_Buffer.m_Data ) == -1 )
+ if( GrowBuffer( neededSize, m_GrowthFactor, m_Buffer.m_DataPosition, m_Buffer.m_Data ) == -1 )
doTransportsFlush = true;
- GrowBuffer( indexSize, m_GrowthFactor, m_MetadataSet.VarsIndexPosition, m_MetadataSet.VarsIndex ); //not checking for metadata
+ GrowBuffer( indexSize, m_GrowthFactor, m_MetadataSet.VarsIndexPosition, m_MetadataSet.VarsIndex );
return doTransportsFlush;
}
diff --git a/src/format/BP1Writer.cpp b/src/format/BP1Writer.cpp
index da5044d49..219c1419f 100644
--- a/src/format/BP1Writer.cpp
+++ b/src/format/BP1Writer.cpp
@@ -32,86 +32,88 @@ void BP1Writer::WriteProcessGroupIndex( const bool isFortran, const std::string
metadataSet.DataPGLengthPosition = buffer.m_DataPosition;
// adapt this part to local variables
- std::vector<char*> dataBuffers{ buffer.m_Data.data() };
- std::vector<size_t> dataPositions{ buffer.m_DataPosition };
- std::vector<size_t> dataAbsolutePositions{ buffer.m_DataAbsolutePosition };
+ const std::vector<int> methodIDs = GetMethodIDs( transports );
- std::vector<char*> metadataBuffers{ metadataSet.PGIndex.data() };
- std::vector<std::size_t> metadataPositions{ metadataSet.PGIndexPosition };
+ const std::size_t dataPGLengthPosition = buffer.m_DataPosition;
+ const std::size_t metadataPGLengthPosition = metadataSet.PGIndexPosition;
- const std::vector<int> methodIDs = GetMethodIDs( transports );
+ metadataSet.PGIndexPosition += 2; //skip length of pg in metadata, 2 bytes, would write at the end
+ buffer.m_DataPosition += 8; //skip length of pg in data, 8 bytes, would write at the end
- WriteProcessGroupIndexCommon( isFortran, name, processID, timeStepName, timeStep, methodIDs,
- dataBuffers, dataPositions, dataAbsolutePositions,
- metadataBuffers, metadataPositions );
+ //write name to metadata
+ const std::uint16_t lengthOfName = name.length();
+ WriteNameRecord( name, lengthOfName, metadataSet.PGIndex, metadataSet.PGIndexPosition );
- metadataSet.PGIndexPosition = metadataPositions[0];
- metadataSet.DataVarsCountPosition = dataPositions[0];
- buffer.m_DataPosition = dataPositions[0] + 12; //add 12 for vars count and vars length
- buffer.m_DataAbsolutePosition = dataAbsolutePositions[0] + 12; //add 12 for vars count and vars length
- metadataSet.PGCount += 1;
+ //write if host language Fortran in metadata and data
+ const char hostFortran = ( isFortran ) ? 'y' : 'n'; //if host language is fortran
+ MemcpyToBuffer( metadataSet.PGIndex, metadataSet.PGIndexPosition, &hostFortran, 1 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &hostFortran, 1 );
-}
+ //name in data
+ WriteNameRecord( name, lengthOfName, buffer.m_Data, buffer.m_DataPosition );
+ //processID,
+ MemcpyToBuffer( metadataSet.PGIndex, metadataSet.PGIndexPosition, &processID, 4 );
+ //skip coordination var in data ....what is coordination var?
+ buffer.m_DataPosition += 4;
-void BP1Writer::WriteProcessGroupIndex( const bool isFortran, const std::string name, const unsigned int processID,
- const std::string timeStepName, const unsigned int timeStep,
- const std::vector< std::shared_ptr<Transport> >& transports,
- std::vector< std::shared_ptr<Capsule> >& capsules,
- std::vector<BP1MetadataSet>& metadataSets ) const noexcept
-{
- // adapt this part to local variables
- std::vector<char*> metadataBuffers, dataBuffers;
- std::vector<std::size_t> metadataPositions, dataPositions, dataAbsolutePositions;
+ //time step name to metadata and data
+ const std::uint16_t lengthOfTimeStep = timeStepName.length();
+ WriteNameRecord( timeStepName, lengthOfTimeStep, metadataSet.PGIndex, metadataSet.PGIndexPosition );
+ WriteNameRecord( timeStepName, lengthOfTimeStep, buffer.m_Data, buffer.m_DataPosition );
- for( auto& metadataSet : metadataSets )
- {
- metadataBuffers.push_back( metadataSet.PGIndex.data() );
- metadataPositions.push_back( metadataSet.PGIndexPosition );
- }
+ //time step to metadata and data
+ MemcpyToBuffer( metadataSet.PGIndex, metadataSet.PGIndexPosition, &timeStep, 4 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &timeStep, 4 );
- unsigned int index = 0;
- for( auto& capsule : capsules )
- {
- dataBuffers.push_back( capsule->GetData( ) );
- dataPositions.push_back( capsule->m_DataPosition );
- dataAbsolutePositions.push_back( capsule->m_DataAbsolutePosition );
+ //offset to pg in data in metadata which is the current absolute position
+ MemcpyToBuffer( metadataSet.PGIndex, metadataSet.PGIndexPosition, &buffer.m_DataAbsolutePosition, 8 );
- metadataSets[index].DataPGLengthPosition = capsule->m_DataPosition;
- ++index;
- }
+ //Back to writing metadata pg index length (length of group)
+ const std::uint16_t metadataPGIndexLength = metadataSet.PGIndexPosition - metadataPGLengthPosition - 2; //without length of group record
+ std::memcpy( &metadataSet.PGIndex[metadataPGLengthPosition], &metadataPGIndexLength, 2 );
- const std::vector<int> methodIDs = GetMethodIDs( transports );
- WriteProcessGroupIndexCommon( isFortran, name, processID, timeStepName, timeStep, methodIDs,
- dataBuffers, dataPositions, dataAbsolutePositions,
- metadataBuffers, metadataPositions );
+ //here write method in data
+ const std::uint8_t methodsSize = methodIDs.size();
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &methodsSize, 1 ); //method count
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &methodsSize, 2 ); //method length, assume one byte for methodID for now
- //update positions and varsCount originally passed by value
- const unsigned int buffersSize = static_cast<unsigned int>( capsules.size() );
- for( unsigned int i = 0; i < buffersSize; ++i )
+ for( auto& methodID : methodIDs )
{
- metadataSets[i].PGIndexPosition = metadataPositions[i];
- metadataSets[i].DataVarsCountPosition = dataPositions[i];
- capsules[i]->m_DataPosition = dataPositions[i] + 12; //add 12 for vars count and vars length
- capsules[i]->m_DataAbsolutePosition = dataAbsolutePositions[i] + 12; //add 12 for vars count and vars length
- metadataSets[i].PGCount += 1;
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &methodID, 1 ); //method ID, unknown for now
+ buffer.m_DataPosition += 2; //skip method params length = 0 (2 bytes) for now
}
+
+ buffer.m_DataAbsolutePosition = buffer.m_DataPosition - dataPGLengthPosition; //update aboslute position
+
+ buffer.m_DataPosition += 12; //add vars count and length
+ buffer.m_DataAbsolutePosition += 12; //add vars count and length
+
+ metadataSet.PGCount += 1;
}
-void BP1Writer::Close( BP1MetadataSet& metadataSet, Capsule& capsule, Transport& transport, bool& isFirstClose,
+
+
+
+void BP1Writer::Close( BP1MetadataSet& metadataSet, capsule::STLVector& buffer, Transport& transport, bool& isFirstClose,
const bool haveMetadata, const bool haveTiming ) const noexcept
{
if( isFirstClose == true )
{
- FlattenData( metadataSet, capsule );
- FlattenMetadata( metadataSet, capsule );
+ FlattenData( metadataSet, buffer );
+ FlattenMetadata( metadataSet, buffer );
isFirstClose = false;
}
//implementing N-to-N for now, no aggregation
- transport.Write( capsule.GetData(), capsule.m_DataPosition );
- transport.Write( capsule.GetMetadata(), capsule.m_MetadataPosition ); //we can improve this by copying metadata to data
+// PrintValues<double>( "myDoubles", capsule.GetData(), 155, 10 );
+// PrintValues<float>( "myMatrix", capsule.GetData(), 391, 9 );
+// PrintValues<float>( "myMatrix2", capsule.GetData(), 584, 9 );
+// PrintValues<float>( "myMatrix3", capsule.GetData(), 777, 9 );
+
+ transport.Write( buffer.m_Data.data(), buffer.m_DataPosition ); //single write
+
if( haveMetadata == true )
{
@@ -123,82 +125,16 @@ void BP1Writer::Close( BP1MetadataSet& metadataSet, Capsule& capsule, Transport&
-//PRIVATE FUNCTIONS
-void BP1Writer::WriteProcessGroupIndexCommon( const bool isFortran, const std::string name, const unsigned int processID,
- const std::string timeStepName, const unsigned int timeStep,
- const std::vector<int>& methodIDs,
- std::vector<char*>& dataBuffers, std::vector<std::size_t>& dataPositions,
- std::vector<std::size_t>& dataAbsolutePositions,
- std::vector<char*>& metadataBuffers,
- std::vector<std::size_t>& metadataPositions ) const noexcept
-{
- const std::vector<std::size_t> pgLengthDataPositions( dataPositions ); //needed to update dataAbsolutePositions
- std::vector<std::size_t> pgLengthPositions( metadataPositions ); //get length of pg position
-
- MovePositions( 2, metadataPositions ); //skip length of pg in metadata, 2 bytes, would write at the end
- MovePositions( 8, dataPositions ); //skip length of pg including data, 8 bytes, would write at the end
-
- //write name to metadata
- const std::uint16_t lengthOfName = name.length();
- WriteNameRecord( name, lengthOfName, metadataBuffers, metadataPositions );
-
- //write is host language Fortran in metadata and data
- const char hostFortran = ( isFortran ) ? 'y' : 'n'; //if host language is fortran
- MemcpyToBuffers( metadataBuffers, metadataPositions, &hostFortran, 1 );
- MemcpyToBuffers( dataBuffers, dataPositions, &hostFortran, 1 );
-
- //name in data
- WriteNameRecord( name, lengthOfName, dataBuffers, dataPositions );
-
- //processID,
- MemcpyToBuffers( metadataBuffers, metadataPositions, &processID, 4 );
- //skip coordination var in data ....what is coordination var?
- MovePositions( 4, dataPositions );
-
- //time step name to metadata and data
- const std::uint16_t lengthOfTimeStep = timeStepName.length();
- WriteNameRecord( timeStepName, lengthOfTimeStep, metadataBuffers, metadataPositions );
- WriteNameRecord( timeStepName, lengthOfTimeStep, dataBuffers, dataPositions );
- //time step to metadata and data
- MemcpyToBuffers( metadataBuffers, metadataPositions, &timeStep, 4 );
- MemcpyToBuffers( dataBuffers, dataPositions, &timeStep, 4 );
- //write offset to pg in data on metadata which is the current absolute position
- MemcpyToBuffers( metadataBuffers, metadataPositions, dataAbsolutePositions, 8 );
-
- //get pg index length
- std::vector<std::uint16_t> metadataPGIndexLengths( metadataPositions.size() );
- for( unsigned int i = 0; i < metadataPositions.size(); ++i )
- metadataPGIndexLengths[i] = metadataPositions[i] - pgLengthPositions[i] - 2; //without length of group record
-
- //write to metadata length position the pgIndex length
- MemcpyToBuffers( metadataBuffers, pgLengthPositions, metadataPGIndexLengths, 2 );
-
- //here write method in data
- const std::uint8_t methodsSize = methodIDs.size();
- MemcpyToBuffers( dataBuffers, dataPositions, &methodsSize, 1 ); //method count
- MemcpyToBuffers( dataBuffers, dataPositions, &methodsSize, 2 ); //method length, assume one byte for methodID for now
-
- for( auto& methodID : methodIDs )
- {
- MemcpyToBuffers( dataBuffers, dataPositions, &methodID, 1 ); //method ID, unknown for now
- MovePositions( 2, dataPositions ); //method params length = 0 for now
- }
-
- //dataAbsolutePositions need to be updated
- for( unsigned int i = 0; i < dataPositions.size(); ++i )
- dataAbsolutePositions[i] += dataPositions[i] - pgLengthDataPositions[i];
-}
-
//PRIVATE FUNCTIONS
void BP1Writer::WriteNameRecord( const std::string name, const std::uint16_t length,
- std::vector<char*>& buffers, std::vector<std::size_t>& positions ) const noexcept
+ std::vector<char>& buffer, std::size_t& position ) const noexcept
{
- MemcpyToBuffers( buffers, positions, &length, 2 );
- MemcpyToBuffers( buffers, positions, name.c_str( ), length );
+ MemcpyToBuffer( buffer, position, &length, 2 );
+ MemcpyToBuffer( buffer, position, name.c_str( ), length );
}
-void BP1Writer::WriteDimensionRecord( std::vector<char*>& buffers, std::vector<std::size_t>& positions,
+void BP1Writer::WriteDimensionRecord( std::vector<char>& buffer, std::size_t& position,
const std::vector<std::size_t>& localDimensions,
const std::vector<std::size_t>& globalDimensions,
const std::vector<std::size_t>& globalOffsets,
@@ -207,28 +143,28 @@ void BP1Writer::WriteDimensionRecord( std::vector<char*>& buffers, std::vector<s
if( addType == true )
{
constexpr char no = 'n'; //dimension format unsigned int value for now
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ for( unsigned int d = 0; d < localDimensions.size(); ++d )
{
- MemcpyToBuffers( buffers, positions, &no, 1 );
- MemcpyToBuffers( buffers, positions, &localDimensions[d], 8 );
- MemcpyToBuffers( buffers, positions, &no, 1 );
- MemcpyToBuffers( buffers, positions, &globalDimensions[d], 8 );
- MemcpyToBuffers( buffers, positions, &no, 1 );
- MemcpyToBuffers( buffers, positions, &globalOffsets[d], 8 );
+ MemcpyToBuffer( buffer, position, &no, 1 );
+ MemcpyToBuffer( buffer, position, &localDimensions[d], 8 );
+ MemcpyToBuffer( buffer, position, &no, 1 );
+ MemcpyToBuffer( buffer, position, &globalDimensions[d], 8 );
+ MemcpyToBuffer( buffer, position, &no, 1 );
+ MemcpyToBuffer( buffer, position, &globalOffsets[d], 8 );
}
}
else
{
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ for( unsigned int d = 0; d < localDimensions.size(); ++d )
{
- MemcpyToBuffers( buffers, positions, &localDimensions[d], 8 );
- MemcpyToBuffers( buffers, positions, &globalDimensions[d], 8 );
- MemcpyToBuffers( buffers, positions, &globalOffsets[d], 8 );
+ MemcpyToBuffer( buffer, position, &localDimensions[d], 8 );
+ MemcpyToBuffer( buffer, position, &globalDimensions[d], 8 );
+ MemcpyToBuffer( buffer, position, &globalOffsets[d], 8 );
}
}
}
-void BP1Writer::WriteDimensionRecord( std::vector<char*>& buffers, std::vector<std::size_t>& positions,
+void BP1Writer::WriteDimensionRecord( std::vector<char>& buffer, std::size_t& position,
const std::vector<std::size_t>& localDimensions,
const unsigned int skip,
const bool addType ) const noexcept
@@ -238,96 +174,88 @@ void BP1Writer::WriteDimensionRecord( std::vector<char*>& buffers, std::vector<s
constexpr char no = 'n'; //dimension format unsigned int value (not using memberID for now)
for( const auto& localDimension : localDimensions )
{
- MemcpyToBuffers( buffers, positions, &no, 1 );
- MemcpyToBuffers( buffers, positions, &localDimension, 8 );
- MovePositions( skip, positions );
+ MemcpyToBuffer( buffer, position, &no, 1 );
+ MemcpyToBuffer( buffer, position, &localDimension, 8 );
+ position += skip;
}
}
else
{
for( const auto& localDimension : localDimensions )
{
- MemcpyToBuffers( buffers, positions, &localDimension, 8 );
- MovePositions( skip, positions );
+ MemcpyToBuffer( buffer, position, &localDimension, 8 );
+ position += skip;
}
}
}
-void BP1Writer::FlattenData( BP1MetadataSet& metadataSet, Capsule& capsule ) const noexcept
+void BP1Writer::FlattenData( BP1MetadataSet& metadataSet, capsule::STLVector& buffer ) const noexcept
{
- //Finish writing pg group length and, vars count and length in Data
- char* data = capsule.GetData();
- const std::uint64_t dataPGLength = capsule.m_DataPosition - metadataSet.DataPGLengthPosition - 8 + 12; //without record itself, 12 due to empty attributes
- std::memcpy( &data[metadataSet.DataPGLengthPosition], &dataPGLength, 8 );
-
- //vars count
- std::memcpy( &data[metadataSet.DataVarsCountPosition], &metadataSet.VarsCount, 4 );
+ //vars count and Length
+ std::memcpy( &buffer.m_Data[metadataSet.DataVarsCountPosition], &metadataSet.VarsCount, 4 ); //count
+ const std::uint64_t dataVarsLength = buffer.m_DataPosition - metadataSet.DataVarsCountPosition - 8 - 4; //without record itself and vars count
+ std::memcpy( &buffer.m_Data[metadataSet.DataVarsCountPosition+4], &dataVarsLength, 8 ); //length
- //vars length
- const std::uint64_t dataVarsLength = capsule.m_DataPosition - metadataSet.DataVarsCountPosition - 8 - 4; //without record itself and vars count
- std::memcpy( &data[metadataSet.DataVarsCountPosition+4], &dataVarsLength, 8 );
+ //attributes (empty for now) count and length by moving positions
+ buffer.m_DataPosition += 12;
+ buffer.m_DataAbsolutePosition += 12;
- //here add empty attributes
- capsule.m_DataPosition += 12;
- capsule.m_DataAbsolutePosition += 12;
+ //Finish writing pg group length and, vars count and length in Data
+ const std::uint64_t dataPGLength = buffer.m_DataPosition - metadataSet.DataPGLengthPosition - 8; //without record itself, 12 due to empty attributes
+ std::memcpy( &buffer.m_Data[metadataSet.DataPGLengthPosition], &dataPGLength, 8 );
}
-void BP1Writer::FlattenMetadata( BP1MetadataSet& metadataSet, Capsule& capsule ) const noexcept
+void BP1Writer::FlattenMetadata( BP1MetadataSet& metadataSet, capsule::STLVector& buffer ) const noexcept
{
//Finish writing metadata counts and lengths (IndexPosition)
- const std::uint64_t pgIndexCount = metadataSet.PGCount;
+ //pg index
+ std::memcpy( &metadataSet.PGIndex[0], &metadataSet.PGCount, 8 ); //count
const std::uint64_t pgIndexLength = metadataSet.PGIndexPosition - 16; //without record itself
- std::size_t pgIndexPosition = 0;
- MemcpyToBuffer( metadataSet.PGIndex.data(), pgIndexPosition, &pgIndexCount, 8 );
- MemcpyToBuffer( metadataSet.PGIndex.data(), pgIndexPosition, &pgIndexLength, 8 );
-
- const std::uint32_t varsIndexCount = metadataSet.VarsCount;
+ std::memcpy( &metadataSet.PGIndex[8], &pgIndexLength, 8 );
+ //vars index
+ std::memcpy( &metadataSet.VarsIndex[0], &metadataSet.VarsCount, 4 ); //count
const std::uint64_t varsIndexLength = metadataSet.VarsIndexPosition - 12; //without record itself
- std::size_t varsIndexPosition = 0;
- MemcpyToBuffer( metadataSet.VarsIndex.data(), varsIndexPosition, &varsIndexCount, 4 );
- MemcpyToBuffer( metadataSet.VarsIndex.data(), varsIndexPosition, &varsIndexLength, 8 );
-
- const std::uint32_t attributesIndexCount = metadataSet.AttributesCount;
+ std::memcpy( &metadataSet.VarsIndex[4], &varsIndexLength, 8 );
+ //attributes index
+ std::memcpy( &metadataSet.AttributesIndex[0], &metadataSet.AttributesCount, 4 ); //count
const std::uint64_t attributesIndexLength = metadataSet.AttributesIndexPosition - 12; //without record itself
- std::size_t attributesIndexPosition = 0;
- MemcpyToBuffer( metadataSet.AttributesIndex.data(), attributesIndexPosition, &attributesIndexCount, 4 );
- MemcpyToBuffer( metadataSet.AttributesIndex.data(), attributesIndexPosition, &attributesIndexLength, 8 );
+ std::memcpy( &metadataSet.AttributesIndex[4], &attributesIndexLength, 8 );
+
+ const std::size_t metadataSize = metadataSet.PGIndexPosition + metadataSet.VarsIndexPosition +
+ metadataSet.AttributesIndexPosition + metadataSet.MiniFooterSize;
- const std::size_t metadataSize = pgIndexLength + varsIndexLength + attributesIndexLength + metadataSet.MiniFooterSize + 40;
- capsule.ResizeMetadata( metadataSize );
- char* metadata = capsule.GetMetadata();
+ buffer.m_Data.resize( buffer.m_DataPosition + metadataSize ); //resize data to fit metadata, must replace with growth buffer strategy
- std::size_t position = 0;
- MemcpyToBuffer( metadata, position, metadataSet.PGIndex.data(), metadataSet.PGIndexPosition );
- MemcpyToBuffer( metadata, position, metadataSet.VarsIndex.data(), metadataSet.VarsIndexPosition );
- MemcpyToBuffer( metadata, position, metadataSet.AttributesIndex.data(), metadataSet.AttributesIndexPosition );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, metadataSet.PGIndex.data(), metadataSet.PGIndexPosition );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, metadataSet.VarsIndex.data(), metadataSet.VarsIndexPosition );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, metadataSet.AttributesIndex.data(), metadataSet.AttributesIndexPosition );
//getting absolute offsets, minifooter is 28 bytes for now
- const std::uint64_t offsetPGIndex = capsule.m_DataAbsolutePosition;
- MemcpyToBuffer( metadata, position, &offsetPGIndex, 8 );
+ const std::uint64_t offsetPGIndex = buffer.m_DataAbsolutePosition;
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &offsetPGIndex, 8 );
const std::uint64_t offsetVarsIndex = offsetPGIndex + metadataSet.PGIndexPosition;
- MemcpyToBuffer( metadata, position, &offsetVarsIndex, 8 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &offsetVarsIndex, 8 );
const std::uint64_t offsetAttributeIndex = offsetVarsIndex + metadataSet.VarsIndexPosition;
- MemcpyToBuffer( metadata, position, &offsetAttributeIndex, 8 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &offsetAttributeIndex, 8 );
//version
if( IsLittleEndian( ) )
{
const std::uint8_t endian = 0;
- MemcpyToBuffer( metadata, position, &endian, 1 );
- position += 2;
- MemcpyToBuffer( metadata, position, &m_Version, 1 );
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &endian, 1 );
+ buffer.m_DataPosition += 2;
+ MemcpyToBuffer( buffer.m_Data, buffer.m_DataPosition, &m_Version, 1 );
}
else
{
}
- capsule.m_MetadataPosition = position;
+ buffer.m_DataAbsolutePosition += metadataSize;
}
diff --git a/src/functions/adiosFunctions.cpp b/src/functions/adiosFunctions.cpp
index 641d5b81d..07461bfb3 100644
--- a/src/functions/adiosFunctions.cpp
+++ b/src/functions/adiosFunctions.cpp
@@ -571,13 +571,6 @@ int GrowBuffer( const std::size_t incomingDataSize, const float growthFactor, co
}
-void MovePositions( const int bytes, std::vector<std::size_t>& positions ) noexcept
-{
- for( auto& position : positions )
- position += bytes;
-}
-
-
bool IsLittleEndian( ) noexcept
{
diff --git a/src/transport/file/FD.cpp b/src/transport/file/FD.cpp
index a88d0105c..06e3f21ea 100644
--- a/src/transport/file/FD.cpp
+++ b/src/transport/file/FD.cpp
@@ -59,7 +59,7 @@ void FD::Open( const std::string name, const std::string accessMode )
{
if( m_FileDescriptor == -1 )
throw std::ios_base::failure( "ERROR: couldn't open file " + m_Name +
- ", from call to Open in FD transport using POSIX Open\n" );
+ ", from call to Open in FD transport using POSIX open. Does file exists?\n" );
}
}
--
GitLab