From db587112e1f90ba5a8bf01f6f471e835dcd50f80 Mon Sep 17 00:00:00 2001
From: wfg <wfg@pc0098504.ornl.gov>
Date: Thu, 23 Feb 2017 09:40:58 -0500
Subject: [PATCH] Updates to BP1Writer and complex support
Added BP1Writer pgindex and metadata functionality. Need to hook it with
BPWriter engine
Added complex support in ADIOS class and DataMan engine and example for
DataManWriter
---
.gitignore | 2 +
buildDataMan.sh | 11 +--
.../datamanWriter/helloDataManWriter.cpp | 8 ++
.../helloDataManWriter_nompi.cpp | 9 ++-
examples/hello/writer/helloWriter_nompi.cpp | 2 +-
include/ADIOS.h | 45 +++++++++++
include/core/Variable.h | 15 +++-
include/engine/bp/BPWriter.h | 16 ++--
include/engine/dataman/DataManWriter.h | 7 ++
include/format/BP1.h | 11 ++-
include/format/BP1Writer.h | 35 ++++++++-
include/functions/adiosFunctions.h | 9 ++-
include/functions/adiosTemplates.h | 2 +-
src/engine/bp/BPWriter.cpp | 51 ++++++-------
src/engine/dataman/DataManWriter.cpp | 18 +++++
src/format/BP1Writer.cpp | 75 ++++++++++++++-----
src/functions/adiosFunctions.cpp | 9 ++-
17 files changed, 246 insertions(+), 79 deletions(-)
diff --git a/.gitignore b/.gitignore
index 45f236b25..66a3c55a3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,8 @@
*.a
*.so
*.tmp
+*.exe
+*.bp
#Eclipse
.cproject
diff --git a/buildDataMan.sh b/buildDataMan.sh
index 9a298da4b..d629be60a 100755
--- a/buildDataMan.sh
+++ b/buildDataMan.sh
@@ -6,12 +6,11 @@
# Author: wfg
DATAMAN_LOCATION=/home/wfg/Applications/DataMan
-echo "#################################################################"
-echo "Start building ADIOS ./lib/libadios.a ./libadios_nompi.a"
-make HAVE_DATAMAN=yes DATAMAN_LOC=$DATAMAN_LOCATION #build the ./lib/libadios.a and ./libadios_nompi.a
-echo "#################################################################"
+echo "######################################################################################"
+echo "Start building ADIOS ./lib/libadios.a ./libadios_nompi.a with DataMan library"
+echo "######################################################################################"
echo
-
+make HAVE_DATAMAN=yes DATAMAN_LOC=$DATAMAN_LOCATION #build the ./lib/libadios.a and ./libadios_nompi.a
echo
echo "#################################################################"
echo "Building Dataman Reader and Writer examples"
@@ -25,6 +24,8 @@ echo
echo "#################################################################"
echo "Running nompi.exe examples"
echo "#################################################################"
+echo
+echo
echo "#################################################################"
echo "DataMan writer"
echo "#################################################################"
diff --git a/examples/hello/datamanWriter/helloDataManWriter.cpp b/examples/hello/datamanWriter/helloDataManWriter.cpp
index 7eff00a7e..689217ec5 100644
--- a/examples/hello/datamanWriter/helloDataManWriter.cpp
+++ b/examples/hello/datamanWriter/helloDataManWriter.cpp
@@ -29,10 +29,17 @@ int main( int argc, char* argv [] )
std::vector<double> myDoubles = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
const std::size_t Nx = myDoubles.size();
+ std::vector<std::complex<float>> myCFloats;
+ myCFloats.reserve( 3 );
+ myCFloats.emplace_back( 1, 3 );
+ myCFloats.emplace_back( 2, 2 );
+ myCFloats.emplace_back( 3, 1 );
+
try
{
//Define variable and local size
auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", {Nx} );
+ auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>( "myCFloats", {3} );
//Define method for engine creation, it is basically straight-forward parameters
adios::Method& datamanSettings = adios.DeclareMethod( "WAN", "DataManWriter" ); //default method type is Writer
@@ -52,6 +59,7 @@ int main( int argc, char* argv [] )
throw std::ios_base::failure( "ERROR: failed to create DataMan I/O engine at Open\n" );
datamanWriter->Write( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
+ datamanWriter->Write( ioMyCFloats, myCFloats.data() );
datamanWriter->Close( );
}
diff --git a/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp b/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp
index 9e591367c..0ee05f1ba 100644
--- a/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp
+++ b/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp
@@ -20,11 +20,18 @@ int main( int argc, char* argv [] )
std::vector<double> myDoubles = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
const std::size_t Nx = myDoubles.size();
+ std::vector<std::complex<float>> myCFloats;
+ myCFloats.reserve( 3 );
+ myCFloats.emplace_back( 1, 3 );
+ myCFloats.emplace_back( 2, 2 );
+ myCFloats.emplace_back( 3, 1 );
+
try
{
//Define variable and local size
//Define variable and local size
auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", adios::Dims{Nx} );
+ auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>( "myCFloats", {3} );
//Define method for engine creation, it is basically straight-forward parameters
adios::Method& datamanSettings = adios.DeclareMethod( "WAN", "DataManWriter" ); //default method type is Writer
@@ -39,8 +46,8 @@ int main( int argc, char* argv [] )
if( datamanWriter == nullptr )
throw std::ios_base::failure( "ERROR: failed to create DataMan I/O engine at Open\n" );
- //datamanWriter->Write( "myDoubles", myDoubles.data() ); //you can write either by string or by object
datamanWriter->Write( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
+ datamanWriter->Write( ioMyCFloats, myCFloats.data() );
datamanWriter->Close( );
}
catch( std::invalid_argument& e )
diff --git a/examples/hello/writer/helloWriter_nompi.cpp b/examples/hello/writer/helloWriter_nompi.cpp
index 7438715cc..b15518a2e 100644
--- a/examples/hello/writer/helloWriter_nompi.cpp
+++ b/examples/hello/writer/helloWriter_nompi.cpp
@@ -27,7 +27,7 @@ int main( int argc, char* argv [] )
//Define method for engine creation, it is basically straight-forward parameters
adios::Method& bpWriterSettings = adios.DeclareMethod( "SinglePOSIXFile" ); //default method type is Writer
- bpWriterSettings.AddTransport( "POSIX", "have_metadata_file=yes" );
+ bpWriterSettings.AddTransport( "File", "have_metadata_file=yes" );
//Create engine smart pointer due to polymorphism,
//Open returns a smart pointer to Engine containing the Derived class Writer
diff --git a/include/ADIOS.h b/include/ADIOS.h
index 93a152e9a..939bdf3c0 100644
--- a/include/ADIOS.h
+++ b/include/ADIOS.h
@@ -406,6 +406,39 @@ Variable<long double>& ADIOS::DefineVariable( const std::string name, const Dims
}
+template<> inline
+Variable<std::complex<float>>& ADIOS::DefineVariable( const std::string name, const Dims dimensions,
+ const Dims globalDimensions, const Dims globalOffsets )
+{
+ CheckVariableInput( name, dimensions );
+ m_CFloat.emplace_back( name, dimensions, globalDimensions, globalOffsets, m_DebugMode );
+ m_Variables.emplace( name, std::make_pair( GetType<std::complex<float>>(), m_CFloat.size()-1 ) );
+ return m_CFloat.back();
+}
+
+
+template<> inline
+Variable<std::complex<double>>& ADIOS::DefineVariable( const std::string name, const Dims dimensions,
+ const Dims globalDimensions, const Dims globalOffsets )
+{
+ CheckVariableInput( name, dimensions );
+ m_CDouble.emplace_back( name, dimensions, globalDimensions, globalOffsets, m_DebugMode );
+ m_Variables.emplace( name, std::make_pair( GetType<std::complex<double>>(), m_CDouble.size()-1 ) );
+ return m_CDouble.back();
+}
+
+
+template<> inline
+Variable<std::complex<long double>>& ADIOS::DefineVariable( const std::string name, const Dims dimensions,
+ const Dims globalDimensions, const Dims globalOffsets )
+{
+ CheckVariableInput( name, dimensions );
+ m_CLDouble.emplace_back( name, dimensions, globalDimensions, globalOffsets, m_DebugMode );
+ m_Variables.emplace( name, std::make_pair( GetType<std::complex<double>>(), m_CLDouble.size()-1 ) );
+ return m_CLDouble.back();
+}
+
+
//Get template specialization
template<> inline
Variable<char>& ADIOS::GetVariable( const std::string name )
@@ -459,6 +492,18 @@ template<> inline
Variable<long double>& ADIOS::GetVariable( const std::string name )
{ return m_LDouble[ GetVariableIndex<long double>(name) ]; }
+template<> inline
+Variable<std::complex<float>>& ADIOS::GetVariable( const std::string name )
+{ return m_CFloat[ GetVariableIndex<std::complex<float>>(name) ]; }
+
+template<> inline
+Variable<std::complex<double>>& ADIOS::GetVariable( const std::string name )
+{ return m_CDouble[ GetVariableIndex<std::complex<double>>(name) ]; }
+
+template<> inline
+Variable<std::complex<long double>>& ADIOS::GetVariable( const std::string name )
+{ return m_CLDouble[ GetVariableIndex<std::complex<long double>>(name) ]; }
+
} //end namespace
diff --git a/include/core/Variable.h b/include/core/Variable.h
index 036e68e3d..c6f6b3c8c 100644
--- a/include/core/Variable.h
+++ b/include/core/Variable.h
@@ -72,10 +72,21 @@ public:
if( size > 10 )
size = 10;
- for( std::size_t i = 0; i < size; ++i )
+ if( m_Type.find("complex") != m_Type.npos ) //it's complex
{
- logInfo << m_AppValues[i] << " ";
+ for( unsigned int i = 0; i < size; ++i )
+ {
+ logInfo << "( " << std::real( m_AppValues[i] ) << " , " << std::imag( m_AppValues[i] ) << " ) ";
+ }
}
+ else
+ {
+ for( unsigned int i = 0; i < size; ++i )
+ {
+ logInfo << m_AppValues[i] << " ";
+ }
+ }
+
logInfo << " ...";
}
logInfo << "\n";
diff --git a/include/engine/bp/BPWriter.h b/include/engine/bp/BPWriter.h
index 105361c58..1e603c885 100644
--- a/include/engine/bp/BPWriter.h
+++ b/include/engine/bp/BPWriter.h
@@ -79,10 +79,14 @@ public:
private:
capsule::STLVector m_Buffer; ///< heap capsule using STL std::vector<char>
+ std::size_t m_BufferVariableCountPosition = 0; ///< needs to be updated in every advance step
+ bool m_IsFirstClose = true; ///< set to false after first Close is reached so metadata doesn't have to be accommodated for a subsequent Close
+ std::size_t m_MaxBufferSize;
+ float m_GrowthFactor = 1.5; ///< capsule memory growth factor, new_memory = m_GrowthFactor * current_memory
+
format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
format::BP1MetadataSet m_MetadataSet; ///< metadata set accompanying the heap buffer data in bp format. Needed by m_BP1Writer
- std::size_t m_MaxBufferSize;
- float m_GrowthFactor = 1.5;
+
bool m_TransportFlush = false; ///< true: transport flush happened, buffer must be reset
void Init( );
@@ -105,10 +109,10 @@ private:
//set variable
variable.m_AppValues = values;
m_WrittenVariables.insert( variable.m_Name );
+
//precalculate new metadata and payload sizes
const std::size_t indexSize = m_BP1Writer.GetVariableIndexSize( variable );
const std::size_t payloadSize = variable.PayLoadSize(); //will change if compression is applied
-
//Buffer reallocation, expensive part
m_TransportFlush = CheckBuffersAllocation( indexSize, payloadSize );
@@ -126,11 +130,7 @@ private:
}
else //Write data to buffer
{
- //EXPENSIVE part, might want to use threads if large.
- MemcpyThreads( m_Buffer.m_Data.data(), variable.m_AppValues, payloadSize, m_Cores );
- //update indices
- m_Buffer.m_DataPosition += payloadSize;
- m_Buffer.m_DataAbsolutePosition += payloadSize;
+ m_BP1Writer.WriteVariablePayload( variable, m_Buffer, m_Cores );
}
}
diff --git a/include/engine/dataman/DataManWriter.h b/include/engine/dataman/DataManWriter.h
index be07b1aba..81c9fbb87 100644
--- a/include/engine/dataman/DataManWriter.h
+++ b/include/engine/dataman/DataManWriter.h
@@ -56,6 +56,9 @@ public:
void Write( Variable<float>& variable, const float* values );
void Write( Variable<double>& variable, const double* values );
void Write( Variable<long double>& variable, const long double* values );
+ void Write( Variable<std::complex<float>>& variable, const std::complex<float>* values );
+ void Write( Variable<std::complex<double>>& variable, const std::complex<double>* values );
+ void Write( Variable<std::complex<long double>>& variable, const std::complex<long double>* values );
void Write( const std::string variableName, const char* values );
void Write( const std::string variableName, const unsigned char* values );
@@ -70,6 +73,10 @@ public:
void Write( const std::string variableName, const float* values );
void Write( const std::string variableName, const double* values );
void Write( const std::string variableName, const long double* values );
+ void Write( const std::string variableName, const std::complex<float>* values );
+ void Write( const std::string variableName, const std::complex<double>* values );
+ void Write( const std::string variableName, const std::complex<long double>* values );
+
void Close( const int transportIndex = -1 );
diff --git a/include/format/BP1.h b/include/format/BP1.h
index b4139cd2f..1696c143b 100644
--- a/include/format/BP1.h
+++ b/include/format/BP1.h
@@ -33,24 +33,23 @@ struct BP1MetadataSet
std::uint32_t TimeStep = 0; ///< current time step, updated with advance step, if append it will be updated to last
std::uint64_t PGCount = 0; ///< number of process groups
- std::uint64_t PGLength = 0; ///< length in bytes of process groups
std::size_t PGIndexPosition = 16;
std::vector<char> PGIndex = std::vector<char>( 102400 ); ///< process group index metadata
std::uint32_t VarsCount = 0; ///< number of written Variables
- std::uint64_t VarsLength = 0; ///< length in bytes of written Variables
std::size_t VarsIndexPosition = 12; ///< initial position in bytes
std::vector<char> VarsIndex = std::vector<char>( 102400 ); ///< metadata variable index, start with 1Kb
std::uint32_t AttributesCount = 0; ///< number of Attributes
- std::uint64_t AttributesLength = 0; ///< length in bytes of Attributes
std::size_t AttributesIndexPosition = 12; ///< initial position in bytes
- std::vector<char> AttributeIndex = std::vector<char>( 102400 ); ///< metadata attribute index, start with 1Kb
+ std::vector<char> AttributesIndex = std::vector<char>( 102400 ); ///< metadata attribute index, start with 1Kb
- std::vector<char> MiniFooter = std::vector<char>( 28 );
+ std::vector<char> MiniFooter = std::vector<char>( 28 ); ///< 56?
};
-
+/**
+ * Base class for BP1Writer and BP1Reader format
+ */
class BP1
{
diff --git a/include/format/BP1Writer.h b/include/format/BP1Writer.h
index 1a1e5653f..7509b4fee 100644
--- a/include/format/BP1Writer.h
+++ b/include/format/BP1Writer.h
@@ -39,6 +39,7 @@ public:
unsigned int m_Cores = 1; ///< number of cores for thread operations in large array (min,max)
unsigned int m_Verbosity = 0; ///< statistics verbosity, can change if redefined in Engine method.
float m_GrowthFactor = 1.5; ///< memory growth factor, can change if redefined in Engine method.
+ const std::uint8_t m_Version = 3;
/**
* Calculates the Process Index size in bytes according to the BP format, including list of method with no parameters (for now)
@@ -212,7 +213,31 @@ public:
}
}
- void Close( const BP1MetadataSet& metadataSet, Capsule& capsule, Transport& transport ) const;
+ /**
+ * Expensive part this is only for heap buffers need to adapt to vector of capsules
+ * @param variable
+ * @param buffer
+ */
+ template< class T >
+ void WriteVariablePayload( const Variable<T>& variable, capsule::STLVector& buffer, const unsigned int cores = 1 ) const noexcept
+ {
+ std::size_t payloadSize = variable.PayLoadSize();
+ MemcpyThreads( buffer.m_Data.data(), variable.m_AppValues, payloadSize, cores ); //EXPENSIVE part, might want to use threads if large.
+ //update indices
+ buffer.m_DataPosition += payloadSize;
+ buffer.m_DataAbsolutePosition += payloadSize;
+ }
+
+
+ /**
+ * Function that collects metadata (if first close) and writes to a single transport
+ * @param metadataSet
+ * @param capsule
+ * @param transport
+ * @param isFirstClose
+ */
+ void Close( BP1MetadataSet& metadataSet, Capsule& capsule, Transport& transport, bool& isFirstClose ) const noexcept;
+
private:
@@ -525,8 +550,12 @@ private:
*/
void CloseRankFile( Capsule& capsule, Transport& transport ) const;
- void SetMetadata( const BP1MetadataSet& metadataSet, Capsule& capsule ) const; ///< sets the metadata buffer in capsule with indices and minifooter
- void SetMiniFooter( BP1MetadataSet& metadataSet ) const; ///< sets the minifooter
+ /**
+ * Flattens the metadata indices into a single metadata buffer in capsule
+ * @param metadataSet
+ * @param capsule
+ */
+ void FlattenMetadata( BP1MetadataSet& metadataSet, Capsule& capsule ) const noexcept; ///< sets the metadata buffer in capsule with indices and minifooter
};
diff --git a/include/functions/adiosFunctions.h b/include/functions/adiosFunctions.h
index 017816a66..5e00002b8 100644
--- a/include/functions/adiosFunctions.h
+++ b/include/functions/adiosFunctions.h
@@ -175,7 +175,14 @@ int GrowBuffer( const std::size_t incomingDataSize, const float growthFactor, co
* @param bytes input number of bytes
* @param positions += bytes
*/
-void MovePositions( const int bytes, std::vector<std::size_t>& positions );
+void MovePositions( const int bytes, std::vector<std::size_t>& positions ) noexcept;
+
+
+/**
+ * Check if system is little endian
+ * @return true: little endian, false: big endian
+ */
+bool IsLittleEndian( ) noexcept;
} //end namespace
diff --git a/include/functions/adiosTemplates.h b/include/functions/adiosTemplates.h
index 89036f38b..4f34ca386 100644
--- a/include/functions/adiosTemplates.h
+++ b/include/functions/adiosTemplates.h
@@ -131,7 +131,7 @@ void GetMinMax( const std::complex<T>* values, const std::size_t size, T& min, T
* threaded version of std::memcpy
* @param dest
* @param source
- * @param count
+ * @param count total number of bytest (as in memcpy)
* @param cores
*/
template<class T, class U>
diff --git a/src/engine/bp/BPWriter.cpp b/src/engine/bp/BPWriter.cpp
index 01c7b7fad..e5e53a319 100644
--- a/src/engine/bp/BPWriter.cpp
+++ b/src/engine/bp/BPWriter.cpp
@@ -161,15 +161,14 @@ void BPWriter::Write( const std::string variableName, const void* values )
void BPWriter::AdvanceStep( )
{
-
-
-
+ //first close current pg
}
void BPWriter::Close( const int transportIndex )
{
- //BP1BPWriter to update the metadata indices
+ //BP1Writer to update the metadata indices
+
//merge all metadata indices in capsule.m_Metadata buffer or capsule.m_Data buffer (depends on transport predefined functionality)
@@ -177,18 +176,7 @@ void BPWriter::Close( const int transportIndex )
//BP1BPWriter to write to corresponding transport
- if( transportIndex == -1 ) // all transports
- {
- for( auto& transport : m_Transports )
- transport->Write( m_Buffer.m_Data.data(), m_Buffer.m_DataPosition );
- for( auto& transport : m_Transports )
- transport->Close( ); //actually no need, close is in destructor (like fstream)
- }
- else
- {
- m_Transports[ transportIndex ]->Write( m_Buffer.m_Data.data(), m_Buffer.m_DataPosition );
- }
//Close the corresponding transport
}
@@ -239,10 +227,16 @@ void BPWriter::InitTransports( )
else
{
if( m_DebugMode == true )
- throw std::invalid_argument( "ERROR: transport + " + itTransport->second + " not supported, in " +
+ throw std::invalid_argument( "ERROR: file transport library " + itLibrary->second + " not supported, in " +
m_Name + m_EndMessage );
}
}
+ else
+ {
+ if( m_DebugMode == true )
+ throw std::invalid_argument( "ERROR: transport " + itTransport->second + " (you mean File?) not supported, in " +
+ m_Name + m_EndMessage );
+ }
}
}
@@ -261,10 +255,10 @@ void BPWriter::InitProcessGroup( )
void BPWriter::WriteProcessGroupIndex( )
{
//pg = process group
- const std::string pgName( std::to_string( m_RankMPI ) ); //using rank as name
+ const std::string name( std::to_string( m_RankMPI ) ); //using rank as name
const unsigned int timeStep = m_MetadataSet.TimeStep;
const std::string timeStepName( std::to_string( timeStep ) );
- const std::size_t pgIndexSize = m_BP1Writer.GetProcessGroupIndexSize( pgName, timeStepName, m_Transports.size() );
+ const std::size_t pgIndexSize = m_BP1Writer.GetProcessGroupIndexSize( name, timeStepName, m_Transports.size() );
//metadata
GrowBuffer( pgIndexSize, m_GrowthFactor, m_MetadataSet.PGIndexPosition, m_MetadataSet.PGIndex );
@@ -272,22 +266,19 @@ void BPWriter::WriteProcessGroupIndex( )
//data? Need to be careful, maybe add some trailing tolerance in variable ????
GrowBuffer( pgIndexSize, m_GrowthFactor, m_Buffer.m_DataPosition, m_Buffer.m_Data );
-// const bool isFortran = ( m_HostLanguage == "Fortran" ) ? true : false;
-// const unsigned int processID = static_cast<unsigned int> ( m_RankMPI );
+ const bool isFortran = ( m_HostLanguage == "Fortran" ) ? true : false;
+ const unsigned int processID = static_cast<unsigned int> ( m_RankMPI );
-// m_BP1BPWriter.WriteProcessGroupIndex( isFortran, name, processID, timeStepName, timeStep, m_Transports,
-// m_Buffer.m_Data, m_Buffer.m_DataPosition, m_Buffer.m_DataAbsolutePosition,
-// m_MetadataSet.PGIndex, m_MetadataSet.PGIndexPosition );
-
-// const bool isFortran, const std::string name, const unsigned int processID,
-// const std::string timeStepName, const unsigned int timeStep,
-// std::vector<char*>& dataBuffers, std::vector<std::size_t>& dataPositions,
-// std::vector<std::size_t>& dataAbsolutePositions,
-// std::vector<char*>& metadataBuffers,
-// std::vector<std::size_t>& metadataPositions
+ m_BP1Writer.WriteProcessGroupIndex( isFortran, name, processID, timeStepName, timeStep, m_Transports,
+ m_Buffer, m_MetadataSet );
+ m_BufferVariableCountPosition = m_Buffer.m_DataPosition; //fixed for every PG
}
+
+
+
+
bool BPWriter::CheckBuffersAllocation( const std::size_t indexSize, const std::size_t payloadSize )
{
//Check if data in buffer needs to be reallocated
diff --git a/src/engine/dataman/DataManWriter.cpp b/src/engine/dataman/DataManWriter.cpp
index 13d1e41bb..af9849a5c 100644
--- a/src/engine/dataman/DataManWriter.cpp
+++ b/src/engine/dataman/DataManWriter.cpp
@@ -99,6 +99,16 @@ void DataManWriter::Write( Variable<double>& variable, const double* values )
void DataManWriter::Write( Variable<long double>& variable, const long double* values )
{ WriteVariableCommon( variable, values ); }
+void DataManWriter::Write( Variable<std::complex<float>>& variable, const std::complex<float>* values )
+{ WriteVariableCommon( variable, values ); }
+
+void DataManWriter::Write( Variable<std::complex<double>>& variable, const std::complex<double>* values )
+{ WriteVariableCommon( variable, values ); }
+
+void DataManWriter::Write( Variable<std::complex<long double>>& variable, const std::complex<long double>* values )
+{ WriteVariableCommon( variable, values ); }
+
+//String version
void DataManWriter::Write( const std::string variableName, const char* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<char>( variableName ), values ); }
@@ -138,6 +148,14 @@ void DataManWriter::Write( const std::string variableName, const double* values
void DataManWriter::Write( const std::string variableName, const long double* values )
{ WriteVariableCommon( m_ADIOS.GetVariable<long double>( variableName ), values ); }
+void DataManWriter::Write( const std::string variableName, const std::complex<float>* values )
+{ WriteVariableCommon( m_ADIOS.GetVariable<std::complex<float>>( variableName ), values ); }
+
+void DataManWriter::Write( const std::string variableName, const std::complex<double>* values )
+{ WriteVariableCommon( m_ADIOS.GetVariable<std::complex<double>>( variableName ), values ); }
+
+void DataManWriter::Write( const std::string variableName, const std::complex<long double>* values )
+{ WriteVariableCommon( m_ADIOS.GetVariable<std::complex<long double>>( variableName ), values ); }
void DataManWriter::Close( const int transportIndex )
diff --git a/src/format/BP1Writer.cpp b/src/format/BP1Writer.cpp
index 09a1e8cac..c46161ed0 100644
--- a/src/format/BP1Writer.cpp
+++ b/src/format/BP1Writer.cpp
@@ -46,6 +46,8 @@ void BP1Writer::WriteProcessGroupIndex( const bool isFortran, const std::string
buffer.m_DataPosition = dataPositions[0];
buffer.m_DataAbsolutePosition = dataAbsolutePositions[0];
metadataSet.PGIndexPosition = metadataPositions[0];
+
+ ++metadataSet.PGCount;
}
@@ -91,17 +93,21 @@ void BP1Writer::WriteProcessGroupIndex( const bool isFortran, const std::string
}
-
-void BP1Writer::Close( const BP1MetadataSet& metadataSet, Capsule& capsule, Transport& transport ) const
+void BP1Writer::Close( BP1MetadataSet& metadataSet, Capsule& capsule,
+ Transport& transport, bool& isFirstClose ) const noexcept
{
-
-
-
+ if( isFirstClose == true )
+ {
+ FlattenMetadata( metadataSet, capsule ); //now capsule
+ isFirstClose = false;
+ }
+ //implementing N-to-N for now, no aggregation
+ transport.Write( capsule.GetData(), capsule.m_DataPosition );
+ transport.Write( capsule.GetMetadata(), capsule.GetMetadataSize() ); //we can improve this by copying metadata to data
+ transport.Close();
}
-
-
void BP1Writer::WriteProcessGroupIndexCommon( const bool isFortran, const std::string name, const unsigned int processID,
const std::string timeStepName, const unsigned int timeStep,
const std::vector<int>& methodIDs,
@@ -225,30 +231,59 @@ void BP1Writer::CloseRankFile( Capsule& capsule, Transport& transport ) const
-
-void BP1Writer::SetMiniFooter( BP1MetadataSet& metadataSet ) const
+void BP1Writer::FlattenMetadata( BP1MetadataSet& metadataSet, Capsule& capsule ) const noexcept
{
+ //Finish writing metadata counts and lengths (IndexPosition)
+ const std::size_t pgLength = metadataSet.PGIndexPosition;
+ std::memcpy( &metadataSet.PGIndex[0], &metadataSet.PGCount, 8 );
+ std::memcpy( &metadataSet.PGIndex[8], &pgLength, 8 );
+ const std::size_t varsIndexLength = metadataSet.VarsIndexPosition;
+ std::memcpy( &metadataSet.VarsIndex[0], &metadataSet.VarsCount, 4 );
+ std::memcpy( &metadataSet.VarsIndex[4], &varsIndexLength, 8 );
+ const std::size_t attributesIndexLength = metadataSet.AttributesIndexPosition;
+ std::memcpy( &metadataSet.AttributesIndex[0], &metadataSet.AttributesCount, 4 );
+ std::memcpy( &metadataSet.AttributesIndex[4], &attributesIndexLength, 8 );
-}
+ const std::size_t metadataSize = pgLength + varsIndexLength + attributesIndexLength + metadataSet.MiniFooter.size();
+ capsule.ResizeMetadata( metadataSize );
+ char* metadata = capsule.GetMetadata();
-void BP1Writer::SetMetadata( const BP1MetadataSet& metadataSet, Capsule& capsule ) const
-{
+ std::memcpy( &metadata[0], metadataSet.PGIndex.data(), pgLength );
+ std::memcpy( &metadata[pgLength], metadataSet.VarsIndex.data(), varsIndexLength );
+ std::memcpy( &metadata[varsIndexLength], metadataSet.AttributesIndex.data(), attributesIndexLength );
+
+ //getting absolute offsets, minifooter is 28 bytes for now
+ const std::uint64_t offsetPGIndex = capsule.m_DataAbsolutePosition;
+ const std::uint64_t offsetVarsIndex = offsetPGIndex + pgLength;
+ const std::uint64_t offsetAttributeIndex = offsetVarsIndex + varsIndexLength;
+ std::size_t position = attributesIndexLength;
- //setup metadata to capsule metadata buffer
- // const std::size_t processGroupIndexSize = m_ProcessGroupsLength + 16;
- // const std::size_t variableIndexSize = m_VariablesLength + 12;
- // const std::size_t attributeIndexSize = m_AttributesLength + 12;
- // const std::size_t minifooterSize = 28; //28
- // const std::size_t metadataSize = processGroupIndexSize + variableIndexSize + attributeIndexSize + minifooterSize;
- //
- // capsule.ResizeMetadata( metadataSize );
+ //offsets
+ std::memcpy( &metadata[position], &offsetPGIndex, 8 );
+ std::memcpy( &metadata[position+8], &offsetVarsIndex, 8 );
+ std::memcpy( &metadata[position+16], &offsetAttributeIndex, 8 );
+
+ position += 24; //position position to version record
+ if( IsLittleEndian() == true )//little endian machine
+ {
+ constexpr std::uint8_t littleEndian = 0;
+ std::memcpy( &metadata[position], &littleEndian, 1 );
+ }
+ else //big endian
+ {
+ constexpr std::uint8_t bigEndian = 1;
+ std::memcpy( &metadata[position], &bigEndian, 1 );
+ }
+ position += 3;
+ std::memcpy( &metadata[position], &m_Version, 1 );
}
+
} //end namespace format
} //end namespace adios
diff --git a/src/functions/adiosFunctions.cpp b/src/functions/adiosFunctions.cpp
index fa18131de..8af45108a 100644
--- a/src/functions/adiosFunctions.cpp
+++ b/src/functions/adiosFunctions.cpp
@@ -571,11 +571,18 @@ int GrowBuffer( const std::size_t incomingDataSize, const float growthFactor, co
}
-void MovePositions( const int bytes, std::vector<std::size_t>& positions )
+void MovePositions( const int bytes, std::vector<std::size_t>& positions ) noexcept
{
for( auto& position : positions )
position += bytes;
}
+bool IsLittleEndian( ) noexcept
+{
+ std::uint16_t hexa = 0x1234;
+ return *reinterpret_cast<std::uint8_t*>(&hexa) != 0x12;
+}
+
+
} //end namespace
--
GitLab