From fc27506e642d4e7e564bc7ee42b43437c19221b5 Mon Sep 17 00:00:00 2001
From: wfg <wfg@pc0098504.ornl.gov>
Date: Fri, 3 Feb 2017 16:50:11 -0500
Subject: [PATCH] Added more changes to BP1Writer and Writer engine
Corrected bug by including cmath in adiosFunctions.cpp
Removed helloWriter.cpp and helloDataMan.cpp, now using
helloDataMan_OOP.cpp and helloWriter_OOP.cpp
---
examples/hello/dataman/helloDataMan.cpp | 100 ----------------------
examples/hello/writer/helloWriter.cpp | 88 -------------------
examples/hello/writer/helloWriter_OOP.cpp | 10 +--
include/engine/writer/WriterTemplates.h | 2 +-
include/format/BP1Writer.h | 77 +++++------------
include/functions/adiosTemplates.h | 2 +-
src/format/BP1Writer.cpp | 57 ++++++++++++
src/functions/adiosFunctions.cpp | 3 +-
8 files changed, 87 insertions(+), 252 deletions(-)
delete mode 100644 examples/hello/dataman/helloDataMan.cpp
delete mode 100644 examples/hello/writer/helloWriter.cpp
diff --git a/examples/hello/dataman/helloDataMan.cpp b/examples/hello/dataman/helloDataMan.cpp
deleted file mode 100644
index 34aebe20f..000000000
--- a/examples/hello/dataman/helloDataMan.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * helloADIOSNoXML_OOP.cpp
- *
- * Created on: Jan 9, 2017
- * Author: wfg
- */
-
-#include <vector>
-#include <iostream>
-
-#ifdef HAVE_MPI
- #include <mpi.h>
-#else
- #include "mpidummy.h"
- using adios::MPI_Init;
- using adios::MPI_Comm_rank;
- using adios::MPI_Finalize;
-#endif
-
-
-#include "ADIOS.h"
-
-
-int main( int argc, char* argv [] )
-{
- MPI_Init( &argc, &argv );
- int rank;
- MPI_Comm_rank( MPI_COMM_WORLD, &rank );
- const bool adiosDebug = true;
-
- //Application variable
- std::vector<int> myInts = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
- unsigned int myIntsSize = myInts.size();
-
- try
- {
-
- //Define group and variables
- adios::ADIOS adios( MPI_COMM_WORLD, adiosDebug );
- const std::string groupName( "ints" );
- adios.DeclareGroup( groupName );
- adios.DefineVariable( groupName, "myIntsSize", "int" );
- adios.DefineVariable( groupName, "myInts", "double", "myIntsSize" );
-
-
- //Define method...
- const std::string methodName( "DataManSend" );
- adios.DeclareMethod( methodName, "DataMan" ); //2nd item is type and must be supported e.g. Writer (empty default), DataMan, Sirius, etc.
- adios.AddTransport( methodName, "ZeroMQ", "format=json", "localip=128.11.1.1.2", "remoteip=128.11.1.1.1", "real_time=yes" );
- adios.AddTransport( methodName, "MDTM", "localip=128.11.1.1.2", "remoteip=128.11.1.1.1", "pipes=4", "prefix=ornl" );
- adios.AddTransport( methodName, "POSIX", "fname=myMessage.bp" ); //you can write things to file as well
-
-
- //this illustrates method uniqueness
- const std::string methodName2( "DataManSend2" );
- adios.DeclareMethod( methodName2, "DataMan" ); //should this be variadic?
- adios.AddTransport( methodName2, "ZeroMQ", "format=json", "tcp=128.11.1.1.2", "realtime_change=yes" );
-
-
- //Create engine handler and Write
- int handler = adios.Open( "myInts.bp", "w", methodName );
- adios.SetDefaultGroup( handler, groupName );
- adios.Write<unsigned int>( handler, "myIntsSize", &myIntsSize );
- adios.Write<int>( handler, "myInts", &myInts.front() );
- adios.Close( handler );
-
- }
- catch( std::invalid_argument& e )
- {
- if( rank == 0 )
- {
- std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
- std::cout << e.what() << "\n";
- }
- }
- catch( std::ios_base::failure& e )
- {
- if( rank == 0 )
- {
- std::cout << "System exception, STOPPING PROGRAM\n";
- std::cout << e.what() << "\n";
- }
- }
- catch( std::exception& e )
- {
- if( rank == 0 )
- {
- std::cout << "Exception, STOPPING PROGRAM\n";
- std::cout << e.what() << "\n";
- }
- }
-
- MPI_Finalize( );
-
- return 0;
-
-}
-
-
-
diff --git a/examples/hello/writer/helloWriter.cpp b/examples/hello/writer/helloWriter.cpp
deleted file mode 100644
index dc58d474f..000000000
--- a/examples/hello/writer/helloWriter.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * helloADIOSNoXML_OOP.cpp
- *
- * Created on: Jan 9, 2017
- * Author: wfg
- */
-
-#include <vector>
-#include <iostream>
-
-#ifdef HAVE_MPI
- #include <mpi.h>
-#else
- #include "mpidummy.h"
- using adios::MPI_Init;
- using adios::MPI_Comm_rank;
- using adios::MPI_Finalize;
-#endif
-
-
-#include "ADIOS.h"
-
-
-int main( int argc, char* argv [] )
-{
- MPI_Init( &argc, &argv );
- int rank;
- MPI_Comm_rank( MPI_COMM_WORLD, &rank );
- const bool adiosDebug = true;
-
- //Application variable
- std::vector<double> myInts = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
- int myIntsSize = 10;
-
- try
- {
- //Define group and variables
- adios::ADIOS adios( MPI_COMM_WORLD, adiosDebug );
- const std::string groupName( "ints" );
- adios.DeclareGroup( groupName );
- adios.DefineVariable( groupName, "myIntsSize", "int" );
- adios.DefineVariable( groupName, "myInts", "double", "myIntsSize" );
-
- //Define method
- const std::string methodName( "IntsWriter" );
- adios.DeclareMethod( methodName, "Writer" );
- adios.AddTransport( methodName, "POSIX" );
-
- //Create engine handler and Write
- int handler = adios.Open( "myInts.bp", "w", methodName );
- adios.SetDefaultGroup( handler, groupName );
- adios.Write<int>( handler, "myIntsSize", &myIntsSize );
- adios.Write<double>( handler, "myInts", &myInts.front() );
- adios.Close( handler );
- }
- catch( std::invalid_argument& e )
- {
- if( rank == 0 )
- {
- std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
- std::cout << e.what() << "\n";
- }
- }
- catch( std::ios_base::failure& e )
- {
- if( rank == 0 )
- {
- std::cout << "System exception, STOPPING PROGRAM\n";
- std::cout << e.what() << "\n";
- }
- }
- catch( std::exception& e )
- {
- if( rank == 0 )
- {
- std::cout << "Exception, STOPPING PROGRAM\n";
- std::cout << e.what() << "\n";
- }
- }
-
- MPI_Finalize( );
-
- return 0;
-
-}
-
-
-
diff --git a/examples/hello/writer/helloWriter_OOP.cpp b/examples/hello/writer/helloWriter_OOP.cpp
index 35b526f9c..31b07a80f 100644
--- a/examples/hello/writer/helloWriter_OOP.cpp
+++ b/examples/hello/writer/helloWriter_OOP.cpp
@@ -28,6 +28,7 @@ int main( int argc, char* argv [] )
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
const bool adiosDebug = true;
adios::ADIOS adios( MPI_COMM_WORLD, adiosDebug );
+ //adios::ADIOS adios( "xmlFile.adios", MPI_COMM_WORLD, adiosDebug );
//Application variable
std::vector<double> myDoubles = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
@@ -38,17 +39,13 @@ int main( int argc, char* argv [] )
{
//Define group and variables with transforms, variables don't have functions, only group can access variables
adios::Group& ioGroup = adios.DeclareGroup( "ioGroup" );
-
adios::Var ioNx = ioGroup.DefineVariable<unsigned int>( "Nx" );
adios::Dims dimNx = ioGroup.SetDimensions( {ioNx} );
-
- adios::Var ioMyDoubles = ioGroup.DefineVariable<double>( "myDoubles", dimNx );
- //adios::Var ioMyFloats = ioGroup.DefineVariable<float>( "myFloats", dimNx ) ;
+ adios::Var ioMyDoubles = ioGroup.DefineVariable<double>( "myDoubles" );
//add transform to variable in group...not executed (just testing API)
adios::Transform bzip2 = adios::transform::BZIP2( );
ioGroup.AddTransform( ioMyDoubles, bzip2, 1 );
- //ioGroup.AddTransform( ioMyFloats, bzip2, 1 );
//Define method for engine creation, it is basically straight-forward parameters
adios::Method& bpWriterSettings = adios.DeclareMethod( "SinglePOSIXFile" ); //default method type is Writer
@@ -63,8 +60,7 @@ int main( int argc, char* argv [] )
throw std::ios_base::failure( "ERROR: failed to open ADIOS bpWriter\n" );
bpWriter->Write<unsigned int>( ioNx, &Nx );
- //bpWriter->Write<double>( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
- //bpWriter->Write<float>( ioMyFloats, myFloats.data() );
+ bpWriter->Write<double>( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
bpWriter->Close( );
}
catch( std::invalid_argument& e )
diff --git a/include/engine/writer/WriterTemplates.h b/include/engine/writer/WriterTemplates.h
index 0ce6fd954..3d864ddd6 100644
--- a/include/engine/writer/WriterTemplates.h
+++ b/include/engine/writer/WriterTemplates.h
@@ -44,7 +44,7 @@ void WriterWriteVariable( const Group& group, const Var variableName, const Vari
Heap& buffer, std::vector< std::shared_ptr<Transport> >& transports,
format::BP1Writer& bp1Writer, const unsigned int cores = 1 )
{
- auto lf_CheckAllocationResult = [ ]( const int result, const std::string variableName, const int rankMPI )
+ auto lf_CheckAllocationResult = []( const int result, const std::string variableName, const int rankMPI )
{
if( result == -1 )
throw std::runtime_error( "ERROR: bad_alloc when writing variable " + variableName +
diff --git a/include/format/BP1Writer.h b/include/format/BP1Writer.h
index 29c52f90c..f984c2748 100644
--- a/include/format/BP1Writer.h
+++ b/include/format/BP1Writer.h
@@ -178,80 +178,38 @@ public:
if( variable.GlobalBoundsIndex == -1 ) //local variable
{
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- //metadata characteristics
- MemcpyToBuffers( metadataBuffers, metadataPositions, &localDimensions[d], 8 );
- MovePositions( 16, metadataPositions ); //skipping global dimension(8), global offset (8)
- }
+ WriteDimensionRecord( metadataBuffers, metadataPositions, localDimensions, 16 );
+ WriteDimensionRecord( dataBuffers, dataPositions, localDimensions, 18, true ); //not using memberID for now
- constexpr char no = 'n'; //dimension format unsigned int value (not using memberID for now)
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- //data dimensions
- MemcpyToBuffers( dataBuffers, dataPositions, &no, 1 );
- MemcpyToBuffers( dataBuffers, dataPositions, &localDimensions[d], 8 );
- MovePositions( 18, dataPositions ); //skipping var no + global dimension(8), var no + global offset (8)
- }
+ dataCharacteristicsCountPositions = dataPositions; //very important to track as writer is going back to this position
+ MovePositions( 5, dataPositions ); //skip characteristics count(1) + length (4)
//dimensions in data characteristic entry
- dataCharacteristicsCountPositions = dataPositions; //very important
-
- MovePositions( 5, dataPositions ); //skip characteristics count(1) + length (4)
MemcpyToBuffers( dataBuffers, dataPositions, &characteristicID, 1 );
-
const std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
MemcpyToBuffers( dataBuffers, dataPositions, &lengthOfDimensionsCharacteristic, 2 );
MemcpyToBuffers( dataBuffers, dataPositions, &dimensions, 1 );
MemcpyToBuffers( dataBuffers, dataPositions, &dimensionsLength, 2 );
-
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- MemcpyToBuffers( dataBuffers, dataPositions, &localDimensions[d], 8 );
- MovePositions( 16, dataPositions );
- }
+ WriteDimensionRecord( dataBuffers, dataPositions, localDimensions, 16 );
}
else //global variable
{
const std::vector<unsigned long long int> globalDimensions = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].first );
const std::vector<unsigned long long int> globalOffsets = group.GetDimensions( group.m_GlobalBounds[variable.GlobalBoundsIndex].second );
- //metadata, writing in characteristics
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- MemcpyToBuffers( metadataBuffers, metadataPositions, &localDimensions[d], 8 );
- MemcpyToBuffers( metadataBuffers, metadataPositions, &globalDimensions[d], 8 );
- MemcpyToBuffers( metadataBuffers, metadataPositions, &globalOffsets[d], 8 );
- }
+ WriteDimensionRecord( metadataBuffers, metadataPositions, localDimensions, globalDimensions, globalOffsets );
+ WriteDimensionRecord( dataBuffers, dataPositions, localDimensions, globalDimensions, globalOffsets, true );
- //data dimensions entry
- constexpr char no = 'n'; //dimension format unsigned int value for now
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- MemcpyToBuffers( dataBuffers, dataPositions, &no, 1 );
- MemcpyToBuffers( dataBuffers, dataPositions, &localDimensions[d], 8 );
- MemcpyToBuffers( dataBuffers, dataPositions, &no, 1 );
- MemcpyToBuffers( dataBuffers, dataPositions, &globalDimensions[d], 8 );
- MemcpyToBuffers( dataBuffers, dataPositions, &no, 1 );
- MemcpyToBuffers( dataBuffers, dataPositions, &globalOffsets[d], 8 );
- }
+ dataCharacteristicsCountPositions = dataPositions; //very important, going back to these positions
+ MovePositions( 5, dataPositions ); //skip characteristics count(1) + length (4)
//dimensions in data characteristic entry
- dataCharacteristicsCountPositions = dataPositions;
- MovePositions( 5, dataPositions ); //skip characteristics count(1) + length (4)
MemcpyToBuffers( dataBuffers, dataPositions, &characteristicID, 1 ); //id
-
const std::int16_t lengthOfDimensionsCharacteristic = 3 + 24 * dimensions; // 3 = dimension(1) + length(2) ; 24 = 3 local, global, global offset x 8 bytes/each
MemcpyToBuffers( dataBuffers, dataPositions, &lengthOfDimensionsCharacteristic, 2 );
MemcpyToBuffers( dataBuffers, dataPositions, &dimensions, 1 );
MemcpyToBuffers( dataBuffers, dataPositions, &dimensionsLength, 2 );
-
- for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
- {
- MemcpyToBuffers( dataBuffers, dataPositions, &localDimensions[d], 8 );
- MemcpyToBuffers( dataBuffers, dataPositions, &globalDimensions[d], 8 );
- MemcpyToBuffers( dataBuffers, dataPositions, &globalOffsets[d], 8 );
- }
+ WriteDimensionRecord( dataBuffers, dataPositions, localDimensions, globalDimensions, globalOffsets );
}
++characteristicsCounter;
@@ -268,7 +226,7 @@ public:
//data
MemcpyToBuffers( dataBuffers, dataPositions, &characteristicID, 1 );
const std::int16_t lengthOfCharacteristic = 2 + lengthOfName;
- MemcpyToBuffers( dataBuffers, dataPositions, &lengthOfCharacteristic, 2 );
+ MemcpyToBuffers( dataBuffers, dataPositions, &lengthOfCharacteristic, 2 ); //added in data
WriteNameRecord( variableName, lengthOfName, dataBuffers, dataPositions );
}
else // Stat -> Min, Max for arrays,
@@ -349,6 +307,17 @@ private:
void WriteNameRecord( const std::string& name, const std::uint16_t& length,
std::vector<char*>& buffers, std::vector<std::size_t>& positions );
+ void WriteDimensionRecord( std::vector<char*>& buffers, std::vector<std::size_t>& positions,
+ const std::vector<unsigned long long int>& localDimensions,
+ const std::vector<unsigned long long int>& globalDimensions,
+ const std::vector<unsigned long long int>& globalOffsets,
+ const bool addType = false );
+
+ void WriteDimensionRecord( std::vector<char*>& buffers, std::vector<std::size_t>& positions,
+ const std::vector<unsigned long long int>& localDimensions,
+ const unsigned int skip,
+ const bool addType = false );
+
/**
*
* @param id
@@ -357,7 +326,7 @@ private:
* @param positions
* @param addLength true for data, false for metadata
*/
- template< class T>
+ template<class T>
void WriteStatisticsRecord( const std::uint8_t& id, const T& value,
std::vector<char*>& buffers, std::vector<std::size_t>& positions, const bool addLength = false )
{
diff --git a/include/functions/adiosTemplates.h b/include/functions/adiosTemplates.h
index 831835e56..9c97f2215 100644
--- a/include/functions/adiosTemplates.h
+++ b/include/functions/adiosTemplates.h
@@ -68,7 +68,7 @@ bool IsTypeAlias( const std::string type,
* @param max
*/
template<class T>
-void GetMinMax( const T* values, const size_t size, T& min, T& max, const unsigned int cores = 1 ) noexcept
+void GetMinMax( const T* values, const std::size_t size, T& min, T& max, const unsigned int cores = 1 ) noexcept
{
min = values[0];
max = values[0];
diff --git a/src/format/BP1Writer.cpp b/src/format/BP1Writer.cpp
index 55449a49d..b48a229bb 100644
--- a/src/format/BP1Writer.cpp
+++ b/src/format/BP1Writer.cpp
@@ -42,6 +42,63 @@ void BP1Writer::WriteNameRecord( const std::string& name, const std::uint16_t& l
}
+void BP1Writer::WriteDimensionRecord( std::vector<char*>& buffers, std::vector<std::size_t>& positions,
+ const std::vector<unsigned long long int>& localDimensions,
+ const std::vector<unsigned long long int>& globalDimensions,
+ const std::vector<unsigned long long int>& globalOffsets,
+ const bool addType )
+{
+ if( addType == true )
+ {
+ constexpr char no = 'n'; //dimension format unsigned int value for now
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ MemcpyToBuffers( buffers, positions, &no, 1 );
+ MemcpyToBuffers( buffers, positions, &localDimensions[d], 8 );
+ MemcpyToBuffers( buffers, positions, &no, 1 );
+ MemcpyToBuffers( buffers, positions, &globalDimensions[d], 8 );
+ MemcpyToBuffers( buffers, positions, &no, 1 );
+ MemcpyToBuffers( buffers, positions, &globalOffsets[d], 8 );
+ }
+ }
+ else
+ {
+ for( unsigned int d = 0; d < (unsigned int)localDimensions.size(); ++d )
+ {
+ MemcpyToBuffers( buffers, positions, &localDimensions[d], 8 );
+ MemcpyToBuffers( buffers, positions, &globalDimensions[d], 8 );
+ MemcpyToBuffers( buffers, positions, &globalOffsets[d], 8 );
+ }
+ }
+}
+
+void BP1Writer::WriteDimensionRecord( std::vector<char*>& buffers, std::vector<std::size_t>& positions,
+ const std::vector<unsigned long long int>& localDimensions,
+ const unsigned int skip,
+ const bool addType )
+{
+
+ if( addType == true )
+ {
+ constexpr char no = 'n'; //dimension format unsigned int value (not using memberID for now)
+ for( const auto& localDimension : localDimensions )
+ {
+ MemcpyToBuffers( buffers, positions, &no, 1 );
+ MemcpyToBuffers( buffers, positions, &localDimension, 8 );
+ MovePositions( skip, positions );
+ }
+ }
+ else
+ {
+ for( const auto& localDimension : localDimensions )
+ {
+ MemcpyToBuffers( buffers, positions, &localDimension, 8 );
+ MovePositions( skip, positions );
+ }
+ }
+}
+
+
void BP1Writer::ClosePOSIX( Capsule& capsule, Transport& transport )
{
diff --git a/src/functions/adiosFunctions.cpp b/src/functions/adiosFunctions.cpp
index 155e10fef..d1eb5be1d 100644
--- a/src/functions/adiosFunctions.cpp
+++ b/src/functions/adiosFunctions.cpp
@@ -9,10 +9,10 @@
#include <fstream>
#include <sstream>
#include <stdexcept>
-#include <iostream>
#include <thread> //std::thread
#include <cstring> //std::memcpy
#include <algorithm> //std::count
+#include <cmath> // std::ceil, std::pow, std::log
#include <sys/types.h> //CreateDirectory
#include <sys/stat.h> //stat
@@ -576,4 +576,5 @@ void MovePositions( const int bytes, std::vector<std::size_t>& positions )
position += bytes;
}
+
} //end namespace
--
GitLab