From 53eacc4b7652b412cb728309899bf117d8854f6a Mon Sep 17 00:00:00 2001
From: wfg <wfg@pc0098504.ornl.gov>
Date: Mon, 27 Feb 2017 17:06:39 -0500
Subject: [PATCH] Started working on BPReader
examples/hello/bpWriter working
To do:
BPReader examples and BP1Reader class
---
buildDataMan.sh | 1 -
examples/hello/{writer => bpReader}/Makefile | 0
examples/hello/bpReader/helloBPReader.cpp | 72 +++++++
.../helloBPReader_nompi.cpp} | 0
examples/hello/bpWriter/Makefile | 31 +++
.../helloBPWriter.cpp} | 0
.../hello/bpWriter/helloBPWriter_nompi.cpp | 62 ++++++
include/engine/bp/BPReader.h | 101 ++++++++++
src/ADIOS.cpp | 10 +-
src/engine/bp/BPReader.cpp | 177 ++++++++++++++++++
src/engine/bp/BPWriter.cpp | 6 +-
11 files changed, 454 insertions(+), 6 deletions(-)
rename examples/hello/{writer => bpReader}/Makefile (100%)
create mode 100644 examples/hello/bpReader/helloBPReader.cpp
rename examples/hello/{writer/helloWriter_nompi.cpp => bpReader/helloBPReader_nompi.cpp} (100%)
create mode 100644 examples/hello/bpWriter/Makefile
rename examples/hello/{writer/helloWriter.cpp => bpWriter/helloBPWriter.cpp} (100%)
create mode 100644 examples/hello/bpWriter/helloBPWriter_nompi.cpp
create mode 100644 include/engine/bp/BPReader.h
create mode 100644 src/engine/bp/BPReader.cpp
diff --git a/buildDataMan.sh b/buildDataMan.sh
index dd5005852..0e65f3f0c 100755
--- a/buildDataMan.sh
+++ b/buildDataMan.sh
@@ -7,7 +7,6 @@
DATAMAN_LOCATION=/Users/w4g/Dropbox/lib/DataMan
MPICOMPILER=mpic++
-export DYLD_LIBRARY_PATH=$DATAMAN_LOCATION/lib:$DYLD_LIBRARY_PATH
if [ "$(uname)" == "Darwin" ]; then
CCOMPILER=clang++
diff --git a/examples/hello/writer/Makefile b/examples/hello/bpReader/Makefile
similarity index 100%
rename from examples/hello/writer/Makefile
rename to examples/hello/bpReader/Makefile
diff --git a/examples/hello/bpReader/helloBPReader.cpp b/examples/hello/bpReader/helloBPReader.cpp
new file mode 100644
index 000000000..6bb1ef383
--- /dev/null
+++ b/examples/hello/bpReader/helloBPReader.cpp
@@ -0,0 +1,72 @@
+/*
+ * helloWriter.cpp
+ *
+ * Created on: Feb 16, 2017
+ * Author: wfg
+ */
+
+
+
+#include <vector>
+#include <iostream>
+
+
+#include <mpi.h>
+
+
+#include "ADIOS_CPP.h"
+
+
+int main( int argc, char* argv [] )
+{
+ MPI_Init( &argc, &argv );
+ int rank;
+ MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+ const bool adiosDebug = true;
+ adios::ADIOS adios( MPI_COMM_WORLD, adiosDebug );
+
+ try
+ {
+ //Define method for engine creation, it is basically straight-forward parameters
+ adios::Method& bpWriterSettings = adios.DeclareMethod( "SingleFile" ); //default method type is BPWriter
+ bpWriterSettings.AddTransport( "File" ); //uses default POSIX library
+
+ //Create engine smart pointer due to polymorphism,
+ //Open returns a smart pointer to Engine containing the Derived class Writer
+ auto bpWriter = adios.Open( "myDoubles.bp", "r", bpWriterSettings );
+
+ if( bpWriter == nullptr )
+ throw std::ios_base::failure( "ERROR: couldn't create bpWriter at Open\n" );
+
+
+ }
+ catch( std::invalid_argument& e )
+ {
+ if( rank == 0 )
+ {
+ std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ }
+ catch( std::ios_base::failure& e )
+ {
+ if( rank == 0 )
+ {
+ std::cout << "System exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ }
+ catch( std::exception& e )
+ {
+ if( rank == 0 )
+ {
+ std::cout << "Exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ }
+
+ MPI_Finalize( );
+
+ return 0;
+
+}
diff --git a/examples/hello/writer/helloWriter_nompi.cpp b/examples/hello/bpReader/helloBPReader_nompi.cpp
similarity index 100%
rename from examples/hello/writer/helloWriter_nompi.cpp
rename to examples/hello/bpReader/helloBPReader_nompi.cpp
diff --git a/examples/hello/bpWriter/Makefile b/examples/hello/bpWriter/Makefile
new file mode 100644
index 000000000..2fadc8004
--- /dev/null
+++ b/examples/hello/bpWriter/Makefile
@@ -0,0 +1,31 @@
+# Makefile for testing purposes, will build helloWriter_OOP_mpi (make or make mpi) or helloWriter_OOP_nompi (make nompi)
+# Created on: Oct 4, 2016
+# Author: wfg
+
+BASE_NAME=helloBPWriter
+
+#COMPILERS
+CC=g++
+MPICC=mpic++
+
+#ADIOS LOCATION
+ADIOS_DIR=../../..
+ADIOS_INCLUDE=-I$(ADIOS_DIR)/include
+ADIOS_LIB=$(ADIOS_DIR)/lib/libadios.a
+ADIOS_NOMPI_LIB=$(ADIOS_DIR)/lib/libadios_nompi.a
+
+#FLAGS
+CFLAGS=-Wall -Wpedantic -std=c++11 -O0 -g
+LDFLAGS=
+
+all: mpi nompi
+
+mpi: $(ADIOS_LIB) $(ADIOS_HFiles)
+ $(MPICC) $(CFLAGS) $(ADIOS_INCLUDE) -DHAVE_MPI $(BASE_NAME).cpp -o $(BASE_NAME).exe $(ADIOS_LIB) $(LDFLAGS) -lpthread
+
+nompi: $(ADIOS_NOMPI_LIB) $(NoMPI_HFiles)
+ $(CC) $(CFLAGS) $(ADIOS_INCLUDE) $(BASE_NAME)_nompi.cpp -o $(BASE_NAME)_nompi.exe $(ADIOS_NOMPI_LIB) $(LDFLAGS) -lpthread
+
+clean:
+ rm *.exe
+
\ No newline at end of file
diff --git a/examples/hello/writer/helloWriter.cpp b/examples/hello/bpWriter/helloBPWriter.cpp
similarity index 100%
rename from examples/hello/writer/helloWriter.cpp
rename to examples/hello/bpWriter/helloBPWriter.cpp
diff --git a/examples/hello/bpWriter/helloBPWriter_nompi.cpp b/examples/hello/bpWriter/helloBPWriter_nompi.cpp
new file mode 100644
index 000000000..b15518a2e
--- /dev/null
+++ b/examples/hello/bpWriter/helloBPWriter_nompi.cpp
@@ -0,0 +1,62 @@
+/*
+ * helloADIOSNoXML_OOP.cpp
+ *
+ * Created on: Jan 9, 2017
+ * Author: wfg
+ */
+
+#include <vector>
+#include <iostream>
+
+#include "ADIOS_CPP.h"
+
+
+int main( int argc, char* argv [] )
+{
+ const bool adiosDebug = true;
+ adios::ADIOS adios( adiosDebug );
+
+ //Application variable
+ std::vector<double> myDoubles = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+ const std::size_t Nx = myDoubles.size();
+
+ try
+ {
+ //Define variable and local size
+ auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", adios::Dims{Nx} );
+
+ //Define method for engine creation, it is basically straight-forward parameters
+ adios::Method& bpWriterSettings = adios.DeclareMethod( "SinglePOSIXFile" ); //default method type is Writer
+ bpWriterSettings.AddTransport( "File", "have_metadata_file=yes" );
+
+ //Create engine smart pointer due to polymorphism,
+ //Open returns a smart pointer to Engine containing the Derived class Writer
+ auto bpWriter = adios.Open( "myDoubles_nompi.bp", "w", bpWriterSettings );
+
+ if( bpWriter == nullptr )
+ throw std::ios_base::failure( "ERROR: couldn't create bpWriter at Open\n" );
+
+ bpWriter->Write<double>( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
+ bpWriter->Close( );
+ }
+ catch( std::invalid_argument& e )
+ {
+ std::cout << "Invalid argument exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ catch( std::ios_base::failure& e )
+ {
+ std::cout << "System exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+ catch( std::exception& e )
+ {
+ std::cout << "Exception, STOPPING PROGRAM\n";
+ std::cout << e.what() << "\n";
+ }
+
+ return 0;
+}
+
+
+
diff --git a/include/engine/bp/BPReader.h b/include/engine/bp/BPReader.h
new file mode 100644
index 000000000..784d5703f
--- /dev/null
+++ b/include/engine/bp/BPReader.h
@@ -0,0 +1,101 @@
+/*
+ * BPReader.h
+ *
+ * Created on: Feb 27, 2017
+ * Author: wfg
+ */
+
+#ifndef BPREADER_H_
+#define BPREADER_H_
+
+#include <iostream> //this must go away
+
+#include "core/Engine.h"
+
+//supported capsules
+#include "capsule/heap/STLVector.h"
+
+
+namespace adios
+{
+
+
+class BPReader : public Engine
+{
+
+public:
+
+ /**
+ * Constructor for single BP capsule engine, writes in BP format into a single heap capsule
+ * @param name unique name given to the engine
+ * @param accessMode
+ * @param mpiComm
+ * @param method
+ * @param debugMode
+ * @param hostLanguage
+ */
+ BPReader( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
+ const Method& method, const bool debugMode = false, const unsigned int cores = 1 );
+
+ ~BPReader( );
+
+
+ Variable<void>* InquireVariable( const std::string name, const bool readIn = true );
+ Variable<char>* InquireVariableChar( const std::string name, const bool readIn = true );
+ Variable<unsigned char>* InquireVariableUChar( const std::string name, const bool readIn = true );
+ Variable<short>* InquireVariableShort( const std::string name, const bool readIn = true );
+ Variable<unsigned short>* InquireVariableUShort( const std::string name, const bool readIn = true );
+ Variable<int>* InquireVariableInt( const std::string name, const bool readIn = true );
+ Variable<unsigned int>* InquireVariableUInt( const std::string name, const bool readIn = true );
+ Variable<long int>* InquireVariableLInt( const std::string name, const bool readIn = true );
+ Variable<unsigned long int>* InquireVariableULInt( const std::string name, const bool readIn = true );
+ Variable<long long int>* InquireVariableLLInt( const std::string name, const bool readIn = true );
+ Variable<unsigned long long int>* InquireVariableULLInt( const std::string name, const bool readIn = true );
+ Variable<float>* InquireVariableFloat( const std::string name, const bool readIn = true );
+ Variable<double>* InquireVariableDouble( const std::string name, const bool readIn = true );
+ Variable<long double>* InquireVariableLDouble( const std::string name, const bool readIn = true );
+ Variable<std::complex<float>>* InquireVariableCFloat( const std::string name, const bool readIn = true );
+ Variable<std::complex<double>>* InquireVariableCDouble( const std::string name, const bool readIn = true );
+ Variable<std::complex<long double>>* InquireVariableCLDouble( const std::string name, const bool readIn = true );
+
+ /**
+ * Not implemented
+ * @param name
+ * @param readIn
+ * @return
+ */
+ VariableCompound* InquireVariableCompound( const std::string name, const bool readIn = true );
+
+
+ void Close( const int transportIndex = -1 );
+
+private:
+
+ capsule::STLVector m_Buffer; ///< heap capsule, contains data and metadata buffers
+ //format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
+
+ void Init( ); ///< calls InitCapsules and InitTransports based on Method, called from constructor
+ void InitCapsules( );
+ void InitTransports( ); ///< from Transports
+
+ std::string GetMdtmParameter( const std::string parameter, const std::map<std::string,std::string>& mdtmParameters );
+
+
+ template< class T >
+ Variable<T>* InquireVariableCommon( const std::string name, const bool readIn )
+ {
+ std::cout << "Hello BPReaderCommon\n";
+
+ //here read variable metadata (dimensions, type, etc.)...then create a Variable like below:
+ //Variable<T>& variable = m_ADIOS.DefineVariable<T>( m_Name + "/" + name, )
+ //return &variable; //return address if success
+ return nullptr; //on failure
+ }
+};
+
+
+
+} //end namespace adios
+
+
+#endif /* BPREADER_H_ */
diff --git a/src/ADIOS.cpp b/src/ADIOS.cpp
index 480645d81..0b05d9de8 100644
--- a/src/ADIOS.cpp
+++ b/src/ADIOS.cpp
@@ -19,6 +19,7 @@
//Engines
#include "engine/bp/BPWriter.h"
+#include "engine/bp/BPReader.h"
#ifdef HAVE_DATAMAN //external dependencies
#include "engine/dataman/DataManWriter.h"
@@ -98,10 +99,17 @@ std::shared_ptr<Engine> ADIOS::Open( const std::string name, const std::string a
const std::string type( method.m_Type );
- if( type == "BPWriter" || type == "bpwriter" )
+ const bool isDefaultWriter = ( accessMode == "w" || accessMode == "write" ) && type.empty() ? true : false;
+ const bool isDefaultReader = ( accessMode == "r" || accessMode == "read" ) && type.empty() ? true : false;
+
+ if( isDefaultWriter || type == "BPWriter" || type == "bpwriter" )
{
return std::make_shared<BPWriter>( *this, name, accessMode, mpiComm, method, m_DebugMode, cores );
}
+ else if( isDefaultReader || type == "BPReader" || type == "bpreader" )
+ {
+ return std::make_shared<BPReader>( *this, name, accessMode, mpiComm, method, m_DebugMode, cores );
+ }
else if( type == "SIRIUS" || type == "sirius" || type == "Sirius" )
{
//not yet supported
diff --git a/src/engine/bp/BPReader.cpp b/src/engine/bp/BPReader.cpp
new file mode 100644
index 000000000..c893d0047
--- /dev/null
+++ b/src/engine/bp/BPReader.cpp
@@ -0,0 +1,177 @@
+/*
+ * BPReader.cpp
+ *
+ * Created on: Feb 27, 2017
+ * Author: wfg
+ */
+
+
+
+#include "engine/bp/BPReader.h"
+
+#include "core/Support.h"
+#include "functions/adiosFunctions.h" //CSVToVector
+
+//supported transports
+#include "transport/file/FD.h" // uses POSIX
+#include "transport/file/FP.h" // uses C FILE*
+#include "transport/file/FStream.h" // uses C++ fstream
+
+
+namespace adios
+{
+
+BPReader::BPReader( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
+ const Method& method, const bool debugMode, const unsigned int cores ):
+ Engine( adios, "BPReader", name, accessMode, mpiComm, method, debugMode, cores, " BPReader constructor (or call to ADIOS Open).\n" ),
+ m_Buffer( accessMode, m_RankMPI, m_DebugMode )
+{
+ Init( );
+}
+
+BPReader::~BPReader( )
+{ }
+
+
+Variable<void>* BPReader::InquireVariable( const std::string name, const bool readIn ) //not yet implemented
+{ return nullptr; }
+
+Variable<char>* BPReader::InquireVariableChar( const std::string name, const bool readIn )
+{ return InquireVariableCommon<char>( name, readIn ); }
+
+Variable<unsigned char>* BPReader::InquireVariableUChar( const std::string name, const bool readIn )
+{ return InquireVariableCommon<unsigned char>( name, readIn ); }
+
+Variable<short>* BPReader::InquireVariableShort( const std::string name, const bool readIn )
+{ return InquireVariableCommon<short>( name, readIn ); }
+
+Variable<unsigned short>* BPReader::InquireVariableUShort( const std::string name, const bool readIn )
+{ return InquireVariableCommon<unsigned short>( name, readIn ); }
+
+Variable<int>* BPReader::InquireVariableInt( const std::string name, const bool readIn )
+{ return InquireVariableCommon<int>( name, readIn ); }
+
+Variable<unsigned int>* BPReader::InquireVariableUInt( const std::string name, const bool readIn )
+{ return InquireVariableCommon<unsigned int>( name, readIn ); }
+
+Variable<long int>* BPReader::InquireVariableLInt( const std::string name, const bool readIn )
+{ return InquireVariableCommon<long int>( name, readIn ); }
+
+Variable<unsigned long int>* BPReader::InquireVariableULInt( const std::string name, const bool readIn )
+{ return InquireVariableCommon<unsigned long int>( name, readIn ); }
+
+Variable<long long int>* BPReader::InquireVariableLLInt( const std::string name, const bool readIn )
+{ return InquireVariableCommon<long long int>( name, readIn ); }
+
+Variable<unsigned long long int>* BPReader::InquireVariableULLInt( const std::string name, const bool readIn )
+{ return InquireVariableCommon<unsigned long long int>( name, readIn ); }
+
+Variable<float>* BPReader::InquireVariableFloat( const std::string name, const bool readIn )
+{ return InquireVariableCommon<float>( name, readIn ); }
+
+Variable<double>* BPReader::InquireVariableDouble( const std::string name, const bool readIn )
+{ return InquireVariableCommon<double>( name, readIn ); }
+
+Variable<long double>* BPReader::InquireVariableLDouble( const std::string name, const bool readIn )
+{ return InquireVariableCommon<long double>( name, readIn ); }
+
+Variable<std::complex<float>>* BPReader::InquireVariableCFloat( const std::string name, const bool readIn )
+{ return InquireVariableCommon<std::complex<float>>( name, readIn ); }
+
+Variable<std::complex<double>>* BPReader::InquireVariableCDouble( const std::string name, const bool readIn )
+{ return InquireVariableCommon<std::complex<double>>( name, readIn ); }
+
+Variable<std::complex<long double>>* BPReader::InquireVariableCLDouble( const std::string name, const bool readIn )
+{ return InquireVariableCommon<std::complex<long double>>( name, readIn ); }
+
+VariableCompound* BPReader::InquireVariableCompound( const std::string name, const bool readIn )
+{ return nullptr; }
+
+
+void BPReader::Close( const int transportIndex )
+{
+
+}
+
+
+//PRIVATE
+void BPReader::Init( )
+{
+ if( m_DebugMode == true )
+ {
+ if( m_AccessMode != "r" && m_AccessMode != "read" )
+ throw std::invalid_argument( "ERROR: BPReader doesn't support access mode " + m_AccessMode +
+ ", in call to ADIOS Open or BPReader constructor\n" );
+ }
+
+ InitCapsules( );
+ InitTransports( );
+}
+
+
+void BPReader::InitCapsules( )
+{
+ //here init memory capsules
+}
+
+
+void BPReader::InitTransports( ) //maybe move this?
+{
+ if( m_DebugMode == true )
+ {
+ if( TransportNamesUniqueness( ) == false )
+ {
+ throw std::invalid_argument( "ERROR: two transports of the same kind (e.g file IO) "
+ "can't have the same name, modify with name= in Method AddTransport\n" );
+ }
+ }
+
+ for( const auto& parameters : m_Method.m_TransportParameters )
+ {
+ auto itTransport = parameters.find( "transport" );
+ if( itTransport->second == "file" || itTransport->second == "File" )
+ {
+ auto itLibrary = parameters.find( "library" );
+ if( itLibrary == parameters.end() || itLibrary->second == "POSIX" ) //use default POSIX
+ {
+ auto file = std::make_shared<transport::FD>( m_MPIComm, m_DebugMode );
+ //m_BP1Writer.OpenRankFiles( m_Name, m_AccessMode, *file );
+ m_Transports.push_back( std::move( file ) );
+ }
+ else if( itLibrary->second == "FILE*" || itLibrary->second == "stdio.h" )
+ {
+ auto file = std::make_shared<transport::FP>( m_MPIComm, m_DebugMode );
+ //m_BP1Writer.OpenRankFiles( m_Name, m_AccessMode, *file );
+ m_Transports.push_back( std::move( file ) );
+
+ }
+ else if( itLibrary->second == "fstream" || itLibrary->second == "std::fstream" )
+ {
+ auto file = std::make_shared<transport::FStream>( m_MPIComm, m_DebugMode );
+ //m_BP1Writer.OpenRankFiles( m_Name, m_AccessMode, *file );
+ m_Transports.push_back( std::move( file ) );
+ }
+ else if( itLibrary->second == "MPI-IO" )
+ {
+
+ }
+ else
+ {
+ if( m_DebugMode == true )
+ throw std::invalid_argument( "ERROR: file transport library " + itLibrary->second + " not supported, in " +
+ m_Name + m_EndMessage );
+ }
+ }
+ else
+ {
+ if( m_DebugMode == true )
+ throw std::invalid_argument( "ERROR: transport " + itTransport->second + " (you mean File?) not supported, in " +
+ m_Name + m_EndMessage );
+ }
+ }
+}
+
+
+} //end namespace
+
+
diff --git a/src/engine/bp/BPWriter.cpp b/src/engine/bp/BPWriter.cpp
index 686d748f1..9577202b9 100644
--- a/src/engine/bp/BPWriter.cpp
+++ b/src/engine/bp/BPWriter.cpp
@@ -180,7 +180,6 @@ void BPWriter::Close( const int transportIndex )
}
-
//PRIVATE FUNCTIONS
void BPWriter::InitTransports( )
{
@@ -205,7 +204,7 @@ void BPWriter::InitTransports( )
m_BP1Writer.OpenRankFiles( m_Name, m_AccessMode, *file );
m_Transports.push_back( std::move( file ) );
}
- else if( itLibrary->second == "FILE*" )
+ else if( itLibrary->second == "FILE*" || itLibrary->second == "stdio.h" )
{
auto file = std::make_shared<transport::FP>( m_MPIComm, m_DebugMode );
m_BP1Writer.OpenRankFiles( m_Name, m_AccessMode, *file );
@@ -280,8 +279,7 @@ void BPWriter::WriteProcessGroupIndex( )
bool BPWriter::CheckBuffersAllocation( const std::size_t indexSize, const std::size_t payloadSize )
{
//Check if data in buffer needs to be reallocated
- const std::size_t dataSize = payloadSize + indexSize + 10; //adding some bytes tolerance
- const std::size_t neededSize = dataSize + m_Buffer.m_DataPosition;
+ const std::size_t neededSize = m_Buffer.m_DataPosition + payloadSize + indexSize + 60; //adding some bytes tolerance
// might need to write payload in batches
bool doTransportsFlush = ( neededSize > m_MaxBufferSize )? true : false;
--
GitLab