diff --git a/Makefile b/Makefile
index 0581afa17e57133858bfc748cb81c681fb987621..33d2851300fbcc098b34b05050d8472c7bbf1fba 100644
--- a/Makefile
+++ b/Makefile
@@ -44,7 +44,7 @@ mpi: $(HFiles) $(OBJMPI)
./bin/mpi/%.o: %.cpp $(HFiles)
@( mkdir -p ./bin/mpi );
- $(MPICC) $(CFLAGS) -DHAVE_MPI $(INC) -o $@ $<
+ $(MPICC) $(CFLAGS) $(INC) -o $@ $<
nompi: $(HFiles) $(OBJNoMPI)
@( mkdir -p ./lib );
@@ -54,7 +54,7 @@ nompi: $(HFiles) $(OBJNoMPI)
./bin/nompi/%.o: %.cpp $(HFiles)
@( mkdir -p ./bin/nompi );
- $(CC) $(CFLAGS) $(INC) -o $@ $<
+ $(CC) $(CFLAGS) $(INC) -DADIOS_NOMPI -o $@ $<
clean:
rm ./bin/mpi/*.o ./lib/libadios.a ./bin/nompi/*.o ./lib/libadios_nompi.a
diff --git a/examples/hello/bpWriter/Makefile b/examples/hello/bpWriter/Makefile
index 2fadc80043069b6fbd428e79f6b22d0352a7fd44..215bdf5e6d89f8efe944762a7afcd28e74a3029b 100644
--- a/examples/hello/bpWriter/Makefile
+++ b/examples/hello/bpWriter/Makefile
@@ -21,10 +21,10 @@ LDFLAGS=
all: mpi nompi
mpi: $(ADIOS_LIB) $(ADIOS_HFiles)
- $(MPICC) $(CFLAGS) $(ADIOS_INCLUDE) -DHAVE_MPI $(BASE_NAME).cpp -o $(BASE_NAME).exe $(ADIOS_LIB) $(LDFLAGS) -lpthread
+ $(MPICC) $(CFLAGS) $(ADIOS_INCLUDE) $(BASE_NAME).cpp -o $(BASE_NAME).exe $(ADIOS_LIB) $(LDFLAGS) -lpthread
nompi: $(ADIOS_NOMPI_LIB) $(NoMPI_HFiles)
- $(CC) $(CFLAGS) $(ADIOS_INCLUDE) $(BASE_NAME)_nompi.cpp -o $(BASE_NAME)_nompi.exe $(ADIOS_NOMPI_LIB) $(LDFLAGS) -lpthread
+ $(CC) $(CFLAGS) $(ADIOS_INCLUDE) -DADIOS_NOMPI $(BASE_NAME)_nompi.cpp -o $(BASE_NAME)_nompi.exe $(ADIOS_NOMPI_LIB) $(LDFLAGS) -lpthread
clean:
rm *.exe
diff --git a/examples/hello/bpWriter/helloBPWriter.cpp b/examples/hello/bpWriter/helloBPWriter.cpp
index 3bc89f0c1d4d8481bc431e274ec9c41f85aaa343..c3c3b81c55c7ad7cc0f002ea93a3cf578335b909 100644
--- a/examples/hello/bpWriter/helloBPWriter.cpp
+++ b/examples/hello/bpWriter/helloBPWriter.cpp
@@ -32,7 +32,7 @@ int main( int argc, char* argv [] )
try
{
//Define variable and local size
- auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", adios::Dims{Nx} );
+ auto ioMyDoubles = adios.DefineVariable<double>( "myDoubles", adios::Dims{Nx} );
//Define method for engine creation, it is basically straight-forward parameters
adios::Method& bpWriterSettings = adios.DeclareMethod( "SingleFile" ); //default method type is BPWriter
@@ -45,7 +45,7 @@ int main( int argc, char* argv [] )
if( bpWriter == nullptr )
throw std::ios_base::failure( "ERROR: couldn't create bpWriter at Open\n" );
- bpWriter->Write( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
+ bpWriter->Write<double>( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
bpWriter->Close( );
}
catch( std::invalid_argument& e )
diff --git a/examples/hello/bpWriter/helloBPWriter_nompi.cpp b/examples/hello/bpWriter/helloBPWriter_nompi.cpp
index b15518a2e1a4323f123047a83f36431e848b1c56..718d76450348236b48f1deb85737e1d4b3fade88 100644
--- a/examples/hello/bpWriter/helloBPWriter_nompi.cpp
+++ b/examples/hello/bpWriter/helloBPWriter_nompi.cpp
@@ -12,7 +12,7 @@
int main( int argc, char* argv [] )
-{
+ {
const bool adiosDebug = true;
adios::ADIOS adios( adiosDebug );
@@ -23,7 +23,7 @@ int main( int argc, char* argv [] )
try
{
//Define variable and local size
- auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", adios::Dims{Nx} );
+ adios::Variable<double>& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", adios::Dims{Nx} );
//Define method for engine creation, it is basically straight-forward parameters
adios::Method& bpWriterSettings = adios.DeclareMethod( "SinglePOSIXFile" ); //default method type is Writer
@@ -38,6 +38,7 @@ int main( int argc, char* argv [] )
bpWriter->Write<double>( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
bpWriter->Close( );
+ //
}
catch( std::invalid_argument& e )
{
diff --git a/examples/hello/datamanReader/Makefile b/examples/hello/datamanReader/Makefile
index e07e52144eec8acd4cb381f8912feeac0ffb53f7..3a5d6007d5ea33aaaf29ab8e23949110dce3c984 100644
--- a/examples/hello/datamanReader/Makefile
+++ b/examples/hello/datamanReader/Makefile
@@ -34,10 +34,10 @@ LIB_NOMPI+= -ldl -lpthread
all: mpi nompi
mpi: $(ADIOS_LIB) $(ADIOS_HFiles)
- $(MPICC) $(CFLAGS) $(INC) -DHAVE_MPI $(BASE_NAME).cpp -o $(BASE_NAME).exe $(LIB)
+ $(MPICC) $(CFLAGS) $(INC) $(BASE_NAME).cpp -o $(BASE_NAME).exe $(LIB)
nompi: $(ADIOS_NOMPI_LIB) $(NoMPI_HFiles)
- $(CC) $(CFLAGS) $(INC) $(BASE_NAME)_nompi.cpp -o $(BASE_NAME)_nompi.exe $(LIB_NOMPI)
+ $(CC) $(CFLAGS) $(INC) -DADIOS_NOMPI $(BASE_NAME)_nompi.cpp -o $(BASE_NAME)_nompi.exe $(LIB_NOMPI)
clean:
rm *.exe;
diff --git a/examples/hello/datamanReader/helloDataManReader.cpp b/examples/hello/datamanReader/helloDataManReader.cpp
index 0943c6c402f32867ac5f966a1bf4ee2f496bbe3e..7ae42ff2ddb7ba85c04e5ea5da3a776fed2d754f 100644
--- a/examples/hello/datamanReader/helloDataManReader.cpp
+++ b/examples/hello/datamanReader/helloDataManReader.cpp
@@ -27,7 +27,7 @@ void getcb( const void *data, std::string doid, std::string var, std::string dty
std::size_t varsize = std::accumulate(varshape.begin(), varshape.end(), 1, std::multiplies<std::size_t>());
- for (int i=0; i<varsize; i++)
+ for( unsigned int i=0; i < varsize; ++i )
std::cout << ((float*)data)[i] << " ";
std::cout << std::endl;
}
diff --git a/examples/hello/datamanReader/helloDataManReader_nompi.cpp b/examples/hello/datamanReader/helloDataManReader_nompi.cpp
index be16f1149ad4a54b89ae951020bb019f9bd9ffbe..c08ce009176755732c088366607c9de58c2a6f9b 100644
--- a/examples/hello/datamanReader/helloDataManReader_nompi.cpp
+++ b/examples/hello/datamanReader/helloDataManReader_nompi.cpp
@@ -19,7 +19,7 @@ void getcb( const void *data, std::string doid, std::string var, std::string dty
std::size_t varsize = std::accumulate(varshape.begin(), varshape.end(), 1, std::multiplies<std::size_t>());
- for (int i=0; i<varsize; i++)
+ for( unsigned int i=0; i < varsize; ++i )
std::cout << ((float*)data)[i] << " ";
std::cout << std::endl;
}
diff --git a/examples/hello/datamanWriter/Makefile b/examples/hello/datamanWriter/Makefile
index 2866c4029ff64f18eb21cbee5c8d32e2a11803ca..45f28cd17eed3abf2e4ee643f5de303f74210cd5 100644
--- a/examples/hello/datamanWriter/Makefile
+++ b/examples/hello/datamanWriter/Makefile
@@ -34,10 +34,10 @@ LIB_NOMPI+= -ldl -lpthread
all: mpi nompi
mpi: $(ADIOS_LIB) $(ADIOS_HFiles)
- $(MPICC) $(CFLAGS) $(INC) -DHAVE_MPI $(BASE_NAME).cpp -o $(BASE_NAME).exe $(LIB)
+ $(MPICC) $(CFLAGS) $(INC) $(BASE_NAME).cpp -o $(BASE_NAME).exe $(LIB)
nompi: $(ADIOS_NOMPI_LIB) $(NoMPI_HFiles)
- $(CC) $(CFLAGS) $(INC) $(BASE_NAME)_nompi.cpp -o $(BASE_NAME)_nompi.exe $(LIB_NOMPI)
+ $(CC) $(CFLAGS) $(INC) -DADIOS_NOMPI $(BASE_NAME)_nompi.cpp -o $(BASE_NAME)_nompi.exe $(LIB_NOMPI)
clean:
rm *.exe;
diff --git a/examples/hello/datamanWriter/helloDataManWriter.cpp b/examples/hello/datamanWriter/helloDataManWriter.cpp
index 574308aa449a20cfdf3bac4440793a626361118d..4b7e2d3eb28177e63ceaa3f0bcc09a234b2e2709 100644
--- a/examples/hello/datamanWriter/helloDataManWriter.cpp
+++ b/examples/hello/datamanWriter/helloDataManWriter.cpp
@@ -61,8 +61,8 @@ int main( int argc, char* argv [] )
if( datamanWriter == nullptr )
throw std::ios_base::failure( "ERROR: failed to create DataMan I/O engine at Open\n" );
- datamanWriter->Write( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
- datamanWriter->Write( ioMyCFloats, myCFloats.data() );
+ datamanWriter->Write<double>( ioMyDoubles, myDoubles.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
+ datamanWriter->Write<std::complex<float>>( ioMyCFloats, myCFloats.data() );
datamanWriter->Close( );
}
diff --git a/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp b/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp
index c5c68a96b5aa7cffb40eae9cfb52bbde83ae4620..eef6c02d718416240a5d8d99ac3989c2c090a931 100644
--- a/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp
+++ b/examples/hello/datamanWriter/helloDataManWriter_nompi.cpp
@@ -31,14 +31,16 @@ int main( int argc, char* argv [] )
{
//Define variable and local size
//Define variable and local size
- auto& ioMyFloats = adios.DefineVariable<float>( "myfloats", adios::Dims{Nx} );
- auto& ioMyFloat = adios.DefineVariable<float>( "myfloat", adios::Dims{1} );
+ adios::ADIOS adios( "config.xml", adiosDebug );
+
+ auto ioMyFloats = adios.DefineVariable<float>( "myfloats", adios::Dims{Nx} );
+ auto ioMyFloat = adios.DefineVariable<float>( "myfloat", adios::Dims{1} );
// auto& ioMyDoubles = adios.DefineVariable<double>( "myDoubles", adios::Dims{Nx} );
// auto& ioMyCFloats = adios.DefineVariable<std::complex<float>>( "myCFloats", {3} );
//Define method for engine creation, it is basically straight-forward parameters
adios::Method& datamanSettings = adios.DeclareMethod( "WAN", "DataManWriter" ); //default method type is Writer
- datamanSettings.SetParameters( "real_time=yes", "method_type=stream", "method=dump", "local_ip=127.0.0.1", "remote_ip=127.0.0.1", "local_port=12306", "remote_port=12307" );
+ datamanSettings.SetParameters( "real_time=yes", "method_type=stream", "method=dump", "application=XGC1", "local_ip=127.0.0.1", "remote_ip=127.0.0.1", "local_port=12306", "remote_port=12307" );
// datamanSettings.AddTransport( "Mdtm", "localIP=128.0.0.0.1", "remoteIP=128.0.0.0.2", "tolerances=1,2,3" );
//datamanSettings.AddTransport( "ZeroMQ", "localIP=128.0.0.0.1.1", "remoteIP=128.0.0.0.2.1", "tolerances=1,2,3" ); not yet supported , will throw an exception
@@ -49,8 +51,9 @@ int main( int argc, char* argv [] )
if( datamanWriter == nullptr )
throw std::ios_base::failure( "ERROR: failed to create DataMan I/O engine at Open\n" );
- datamanWriter->Write( ioMyFloats, myFloats.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
- datamanWriter->Write( ioMyFloat, (float) 1.12 ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
+ datamanWriter->Write<float>( ioMyFloats, myFloats.data() ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
+ const float num = 1.12;
+ datamanWriter->Write<float>( ioMyFloat, &num ); // Base class Engine own the Write<T> that will call overloaded Write from Derived
// datamanWriter->Write( ioMyCFloats, myCFloats.data() );
datamanWriter->Close( );
}
diff --git a/include/ADIOS.h b/include/ADIOS.h
index e0e4e19bff15d9b1f97a61438559a4d907efe677..64b2ef9a51499805e41cfa312af0da5b3dbd091e 100644
--- a/include/ADIOS.h
+++ b/include/ADIOS.h
@@ -18,10 +18,10 @@
#include <complex>
/// \endcond
-#ifdef HAVE_MPI
- #include <mpi.h>
-#else
+#ifdef ADIOS_NOMPI
#include "mpidummy.h"
+#else
+ #include <mpi.h>
#endif
#include "core/Transform.h"
@@ -45,11 +45,7 @@ class ADIOS
public: // PUBLIC Constructors and Functions define the User Interface with ADIOS
- #ifdef HAVE_MPI
- MPI_Comm m_MPIComm = MPI_COMM_NULL; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C
- #else
- MPI_Comm m_MPIComm = 0; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C
- #endif
+ MPI_Comm m_MPIComm = MPI_COMM_SELF; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C
int m_RankMPI = 0; ///< current MPI rank process
int m_SizeMPI = 1; ///< current MPI processes size
@@ -63,20 +59,19 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO
/**
- * @brief Serial constructor for config file
+ * @brief Serial constructor for config file, only allowed and compiled in libadios_nompi.a
* @param configFileName passed to m_ConfigFile
* @param debugMode true: on throws exceptions and do additional checks, false: off (faster, but unsafe)
*/
ADIOS( const std::string configFileName, const bool debugMode = false );
-
/**
* @brief Parallel constructor for XML config file and MPI
* @param configFileName passed to m_XMLConfigFile
* @param mpiComm MPI communicator ...const to be discussed
* @param debugMode true: on, false: off (faster, but unsafe)
*/
- ADIOS( const std::string configFileName, const MPI_Comm mpiComm, const bool debugMode = false );
+ ADIOS( const std::string configFileName, MPI_Comm mpiComm, const bool debugMode = false );
/**
@@ -84,7 +79,7 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO
* @param mpiComm MPI communicator passed to m_MPIComm*
* @param debugMode true: on, false: off (faster)
*/
- ADIOS( const MPI_Comm mpiComm, const bool debugMode = false );
+ ADIOS( MPI_Comm mpiComm, const bool debugMode = false );
~ADIOS( ); ///< empty, using STL containers for memory management
@@ -132,7 +127,7 @@ public: // PUBLIC Constructors and Functions define the User Interface with ADIO
/**
* Declares a new method
* @param name must be unique
- * @param type supported type : "Writer" (default), "DataMan"...future: "Sirius"
+ * @param type supported type : "BPWriter/BPReader" (default), "DataMan"...future: "Sirius"
*/
Method& DeclareMethod( const std::string methodName, const std::string type = "" );
diff --git a/include/ADIOS_C.h b/include/ADIOS_C.h
index e9a20fe84b5cb29360ae7d6342941fcfbd0b5dae..93bf82d3f7cbd907d817cea3deaf218c85a1e7cc 100644
--- a/include/ADIOS_C.h
+++ b/include/ADIOS_C.h
@@ -8,12 +8,12 @@
#ifndef ADIOS_C_H_
#define ADIOS_C_H_
-#ifdef HAVE_MPI
- #include <mpi.h>
-#else
- #include "../mpidummy.h"
+#ifdef ADIOS_NOMPI
+ #include "mpidummy.h"
using adios::MPI_Comm_rank;
using adios::MPI_Comm;
+#else
+ #include <mpi.h>
#endif
@@ -29,13 +29,44 @@ extern "C"
{
#endif
+/**
+ * ADIOS Init
+ * @param mpicomm MPI communicator from user app
+ */
+void adios_init( MPI_Comm mpiComm );
+
+/**
+ * ADIOS Init in debug mode: extra checks
+ * @param mpicomm MPI communicator from user app
+ */
+void adios_init_debug( MPI_Comm mpiComm );
+
+/**
+ * Sequential version (nompi)
+ */
+void adios_init_nompi( );
+
-void adios_init( const char* xmlConfigFile, const MPI_Comm mpicomm );
-void adios_init_debug( const char* xmlConfigFile, const MPI_Comm mpicomm );
+/**
+ *
+ * @param fileName
+ * @param accessMode
+ * @param mpiComm
+ * @return engine handler
+ */
+int adios_open( const char* fileName, const char* accessMode, MPI_Comm mpiComm );
-int adios_open( const char* fileName, const char* accessMode, MPI_Comm );
-void adios_write(const char* groupName, const char* variableName, const void* values );
+/**
+ *
+ * @param variableName
+ * @param values
+ */
+void adios_write( const char* variableName, const void* values );
+/**
+ *
+ * @param handler
+ */
void adios_close( const int handler );
void adios_finalize( const ADIOS* adiosC ); // deallocate ADIOS pointer
diff --git a/include/core/Capsule.h b/include/core/Capsule.h
index e7b54b5558b021883bce7a89f7be03d840a9849a..c04680b028f0ce2789f478495f0a09f4ffbd90da 100644
--- a/include/core/Capsule.h
+++ b/include/core/Capsule.h
@@ -39,6 +39,7 @@ public:
* @param type derived class type
* @param accessMode 'w':write, 'r':read, 'a':append
* @param rankMPI current MPI rank
+ * @param debugMode
*/
Capsule( const std::string type, const std::string accessMode, const int rankMPI, const bool debugMode );
diff --git a/include/core/Engine.h b/include/core/Engine.h
index e5e8f16cac74ea35c72a3e72628d6634ccf20211..aed47b2c3d97d8f8be7640c0204498ac6e8f981e 100644
--- a/include/core/Engine.h
+++ b/include/core/Engine.h
@@ -1,5 +1,5 @@
/*
- * Capsule.h
+ * Engine.h
*
* Created on: Nov 7, 2016
* Author: wfg
@@ -18,10 +18,10 @@
#include <complex> //std::complex
/// \endcond
-#ifdef HAVE_MPI
- #include <mpi.h>
-#else
+#ifdef ADIOS_NOMPI
#include "mpidummy.h"
+#else
+ #include <mpi.h>
#endif
#include "ADIOS.h"
@@ -44,11 +44,7 @@ class Engine
public:
- #ifdef HAVE_MPI
- MPI_Comm m_MPIComm = MPI_COMM_NULL; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C
- #else
- MPI_Comm m_MPIComm = 0; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C
- #endif
+ MPI_Comm m_MPIComm = MPI_COMM_SELF;
const std::string m_EngineType; ///< from derived class
const std::string m_Name; ///< name used for this engine
@@ -58,10 +54,11 @@ public:
int m_RankMPI = 0; ///< current MPI rank process
int m_SizeMPI = 1; ///< current MPI processes size
- const std::string m_HostLanguage = "C++";
+ const std::string m_HostLanguage = "C++"; ///< default host language
/**
* Unique constructor
+ * @param adios
* @param engineType
* @param name
* @param accessMode
@@ -80,7 +77,7 @@ public:
/**
* Needed for DataMan Engine
- * @param callback
+ * @param callback function passed from the user
*/
virtual void SetCallBack( std::function<void( const void*, std::string, std::string, std::string, Dims )> callback );
@@ -114,66 +111,24 @@ public:
* @param values
*/
template< class T >
- void Write( Variable<T>& variable, const T& values )
+ void Write( Variable<T>& variable, const T values )
{
- Write( variable, &values );
+ const T val = values;
+ Write( variable, &val );
}
/**
- * Single value version using string as variable handlers
+ * Single value version using string as variable handlers, allows rvalues to be passed
* @param variableName
* @param values
*/
template< class T >
- void Write( const std::string variableName, const T& values )
+ void Write( const std::string variableName, const T values )
{
- Write( variableName, &values );
+ const T val = values;
+ Write( variableName, &val );
}
- virtual void Write( Variable<char>& variable, const char* values );
- virtual void Write( Variable<unsigned char>& variable, const unsigned char* values );
- virtual void Write( Variable<short>& variable, const short* values );
- virtual void Write( Variable<unsigned short>& variable, const unsigned short* values );
- virtual void Write( Variable<int>& variable, const int* values );
- virtual void Write( Variable<unsigned int>& variable, const unsigned int* values );
- virtual void Write( Variable<long int>& variable, const long int* values );
- virtual void Write( Variable<unsigned long int>& variable, const unsigned long int* values );
- virtual void Write( Variable<long long int>& variable, const long long int* values );
- virtual void Write( Variable<unsigned long long int>& variable, const unsigned long long int* values );
- virtual void Write( Variable<float>& variable, const float* values );
- virtual void Write( Variable<double>& variable, const double* values );
- virtual void Write( Variable<long double>& variable, const long double* values );
- virtual void Write( Variable<std::complex<float>>& variable, const std::complex<float>* values );
- virtual void Write( Variable<std::complex<double>>& variable, const std::complex<double>* values );
- virtual void Write( Variable<std::complex<long double>>& variable, const std::complex<long double>* values );
- virtual void Write( VariableCompound& variable, const void* values );
-
-
- /**
- * @brief Write functions can be overridden by derived classes. Base class behavior is to:
- * 1) Write to Variable values (m_Values) using the pointer to default group *m_Group set with SetDefaultGroup function
- * 2) Transform the data
- * 3) Write to all capsules -> data and metadata
- * @param variableName
- * @param values coming from user app
- */
- virtual void Write( const std::string variableName, const char* values );
- virtual void Write( const std::string variableName, const unsigned char* values );
- virtual void Write( const std::string variableName, const short* values );
- virtual void Write( const std::string variableName, const unsigned short* values );
- virtual void Write( const std::string variableName, const int* values );
- virtual void Write( const std::string variableName, const unsigned int* values );
- virtual void Write( const std::string variableName, const long int* values );
- virtual void Write( const std::string variableName, const unsigned long int* values );
- virtual void Write( const std::string variableName, const long long int* values );
- virtual void Write( const std::string variableName, const unsigned long long int* values );
- virtual void Write( const std::string variableName, const float* values );
- virtual void Write( const std::string variableName, const double* values );
- virtual void Write( const std::string variableName, const long double* values );
- virtual void Write( const std::string variableName, const std::complex<float>* values );
- virtual void Write( const std::string variableName, const std::complex<double>* values );
- virtual void Write( const std::string variableName, const std::complex<long double>* values );
- virtual void Write( const std::string variableName, const void* values );
/**
* Indicates that a new step is going to be written as new variables come in.
@@ -226,6 +181,53 @@ protected:
virtual void InitCapsules( ); ///< Initialize transports from Method, called from Init in constructor.
virtual void InitTransports( ); ///< Initialize transports from Method, called from Init in constructor.
+
+ virtual void Write( Variable<char>& variable, const char* values );
+ virtual void Write( Variable<unsigned char>& variable, const unsigned char* values );
+ virtual void Write( Variable<short>& variable, const short* values );
+ virtual void Write( Variable<unsigned short>& variable, const unsigned short* values );
+ virtual void Write( Variable<int>& variable, const int* values );
+ virtual void Write( Variable<unsigned int>& variable, const unsigned int* values );
+ virtual void Write( Variable<long int>& variable, const long int* values );
+ virtual void Write( Variable<unsigned long int>& variable, const unsigned long int* values );
+ virtual void Write( Variable<long long int>& variable, const long long int* values );
+ virtual void Write( Variable<unsigned long long int>& variable, const unsigned long long int* values );
+ virtual void Write( Variable<float>& variable, const float* values );
+ virtual void Write( Variable<double>& variable, const double* values );
+ virtual void Write( Variable<long double>& variable, const long double* values );
+ virtual void Write( Variable<std::complex<float>>& variable, const std::complex<float>* values );
+ virtual void Write( Variable<std::complex<double>>& variable, const std::complex<double>* values );
+ virtual void Write( Variable<std::complex<long double>>& variable, const std::complex<long double>* values );
+ virtual void Write( VariableCompound& variable, const void* values );
+
+
+ /**
+ * @brief Write functions can be overridden by derived classes. Base class behavior is to:
+ * 1) Write to Variable values (m_Values) using the pointer to default group *m_Group set with SetDefaultGroup function
+ * 2) Transform the data
+ * 3) Write to all capsules -> data and metadata
+ * @param variableName
+ * @param values coming from user app
+ */
+ virtual void Write( const std::string variableName, const char* values );
+ virtual void Write( const std::string variableName, const unsigned char* values );
+ virtual void Write( const std::string variableName, const short* values );
+ virtual void Write( const std::string variableName, const unsigned short* values );
+ virtual void Write( const std::string variableName, const int* values );
+ virtual void Write( const std::string variableName, const unsigned int* values );
+ virtual void Write( const std::string variableName, const long int* values );
+ virtual void Write( const std::string variableName, const unsigned long int* values );
+ virtual void Write( const std::string variableName, const long long int* values );
+ virtual void Write( const std::string variableName, const unsigned long long int* values );
+ virtual void Write( const std::string variableName, const float* values );
+ virtual void Write( const std::string variableName, const double* values );
+ virtual void Write( const std::string variableName, const long double* values );
+ virtual void Write( const std::string variableName, const std::complex<float>* values );
+ virtual void Write( const std::string variableName, const std::complex<double>* values );
+ virtual void Write( const std::string variableName, const std::complex<long double>* values );
+ virtual void Write( const std::string variableName, const void* values );
+
+
/**
* Used to verify parameters in m_Method containers
* @param itParam iterator to a certain parameter
diff --git a/include/core/Method.h b/include/core/Method.h
index 84ed0f95ecc7ff13fa97f971f083c07f4f760aff..2f7070b3a3142d946eda365dfb1682bcb79ed282 100644
--- a/include/core/Method.h
+++ b/include/core/Method.h
@@ -73,5 +73,4 @@ private:
} //end namespace
-
#endif /* METHOD_H_ */
diff --git a/include/core/Transport.h b/include/core/Transport.h
index cb4d3ea08cc26bf5513a0c8932f160277f53e7a4..0347442634ab266a2d727224e5429d269fd3bde4 100644
--- a/include/core/Transport.h
+++ b/include/core/Transport.h
@@ -13,10 +13,10 @@
#include <vector>
/// \endcond
-#ifdef HAVE_MPI
- #include <mpi.h>
+#ifdef ADIOS_NOMPI
+ #include "mpidummy.h"
#else
- #include "mpidummy.h"
+ #include <mpi.h>
#endif
@@ -34,11 +34,7 @@ public:
std::string m_AccessMode; ///< from Open
bool m_IsOpen = false;
- #ifdef HAVE_MPI
- MPI_Comm m_MPIComm = NULL; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C
- #else
- MPI_Comm m_MPIComm = 0; ///< only used as reference to MPI communicator passed from parallel constructor, MPI_Comm is a pointer itself. Public as called from C
- #endif
+ MPI_Comm m_MPIComm = MPI_COMM_SELF;
int m_RankMPI = 0; ///< current MPI rank process
int m_SizeMPI = 1; ///< current MPI processes size
diff --git a/include/engine/bp/BPWriter.h b/include/engine/bp/BPWriter.h
index 4337059fa7cb4c4c9e65fc6b30dc0214464ce590..5030370511bae11f5c14bf8b8641e66a296ab8d2 100644
--- a/include/engine/bp/BPWriter.h
+++ b/include/engine/bp/BPWriter.h
@@ -36,6 +36,35 @@ public:
~BPWriter( );
+
+
+ void Advance( );
+
+ /**
+ * Closes a single transport or all transports
+ * @param transportIndex, if -1 (default) closes all transports, otherwise it closes a transport in m_Transport[transportIndex]. In debug mode the latter is bounds-checked.
+ */
+ void Close( const int transportIndex = -1 );
+
+private:
+
+ capsule::STLVector m_Buffer; ///< heap capsule using STL std::vector<char>
+
+ bool m_IsFirstClose = true; ///< set to false after first Close is reached so metadata doesn't have to be accommodated for a subsequent Close
+ std::size_t m_MaxBufferSize; ///< maximum allowed memory to be allocated
+ float m_GrowthFactor = 1.5; ///< capsule memory growth factor, new_memory = m_GrowthFactor * current_memory
+
+ format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
+ format::BP1MetadataSet m_MetadataSet; ///< metadata set accompanying the heap buffer data in bp format. Needed by m_BP1Writer
+
+ bool m_TransportFlush = false; ///< true: transport flush happened, buffer must be reset
+
+ void Init( );
+ void InitTransports( );
+ void InitProcessGroup( );
+
+ void WriteProcessGroupIndex( );
+
void Write( Variable<char>& variable, const char* values );
void Write( Variable<unsigned char>& variable, const unsigned char* values );
void Write( Variable<short>& variable, const short* values );
@@ -72,34 +101,6 @@ public:
void Write( const std::string variableName, const std::complex<long double>* values );
void Write( const std::string variableName, const void* values );
- void Advance( );
-
- /**
- * Closes a single transport or all transports
- * @param transportIndex, if -1 (default) closes all transports, otherwise it closes a transport in m_Transport[transportIndex]. In debug mode the latter is bounds-checked.
- */
- void Close( const int transportIndex = -1 );
-
-private:
-
- capsule::STLVector m_Buffer; ///< heap capsule using STL std::vector<char>
- std::size_t m_BufferVariableCountPosition = 0; ///< needs to be updated in every advance step
- bool m_IsFirstClose = true; ///< set to false after first Close is reached so metadata doesn't have to be accommodated for a subsequent Close
- std::size_t m_MaxBufferSize; ///< maximum allowed memory to be allocated
- float m_GrowthFactor = 1.5; ///< capsule memory growth factor, new_memory = m_GrowthFactor * current_memory
-
- format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
- format::BP1MetadataSet m_MetadataSet; ///< metadata set accompanying the heap buffer data in bp format. Needed by m_BP1Writer
-
- bool m_TransportFlush = false; ///< true: transport flush happened, buffer must be reset
-
- void Init( );
- void InitTransports( );
- void InitProcessGroup( );
-
-
- void WriteProcessGroupIndex( );
-
/**
* Common function for primitive (including std::complex) writes
@@ -135,8 +136,8 @@ private:
else //Write data to buffer
{
m_BP1Writer.WriteVariablePayload( variable, m_Buffer, m_Cores );
-
}
+
variable.m_AppValues = nullptr; //setting pointer to null as not needed after write
}
diff --git a/include/engine/dataman/DataManReader.h b/include/engine/dataman/DataManReader.h
index 9def2e9f647212e357a19b04e73a30432c861641..0daf7f318737d1caf45145186d8305536a196a52 100644
--- a/include/engine/dataman/DataManReader.h
+++ b/include/engine/dataman/DataManReader.h
@@ -27,13 +27,14 @@ class DataManReader : public Engine
public:
/**
- * Constructor for single BP capsule engine, writes in BP format into a single heap capsule
- * @param name unique name given to the engine
+ * Constructor for dataman engine Reader for WAN communications
+ * @param adios
+ * @param name
* @param accessMode
* @param mpiComm
* @param method
* @param debugMode
- * @param hostLanguage
+ * @param cores
*/
DataManReader( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
const Method& method, const bool debugMode = false, const unsigned int cores = 1 );
diff --git a/include/engine/dataman/DataManWriter.h b/include/engine/dataman/DataManWriter.h
index 2d85c1fb1ba00dcdb00219586366cf61e891a290..58c2728617f1df0f5cdd3dc79cac16af9d6d94d8 100644
--- a/include/engine/dataman/DataManWriter.h
+++ b/include/engine/dataman/DataManWriter.h
@@ -30,13 +30,14 @@ class DataManWriter : public Engine
public:
/**
- * Constructor for single BP capsule engine, writes in BP format into a single heap capsule
+ * Constructor for dataman engine Writer for WAN communications
+ * @param adios
* @param name unique name given to the engine
* @param accessMode
* @param mpiComm
* @param method
* @param debugMode
- * @param hostLanguage
+ * @param cores
*/
DataManWriter( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
const Method& method, const bool debugMode = false, const unsigned int cores = 1 );
@@ -45,6 +46,21 @@ public:
void SetCallBack( std::function<void( const void*, std::string, std::string, std::string, Dims )> callback );
+ void Close( const int transportIndex = -1 );
+
+private:
+
+ capsule::STLVector m_Buffer; ///< heap capsule, contains data and metadata buffers
+ format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
+
+ bool m_DoRealTime = false;
+ DataManager m_Man;
+ std::function<void( const void*, std::string, std::string, std::string, Dims )> m_CallBack; ///< call back function
+
+ void Init( ); ///< calls InitCapsules and InitTransports based on Method, called from constructor
+ void InitCapsules( );
+ void InitTransports( ); ///< from Transports
+
void Write( Variable<char>& variable, const char* values );
void Write( Variable<unsigned char>& variable, const unsigned char* values );
void Write( Variable<short>& variable, const short* values );
@@ -80,21 +96,6 @@ public:
void Write( const std::string variableName, const std::complex<long double>* values );
- void Close( const int transportIndex = -1 );
-
-private:
-
- capsule::STLVector m_Buffer; ///< heap capsule, contains data and metadata buffers
- format::BP1Writer m_BP1Writer; ///< format object will provide the required BP functionality to be applied on m_Buffer and m_Transports
-
- bool m_DoRealTime = false;
- DataManager m_Man;
- std::function<void( const void*, std::string, std::string, std::string, Dims )> m_CallBack; ///< call back function
-
- void Init( ); ///< calls InitCapsules and InitTransports based on Method, called from constructor
- void InitCapsules( );
- void InitTransports( ); ///< from Transports
-
/**
* From transport Mdtm in m_Method
* @param parameter must be an accepted parameter
diff --git a/include/format/BP1.h b/include/format/BP1.h
index cfccc7d47f543e1e432eea561ec5349130498d74..ed1c9e445d23816af7b6646dc9251f8b74990753 100644
--- a/include/format/BP1.h
+++ b/include/format/BP1.h
@@ -8,23 +8,24 @@
#ifndef BP1_H_
#define BP1_H_
-
+/// \cond EXCLUDE_FROM_DOXYGEN
#include <memory> //std::shared_ptr
+/// \endcond
-#ifdef HAVE_MPI
- #include <mpi.h>
-#else
+#if ADIOS_NOMPI
#include "mpidummy.h"
+#else
+ #include <mpi.h>
#endif
#include "core/Transport.h"
+
namespace adios
{
namespace format
{
-
/**
* Struct that tracks metadata indices in bp format
*/
@@ -45,7 +46,11 @@ struct BP1MetadataSet
std::size_t AttributesIndexPosition = 12; ///< initial position in bytes
std::vector<char> AttributesIndex = std::vector<char>( 102400 ); ///< metadata attribute index, start with 1Kb
- std::vector<char> MiniFooter = std::vector<char>( 28 ); ///< 56?
+ const unsigned int MiniFooterSize = 28; ///< 28 for now
+
+ //PG (relative) positions in Data buffer to be updated
+ std::size_t DataPGLengthPosition = 0; ///< current PG initial ( relative ) position, needs to be updated in every advance step or init
+ std::size_t DataVarsCountPosition = 0; ///< current PG variable count ( relative ) position, needs to be updated in every advance step or init
};
/**
diff --git a/include/format/BP1Writer.h b/include/format/BP1Writer.h
index e618b11d06c538e251961c62a56cd5dd7a663895..86842e8385ea3bdc99ebdf21447c2cad0a7c645f 100644
--- a/include/format/BP1Writer.h
+++ b/include/format/BP1Writer.h
@@ -10,7 +10,7 @@
/// \cond EXCLUDE_FROM_DOXYGEN
#include <vector>
-#include <cstdint> //std::intX_t fixed size classes
+#include <cstdint> //std::intX_t fixed size integers
#include <algorithm> //std::count, std::copy, std::for_each
#include <cstring> //std::memcpy
#include <cmath> //std::ceil
@@ -30,7 +30,6 @@ namespace format
{
-
class BP1Writer : public BP1
{
@@ -130,7 +129,7 @@ public:
indexSize += 1 + 1; //id
}
- return indexSize;
+ return indexSize + 12; ///extra 12 bytes in case of attributes
//need to add transform characteristics
}
@@ -318,9 +317,12 @@ private:
std::vector<std::size_t> metadataCharacteristicsCountPositions( metadataPositions ); //very important, can't be const as it is updated by MemcpyToBuffer
std::uint8_t characteristicsCounter = 0; //used for characteristics count, characteristics length will be calculated at the end
+ //here move positions 5 bytes in data and metadata for characteristics count + length
+ MovePositions( 5, metadataPositions );
+
//DIMENSIONS CHARACTERISTIC
- const std::vector<size_t>& localDimensions = variable.m_Dimensions;
+ const std::vector<std::size_t>& localDimensions = variable.m_Dimensions;
//write to metadata characteristic
//characteristic: dimension
@@ -418,7 +420,7 @@ private:
//update absolute positions with dataPositions, this is the payload offset
for( unsigned int i = 0; i < dataAbsolutePositions.size(); ++i )
- dataAbsolutePositions[i] += dataPositions[i];
+ dataAbsolutePositions[i] += dataPositions[i] - dataLengthPositions[i];
characteristicID = characteristic_payload_offset;
MemcpyToBuffers( metadataBuffers, metadataPositions, &characteristicID, 1 ); //variable payload offset id
@@ -550,6 +552,14 @@ private:
*/
void FlattenMetadata( BP1MetadataSet& metadataSet, Capsule& capsule ) const noexcept; ///< sets the metadata buffer in capsule with indices and minifooter
+
+ /**
+ * Flattens the data and fills the pg length, vars count, vars length and attributes
+ * @param metadataSet
+ * @param capsule
+ */
+ void FlattenData( BP1MetadataSet& metadataSet, Capsule& capsule ) const noexcept;
+
};
diff --git a/include/functions/adiosFunctions.h b/include/functions/adiosFunctions.h
index 5e00002b8c28396659f176fbf568acca109a9133..511e267655fc956417b0683c63f04cbe74b05e55 100644
--- a/include/functions/adiosFunctions.h
+++ b/include/functions/adiosFunctions.h
@@ -16,10 +16,10 @@
#include <memory> //std::shared_ptr
/// \endcond
-#ifdef HAVE_MPI
- #include <mpi.h>
-#else
+#ifdef ADIOS_NOMPI
#include "mpidummy.h"
+#else
+ #include <mpi.h>
#endif
#include "core/Transform.h"
diff --git a/include/functions/adiosTemplates.h b/include/functions/adiosTemplates.h
index 4f34ca386fa1b34ca2d3a0df49109d60053d2243..814a43ee3c69e06289a68fa2d77a006da8aaba00 100644
--- a/include/functions/adiosTemplates.h
+++ b/include/functions/adiosTemplates.h
@@ -72,22 +72,23 @@ bool IsTypeAlias( const std::string type,
* @param size of the values array
* @param min from values
* @param max from values
+ * @param cores threaded version not yet implemented
*/
template<class T> inline
void GetMinMax( const T* values, const std::size_t size, T& min, T& max, const unsigned int cores = 1 ) noexcept
{
min = values[0];
- max = values[0];
+ max = min;
for( std::size_t i = 1; i < size; ++i )
{
- if( min < values[i] )
+ if( values[i] < min )
{
min = values[i];
continue;
}
- if( max > values[i] )
+ if( values[i] > max )
max = values[i];
}
}
@@ -111,13 +112,13 @@ void GetMinMax( const std::complex<T>* values, const std::size_t size, T& min, T
{
T norm = std::norm( values[i] );
- if( min < norm )
+ if( norm < min )
{
min = norm;
continue;
}
- if( max > norm )
+ if( norm > max )
{
max = norm;
}
diff --git a/src/ADIOS.cpp b/src/ADIOS.cpp
index 0b05d9de871d9d53977bca51006874b2ab2c658f..bbcf0d20552be1f293800a37d2bd51a3250a6214 100644
--- a/src/ADIOS.cpp
+++ b/src/ADIOS.cpp
@@ -36,7 +36,7 @@ ADIOS::ADIOS( const bool debugMode ):
InitMPI( );
}
-
+#ifdef ADIOS_NOMPI
ADIOS::ADIOS( const std::string configFileName, const bool debugMode ):
m_ConfigFile{ configFileName },
m_DebugMode{ debugMode }
@@ -44,9 +44,10 @@ ADIOS::ADIOS( const std::string configFileName, const bool debugMode ):
InitMPI( );
// InitXML( m_ConfigFile, m_MPIComm, m_DebugMode, m_Transforms );
}
+#endif
-ADIOS::ADIOS( const std::string xmlConfigFile, const MPI_Comm mpiComm, const bool debugMode ):
+ADIOS::ADIOS( const std::string xmlConfigFile, MPI_Comm mpiComm, const bool debugMode ):
m_MPIComm{ mpiComm },
m_ConfigFile{ xmlConfigFile },
m_DebugMode{ debugMode }
@@ -56,7 +57,7 @@ ADIOS::ADIOS( const std::string xmlConfigFile, const MPI_Comm mpiComm, const boo
}
-ADIOS::ADIOS( const MPI_Comm mpiComm, const bool debugMode ):
+ADIOS::ADIOS( MPI_Comm mpiComm, const bool debugMode ):
m_MPIComm{ mpiComm },
m_DebugMode{ debugMode }
{
@@ -70,6 +71,13 @@ ADIOS::~ADIOS( )
void ADIOS::InitMPI( )
{
+ if( m_DebugMode == true )
+ {
+ if( m_MPIComm == MPI_COMM_NULL )
+ throw std::ios_base::failure( "ERROR: engine communicator is MPI_COMM_NULL,"
+ " in call to ADIOS Open or Constructor\n" );
+ }
+
MPI_Comm_rank( m_MPIComm, &m_RankMPI );
MPI_Comm_size( m_MPIComm, &m_SizeMPI );
}
@@ -87,7 +95,8 @@ Method& ADIOS::DeclareMethod( const std::string methodName, const std::string ty
}
-std::shared_ptr<Engine> ADIOS::Open( const std::string name, const std::string accessMode, MPI_Comm mpiComm, const Method& method, const unsigned int cores )
+std::shared_ptr<Engine> ADIOS::Open( const std::string name, const std::string accessMode, MPI_Comm mpiComm,
+ const Method& method, const unsigned int cores )
{
if( m_DebugMode == true )
{
@@ -99,7 +108,9 @@ std::shared_ptr<Engine> ADIOS::Open( const std::string name, const std::string a
const std::string type( method.m_Type );
- const bool isDefaultWriter = ( accessMode == "w" || accessMode == "write" ) && type.empty() ? true : false;
+ const bool isDefaultWriter = ( accessMode == "w" || accessMode == "write" ||
+ accessMode == "a" || accessMode == "append" ) && type.empty() ? true : false;
+
const bool isDefaultReader = ( accessMode == "r" || accessMode == "read" ) && type.empty() ? true : false;
if( isDefaultWriter || type == "BPWriter" || type == "bpwriter" )
@@ -227,6 +238,15 @@ void ADIOS::MonitorVariables( std::ostream& logStream )
else if( type == GetType<long double>() )
GetVariable<long double>( name ).Monitor( logStream );
+
+ else if( type == GetType<std::complex<float>>() )
+ GetVariable<std::complex<float>>( name ).Monitor( logStream );
+
+ else if( type == GetType<std::complex<double>>() )
+ GetVariable<std::complex<double>>( name ).Monitor( logStream );
+
+ else if( type == GetType<std::complex<long double>>() )
+ GetVariable<std::complex<long double>>( name ).Monitor( logStream );
}
}
diff --git a/src/core/Engine.cpp b/src/core/Engine.cpp
index 0ad8d8c946f528e5509db4c60649a8007ee4b792..fbcd99ded10f3df6b9aefbc86ad4b819d01b6c9d 100644
--- a/src/core/Engine.cpp
+++ b/src/core/Engine.cpp
@@ -16,7 +16,7 @@ namespace adios
Engine::Engine( ADIOS& adios, const std::string engineType, const std::string name, const std::string accessMode,
- const MPI_Comm mpiComm, const Method& method, const bool debugMode, const unsigned int cores,
+ MPI_Comm mpiComm, const Method& method, const bool debugMode, const unsigned int cores,
const std::string endMessage ):
m_MPIComm{ mpiComm },
m_EngineType{ engineType },
@@ -28,6 +28,13 @@ Engine::Engine( ADIOS& adios, const std::string engineType, const std::string na
m_Cores{ cores },
m_EndMessage{ endMessage }
{
+ if( m_DebugMode == true )
+ {
+ if( m_MPIComm == MPI_COMM_NULL )
+ throw std::ios_base::failure( "ERROR: engine communicator is MPI_COMM_NULL,"
+ " in call to ADIOS Open or Constructor\n" );
+ }
+
MPI_Comm_rank( m_MPIComm, &m_RankMPI );
MPI_Comm_size( m_MPIComm, &m_SizeMPI );
}
diff --git a/src/engine/bp/BPWriter.cpp b/src/engine/bp/BPWriter.cpp
index 9577202b974957df4aa462166d527640ba8cd51d..49c2dc0bb98f63976cc8ccc826dc9ba8caa64ba3 100644
--- a/src/engine/bp/BPWriter.cpp
+++ b/src/engine/bp/BPWriter.cpp
@@ -19,7 +19,7 @@ namespace adios
{
-BPWriter::BPWriter( ADIOS& adios, const std::string name, const std::string accessMode, const MPI_Comm mpiComm,
+BPWriter::BPWriter( ADIOS& adios, const std::string name, const std::string accessMode, MPI_Comm mpiComm,
const Method& method, const bool debugMode, const unsigned int cores ):
Engine( adios, "BPWriter", name, accessMode, mpiComm, method, debugMode, cores, " BPWriter constructor (or call to ADIOS Open).\n" ),
m_Buffer{ capsule::STLVector( accessMode, m_RankMPI, m_DebugMode ) },
@@ -269,7 +269,6 @@ void BPWriter::WriteProcessGroupIndex( )
m_BP1Writer.WriteProcessGroupIndex( isFortran, name, processID, timeStepName, timeStep, m_Transports,
m_Buffer, m_MetadataSet );
- m_BufferVariableCountPosition = m_Buffer.m_DataPosition; //fixed for every new PG
}
diff --git a/src/format/BP1.cpp b/src/format/BP1.cpp
index 2a29f11de2843e6d2226171edb11d2d07600674a..2c28f28cbc995ce5c8ae49887d14524b5dcf3550 100644
--- a/src/format/BP1.cpp
+++ b/src/format/BP1.cpp
@@ -31,19 +31,14 @@ void BP1::OpenRankFiles( const std::string name, const std::string accessMode, T
baseName = name;
directory = name + ".bp";
}
- CreateDirectory( directory );
+ CreateDirectory( directory ); //creates a directory and sub-directories recursively
std::string fileName( directory + "/" + baseName + ".bp." + std::to_string( file.m_RankMPI ) );
-
- if( file.m_MPIComm == MPI_COMM_SELF )
- fileName = name;
-
file.Open( fileName, accessMode ); // opens a file transport under name.bp.dir/name.bp.rank reserve that location fro writing
}
-
std::vector<int> BP1::GetMethodIDs( const std::vector< std::shared_ptr<Transport> >& transports ) const noexcept
{
auto lf_GetMethodID = []( const std::string method ) -> int
diff --git a/src/format/BP1Writer.cpp b/src/format/BP1Writer.cpp
index a4ac52cf81ad29bd66c9807b8ad19bba28ebf722..5386e37806a704c5ddd59a9c2fc90fd792db261f 100644
--- a/src/format/BP1Writer.cpp
+++ b/src/format/BP1Writer.cpp
@@ -29,6 +29,8 @@ void BP1Writer::WriteProcessGroupIndex( const bool isFortran, const std::string
const std::vector< std::shared_ptr<Transport> >& transports,
capsule::STLVector& buffer, BP1MetadataSet& metadataSet ) const noexcept
{
+ metadataSet.DataPGLengthPosition = buffer.m_DataPosition;
+
// adapt this part to local variables
std::vector<char*> dataBuffers{ buffer.m_Data.data() };
std::vector<size_t> dataPositions{ buffer.m_DataPosition };
@@ -47,7 +49,9 @@ void BP1Writer::WriteProcessGroupIndex( const bool isFortran, const std::string
buffer.m_DataAbsolutePosition = dataAbsolutePositions[0];
metadataSet.PGIndexPosition = metadataPositions[0];
+ metadataSet.DataVarsCountPosition = dataPositions[0];
++metadataSet.PGCount;
+
}
@@ -57,7 +61,6 @@ void BP1Writer::WriteProcessGroupIndex( const bool isFortran, const std::string
std::vector< std::shared_ptr<Capsule> >& capsules,
std::vector<BP1MetadataSet>& metadataSets ) const noexcept
{
-
// adapt this part to local variables
std::vector<char*> metadataBuffers, dataBuffers;
std::vector<std::size_t> metadataPositions, dataPositions, dataAbsolutePositions;
@@ -68,11 +71,15 @@ void BP1Writer::WriteProcessGroupIndex( const bool isFortran, const std::string
metadataPositions.push_back( metadataSet.PGIndexPosition );
}
+ unsigned int index = 0;
for( auto& capsule : capsules )
{
dataBuffers.push_back( capsule->GetData( ) );
dataPositions.push_back( capsule->m_DataPosition );
dataAbsolutePositions.push_back( capsule->m_DataAbsolutePosition );
+
+ metadataSets[index].DataPGLengthPosition = capsule->m_DataPosition;
+ ++index;
}
const std::vector<int> methodIDs = GetMethodIDs( transports );
@@ -89,6 +96,7 @@ void BP1Writer::WriteProcessGroupIndex( const bool isFortran, const std::string
capsules[i]->m_DataPosition = dataPositions[i];
capsules[i]->m_DataAbsolutePosition = dataAbsolutePositions[i];
+ metadataSets[i].DataVarsCountPosition = dataPositions[i];
}
}
@@ -98,16 +106,18 @@ void BP1Writer::Close( BP1MetadataSet& metadataSet, Capsule& capsule, Transport&
{
if( isFirstClose == true )
{
+ FlattenData( metadataSet, capsule );
FlattenMetadata( metadataSet, capsule );
isFirstClose = false;
}
//implementing N-to-N for now, no aggregation
transport.Write( capsule.GetData(), capsule.m_DataPosition );
+ transport.Write( capsule.GetMetadata(), capsule.GetMetadataSize() ); //we can improve this by copying metadata to data
if( haveMetadata == true )
- transport.Write( capsule.GetMetadata(), capsule.GetMetadataSize() ); //we can improve this by copying metadata to data
-
- //here accumulate in metadata file
+ {
+ //here call aggregator
+ }
transport.Close();
}
@@ -123,7 +133,8 @@ void BP1Writer::WriteProcessGroupIndexCommon( const bool isFortran, const std::s
std::vector<char*>& metadataBuffers,
std::vector<std::size_t>& metadataPositions ) const noexcept
{
- std::vector<std::size_t> metadataLengthPositions( metadataPositions ); //get length of pg position
+ const std::vector<std::size_t> initialDataPositions( dataPositions );
+ std::vector<std::size_t> pgLengthPositions( metadataPositions ); //get length of pg position
MovePositions( 2, metadataPositions ); //skip length of pg in metadata, 2 bytes, would write at the end
MovePositions( 8, dataPositions ); //skip length of pg including data, 8 bytes, would write at the end
@@ -152,18 +163,21 @@ void BP1Writer::WriteProcessGroupIndexCommon( const bool isFortran, const std::s
//time step to metadata and data
MemcpyToBuffers( metadataBuffers, metadataPositions, &timeStep, 4 );
MemcpyToBuffers( dataBuffers, dataPositions, &timeStep, 4 );
-
//write offset to pg in data on metadata which is the current absolute position
MemcpyToBuffers( metadataBuffers, metadataPositions, dataAbsolutePositions, 8 );
//get pg index length
std::vector<std::uint16_t> metadataIndexLengths( metadataPositions.size() );
for( unsigned int i = 0; i < metadataPositions.size(); ++i )
- metadataIndexLengths[i] = metadataPositions[i] - metadataLengthPositions[i];
+ metadataIndexLengths[i] = metadataPositions[i] - pgLengthPositions[i] - 2; //without length of group record?
//write to metadata length position the pgIndex length
- MemcpyToBuffers( metadataBuffers, metadataLengthPositions, metadataIndexLengths, 2 );
- MovePositions( -2, metadataLengthPositions ); //back to original position
+ MemcpyToBuffers( metadataBuffers, pgLengthPositions, metadataIndexLengths, 2 );
+ //MovePositions( -2, pgLengthPositions ); //back to original position, not needed for now
+
+ //dataAbsolutePositions need to be updated
+ for( unsigned int i = 0; i < dataPositions.size(); ++i )
+ dataAbsolutePositions[i] += dataPositions[i] - initialDataPositions[i];
}
//PRIVATE FUNCTIONS
@@ -231,43 +245,61 @@ void BP1Writer::WriteDimensionRecord( std::vector<char*>& buffers, std::vector<s
}
+void BP1Writer::FlattenData( BP1MetadataSet& metadataSet, Capsule& capsule ) const noexcept
+{
+ //Finish writing pg group length and, vars count and length in Data
+ char* data = capsule.GetData();
+ const std::uint64_t dataPGLength = capsule.m_DataPosition - metadataSet.DataPGLengthPosition - 8; //without record itself
+ std::memcpy( &data[metadataSet.DataPGLengthPosition], &dataPGLength, 8 );
+
+ //vars count
+ std::memcpy( &data[metadataSet.DataVarsCountPosition], &metadataSet.VarsCount, 4 );
+
+ //vars length
+ const std::uint64_t dataVarsLength = capsule.m_DataPosition - metadataSet.DataVarsCountPosition - 8 - 4; //without record itself
+ std::memcpy( &data[metadataSet.DataVarsCountPosition+4], &dataVarsLength, 8 );
+
+ //here add empty attributes
+ capsule.m_DataPosition += 12;
+}
+
void BP1Writer::FlattenMetadata( BP1MetadataSet& metadataSet, Capsule& capsule ) const noexcept
{
//Finish writing metadata counts and lengths (IndexPosition)
- const std::size_t pgLength = metadataSet.PGIndexPosition;
+ const std::size_t pgLength = metadataSet.PGIndexPosition - 16; //without record itself
std::memcpy( &metadataSet.PGIndex[0], &metadataSet.PGCount, 8 );
std::memcpy( &metadataSet.PGIndex[8], &pgLength, 8 );
- const std::size_t varsIndexLength = metadataSet.VarsIndexPosition;
+ const std::size_t varsIndexLength = metadataSet.VarsIndexPosition - 12; //without record itself
std::memcpy( &metadataSet.VarsIndex[0], &metadataSet.VarsCount, 4 );
std::memcpy( &metadataSet.VarsIndex[4], &varsIndexLength, 8 );
- const std::size_t attributesIndexLength = metadataSet.AttributesIndexPosition;
+ const std::size_t attributesIndexLength = metadataSet.AttributesIndexPosition - 12; //without record itself
std::memcpy( &metadataSet.AttributesIndex[0], &metadataSet.AttributesCount, 4 );
std::memcpy( &metadataSet.AttributesIndex[4], &attributesIndexLength, 8 );
- const std::size_t metadataSize = pgLength + varsIndexLength + attributesIndexLength + metadataSet.MiniFooter.size();
+ const std::size_t metadataSize = pgLength + varsIndexLength + attributesIndexLength + metadataSet.MiniFooterSize + 40;
capsule.ResizeMetadata( metadataSize );
char* metadata = capsule.GetMetadata();
std::memcpy( &metadata[0], metadataSet.PGIndex.data(), pgLength );
std::memcpy( &metadata[pgLength], metadataSet.VarsIndex.data(), varsIndexLength );
- std::memcpy( &metadata[varsIndexLength], metadataSet.AttributesIndex.data(), attributesIndexLength );
+ std::memcpy( &metadata[pgLength+varsIndexLength], metadataSet.AttributesIndex.data(), attributesIndexLength );
//getting absolute offsets, minifooter is 28 bytes for now
const std::uint64_t offsetPGIndex = capsule.m_DataAbsolutePosition;
const std::uint64_t offsetVarsIndex = offsetPGIndex + pgLength;
const std::uint64_t offsetAttributeIndex = offsetVarsIndex + varsIndexLength;
- std::size_t position = pgLength + varsIndexLength + attributesIndexLength;
+ std::size_t position = pgLength + varsIndexLength + attributesIndexLength; //adding 28 to reach 56 bytes?
//offsets
std::memcpy( &metadata[position], &offsetPGIndex, 8 );
std::memcpy( &metadata[position+8], &offsetVarsIndex, 8 );
std::memcpy( &metadata[position+16], &offsetAttributeIndex, 8 );
- position += 24; //position position to version record
+ position += 24; //position to version record
if( IsLittleEndian() == true )//little endian machine
{
constexpr std::uint8_t littleEndian = 0;
@@ -280,6 +312,7 @@ void BP1Writer::FlattenMetadata( BP1MetadataSet& metadataSet, Capsule& capsule )
}
position += 3;
std::memcpy( &metadata[position], &m_Version, 1 );
+ capsule.m_MetadataPosition = position + 1;
}
diff --git a/src/functions/adiosFunctions.cpp b/src/functions/adiosFunctions.cpp
index 8af45108a683752b320a38651a1b19774676e365..641d5b81d246ea481912fecd2fc76cec77cefb9e 100644
--- a/src/functions/adiosFunctions.cpp
+++ b/src/functions/adiosFunctions.cpp
@@ -578,6 +578,7 @@ void MovePositions( const int bytes, std::vector<std::size_t>& positions ) noexc
}
+
bool IsLittleEndian( ) noexcept
{
std::uint16_t hexa = 0x1234;
diff --git a/src/transport/file/FD.cpp b/src/transport/file/FD.cpp
index 8f2ef36e0cd41eee53c24cfb7f8ce73fe7af2c47..a88d0105c8c8b118e7940f19da88599f1e5a25dc 100644
--- a/src/transport/file/FD.cpp
+++ b/src/transport/file/FD.cpp
@@ -59,7 +59,7 @@ void FD::Open( const std::string name, const std::string accessMode )
{
if( m_FileDescriptor == -1 )
throw std::ios_base::failure( "ERROR: couldn't open file " + m_Name +
- ", from call to Open in POSIX transport\n" );
+ ", from call to Open in FD transport using POSIX Open\n" );
}
}