diff --git a/Makefile b/Makefile index ea19887cf2226625ba2b988fb5dcd5ba444f1d96..00e1f0d1dd0e5ce37b9d5675b5193b0a7d7f9e9a 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,9 @@ ARFLAGS:=rcs HFiles:=$(shell find ./include -type f -name "*.h") CPPFiles:=$(shell find ./src -type f -name "*.cpp") INC:=-I./include -VPATH = ./src ./src/core ./src/functions ./src/engine ./src/engine/writer ./src/capsule ./src/transform ./src/transport +VPATH = ./src ./src/core ./src/functions \ + ./src/engine ./src/engine/writer ./src/engine/dataman \ + ./src/capsule ./src/transform ./src/transport #SEPARATE EXTERNAL LIBRARIES HANDLING in Makefile.libs export $(HFiles) $(CPPFiles) $(CFLAGS) $(LIBS) diff --git a/examples/hello/dataman/helloDataMan.cpp b/examples/hello/dataman/helloDataMan.cpp index 2022a21a390c5685138c66e6cc784d8f9f51baf3..34aebe20fb9aa564de2a6d9e8837b124b55789c8 100644 --- a/examples/hello/dataman/helloDataMan.cpp +++ b/examples/hello/dataman/helloDataMan.cpp @@ -30,23 +30,25 @@ int main( int argc, char* argv [] ) //Application variable std::vector<int> myInts = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - int myIntsSize = 10; + unsigned int myIntsSize = myInts.size(); try { - //Define group and variables + + //Define group and variables adios::ADIOS adios( MPI_COMM_WORLD, adiosDebug ); const std::string groupName( "ints" ); adios.DeclareGroup( groupName ); adios.DefineVariable( groupName, "myIntsSize", "int" ); adios.DefineVariable( groupName, "myInts", "double", "myIntsSize" ); + //Define method... const std::string methodName( "DataManSend" ); adios.DeclareMethod( methodName, "DataMan" ); //2nd item is type and must be supported e.g. Writer (empty default), DataMan, Sirius, etc. -// adios.AddTransport( methodName, "ZeroMQ", "format=json", "tcp=128.11.1.1.2", "real_time=yes" ); -// adios.AddTransport( methodName, "MDTM", "format=otherFormat", "tcp=128.11.1.1.2" ); -// adios.AddTransport( methodName, "POSIX", "fname=myfile.bp" ); //you can write things to file as well + adios.AddTransport( methodName, "ZeroMQ", "format=json", "localip=128.11.1.1.2", "remoteip=128.11.1.1.1", "real_time=yes" ); + adios.AddTransport( methodName, "MDTM", "localip=128.11.1.1.2", "remoteip=128.11.1.1.1", "pipes=4", "prefix=ornl" ); + adios.AddTransport( methodName, "POSIX", "fname=myMessage.bp" ); //you can write things to file as well //this illustrates method uniqueness @@ -58,13 +60,10 @@ int main( int argc, char* argv [] ) //Create engine handler and Write int handler = adios.Open( "myInts.bp", "w", methodName ); adios.SetDefaultGroup( handler, groupName ); - - double varDouble = 10.; - adios.Write<double>( handler, "myIntsSize", &varDouble ); + adios.Write<unsigned int>( handler, "myIntsSize", &myIntsSize ); adios.Write<int>( handler, "myInts", &myInts.front() ); adios.Close( handler ); - int handler2 = adios.Open( "somethingelse.bp", "w", methodName2 ); } catch( std::invalid_argument& e ) { diff --git a/include/core/Group.h b/include/core/Group.h index 127685993c1325c303dc53276c13c7503c0365a5..69d0c58ce02e5f63994aa3429c5d84c5a1332192 100644 --- a/include/core/Group.h +++ b/include/core/Group.h @@ -37,6 +37,7 @@ namespace adios */ class Group { + friend class Engine; public: diff --git a/include/engine/dataman/DataMan.h b/include/engine/dataman/DataMan.h index ddbd91b3f7b52dababf079e546f8cd18f4d515e1..69fa69ec23b33f0d8b7b886a99dfbfb92305dd9e 100644 --- a/include/engine/dataman/DataMan.h +++ b/include/engine/dataman/DataMan.h @@ -66,9 +66,9 @@ public: private: - void Init( ); - void InitCapsules( ); - void InitTransports( ); + void Init( ); ///< calls InitCapsules and InitTransports based on Method, called from constructor + void InitCapsules( ); ///< from Method + void InitTransports( ); ///< from Transports }; diff --git a/include/engine/dataman/DataManTemplates.h b/include/engine/dataman/DataManTemplates.h index d25566d560d61fe71cf618049c404b905110c36e..f6a6140533c5c19f44a5381ffd41b6c59df36be7 100644 --- a/include/engine/dataman/DataManTemplates.h +++ b/include/engine/dataman/DataManTemplates.h @@ -8,20 +8,29 @@ #ifndef DATAMANTEMPLATES_H_ #define DATAMANTEMPLATES_H_ +#include <vector> +#include <iostream> + + +#include "core/Group.h" +#include "core/Variable.h" +#include "core/Capsule.h" +#include "core/Transport.h" + namespace adios { template<class T> -void DataManWrite( Group& group, const std::string variableName, Variable<T>& variable, Heap& capsule ) +void DataManWriteVariable( Group& group, const std::string variableName, Variable<T>& variable, std::vector<std::shared_ptr<Capsule> >& capsules, + std::vector<std::shared_ptr<Transport> >& transports ) { - //here write your magic + //here write your magic, this template replaces MACRO + std::cout << "Hello from DataMan Write variable " << variableName << "\n"; } - - } //end namespace diff --git a/include/engine/writer/WriterTemplates.h b/include/engine/writer/WriterTemplates.h index bf9631581ab8654da1bdd4e48628e4c666207aa3..314d6c3a48076844d7a7416b12880b4693257f14 100644 --- a/include/engine/writer/WriterTemplates.h +++ b/include/engine/writer/WriterTemplates.h @@ -8,6 +8,15 @@ #ifndef WRITERTEMPLATES_H_ #define WRITERTEMPLATES_H_ +#include <string> + + +#include "core/Group.h" +#include "core/Variable.h" +#include "capsule/Heap.h" +#include "functions/adiosFunctions.h" +#include "functions/capsuleTemplates.h" + namespace adios @@ -78,13 +87,6 @@ void WriterWriteVariable( Group& group, const std::string variableName, Variable - - - - - - - } //end namespace diff --git a/include/transport/MdtmMan.h b/include/transport/MdtmMan.h index 8dc776ca61d65eed0b0cc7601a4bb50118d31ee4..c6e0285bd4b5fd442c12a1d7e1523b77098d2210 100644 --- a/include/transport/MdtmMan.h +++ b/include/transport/MdtmMan.h @@ -8,16 +8,8 @@ #ifndef MDTMMAN_H_ #define MDTMMAN_H_ -/// \cond EXCLUDE_FROM_DOXYGEN -#include <thread> -#include <queue> -/// \endcond - -#ifdef HAVE_DATAMAN #include "external/json.hpp" -#endif - #include "core/Transport.h" @@ -30,18 +22,27 @@ class MdtmMan : public Transport public: - MdtmMan( ); ///< default empty constructor - - - MdtmMan( const std::string localIP, const std::string remoteIP, const std::string mode, const std::string prefix, - const int numberOfPipes, const std::vector<int> tolerance, const std::vector<int> priority, + /** + * + * @param localIP + * @param remoteIP + * @param mode + * @param prefix + * @param numberOfPipes + * @param tolerances + * @param priorities + * @param mpiComm + * @param debugMode + */ + MdtmMan( const std::string localIP, const std::string remoteIP, const std::string mode, const std::string prefix, + const int numberOfPipes, const std::vector<int> tolerances, const std::vector<int> priorities, MPI_Comm mpiComm, const bool debugMode ); ~MdtmMan( ); - void Open( const std::string name, const std::string accessMode ) final; + void Open( const std::string name, const std::string accessMode ); void SetBuffer( char* buffer, std::size_t size ); @@ -64,9 +65,9 @@ private: std::string m_RemoteIP; ///< remote ip address, can change over time std::string m_Mode; ///< send/write, receive/read std::string m_Prefix; ///< prefix given to message - int m_NumberOfPipes; ///< should it be unsigned int? - std::vector<int> m_Tolerace; - std::vector<int> m_Priority; + int m_NumberOfPipes = -1; ///< should it be unsigned int? + std::vector<int> m_Tolerances; + std::vector<int> m_Priorities; /** * Should we change data to char* ? @@ -87,7 +88,7 @@ private: const std::uint64_t timestep, const int tolerance, const int priority ); /** - * Should we change data to char* ? + * * @param data * @param doid * @param variable diff --git a/src/ADIOS.cpp b/src/ADIOS.cpp index 3932f626711d72f4508a61a7eaf3a3edbd38c6ea..9181a52fe02909e5afce456fa2b2f8214be5b9e5 100644 --- a/src/ADIOS.cpp +++ b/src/ADIOS.cpp @@ -6,7 +6,6 @@ */ /// \cond EXCLUDE_FROM_DOXYGEN -#include <engine/writer/Writer.h> #include <iostream> #include <fstream> #include <sstream> @@ -16,7 +15,10 @@ #include "ADIOS.h" #include "functions/adiosFunctions.h" -//Engine +//Engines +#include "engine/writer/Writer.h" +#include "engine/dataman/DataMan.h" + namespace adios { @@ -123,10 +125,10 @@ unsigned int ADIOS::Open( const std::string name, const std::string accessMode, // { // //here must complete // } -// else if( type == "DataMan" ) -// { -// //here must complete -// } + else if( type == "DataMan" ) + { + m_Engines.emplace( m_EngineCounter, std::make_shared<engine::DataMan>( name, accessMode, mpiComm, itMethod->second, cores ) ); + } else { if( m_DebugMode == true ) diff --git a/src/engine/dataman/DataMan.cpp b/src/engine/dataman/DataMan.cpp index 65261d6dd01b0923be9fd8d0567a80a4c5207f17..c4ac6bc321dfe28290502d625c9fd2ca4889e0e1 100644 --- a/src/engine/dataman/DataMan.cpp +++ b/src/engine/dataman/DataMan.cpp @@ -9,6 +9,7 @@ #include "engine/dataman/DataMan.h" +#include "engine/dataman/DataManTemplates.h" #include "core/Support.h" //supported capsules @@ -46,138 +47,223 @@ void DataMan::Init( ) void DataMan::Write( Group& group, const std::string variableName, const char* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("char"), " from call to Write char*" ); + Variable<char>& variable = group.m_Char[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } + void DataMan::Write( Group& group, const std::string variableName, const unsigned char* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned char"), " from call to Write unsigned char*" ); + Variable<unsigned char>& variable = group.m_UChar[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } + void DataMan::Write( Group& group, const std::string variableName, const short* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("short"), " from call to Write short*" ); + Variable<short>& variable = group.m_Short[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } + void DataMan::Write( Group& group, const std::string variableName, const unsigned short* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned short"), " from call to Write unsigned short*" ); + Variable<unsigned short>& variable = group.m_UShort[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } + void DataMan::Write( Group& group, const std::string variableName, const int* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("int"), " from call to Write int*" ); + Variable<int>& variable = group.m_Int[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } + void DataMan::Write( Group& group, const std::string variableName, const unsigned int* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned int"), " from call to Write unsigned int*" ); + Variable<unsigned int>& variable = group.m_UInt[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } + void DataMan::Write( Group& group, const std::string variableName, const long int* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("long int"), " from call to Write long int*" ); + Variable<long int>& variable = group.m_LInt[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } + void DataMan::Write( Group& group, const std::string variableName, const unsigned long int* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned long int"), " from call to Write unsigned long int*" ); + Variable<unsigned long int>& variable = group.m_ULInt[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } + void DataMan::Write( Group& group, const std::string variableName, const long long int* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("long long int"), " from call to Write long long int*" ); + Variable<long long int>& variable = group.m_LLInt[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } + void DataMan::Write( Group& group, const std::string variableName, const unsigned long long int* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("unsigned long long int"), " from call to Write unsigned long long int*" ); + Variable<unsigned long long int>& variable = group.m_ULLInt[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( Group& group, const std::string variableName, const float* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("float"), " from call to Write float*" ); + Variable<float>& variable = group.m_Float[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( Group& group, const std::string variableName, const double* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("double"), " from call to Write double*" ); + Variable<double>& variable = group.m_Double[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( Group& group, const std::string variableName, const long double* values ) { - + auto index = PreSetVariable( group, variableName, Support::DatatypesAliases.at("long double"), " from call to Write long double*" ); + Variable<long double>& variable = group.m_LDouble[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( group, variableName, variable, m_Capsules, m_Transports ); } - +//USING Preset Group void DataMan::Write( const std::string variableName, const char* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("char"), " from call to Write char*" ); + Variable<char>& variable = m_Group->m_Char[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const unsigned char* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned char"), " from call to Write unsigned char*" ); + Variable<unsigned char>& variable = m_Group->m_UChar[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const short* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("short"), " from call to Write short*" ); + Variable<short>& variable = m_Group->m_Short[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const unsigned short* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned short"), " from call to Write unsigned short*" ); + Variable<unsigned short>& variable = m_Group->m_UShort[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const int* values ) { auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("int"), " from call to Write int*" ); - std::cout << "Hello from DataMan Write integer with index " << index << "\n"; + Variable<int>& variable = m_Group->m_Int[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const unsigned int* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned int"), " from call to Write unsigned int*" ); + Variable<unsigned int>& variable = m_Group->m_UInt[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const long int* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("long int"), " from call to Write long int*" ); + Variable<long int>& variable = m_Group->m_LInt[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const unsigned long int* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned long int"), " from call to Write unsigned long int*" ); + Variable<unsigned long int>& variable = m_Group->m_ULInt[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const long long int* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("long long int"), " from call to Write long long int*" ); + Variable<long long int>& variable = m_Group->m_LLInt[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const unsigned long long int* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("unsigned long long int"), " from call to Write unsigned long long int*" ); + Variable<unsigned long long int>& variable = m_Group->m_ULLInt[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const float* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("float"), " from call to Write float*" ); + Variable<float>& variable = m_Group->m_Float[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const double* values ) { auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("double"), " from call to Write double*" ); - std::cout << "Hello from Data Write double with index " << index << "\n"; + Variable<double>& variable = m_Group->m_Double[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } void DataMan::Write( const std::string variableName, const long double* values ) { - + auto index = PreSetVariable( *m_Group, variableName, Support::DatatypesAliases.at("long double"), " from call to Write long double*" ); + Variable<long double>& variable = m_Group->m_LDouble[index]; //must be a reference + variable.Values = values; + DataManWriteVariable( *m_Group, variableName, variable, m_Capsules, m_Transports ); } diff --git a/src/mpidummy.cpp b/src/mpidummy.cpp index 3a3de22a26c738e462d9acb3fb3b33fb835479c2..7647ab48e9fe33cee87fe9404cfdfd09f7e41769 100644 --- a/src/mpidummy.cpp +++ b/src/mpidummy.cpp @@ -197,7 +197,7 @@ int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_ uint64_t bytes_read; bytes_read = read (fh, buf, bytes_to_read); if (bytes_read != bytes_to_read) { - snprintf(mpierrmsg, MPI_MAX_ERROR_STRING, "could not read %lu bytes. read only: %lu \n", bytes_to_read, bytes_read); + snprintf(mpierrmsg, MPI_MAX_ERROR_STRING, "could not read %llu bytes. read only: %llu \n", bytes_to_read, bytes_read); return -2; } *status = bytes_read; diff --git a/src/transport/MdtmMan.cpp b/src/transport/MdtmMan.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a043647a399437de84a7903878e1ca32489a0791 --- /dev/null +++ b/src/transport/MdtmMan.cpp @@ -0,0 +1,95 @@ +/* + * MdtmMan.cpp + * + * Created on: Jan 22, 2017 + * Author: wfg + */ + + + +#include "transport/MdtmMan.h" + + + +namespace adios +{ + + +MdtmMan::MdtmMan( const std::string localIP, const std::string remoteIP, const std::string mode, const std::string prefix, + const int numberOfPipes, const std::vector<int> tolerances, const std::vector<int> priorities, + MPI_Comm mpiComm, const bool debugMode ): + Transport( "File", mpiComm, debugMode ), + m_LocalIP { localIP }, + m_RemoteIP{ remoteIP }, + m_Mode{ mode }, + m_Prefix { prefix }, + m_NumberOfPipes{ numberOfPipes }, + m_Tolerances{ tolerances }, + m_Priorities{ priorities } +{ } + +MdtmMan::~MdtmMan( ) +{ } + + +void MdtmMan::Open( const std::string name, const std::string accessMode ) +{ } + + +void MdtmMan::SetBuffer( char* buffer, std::size_t size ) +{ } + + +void MdtmMan::Write( const char* buffer, std::size_t size ) +{ } + + +void MdtmMan::Flush( ) +{ } + + +void MdtmMan::Close( ) +{ + + m_NumberOfPipes = -1; +} + + +//PRIVATE Functions +int MdtmMan::Put( const void* data, const std::string doid, const std::string variable, const std::string dType, + const std::vector<std::uint64_t>& putShape, const std::vector<uint64_t>& varShape, const std::vector<uint64_t>& offset, + const std::uint64_t timestep, const int tolerance, const int priority ) +{ + + + return 0; +} + + +int MdtmMan::Get( void* data, const std::string doid, const std::string variable, const std::string dType, + const std::vector<std::uint64_t>& putShape, const std::vector<uint64_t>& varShape, const std::vector<uint64_t>& offset, + const std::uint64_t timestep, const int tolerance, const int priority ) +{ + + return 0; +} + + +int MdtmMan::Get( void *data, const std::string doid, const std::string variable, const std::string dType, + std::vector<std::uint64_t>& varShape, const std::uint64_t timestep ) +{ + + return 0; +} + + + +void MdtmMan::OnReceive( nlohmann::json& jData ) +{ + +} + + + + +} //end namespace