diff --git a/examples/heatTransfer/HeatTransfer.cpp b/examples/heatTransfer/HeatTransfer.cpp
index de71231a5b1895a7805b6c9bf7ef6c68b880dc82..473a5e000fc9b8a2ade2597244dbcff331a712f3 100644
--- a/examples/heatTransfer/HeatTransfer.cpp
+++ b/examples/heatTransfer/HeatTransfer.cpp
@@ -162,14 +162,14 @@ void HeatTransfer::exchange( MPI_Comm comm )
MPI_Status status;
if( m_s->rank_left >= 0 )
{
- std::cout << "Rank " << m_s->rank << " send left to rank " << m_s->rank_left;
+ std::cout << "Rank " << m_s->rank << " send left to rank " << m_s->rank_left << std::endl;
for( int i = 0; i < m_s->ndx+2; ++i)
send_x[i] = m_TCurrent[i][1];
MPI_Send(send_x, m_s->ndx+2, MPI_REAL8, m_s->rank_left, tag, comm);
}
if( m_s->rank_right >= 0 )
{
- std::cout << "Rank " << m_s->rank << " receive from right from rank " << m_s->rank_right;
+ std::cout << "Rank " << m_s->rank << " receive from right from rank " << m_s->rank_right << std::endl;
MPI_Recv(recv_x, m_s->ndx+2, MPI_REAL8, m_s->rank_right, tag, comm, &status);
for( int i = 0; i < m_s->ndx+2; ++i)
m_TCurrent[i][m_s->ndy+1] = recv_x[i];
@@ -179,14 +179,14 @@ void HeatTransfer::exchange( MPI_Comm comm )
tag = 2;
if( m_s->rank_right >= 0 )
{
- std::cout << "Rank " << m_s->rank << " send right to rank " << m_s->rank_right;
+ std::cout << "Rank " << m_s->rank << " send right to rank " << m_s->rank_right << std::endl;
for( int i = 0; i < m_s->ndx+2; ++i)
send_x[i] = m_TCurrent[i][m_s->ndy];
MPI_Send(send_x, m_s->ndx+2, MPI_REAL8, m_s->rank_right, tag, comm);
}
if( m_s->rank_left >= 0 )
{
- std::cout << "Rank " << m_s->rank << " receive from left from rank " << m_s->rank_left;
+ std::cout << "Rank " << m_s->rank << " receive from left from rank " << m_s->rank_left << std::endl;
MPI_Recv(recv_x, m_s->ndx+2, MPI_REAL8, m_s->rank_left, tag, comm, &status);
for( int i = 0; i < m_s->ndx+2; ++i)
m_TCurrent[i][0] = recv_x[i];
@@ -196,12 +196,12 @@ void HeatTransfer::exchange( MPI_Comm comm )
tag = 3;
if( m_s->rank_down >= 0 )
{
- std::cout << "Rank " << m_s->rank << " send down to rank " << m_s->rank_down;
+ std::cout << "Rank " << m_s->rank << " send down to rank " << m_s->rank_down << std::endl;
MPI_Send(m_TCurrent[m_s->ndx], m_s->ndy+2, MPI_REAL8, m_s->rank_down, tag, comm);
}
if ( m_s->rank_up >= 0 )
{
- std::cout << "Rank " << m_s->rank << " receive from above from rank " << m_s->rank_up;
+ std::cout << "Rank " << m_s->rank << " receive from above from rank " << m_s->rank_up << std::endl;
MPI_Recv(m_TCurrent[0], m_s->ndy+2, MPI_REAL8, m_s->rank_up, tag, comm, &status);
}
@@ -209,12 +209,12 @@ void HeatTransfer::exchange( MPI_Comm comm )
tag = 4;
if( m_s->rank_up >= 0 )
{
- std::cout << "Rank " << m_s->rank << " send up to rank " << m_s->rank_up;
+ std::cout << "Rank " << m_s->rank << " send up to rank " << m_s->rank_up << std::endl;
MPI_Send(m_TCurrent[1], m_s->ndy+2, MPI_REAL8, m_s->rank_up, tag, comm);
}
if ( m_s->rank_down >= 0 )
{
- std::cout << "Rank " << m_s->rank << " receive from below from rank " << m_s->rank_down;
+ std::cout << "Rank " << m_s->rank << " receive from below from rank " << m_s->rank_down << std::endl;
MPI_Recv(m_TCurrent[m_s->ndx+1], m_s->ndy+2, MPI_REAL8, m_s->rank_down, tag, comm, &status);
}
@@ -222,4 +222,17 @@ void HeatTransfer::exchange( MPI_Comm comm )
delete[] recv_x;
}
-
+#include <cstring>
+/* Copies the internal ndx*ndy section of the ndx+2 * ndy+2 local array
+ * into a separate contiguous vector and returns it.
+ * @return A vector with ndx*ndy elements
+ */
+std::vector<double> HeatTransfer::data_noghost()
+{
+ std::vector<double>d( m_s->ndx * m_s->ndy );
+ for( int i = 1; i <= m_s->ndx; ++i )
+ {
+ std::memcpy( &d[(i-1)*m_s->ndy], m_TCurrent[i]+1, m_s->ndy*sizeof(double));
+ }
+ return d;
+}
diff --git a/examples/heatTransfer/HeatTransfer.h b/examples/heatTransfer/HeatTransfer.h
index 6008f8ea74aa8292cebc4771489c536837546c1b..1f05505113db73601f60c42820a62a1f472977c9 100644
--- a/examples/heatTransfer/HeatTransfer.h
+++ b/examples/heatTransfer/HeatTransfer.h
@@ -9,6 +9,8 @@
#define HEATTRANSFER_H_
#include <mpi.h>
+#include <vector>
+
#include "Settings.h"
class HeatTransfer
@@ -21,8 +23,10 @@ public:
void heatEdges(); // reset the heat values at the global edge
void exchange( MPI_Comm comm ); // send updates to neighbors
- const double *data() {return m_TCurrent[0];}; // return (1D) pointer to current T data
- const double T(int i, int j) {return m_TCurrent[i][j];}; // return current T value at i,j local coordinate
+ // return (1D) pointer to current T data, ndx+2 * ndy+2 elements
+ const double *data() {return m_TCurrent[0];};
+ // return (1D) pointer to current T data without ghost cells, ndx*ndy elements
+ std::vector<double> data_noghost();
void printT(std::string message, MPI_Comm comm); // debug: print local TCurrent on stdout
diff --git a/examples/heatTransfer/IO.h b/examples/heatTransfer/IO.h
index 1e6539eb3ab71989f6f0525af1c862467ad83166..542112f1c8d9c03c73f126e1f7a5da5e40e4f37b 100644
--- a/examples/heatTransfer/IO.h
+++ b/examples/heatTransfer/IO.h
@@ -16,7 +16,7 @@ class IO
{
public:
- IO( std::string output_basename, MPI_Comm comm );
+ IO( std::shared_ptr<Settings> s, MPI_Comm comm );
~IO();
void write( int step, std::shared_ptr<HeatTransfer> ht, std::shared_ptr<Settings> s, MPI_Comm comm );
diff --git a/examples/heatTransfer/IO_adios1.cpp b/examples/heatTransfer/IO_adios1.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3cd1705811590b27966801a04287cf1d558feb6d
--- /dev/null
+++ b/examples/heatTransfer/IO_adios1.cpp
@@ -0,0 +1,73 @@
+/*
+ * IO_ADIOS1.cpp
+ *
+ * Created on: Feb 2017
+ * Author: Norbert Podhorszki
+ */
+
+#include <string>
+#include <iostream>
+#include <iomanip>
+
+#include "IO.h"
+#include "adios.h"
+
+static int64_t group;
+static int rank_saved;
+
+IO::IO( std::shared_ptr<Settings> s, MPI_Comm comm )
+{
+ rank_saved = s->rank;
+ m_outputfilename = s->outputfile + ".bp";
+ adios_init_noxml( comm );
+ adios_declare_group( &group, "heat", "", adios_stat_default );
+ adios_select_method( group, "MPI", "", "" );
+
+ adios_define_var( group, "gndx", "", adios_integer, "", "", "");
+ adios_define_var( group, "gndy", "", adios_integer, "", "", "");
+
+ std::string ldims( std::to_string( s->ndx ) + "," + std::to_string( s->ndy ));
+ std::string gdims( std::to_string( s->gndx ) + "," + std::to_string( s->gndy ));
+ std::string offs( std::to_string( s->offsx ) + "," + std::to_string( s->offsy ));
+ uint64_t T_id;
+ T_id = adios_define_var( group, "T", "", adios_double,
+ ldims.c_str(), gdims.c_str(), offs.c_str() );
+
+ adios_set_transform( T_id, "none");
+ //adios_set_transform( T_id, "zfp:accuracy=0.001");
+}
+
+IO::~IO()
+{
+ adios_finalize(rank_saved);
+}
+
+void IO::write( int step, std::shared_ptr<HeatTransfer> ht, std::shared_ptr<Settings> s, MPI_Comm comm )
+{
+ char mode[2] = "w";
+ if (step > 0)
+ {
+ mode[0] = 'a';
+ }
+
+ // for time measurements, let's synchronize the processes
+ MPI_Barrier( comm );
+ double time_start = MPI_Wtime();
+
+ int64_t f;
+ adios_open( &f, "heat", m_outputfilename.c_str(), mode, comm);
+ adios_write( f, "gndx", &s->gndx);
+ adios_write( f, "gndy", &s->gndy);
+ adios_write( f, "T", ht->data_noghost().data() );
+ adios_close( f );
+
+ MPI_Barrier( comm );
+ double total_time = MPI_Wtime() - time_start;
+ uint64_t adios_totalsize = 2*sizeof(int) + 2*s->ndx*s->ndy*sizeof(double);
+ uint64_t sizeMB = adios_totalsize * s->nproc/1024/1024/1024; // size in MB
+ uint64_t mbs = sizeMB/total_time;
+ if (s->rank==0)
+ std::cout << "Step " << step << ": " << m_outputfilename
+ << " " << sizeMB << " " << total_time << "" << mbs << std::endl;
+}
+
diff --git a/examples/heatTransfer/IO_adios2.cpp b/examples/heatTransfer/IO_adios2.cpp
index 0617a3f1ca4a8e01cf65eb904304d85aada59258..c6f3615b414c45a1c6103812e28a489f0ce9e9df 100644
--- a/examples/heatTransfer/IO_adios2.cpp
+++ b/examples/heatTransfer/IO_adios2.cpp
@@ -8,16 +8,16 @@
#include "IO.h"
-IO::IO( std::string output_basename )
+IO::IO( std::shared_ptr<Settings> s, MPI_Comm comm )
{
- m_outputfilename = output_basename + ".bp";
+ m_outputfilename = s->outputfile + ".bp";
}
IO::~IO()
{
}
-void IO::write( int step, int curr, Settings& s )
+void IO::write(int step, std::shared_ptr<HeatTransfer> ht, std::shared_ptr<Settings> s, MPI_Comm comm )
{
}
diff --git a/examples/heatTransfer/IO_ascii.cpp b/examples/heatTransfer/IO_ascii.cpp
index ce71ecd5897d2cdb44b144a4c564fb5aa024c72a..4ab36749bd10cdb77c527b6385eb2d29a3fe33f7 100644
--- a/examples/heatTransfer/IO_ascii.cpp
+++ b/examples/heatTransfer/IO_ascii.cpp
@@ -1,5 +1,5 @@
/*
- * IO_ADIOS2.cpp
+ * IO_ascii.cpp
*
* Created on: Feb 2017
* Author: Norbert Podhorszki
@@ -14,9 +14,9 @@
static std::ofstream of;
static std::streambuf *buf;
-IO::IO( std::string output_basename, MPI_Comm comm )
+IO::IO( std::shared_ptr<Settings> s, MPI_Comm comm )
{
- m_outputfilename = output_basename;
+ m_outputfilename = s->outputfile;
if (m_outputfilename == "cout")
{
@@ -27,7 +27,7 @@ IO::IO( std::string output_basename, MPI_Comm comm )
int rank;
MPI_Comm_rank( comm, &rank );
std::string rs = std::to_string(rank);
- of.open(output_basename + rs + ".txt");
+ of.open(m_outputfilename + rs + ".txt");
buf = of.rdbuf();
}
}
@@ -43,7 +43,7 @@ IO::~IO()
void IO::write( int step, std::shared_ptr<HeatTransfer> ht, std::shared_ptr<Settings> s, MPI_Comm comm)
{
std::ostream out(buf);
- if( step == 1)
+ if( step == 0)
{
out << "rank=" << s->rank
<< " size=" << s->ndx << "x" << s->ndy
diff --git a/examples/heatTransfer/Makefile b/examples/heatTransfer/Makefile
index e1ed1b5b88fbbc8b04a1e54493e0afdf9573360c..1d8b8c5acc840678d8b566946c8ed09d000fbecd 100644
--- a/examples/heatTransfer/Makefile
+++ b/examples/heatTransfer/Makefile
@@ -1,20 +1,34 @@
-# Makefile for testing purposes, will build writer_mpi (make or make mpi) or writer_nompi (make nompi)
+# Makefile
# Created on: Feb 13, 2017
# Author: pnorbert
+#
+# SYSTEM SPECIFIC SETTINGS
+#
#COMPILERS
CC=g++
MPICC=mpic++
-#ADIOS LOCATION
-ADIOS_DIR=../..
-ADIOS_INCLUDE=-I$(ADIOS_DIR)/include
-ADIOS_LIB=$(ADIOS_DIR)/lib/libadios.a
-ADIOS_NOMPI_LIB=$(ADIOS_DIR)/lib/libadios_nompi.a
-
#FLAGS
CFLAGS=-Wall -Wpedantic -Woverloaded-virtual -std=c++11 -O0 -g
-LDFLAGS=
+LDFLAGS=-lpthread
+
+#ADIOS1 LOCATION
+ADIOS1_DIR=/opt/adios/1.11
+
+#ADIOS2 LOCATION
+ADIOS2_DIR=../..
+
+#
+# SHOULD NOT NEED TO CHANGE ANYTHING BELOW
+#
+
+ADIOS1_INCLUDE=-I$(ADIOS1_DIR)/include
+ADIOS1_LIB=`${ADIOS1_DIR}/bin/adios_config -l`
+
+
+ADIOS2_INCLUDE= -DHAVE_MPI -I$(ADIOS2_DIR)/include
+ADIOS2_LIB=$(ADIOS2_DIR)/lib/libadios.a
default:
@echo "Make targets: ascii hdf5_a hdf5_b phdf5 adios1 adios2"
@@ -31,11 +45,14 @@ help: default
ascii: main.cpp HeatTransfer.cpp Settings.cpp IO_ascii.cpp
- $(MPICC) $(CFLAGS) $(ADIOS_INCLUDE) -DHAVE_MPI -o heatTransfer_ascii $^ $(ADIOS_LIB) $(LDFLAGS) -lpthread
+ $(MPICC) $(CFLAGS) -o heatTransfer_ascii $^ $(LDFLAGS)
+
+adios1: main.cpp HeatTransfer.cpp Settings.cpp IO_adios1.cpp
+ $(MPICC) $(CFLAGS) $(ADIOS1_INCLUDE) -o heatTransfer_adios1 $^ $(ADIOS1_LIB) $(LDFLAGS)
adios2: main.cpp HeatTransfer.cpp Settings.cpp IO_adios2.cpp
- $(MPICC) $(CFLAGS) $(ADIOS_INCLUDE) -DHAVE_MPI -o heatTransfer_adios2 $^ $(ADIOS_LIB) $(LDFLAGS) -lpthread
+ $(MPICC) $(CFLAGS) $(ADIOS2_INCLUDE) -o heatTransfer_adios2 $^ $(ADIOS2_LIB) $(LDFLAGS)
clean:
- rm -f heatTransfer_ascii heatTransfer_adios2
+ rm -f heatTransfer_ascii heatTransfer_adios1 heatTransfer_adios2
diff --git a/examples/heatTransfer/main.cpp b/examples/heatTransfer/main.cpp
index 0fcde73ffa3f1c2e0e4300ea6225bca65d0c2319..c4978446b218fe326419f83fa2a48ccf85ef69c5 100644
--- a/examples/heatTransfer/main.cpp
+++ b/examples/heatTransfer/main.cpp
@@ -59,13 +59,15 @@ int main( int argc, char* argv [] )
double timeStart = MPI_Wtime();
std::shared_ptr<Settings> settings( new Settings( argc, argv, rank, nproc ));
std::shared_ptr<HeatTransfer> ht( new HeatTransfer( settings ));
- std::shared_ptr<IO> io( new IO( settings->outputfile, mpiHeatTransferComm ));
+ std::shared_ptr<IO> io( new IO( settings, mpiHeatTransferComm ));
-
- ht->init(false);
+ ht->init(true);
ht->printT("Initialized T:", mpiHeatTransferComm);
ht->heatEdges();
+ //ht->exchange( mpiHeatTransferComm );
ht->printT("Heated T:", mpiHeatTransferComm);
+ io->write( 0, ht, settings, mpiHeatTransferComm );
+
for( int t = 1; t <= settings->steps; ++t )
{
if( rank == 0 )
@@ -73,8 +75,8 @@ int main( int argc, char* argv [] )
for( int iter = 1; iter <= settings->iterations; ++iter )
{
ht->iterate();
- ht->heatEdges();
ht->exchange( mpiHeatTransferComm );
+ ht->heatEdges();
}
io->write( t, ht, settings, mpiHeatTransferComm );
}
@@ -101,6 +103,7 @@ int main( int argc, char* argv [] )
std::cout << e.what() << std::endl;
}
+ MPI_Finalize();
return 0;
}