From da6301eec637d3da8f0bc821aa611891072320f0 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Mon, 12 Aug 2019 15:58:04 -0400
Subject: [PATCH 1/4] Basic support for BlocksInfo in SST with FFS marshaling. 
 Test to follow BP implementation.

---
 source/adios2/engine/sst/SstReader.cpp     | 62 ++++++++++++++++++++--
 source/adios2/toolkit/sst/cp/cp_internal.h |  1 +
 source/adios2/toolkit/sst/cp/cp_writer.c   |  1 -
 source/adios2/toolkit/sst/cp/ffs_marshal.c |  8 ++-
 source/adios2/toolkit/sst/sst.h            | 12 +++--
 5 files changed, 75 insertions(+), 9 deletions(-)

diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp
index d8bc7d799..64e9f63e2 100644
--- a/source/adios2/engine/sst/SstReader.cpp
+++ b/source/adios2/engine/sst/SstReader.cpp
@@ -167,8 +167,65 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode,
         return (void *)NULL;
     };
 
+    auto arrayBlocksInfoCallback =
+        [](void *reader, void *variable, const char *type, int WriterRank,
+           int DimCount, size_t *Shape, size_t *Start, size_t *Count) {
+            std::vector<size_t> VecShape;
+            std::vector<size_t> VecStart;
+            std::vector<size_t> VecCount;
+            std::string Type(type);
+            class SstReader::SstReader *Reader =
+                reinterpret_cast<class SstReader::SstReader *>(reader);
+            size_t currentStep = SstCurrentStep(Reader->m_Input);
+            /*
+             * setup shape of array variable as global (I.E. Count == Shape,
+             * Start == 0)
+             */
+            if (Shape)
+            {
+                for (int i = 0; i < DimCount; i++)
+                {
+                    VecShape.push_back(Shape[i]);
+                    VecStart.push_back(Start[i]);
+                    VecCount.push_back(Count[i]);
+                }
+            }
+            else
+            {
+                VecShape = {};
+                VecStart = {};
+                for (int i = 0; i < DimCount; i++)
+                {
+                    VecCount.push_back(Count[i]);
+                }
+            }
+
+            if (Type == "compound")
+            {
+                return;
+            }
+#define declare_type(T)                                                        \
+    else if (Type == helper::GetType<T>())                                     \
+    {                                                                          \
+        Variable<T> *Var = reinterpret_cast<class Variable<T> *>(variable);    \
+        auto savedShape = Var->m_Shape;                                        \
+        auto savedCount = Var->m_Count;                                        \
+        auto savedStart = Var->m_Start;                                        \
+        Var->m_Shape = VecShape;                                               \
+        Var->m_Count = VecCount;                                               \
+        Var->m_Start = VecStart;                                               \
+        Var->SetBlockInfo((T *)NULL, currentStep);                             \
+        Var->m_Shape = savedShape;                                             \
+        Var->m_Count = savedCount;                                             \
+        Var->m_Start = savedStart;                                             \
+    }
+            ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
+#undef declare_type
+            return;
+        };
+
     SstReaderInitFFSCallback(m_Input, this, varFFSCallback, arrayFFSCallback,
-                             attrFFSCallback);
+                             attrFFSCallback, arrayBlocksInfoCallback);
 
     delete[] cstr;
 }
@@ -525,8 +582,7 @@ void SstReader::DoClose(const int transportIndex) { SstReaderClose(m_Input); }
     {                                                                          \
         if (m_WriterMarshalMethod == SstMarshalFFS)                            \
         {                                                                      \
-            throw std::invalid_argument("ERROR: SST Engine doesn't implement " \
-                                        "function DoAllStepsBlocksInfo\n");    \
+            return variable.m_BlocksInfo;                                      \
         }                                                                      \
         else if (m_WriterMarshalMethod == SstMarshalBP)                        \
         {                                                                      \
diff --git a/source/adios2/toolkit/sst/cp/cp_internal.h b/source/adios2/toolkit/sst/cp/cp_internal.h
index 33b126ef7..fc1c9aee9 100644
--- a/source/adios2/toolkit/sst/cp/cp_internal.h
+++ b/source/adios2/toolkit/sst/cp/cp_internal.h
@@ -200,6 +200,7 @@ struct _SstStream
     VarSetupUpcallFunc VarSetupUpcall;
     ArraySetupUpcallFunc ArraySetupUpcall;
     AttrSetupUpcallFunc AttrSetupUpcall;
+    ArrayBlocksInfoUpcallFunc ArrayBlocksInfoUpcall;
     void *SetupUpcallReader;
     void *ReaderMarshalData;
 
diff --git a/source/adios2/toolkit/sst/cp/cp_writer.c b/source/adios2/toolkit/sst/cp/cp_writer.c
index 9cca46101..beee14e0a 100644
--- a/source/adios2/toolkit/sst/cp/cp_writer.c
+++ b/source/adios2/toolkit/sst/cp/cp_writer.c
@@ -2184,7 +2184,6 @@ extern void CP_LockReaderDefinitionsHandler(CManager cm, CMConnection conn,
                "for timestep %d from reader cohort %d\n",
                Msg->Timestep, ReaderNum);
 
-    /* decrement the reference count for the released timestep */
     PTHREAD_MUTEX_LOCK(&ParentStream->DataLock);
     if ((ParentStream->Rank == 0) &&
         (ParentStream->ConfigParams->CPCommPattern == SstCPCommMin))
diff --git a/source/adios2/toolkit/sst/cp/ffs_marshal.c b/source/adios2/toolkit/sst/cp/ffs_marshal.c
index d8fa9b7bf..5f5fc1561 100644
--- a/source/adios2/toolkit/sst/cp/ffs_marshal.c
+++ b/source/adios2/toolkit/sst/cp/ffs_marshal.c
@@ -715,11 +715,13 @@ extern int SstFFSWriterBeginStep(SstStream Stream, int mode,
 void SstReaderInitFFSCallback(SstStream Stream, void *Reader,
                               VarSetupUpcallFunc VarCallback,
                               ArraySetupUpcallFunc ArrayCallback,
-                              AttrSetupUpcallFunc AttrCallback)
+                              AttrSetupUpcallFunc AttrCallback,
+                              ArrayBlocksInfoUpcallFunc BlocksInfoCallback)
 {
     Stream->VarSetupUpcall = VarCallback;
     Stream->ArraySetupUpcall = ArrayCallback;
     Stream->AttrSetupUpcall = AttrCallback;
+    Stream->ArrayBlocksInfoUpcall = BlocksInfoCallback;
     Stream->SetupUpcallReader = Reader;
 }
 
@@ -1884,6 +1886,10 @@ static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData,
             VarRec->PerWriterCounts[WriterRank] = meta_base->Count;
             VarRec->PerWriterMetaFieldDesc[WriterRank] = &FieldList[i];
             VarRec->PerWriterDataFieldDesc[WriterRank] = NULL;
+            Stream->ArrayBlocksInfoUpcall(Stream->SetupUpcallReader,
+                                          VarRec->Variable, Type, WriterRank,
+                                          meta_base->Dims, meta_base->Shape,
+                                          meta_base->Offsets, meta_base->Count);
             i += 4;
             free(ArrayName);
         }
diff --git a/source/adios2/toolkit/sst/sst.h b/source/adios2/toolkit/sst/sst.h
index 6cdfe22d8..a9bf4ff2d 100644
--- a/source/adios2/toolkit/sst/sst.h
+++ b/source/adios2/toolkit/sst/sst.h
@@ -135,10 +135,14 @@ typedef void *(*ArraySetupUpcallFunc)(void *Reader, const char *Name,
                                       const char *Type, int DimsCount,
                                       size_t *Shape, size_t *Start,
                                       size_t *Count);
-extern void SstReaderInitFFSCallback(SstStream stream, void *Reader,
-                                     VarSetupUpcallFunc VarCallback,
-                                     ArraySetupUpcallFunc ArrayCallback,
-                                     AttrSetupUpcallFunc AttrCallback);
+typedef void (*ArrayBlocksInfoUpcallFunc)(void *Reader, void *Variable,
+                                          const char *Type, int WriterRank,
+                                          int DimsCount, size_t *Shape,
+                                          size_t *Start, size_t *Count);
+extern void SstReaderInitFFSCallback(
+    SstStream stream, void *Reader, VarSetupUpcallFunc VarCallback,
+    ArraySetupUpcallFunc ArrayCallback, AttrSetupUpcallFunc AttrCallback,
+    ArrayBlocksInfoUpcallFunc BlocksInfoCallback);
 
 /*
  *  Calls that support SST-external writer-side aggregation of metadata
-- 
GitLab


From 03e454dadcf30700102777a114067b8b03e519f8 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Mon, 12 Aug 2019 16:32:39 -0400
Subject: [PATCH 2/4] Patch BP3 BlocksInfo call to work

---
 source/adios2/engine/sst/SstReader.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp
index 64e9f63e2..55414df2b 100644
--- a/source/adios2/engine/sst/SstReader.cpp
+++ b/source/adios2/engine/sst/SstReader.cpp
@@ -586,7 +586,7 @@ void SstReader::DoClose(const int transportIndex) { SstReaderClose(m_Input); }
         }                                                                      \
         else if (m_WriterMarshalMethod == SstMarshalBP)                        \
         {                                                                      \
-            return m_BP3Deserializer->BlocksInfo(variable, step);              \
+            return m_BP3Deserializer->BlocksInfo(variable, 0);	               \
         }                                                                      \
         throw std::invalid_argument(                                           \
             "ERROR: Unknown marshal mechanism in DoBlocksInfo\n");             \
-- 
GitLab


From 5fd0daf3d01c71b8d9f29218913a290f0551bf28 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Mon, 12 Aug 2019 16:58:57 -0400
Subject: [PATCH 3/4] Make complex tests conditional on presence

---
 .../engine/staging-common/TestCommonRead.cpp  | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/testing/adios2/engine/staging-common/TestCommonRead.cpp b/testing/adios2/engine/staging-common/TestCommonRead.cpp
index 63fa88ca0..396af2ab7 100644
--- a/testing/adios2/engine/staging-common/TestCommonRead.cpp
+++ b/testing/adios2/engine/staging-common/TestCommonRead.cpp
@@ -146,6 +146,50 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8)
         ASSERT_EQ(var_time.ShapeID(), adios2::ShapeID::GlobalArray);
         ASSERT_EQ(var_time.Shape()[0], writerSize);
 
+        const std::vector<adios2::Variable<int8_t>::Info> i8Info =
+            engine.BlocksInfo(var_i8, engine.CurrentStep());
+        const std::vector<adios2::Variable<int16_t>::Info> i16Info =
+            engine.BlocksInfo(var_i16, engine.CurrentStep());
+        const std::vector<adios2::Variable<int32_t>::Info> i32Info =
+            engine.BlocksInfo(var_i32, engine.CurrentStep());
+        const std::vector<adios2::Variable<int64_t>::Info> i64Info =
+            engine.BlocksInfo(var_i64, engine.CurrentStep());
+        const std::vector<adios2::Variable<float>::Info> r32Info =
+            engine.BlocksInfo(var_r32, engine.CurrentStep());
+        const std::vector<adios2::Variable<double>::Info> r64Info =
+            engine.BlocksInfo(var_r64, engine.CurrentStep());
+        EXPECT_EQ(i8Info.size(), writerSize);
+        EXPECT_EQ(i16Info.size(), writerSize);
+        EXPECT_EQ(i32Info.size(), writerSize);
+        EXPECT_EQ(i64Info.size(), writerSize);
+        EXPECT_EQ(r32Info.size(), writerSize);
+        EXPECT_EQ(r64Info.size(), writerSize);
+
+        for (size_t i = 0; i < writerSize; ++i)
+        {
+            EXPECT_FALSE(i8Info[0].IsValue);
+            EXPECT_FALSE(i16Info[0].IsValue);
+            EXPECT_FALSE(i32Info[0].IsValue);
+            EXPECT_FALSE(i64Info[0].IsValue);
+            EXPECT_FALSE(r32Info[0].IsValue);
+            EXPECT_FALSE(r64Info[0].IsValue);
+        }
+
+        if (var_c32)
+        {
+            const std::vector<adios2::Variable<std::complex<float>>::Info>
+                c32Info = engine.BlocksInfo(var_c32, engine.CurrentStep());
+            const std::vector<adios2::Variable<std::complex<double>>::Info>
+                c64Info = engine.BlocksInfo(var_c64, engine.CurrentStep());
+            EXPECT_EQ(c32Info.size(), writerSize);
+            EXPECT_EQ(c64Info.size(), writerSize);
+            for (size_t i = 0; i < writerSize; ++i)
+            {
+                EXPECT_FALSE(c32Info[0].IsValue);
+                EXPECT_FALSE(c64Info[0].IsValue);
+            }
+        }
+
         long unsigned int myStart =
             (long unsigned int)(writerSize * Nx / mpiSize) * mpiRank;
         long unsigned int myLength =
-- 
GitLab


From b069494ee0bdf5bf4680e808831f35c5000cfaf8 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Mon, 12 Aug 2019 18:29:19 -0400
Subject: [PATCH 4/4] clang-format

---
 source/adios2/engine/sst/SstReader.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp
index 55414df2b..e9947da43 100644
--- a/source/adios2/engine/sst/SstReader.cpp
+++ b/source/adios2/engine/sst/SstReader.cpp
@@ -586,7 +586,7 @@ void SstReader::DoClose(const int transportIndex) { SstReaderClose(m_Input); }
         }                                                                      \
         else if (m_WriterMarshalMethod == SstMarshalBP)                        \
         {                                                                      \
-            return m_BP3Deserializer->BlocksInfo(variable, 0);	               \
+            return m_BP3Deserializer->BlocksInfo(variable, 0);                 \
         }                                                                      \
         throw std::invalid_argument(                                           \
             "ERROR: Unknown marshal mechanism in DoBlocksInfo\n");             \
-- 
GitLab