Testing memory growth

examples/heatTransfer/write/config.xml

@@ -15,11 +15,11 @@
 
             <!-- XXKb, XXMb, or XXXGb supported, default=16Kb 
                 (applications might choose an optimal value) -->
-            <parameter key="InitialBufferSize" value="1Mb"/>
+            <!--<parameter key="InitialBufferSize" value="16Mb"/>  -->
             
             <!-- XXKb, XXMb, or XXXGb supported, default=200Mb 
                 (applications might choose an optimal value) -->
-            <parameter key="MaxBufferSize" value="20Mb"/>
+            <!-- <parameter key="MaxBufferSize" value="20Mb"/> -->
 
             <!-- exponential growth factor > 1, default = 2 (as in the STL) 
                  for this case: 1, 2, 4, 8, 16, 20 Mb -->

source/adios2/ADIOSTypes.h

@@ -152,19 +152,19 @@ const std::string DefaultTimeUnit("Microseconds");
 constexpr TimeUnit DefaultTimeUnitEnum(TimeUnit::Microseconds);
 
 /** default initial bp buffer size, 16Kb, in bytes */
-constexpr size_t DefaultInitialBufferSize(16384);
+constexpr size_t DefaultInitialBufferSize(16 * 1024);
 
-/** default maximum bp buffer size, 250Mb, in bytes.
+/** default maximum bp buffer size, unlimited, in bytes.
  *  Needs to be studied for optimizing applications */
-constexpr size_t DefaultMaxBufferSize(268435456000);
+constexpr size_t DefaultMaxBufferSize(std::numeric_limits<size_t>::max() - 1);
 
-/** default buffer growth factor (from STL vector = 2.). Needs to be studied
+/** default buffer growth factor. Needs to be studied
  * for optimizing applications*/
-constexpr float DefaultBufferGrowthFactor(2.);
+constexpr float DefaultBufferGrowthFactor(1.05);
 
 /** default size for writing/reading files using POSIX/fstream/stdio write
  *  2Gb - 100Kb (tolerance)*/
-constexpr size_t DefaultMaxFileBatchSize(2 * 1024 * 1024 * 1024 - 100 * 1024);
+constexpr size_t DefaultMaxFileBatchSize(2147381248);
 
 // adios alias values and types
 constexpr bool DebugON = true;

source/adios2/toolkit/format/bp1/BP1Base.cpp

@@ -182,7 +182,7 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
         if (!success || m_GrowthFactor <= 1.f)
         {
             throw std::invalid_argument(
-                "ERROR: IO SetParameter buffer_growth value "
+                "ERROR: BufferGrowthFactor value "
                 "can't be less or equal than 1 (default = 1.5), or couldn't "
                 "convert number, in call to Open\n");
         }
@@ -196,10 +196,8 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
 void BP1Base::InitParameterInitBufferSize(const std::string value)
 {
     const std::string errorMessage(
-        "ERROR: couldn't convert value of init_buffer_size IO "
-        "SetParameter, valid syntax: InitialBufferSize=10Gb, "
-        "InitialBufferSize=1000Mb, InitialBufferSize=16Kb (minimum default), "
-        " in call to Open");
+        "ERROR: wrong value for InitialBufferSize, it must be larger than "
+        "16Kb (minimum default), in call to Open\n");
 
     if (m_DebugMode)
     {

source/adios2/toolkit/format/bp1/BP1Writer.cpp

@@ -117,6 +117,15 @@ void BP1Writer::WriteProcessGroupIndex(
 
 void BP1Writer::Advance()
 {
+    // enforce memory policy here to restrict buffer size for each timestep
+    // this is flushing
+
+    if (m_MaxBufferSize == DefaultMaxBufferSize)
+    {
+        // current position + 1Kb chunk tolerance
+        m_MaxBufferSize = m_HeapBuffer.m_DataPosition + 64;
+    }
+
     if (m_Profiler.IsActive)
     {
         m_Profiler.Timers.at("buffering").Resume();