Commit f9a8aca2 authored by William F Godoy's avatar William F Godoy
Browse files

Testing memory growth

parent 7d3dd0ee
...@@ -15,11 +15,11 @@ ...@@ -15,11 +15,11 @@
<!-- XXKb, XXMb, or XXXGb supported, default=16Kb <!-- XXKb, XXMb, or XXXGb supported, default=16Kb
(applications might choose an optimal value) --> (applications might choose an optimal value) -->
<parameter key="InitialBufferSize" value="1Mb"/> <!--<parameter key="InitialBufferSize" value="16Mb"/> -->
<!-- XXKb, XXMb, or XXXGb supported, default=200Mb <!-- XXKb, XXMb, or XXXGb supported, default=200Mb
(applications might choose an optimal value) --> (applications might choose an optimal value) -->
<parameter key="MaxBufferSize" value="20Mb"/> <!-- <parameter key="MaxBufferSize" value="20Mb"/> -->
<!-- exponential growth factor > 1, default = 2 (as in the STL) <!-- exponential growth factor > 1, default = 2 (as in the STL)
for this case: 1, 2, 4, 8, 16, 20 Mb --> for this case: 1, 2, 4, 8, 16, 20 Mb -->
......
...@@ -152,19 +152,19 @@ const std::string DefaultTimeUnit("Microseconds"); ...@@ -152,19 +152,19 @@ const std::string DefaultTimeUnit("Microseconds");
constexpr TimeUnit DefaultTimeUnitEnum(TimeUnit::Microseconds); constexpr TimeUnit DefaultTimeUnitEnum(TimeUnit::Microseconds);
/** default initial bp buffer size, 16Kb, in bytes */ /** default initial bp buffer size, 16Kb, in bytes */
constexpr size_t DefaultInitialBufferSize(16384); constexpr size_t DefaultInitialBufferSize(16 * 1024);
/** default maximum bp buffer size, 250Mb, in bytes. /** default maximum bp buffer size, unlimited, in bytes.
* Needs to be studied for optimizing applications */ * Needs to be studied for optimizing applications */
constexpr size_t DefaultMaxBufferSize(268435456000); constexpr size_t DefaultMaxBufferSize(std::numeric_limits<size_t>::max() - 1);
/** default buffer growth factor (from STL vector = 2.). Needs to be studied /** default buffer growth factor. Needs to be studied
* for optimizing applications*/ * for optimizing applications*/
constexpr float DefaultBufferGrowthFactor(2.); constexpr float DefaultBufferGrowthFactor(1.05);
/** default size for writing/reading files using POSIX/fstream/stdio write /** default size for writing/reading files using POSIX/fstream/stdio write
* 2Gb - 100Kb (tolerance)*/ * 2Gb - 100Kb (tolerance)*/
constexpr size_t DefaultMaxFileBatchSize(2 * 1024 * 1024 * 1024 - 100 * 1024); constexpr size_t DefaultMaxFileBatchSize(2147381248);
// adios alias values and types // adios alias values and types
constexpr bool DebugON = true; constexpr bool DebugON = true;
......
...@@ -182,7 +182,7 @@ void BP1Base::InitParameterBufferGrowth(const std::string value) ...@@ -182,7 +182,7 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
if (!success || m_GrowthFactor <= 1.f) if (!success || m_GrowthFactor <= 1.f)
{ {
throw std::invalid_argument( throw std::invalid_argument(
"ERROR: IO SetParameter buffer_growth value " "ERROR: BufferGrowthFactor value "
"can't be less or equal than 1 (default = 1.5), or couldn't " "can't be less or equal than 1 (default = 1.5), or couldn't "
"convert number, in call to Open\n"); "convert number, in call to Open\n");
} }
...@@ -196,10 +196,8 @@ void BP1Base::InitParameterBufferGrowth(const std::string value) ...@@ -196,10 +196,8 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
void BP1Base::InitParameterInitBufferSize(const std::string value) void BP1Base::InitParameterInitBufferSize(const std::string value)
{ {
const std::string errorMessage( const std::string errorMessage(
"ERROR: couldn't convert value of init_buffer_size IO " "ERROR: wrong value for InitialBufferSize, it must be larger than "
"SetParameter, valid syntax: InitialBufferSize=10Gb, " "16Kb (minimum default), in call to Open\n");
"InitialBufferSize=1000Mb, InitialBufferSize=16Kb (minimum default), "
" in call to Open");
if (m_DebugMode) if (m_DebugMode)
{ {
......
...@@ -117,6 +117,15 @@ void BP1Writer::WriteProcessGroupIndex( ...@@ -117,6 +117,15 @@ void BP1Writer::WriteProcessGroupIndex(
void BP1Writer::Advance() void BP1Writer::Advance()
{ {
// enforce memory policy here to restrict buffer size for each timestep
// this is flushing
if (m_MaxBufferSize == DefaultMaxBufferSize)
{
// current position + 1Kb chunk tolerance
m_MaxBufferSize = m_HeapBuffer.m_DataPosition + 64;
}
if (m_Profiler.IsActive) if (m_Profiler.IsActive)
{ {
m_Profiler.Timers.at("buffering").Resume(); m_Profiler.Timers.at("buffering").Resume();
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment