Skip to content
Snippets Groups Projects
Commit f9a8aca2 authored by William F Godoy's avatar William F Godoy
Browse files

Testing memory growth

parent 7d3dd0ee
No related branches found
No related tags found
1 merge request!190Memory
......@@ -15,11 +15,11 @@
<!-- XXKb, XXMb, or XXXGb supported, default=16Kb
(applications might choose an optimal value) -->
<parameter key="InitialBufferSize" value="1Mb"/>
<!--<parameter key="InitialBufferSize" value="16Mb"/> -->
<!-- XXKb, XXMb, or XXXGb supported, default=200Mb
(applications might choose an optimal value) -->
<parameter key="MaxBufferSize" value="20Mb"/>
<!-- <parameter key="MaxBufferSize" value="20Mb"/> -->
<!-- exponential growth factor > 1, default = 2 (as in the STL)
for this case: 1, 2, 4, 8, 16, 20 Mb -->
......
......@@ -152,19 +152,19 @@ const std::string DefaultTimeUnit("Microseconds");
constexpr TimeUnit DefaultTimeUnitEnum(TimeUnit::Microseconds);
/** default initial bp buffer size, 16Kb, in bytes */
constexpr size_t DefaultInitialBufferSize(16384);
constexpr size_t DefaultInitialBufferSize(16 * 1024);
/** default maximum bp buffer size, 250Mb, in bytes.
/** default maximum bp buffer size, unlimited, in bytes.
* Needs to be studied for optimizing applications */
constexpr size_t DefaultMaxBufferSize(268435456000);
constexpr size_t DefaultMaxBufferSize(std::numeric_limits<size_t>::max() - 1);
/** default buffer growth factor (from STL vector = 2.). Needs to be studied
/** default buffer growth factor. Needs to be studied
* for optimizing applications*/
constexpr float DefaultBufferGrowthFactor(2.);
constexpr float DefaultBufferGrowthFactor(1.05);
/** default size for writing/reading files using POSIX/fstream/stdio write
* 2Gb - 100Kb (tolerance)*/
constexpr size_t DefaultMaxFileBatchSize(2 * 1024 * 1024 * 1024 - 100 * 1024);
constexpr size_t DefaultMaxFileBatchSize(2147381248);
// adios alias values and types
constexpr bool DebugON = true;
......
......@@ -182,7 +182,7 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
if (!success || m_GrowthFactor <= 1.f)
{
throw std::invalid_argument(
"ERROR: IO SetParameter buffer_growth value "
"ERROR: BufferGrowthFactor value "
"can't be less or equal than 1 (default = 1.5), or couldn't "
"convert number, in call to Open\n");
}
......@@ -196,10 +196,8 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
void BP1Base::InitParameterInitBufferSize(const std::string value)
{
const std::string errorMessage(
"ERROR: couldn't convert value of init_buffer_size IO "
"SetParameter, valid syntax: InitialBufferSize=10Gb, "
"InitialBufferSize=1000Mb, InitialBufferSize=16Kb (minimum default), "
" in call to Open");
"ERROR: wrong value for InitialBufferSize, it must be larger than "
"16Kb (minimum default), in call to Open\n");
if (m_DebugMode)
{
......
......@@ -117,6 +117,15 @@ void BP1Writer::WriteProcessGroupIndex(
void BP1Writer::Advance()
{
// enforce memory policy here to restrict buffer size for each timestep
// this is flushing
if (m_MaxBufferSize == DefaultMaxBufferSize)
{
// current position + 1Kb chunk tolerance
m_MaxBufferSize = m_HeapBuffer.m_DataPosition + 64;
}
if (m_Profiler.IsActive)
{
m_Profiler.Timers.at("buffering").Resume();
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment