Commit 1a91dd77 authored by williamfgc's avatar williamfgc Committed by GitHub
Browse files

Merge pull request #190 from williamfgc/memory

Memory
parents b902024a 30ea62d4
......@@ -16,6 +16,8 @@ if(ADIOS2_HAVE_MPI)
PRIVATE ${MPI_C_INCLUDE_PATH}
)
target_link_libraries(heatTransfer_write_adios2 adios2 ${MPI_C_LIBRARIES})
target_compile_definitions(heatTransfer_write_adios2 PRIVATE
-DDEFAULT_CONFIG=${CMAKE_CURRENT_SOURCE_DIR}/config.xml)
if(ADIOS2_HAVE_ADIOS1)
find_package(ADIOS1 REQUIRED)
......
......@@ -14,6 +14,13 @@
#include <adios2.h>
#define str_helper(X) #X
#define str(X) str_helper(X)
#ifndef DEFAULT_CONFIG
#define DEFAULT_CONFIG config.xml
#endif
#define DEFAULT_CONFIG_STR str(DEFAULT_CONFIG)
static int rank_saved;
adios2::ADIOS *ad = nullptr;
std::shared_ptr<adios2::Engine> bpWriter;
......@@ -24,7 +31,8 @@ IO::IO(const Settings &s, MPI_Comm comm)
{
rank_saved = s.rank;
m_outputfilename = s.outputfile + ".bp";
ad = new adios2::ADIOS("config.xml", comm, adios2::DebugON);
ad = new adios2::ADIOS(std::string(DEFAULT_CONFIG_STR), comm,
adios2::DebugON);
// Define method for engine creation
......
<?xml version="1.0"?>
<!-- Config XML file fo the heatTransfer_write_adios2 executable in .
build/bin from IO_adios2.cpp -->
<adios-config>
<io name="output">
<engine type="BPFileWriter">
<!-- for vectorized memory operations, make sure your system
enables threads-->
<parameter key="Threads" value="2"/>
<!-- Microseconds (default), Milliseconds, Seconds,
Minutes, Hours -->
<parameter key="ProfileUnits" value="Microseconds"/>
<!-- XXKb, XXMb, or XXXGb supported, default=16Kb
(applications might choose an optimal value) -->
<!--<parameter key="InitialBufferSize" value="16Kb"/> -->
<!-- XXKb, XXMb, or XXXGb supported, default=Unlimited (until
fails), maximum at each time step
(applications might choose an optimal value) -->
<!-- <parameter key="MaxBufferSize" value="2Gb"/> -->
<!-- exponential growth factor > 1, default = 1.05
1.05 is good for a few large variables, for many small
variables increase the value to 1.5 to 2
(optimal value is application dependent)-->
<parameter key="BufferGrowthFactor" value="1.05"/>
</engine>
<transport type="File">
<!-- POSIX, stdio (C FILE*), fstream (C++) -->
<parameter key="Library" value="POSIX"/>
<!-- For read/write, Microseconds (default), Milliseconds, Seconds,
Minutes, Hours. open/close always in Microseconds -->
<parameter key="ProfileUnits" value="Microseconds"/>
</transport>
</io>
</adios-config>
......@@ -2,7 +2,8 @@
* Distributed under the OSI-approved Apache License, Version 2.0. See
* accompanying file Copyright.txt for details.
*
* ADIOSTypes.h
* ADIOSTypes.h : public header that contains "using/typedef" alias, defaults
* and parameters options as enum classes
*
* Created on: Mar 23, 2017
* Author: Chuck Atkins chuck.atkins@kitware.com
......@@ -31,7 +32,8 @@
namespace adios2
{
/** Variable shape type identifier */
/** Variable shape type identifier, assigned automatically from the signature of
* DefineVariable */
enum class ShapeID
{
GlobalValue, ///< single global value, common case
......@@ -84,12 +86,12 @@ enum class TransportType
WAN
};
/** Just for info purposes */
/** Currently available engines, just for info purposes */
enum class IOEngine
{
Unknown,
BPFileWriter, ///< produces bp files
BPFileReader, ///< read bp files
BPFileReader, ///< read bp files (not yet implemented)
HDF5Writer, ///<
HDF5Reader, ///<
ADIOS1Writer,
......@@ -150,14 +152,19 @@ const std::string DefaultTimeUnit("Microseconds");
constexpr TimeUnit DefaultTimeUnitEnum(TimeUnit::Microseconds);
/** default initial bp buffer size, 16Kb, in bytes */
constexpr size_t DefaultInitialBufferSize(16384);
/** default maximum bp buffer size, 16Mb, in bytes */
constexpr size_t DefaultMaxBufferSize(16777216);
/** default buffer growth factor (from STL vector = 2.) */
constexpr float DefaultBufferGrowthFactor(2.);
constexpr size_t DefaultInitialBufferSize(16 * 1024);
/** default maximum bp buffer size, unlimited, in bytes.
* Needs to be studied for optimizing applications */
constexpr size_t DefaultMaxBufferSize(std::numeric_limits<size_t>::max() - 1);
/** default buffer growth factor. Needs to be studied
* for optimizing applications*/
constexpr float DefaultBufferGrowthFactor(1.05);
/** default size for writing/reading files using POSIX/fstream/stdio write
* 1Gb - 1Kb (tolerance)*/
constexpr size_t DefaultMaxFileBatchSize(1024 * 1024 * 1024 - 1024);
* 2Gb - 100Kb (tolerance)*/
constexpr size_t DefaultMaxFileBatchSize(2147381248);
// adios alias values and types
constexpr bool DebugON = true;
......
......@@ -182,7 +182,7 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
if (!success || m_GrowthFactor <= 1.f)
{
throw std::invalid_argument(
"ERROR: IO SetParameter buffer_growth value "
"ERROR: BufferGrowthFactor value "
"can't be less or equal than 1 (default = 1.5), or couldn't "
"convert number, in call to Open\n");
}
......@@ -196,10 +196,8 @@ void BP1Base::InitParameterBufferGrowth(const std::string value)
void BP1Base::InitParameterInitBufferSize(const std::string value)
{
const std::string errorMessage(
"ERROR: couldn't convert value of init_buffer_size IO "
"SetParameter, valid syntax: InitialBufferSize=10Gb, "
"InitialBufferSize=1000Mb, InitialBufferSize=16Kb (minimum default), "
" in call to Open");
"ERROR: wrong value for InitialBufferSize, it must be larger than "
"16Kb (minimum default), in call to Open\n");
if (m_DebugMode)
{
......
......@@ -117,6 +117,15 @@ void BP1Writer::WriteProcessGroupIndex(
void BP1Writer::Advance()
{
// enforce memory policy here to restrict buffer size for each timestep
// this is flushing
if (m_MaxBufferSize == DefaultMaxBufferSize)
{
// current position + 1Kb chunk tolerance
m_MaxBufferSize = m_HeapBuffer.m_DataPosition + 64;
}
if (m_Profiler.IsActive)
{
m_Profiler.Timers.at("buffering").Resume();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment