Implemented min max non-contiguous memory (68ef6945) · Commits · Podhorszki, Norbert / ADIOS2

source/adios2/engine/bp3/BP3Writer.tcc

+3 −3

Original line number	Diff line number	Diff line
		@@ -51,9 +51,9 @@ void BP3Writer::PutSyncCommon(Variable<T> &variable,
		}

		// WRITE INDEX to data buffer and metadata structure (in memory)//
		m_BP3Serializer.PutVariableMetadata(variable, blockInfo);
		m_BP3Serializer.PutVariablePayload(variable, blockInfo,
		helper::IsRowMajor(m_IO.m_HostLanguage));
		const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage);
		m_BP3Serializer.PutVariableMetadata(variable, blockInfo, sourceRowMajor);
		m_BP3Serializer.PutVariablePayload(variable, blockInfo, sourceRowMajor);
		}

		template <class T>

source/adios2/helper/adiosMath.inl

+46 −13

Original line number	Diff line number	Diff line
		@@ -42,9 +42,7 @@ void GetMinMaxSelection(const T *values, const Dims &shape, const Dims &start,

		Dims currentPoint(start); // current point for contiguous memory
		bool run = true;

		min = std::numeric_limits<T>::max();
		max = std::numeric_limits<T>::min();
		bool firstStep = true;

		while (run)
		{
		@@ -55,14 +53,24 @@ void GetMinMaxSelection(const T *values, const Dims &shape, const Dims &start,
		T minStride, maxStride;
		GetMinMax(values + startOffset, stride, minStride, maxStride);

		if (minStride < min)
		if (firstStep)
		{
		min = minStride;
		max = maxStride;
		firstStep = false;
		}
		else
		{
		if (LessThan(minStride, min))
		{
		min = minStride;
		}
		if (maxStride > max)

		if (GreaterThan(maxStride, max))
		{
		max = maxStride;
		}
		}

		size_t p = startCoord;
		while (true)
		@@ -100,9 +108,7 @@ void GetMinMaxSelection(const T *values, const Dims &shape, const Dims &start,

		Dims currentPoint(start); // current point for contiguous memory
		bool run = true;

		min = std::numeric_limits<T>::max();
		max = std::numeric_limits<T>::min();
		bool firstStep = true;

		while (run)
		{
		@@ -113,14 +119,24 @@ void GetMinMaxSelection(const T *values, const Dims &shape, const Dims &start,
		T minStride, maxStride;
		GetMinMax(values + startOffset, stride, minStride, maxStride);

		if (minStride < min)
		if (firstStep)
		{
		min = minStride;
		max = maxStride;
		firstStep = false;
		}
		if (maxStride > max)
		else
		{
		if (LessThan(minStride, min))
		{
		min = minStride;
		}

		if (GreaterThan(maxStride, max))
		{
		max = maxStride;
		}
		}

		size_t p = startCoord;

		@@ -169,13 +185,30 @@ void GetMinMaxSelection(const T *values, const Dims &shape, const Dims &start,
		}

		template <class T>
		void GetMinMax(const T *values, const size_t size, T &min, T &max) noexcept
		inline void GetMinMax(const T *values, const size_t size, T &min,
		T &max) noexcept
		{
		auto bounds = std::minmax_element(values, values + size);
		min = *bounds.first;
		max = *bounds.second;
		}

		template <>
		inline void GetMinMax(const std::complex<float> *values, const size_t size,
		std::complex<float> &min,
		std::complex<float> &max) noexcept
		{
		GetMinMaxComplex(values, size, min, max);
		}

		template <>
		inline void GetMinMax(const std::complex<double> *values, const size_t size,
		std::complex<double> &min,
		std::complex<double> &max) noexcept
		{
		GetMinMaxComplex(values, size, min, max);
		}

		template <class T>
		void GetMinMaxComplex(const std::complex<T> *values, const size_t size,
		std::complex<T> &min, std::complex<T> &max) noexcept

source/adios2/toolkit/format/bp3/BP3Serializer.cpp

+2 −2

Original line number	Diff line number	Diff line
		@@ -1648,8 +1648,8 @@ size_t BP3Serializer::GetAttributesSizeInData(core::IO &io) const noexcept
		const bool) noexcept; \
		\
		template void BP3Serializer::PutVariableMetadata( \
		const core::Variable<T> &, \
		const typename core::Variable<T>::Info &) noexcept;
		const core::Variable<T> &, const typename core::Variable<T>::Info &, \
		const bool) noexcept;

		ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation)
		#undef declare_template_instantiation

source/adios2/toolkit/format/bp3/BP3Serializer.h

+8 −7

Original line number	Diff line number	Diff line
		@@ -51,9 +51,9 @@ public:
		* @param variable
		*/
		template <class T>
		void PutVariableMetadata(
		const core::Variable<T> &variable,
		const typename core::Variable<T>::Info &blockInfo) noexcept;
		void PutVariableMetadata(const core::Variable<T> &variable,
		const typename core::Variable<T>::Info &blockInfo,
		const bool sourceRowMajor = true) noexcept;

		/**
		* Put in buffer variable payload. Expensive part.
		@@ -220,11 +220,12 @@ private:
		/**
		* Get variable statistics
		* @param variable
		* @param isRowMajor
		* @return stats BP3 Stats
		*/
		template <class T>
		Stats<T>
		GetBPStats(const typename core::Variable<T>::Info &blockInfo) noexcept;
		Stats<T> GetBPStats(const typename core::Variable<T>::Info &blockInfo,
		const bool isRowMajor) noexcept;

		template <class T>
		void
		@@ -423,8 +424,8 @@ private:
		const bool) noexcept; \
		\
		extern template void BP3Serializer::PutVariableMetadata( \
		const core::Variable<T> &, \
		const typename core::Variable<T>::Info &) noexcept;
		const core::Variable<T> &, const typename core::Variable<T>::Info &, \
		const bool) noexcept;

		ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation)
		#undef declare_template_instantiation

source/adios2/toolkit/format/bp3/BP3Serializer.tcc

+11 −11

Original line number	Diff line number	Diff line
		@@ -25,7 +25,8 @@ namespace format
		template <class T>
		inline void BP3Serializer::PutVariableMetadata(
		const core::Variable<T> &variable,
		const typename core::Variable<T>::Info &blockInfo) noexcept
		const typename core::Variable<T>::Info &blockInfo,
		const bool sourceRowMajor) noexcept
		{
		auto lf_SetOffset = [&](uint64_t &offset) {
		if (m_Aggregator.m_IsActive && !m_Aggregator.m_IsConsumer)
		@@ -40,7 +41,7 @@ inline void BP3Serializer::PutVariableMetadata(

		ProfilerStart("buffering");

		Stats<T> stats = GetBPStats<T>(blockInfo);
		Stats<T> stats = GetBPStats<T>(blockInfo, sourceRowMajor);

		// Get new Index or point to existing index
		bool isNew = true; // flag to check if variable is new
		@@ -339,7 +340,8 @@ void BP3Serializer::PutAttributeInIndex(const core::Attribute<T> &attribute,

		template <>
		inline BP3Serializer::Stats<std::string> BP3Serializer::GetBPStats(
		const typename core::Variable<std::string>::Info & /blockInfo/) noexcept
		const typename core::Variable<std::string>::Info & /blockInfo/,
		const bool /isRowMajor/) noexcept
		{
		Stats<std::string> stats;
		stats.Step = m_MetadataSet.TimeStep;
		@@ -348,8 +350,9 @@ inline BP3Serializer::Stats<std::string> BP3Serializer::GetBPStats(
		}

		template <class T>
		BP3Serializer::Stats<T> BP3Serializer::GetBPStats(
		const typename core::Variable<T>::Info &blockInfo) noexcept
		BP3Serializer::Stats<T>
		BP3Serializer::GetBPStats(const typename core::Variable<T>::Info &blockInfo,
		const bool isRowMajor) noexcept
		{
		Stats<T> stats;
		const std::size_t valuesSize = helper::GetTotalSize(blockInfo.Count);
		@@ -365,12 +368,9 @@ BP3Serializer::Stats<T> BP3Serializer::GetBPStats(
		else
		{
		// TODO: need RowMajor bool
		// helper::GetMinMaxSelection(blockInfo.Data,
		// blockInfo.MemoryCount,
		// blockInfo.MemoryStart,
		// blockInfo.Count,
		// true, stats.Min,
		// stats.Max);
		helper::GetMinMaxSelection(blockInfo.Data, blockInfo.MemoryCount,
		blockInfo.MemoryStart, blockInfo.Count,
		isRowMajor, stats.Min, stats.Max);
		}

		// TODO need to implement minmax for non-contiguous memory