From dd7f4712b70f286ccbe0e2b8a7cd7e2196e935a4 Mon Sep 17 00:00:00 2001
From: William F Godoy <wfg@pc0098504.ornl.gov>
Date: Mon, 10 Apr 2017 17:16:04 -0400
Subject: [PATCH] Started working on bpthreads branch
Changed adiosTemplates.h and BP1 functions for STL's insert and copy to
a Buffer in sequential mode
---
include/functions/adiosTemplates.h | 11 +--
include/utilities/format/bp1/BP1Writer.h | 44 ++++++------
source/utilities/format/bp1/BP1Writer.cpp | 84 ++++++++++++-----------
3 files changed, 70 insertions(+), 69 deletions(-)
diff --git a/include/functions/adiosTemplates.h b/include/functions/adiosTemplates.h
index a1a7a2441..5640c5387 100644
--- a/include/functions/adiosTemplates.h
+++ b/include/functions/adiosTemplates.h
@@ -271,15 +271,15 @@ void MemcpyToBuffer(std::vector<char> &raw, std::size_t &position,
* @param elements
*/
template <class T>
-void CopyToBuffer(std::vector<char> &buffer, const T *source,
- const std::size_t elements = 1) noexcept
+void InsertToBuffer(std::vector<char> &buffer, const T *source,
+ const std::size_t elements = 1) noexcept
{
const char *src = reinterpret_cast<const char *>(source);
buffer.insert(buffer.end(), src, src + elements * sizeof(T));
}
/**
- * Overloaded version to copies data to a specific location in the buffer,
+ * Copies data to a specific location in the buffer,
* doesn't update vec.size()
* @param raw
* @param position
@@ -287,8 +287,9 @@ void CopyToBuffer(std::vector<char> &buffer, const T *source,
* @param elements
*/
template <class T>
-void CopyToBuffer(std::vector<char> &buffer, const std::size_t position,
- const T *source, const std::size_t elements = 1) noexcept
+void CopyToBufferPosition(std::vector<char> &buffer, const std::size_t position,
+ const T *source,
+ const std::size_t elements = 1) noexcept
{
const char *src = reinterpret_cast<const char *>(source);
std::copy(src, src + elements * sizeof(T), buffer.begin() + position);
diff --git a/include/utilities/format/bp1/BP1Writer.h b/include/utilities/format/bp1/BP1Writer.h
index 3dc4db2b1..5e126ac00 100644
--- a/include/utilities/format/bp1/BP1Writer.h
+++ b/include/utilities/format/bp1/BP1Writer.h
@@ -228,19 +228,19 @@ private:
WriteNameRecord(variable.m_Name, buffer); // variable name
buffer.insert(buffer.end(), 2, 0); // skip path
const std::uint8_t dataType = GetDataType<T>(); // dataType
- CopyToBuffer(buffer, &dataType);
+ InsertToBuffer(buffer, &dataType);
constexpr char no = 'n'; // isDimension
- CopyToBuffer(buffer, &no);
+ InsertToBuffer(buffer, &no);
// write variable dimensions
const std::uint8_t dimensions = variable.m_LocalDimensions.size();
- CopyToBuffer(buffer, &dimensions); // count
+ InsertToBuffer(buffer, &dimensions); // count
std::uint16_t dimensionsLength =
27 *
dimensions; // 27 is from 9 bytes for each: var y/n + local, var
// y/n + global dimension, var y/n + global offset,
// changed for characteristic
- CopyToBuffer(buffer, &dimensionsLength); // length
+ InsertToBuffer(buffer, &dimensionsLength); // length
WriteDimensionsRecord(buffer, variable.m_LocalDimensions,
variable.m_GlobalDimensions, variable.m_Offsets,
18, true);
@@ -252,7 +252,7 @@ private:
const std::uint64_t varLength = buffer.size() - varLengthPosition +
variable.PayLoadSize() -
8; // remove its own size
- CopyToBuffer(buffer, varLengthPosition, &varLength); // length
+ CopyToBufferPosition(buffer, varLengthPosition, &varLength); // length
heap.m_DataAbsolutePosition +=
buffer.size() - varLengthPosition; // update absolute position to be
@@ -276,19 +276,19 @@ private:
buffer.insert(buffer.end(), 2, 0); // skip path
const std::uint8_t dataType = GetDataType<T>();
- CopyToBuffer(buffer, &dataType);
+ InsertToBuffer(buffer, &dataType);
// Characteristics Sets Count in Metadata
index.Count = 1;
- CopyToBuffer(buffer, &index.Count);
+ InsertToBuffer(buffer, &index.Count);
}
else // update characteristics sets count
{
const std::size_t characteristicsSetsCountPosition =
15 + variable.m_Name.size();
++index.Count;
- CopyToBuffer(buffer, characteristicsSetsCountPosition,
- &index.Count); // test
+ CopyToBufferPosition(buffer, characteristicsSetsCountPosition,
+ &index.Count); // test
}
WriteVariableCharacteristics(variable, stats, buffer);
@@ -310,20 +310,20 @@ private:
// DIMENSIONS
std::uint8_t characteristicID = characteristic_dimensions;
- CopyToBuffer(buffer, &characteristicID);
+ InsertToBuffer(buffer, &characteristicID);
const std::uint8_t dimensions = variable.m_LocalDimensions.size();
if (addLength == true)
{
const std::int16_t lengthOfDimensionsCharacteristic =
24 * dimensions +
- 3; // 24 = 3 local, global, global offset x 8 bytes/each
- CopyToBuffer(buffer, &lengthOfDimensionsCharacteristic);
+ 3; // 24 = 3 local, global, offset x 8 bytes/each
+ InsertToBuffer(buffer, &lengthOfDimensionsCharacteristic);
}
- CopyToBuffer(buffer, &dimensions); // count
+ InsertToBuffer(buffer, &dimensions); // count
const std::uint16_t dimensionsLength = 24 * dimensions;
- CopyToBuffer(buffer, &dimensionsLength); // length
+ InsertToBuffer(buffer, &dimensionsLength); // length
WriteDimensionsRecord(buffer, variable.m_LocalDimensions,
variable.m_GlobalDimensions, variable.m_Offsets,
16, addLength);
@@ -347,15 +347,15 @@ private:
// END OF CHARACTERISTICS
// Back to characteristics count and length
- CopyToBuffer(buffer, characteristicsCountPosition,
- &characteristicsCounter); // count (1)
+ CopyToBufferPosition(buffer, characteristicsCountPosition,
+ &characteristicsCounter); // count (1)
const std::uint32_t characteristicsLength =
buffer.size() - characteristicsCountPosition - 4 -
1; // remove its own length (4 bytes) + characteristic counter ( 1
// byte
// )
- CopyToBuffer(buffer, characteristicsCountPosition + 1,
- &characteristicsLength); // length
+ CopyToBufferPosition(buffer, characteristicsCountPosition + 1,
+ &characteristicsLength); // length
}
/**
@@ -480,12 +480,12 @@ private:
const bool addLength = false) const noexcept
{
const std::uint8_t id = characteristicID;
- CopyToBuffer(buffer, &id);
+ InsertToBuffer(buffer, &id);
if (addLength == true)
{
const std::uint16_t lengthOfCharacteristic = sizeof(T); // id
- CopyToBuffer(buffer, &lengthOfCharacteristic);
+ InsertToBuffer(buffer, &lengthOfCharacteristic);
}
CopyToBuffer(buffer, &value);
@@ -521,9 +521,7 @@ private:
* @param buffer
*/
void FlattenMetadata(BP1MetadataSet &metadataSet,
- capsule::STLVector &buffer) const
- noexcept; ///< sets the metadata buffer in capsule with indices and
- /// minifooter
+ capsule::STLVector &buffer) const noexcept;
};
} // end namespace format
diff --git a/source/utilities/format/bp1/BP1Writer.cpp b/source/utilities/format/bp1/BP1Writer.cpp
index 8cef9fce9..4e0eaae13 100644
--- a/source/utilities/format/bp1/BP1Writer.cpp
+++ b/source/utilities/format/bp1/BP1Writer.cpp
@@ -50,13 +50,13 @@ void BP1Writer::WriteProcessGroupIndex(
// write if host language Fortran in metadata and data
const char hostFortran =
(isFortran) ? 'y' : 'n'; // if host language is fortran
- CopyToBuffer(metadataBuffer, &hostFortran);
- CopyToBuffer(dataBuffer, &hostFortran);
+ InsertToBuffer(metadataBuffer, &hostFortran);
+ InsertToBuffer(dataBuffer, &hostFortran);
// write name in data
WriteNameRecord(name, dataBuffer);
// processID in metadata,
- CopyToBuffer(metadataBuffer, &processID);
+ InsertToBuffer(metadataBuffer, &processID);
// skip coordination var in data ....what is coordination var?
dataBuffer.insert(dataBuffer.end(), 4, 0);
@@ -66,33 +66,33 @@ void BP1Writer::WriteProcessGroupIndex(
WriteNameRecord(timeStepName, dataBuffer);
// time step to metadata and data
- CopyToBuffer(metadataBuffer, &metadataSet.TimeStep);
- CopyToBuffer(dataBuffer, &metadataSet.TimeStep);
+ InsertToBuffer(metadataBuffer, &metadataSet.TimeStep);
+ InsertToBuffer(dataBuffer, &metadataSet.TimeStep);
// offset to pg in data in metadata which is the current absolute position
- CopyToBuffer(metadataBuffer, reinterpret_cast<std::uint64_t *>(
- &heap.m_DataAbsolutePosition));
+ InsertToBuffer(metadataBuffer, reinterpret_cast<std::uint64_t *>(
+ &heap.m_DataAbsolutePosition));
// Back to writing metadata pg index length (length of group)
const std::uint16_t metadataPGIndexLength =
metadataBuffer.size() - metadataPGLengthPosition -
2; // without length of group record
- CopyToBuffer(metadataBuffer, metadataPGLengthPosition,
- &metadataPGIndexLength);
+ CopyToBufferPosition(metadataBuffer, metadataPGLengthPosition,
+ &metadataPGIndexLength);
// DONE With metadataBuffer
// here write method in data
const std::vector<std::uint8_t> methodIDs = GetMethodIDs(transports);
const std::uint8_t methodsCount = methodIDs.size();
- CopyToBuffer(dataBuffer, &methodsCount); // count
+ InsertToBuffer(dataBuffer, &methodsCount); // count
const std::uint16_t methodsLength =
methodIDs.size() *
3; // methodID (1) + method params length(2), no parameters for now
- CopyToBuffer(dataBuffer, &methodsLength); // length
+ InsertToBuffer(dataBuffer, &methodsLength); // length
for (const auto methodID : methodIDs)
{
- CopyToBuffer(dataBuffer, &methodID); // method ID,
+ InsertToBuffer(dataBuffer, &methodID); // method ID,
dataBuffer.insert(dataBuffer.end(), 2,
0); // skip method params length = 0 (2 bytes) for now
}
@@ -213,9 +213,9 @@ void BP1Writer::WriteDimensionsRecord(
{
auto lf_WriteFlaggedDim = [](std::vector<char> &buffer, const char no,
const std::size_t dimension) {
- CopyToBuffer(buffer, &no);
- CopyToBuffer(buffer,
- reinterpret_cast<const std::uint64_t *>(&dimension));
+ InsertToBuffer(buffer, &no);
+ InsertToBuffer(buffer,
+ reinterpret_cast<const std::uint64_t *>(&dimension));
};
// BODY Starts here
@@ -236,8 +236,8 @@ void BP1Writer::WriteDimensionsRecord(
{
for (const auto &localDimension : localDimensions)
{
- CopyToBuffer(buffer, reinterpret_cast<const std::uint64_t *>(
- &localDimension));
+ InsertToBuffer(buffer, reinterpret_cast<const std::uint64_t *>(
+ &localDimension));
buffer.insert(buffer.end(), skip, 0);
}
}
@@ -258,12 +258,12 @@ void BP1Writer::WriteDimensionsRecord(
{
for (unsigned int d = 0; d < localDimensions.size(); ++d)
{
- CopyToBuffer(buffer, reinterpret_cast<const std::uint64_t *>(
- &localDimensions[d]));
- CopyToBuffer(buffer, reinterpret_cast<const std::uint64_t *>(
- &globalDimensions[d]));
- CopyToBuffer(buffer, reinterpret_cast<const std::uint64_t *>(
- &globalOffsets[d]));
+ InsertToBuffer(buffer, reinterpret_cast<const std::uint64_t *>(
+ &localDimensions[d]));
+ InsertToBuffer(buffer, reinterpret_cast<const std::uint64_t *>(
+ &globalDimensions[d]));
+ InsertToBuffer(buffer, reinterpret_cast<const std::uint64_t *>(
+ &globalOffsets[d]));
}
}
}
@@ -273,8 +273,8 @@ void BP1Writer::WriteNameRecord(const std::string name,
std::vector<char> &buffer) const noexcept
{
const std::uint16_t length = name.length();
- CopyToBuffer(buffer, &length);
- CopyToBuffer(buffer, name.c_str(), length);
+ InsertToBuffer(buffer, &length);
+ InsertToBuffer(buffer, name.c_str(), length);
}
BP1Index &
@@ -299,12 +299,13 @@ void BP1Writer::FlattenData(BP1MetadataSet &metadataSet,
{
auto &buffer = heap.m_Data;
// vars count and Length (only for PG)
- CopyToBuffer(buffer, metadataSet.DataPGVarsCountPosition,
- &metadataSet.DataPGVarsCount);
+ CopyToBufferPosition(buffer, metadataSet.DataPGVarsCountPosition,
+ &metadataSet.DataPGVarsCount);
const std::uint64_t varsLength = buffer.size() -
metadataSet.DataPGVarsCountPosition - 8 -
4; // without record itself and vars count
- CopyToBuffer(buffer, metadataSet.DataPGVarsCountPosition + 4, &varsLength);
+ CopyToBufferPosition(buffer, metadataSet.DataPGVarsCountPosition + 4,
+ &varsLength);
// attributes (empty for now) count (4) and length (8) are zero by moving
// positions in time step zero
@@ -315,7 +316,8 @@ void BP1Writer::FlattenData(BP1MetadataSet &metadataSet,
const std::uint64_t dataPGLength =
buffer.size() - metadataSet.DataPGLengthPosition -
8; // without record itself, 12 due to empty attributes
- CopyToBuffer(buffer, metadataSet.DataPGLengthPosition, &dataPGLength);
+ CopyToBufferPosition(buffer, metadataSet.DataPGLengthPosition,
+ &dataPGLength);
++metadataSet.TimeStep;
metadataSet.DataPGIsOpen = false;
@@ -334,7 +336,7 @@ void BP1Writer::FlattenMetadata(BP1MetadataSet &metadataSet,
{
auto &indexBuffer = indexPair.second.Buffer;
const std::uint32_t indexLength = indexBuffer.size() - 4;
- CopyToBuffer(indexBuffer, 0, &indexLength);
+ CopyToBufferPosition(indexBuffer, 0, &indexLength);
length += indexBuffer.size(); // overall length
}
@@ -344,13 +346,13 @@ void BP1Writer::FlattenMetadata(BP1MetadataSet &metadataSet,
[](const std::uint32_t count, const std::uint64_t length,
const std::unordered_map<std::string, BP1Index> &indices,
std::vector<char> &buffer) {
- CopyToBuffer(buffer, &count);
- CopyToBuffer(buffer, &length);
+ InsertToBuffer(buffer, &count);
+ InsertToBuffer(buffer, &length);
for (const auto &indexPair : indices) // set each index length
{
const auto &indexBuffer = indexPair.second.Buffer;
- CopyToBuffer(buffer, indexBuffer.data(), indexBuffer.size());
+ InsertToBuffer(buffer, indexBuffer.data(), indexBuffer.size());
}
};
@@ -378,9 +380,9 @@ void BP1Writer::FlattenMetadata(BP1MetadataSet &metadataSet,
// strategy
// write pg index
- CopyToBuffer(buffer, &pgCount);
- CopyToBuffer(buffer, &pgLength);
- CopyToBuffer(buffer, metadataSet.PGIndex.Buffer.data(), pgLength);
+ InsertToBuffer(buffer, &pgCount);
+ InsertToBuffer(buffer, &pgLength);
+ InsertToBuffer(buffer, metadataSet.PGIndex.Buffer.data(), pgLength);
// Vars indices
lf_FlattenIndices(varsCount, varsLength, metadataSet.VarsIndices, buffer);
// Attribute indices
@@ -393,17 +395,17 @@ void BP1Writer::FlattenMetadata(BP1MetadataSet &metadataSet,
const std::uint64_t offsetAttributeIndex =
offsetVarsIndex + (varsLength + 12);
- CopyToBuffer(buffer, &offsetPGIndex);
- CopyToBuffer(buffer, &offsetVarsIndex);
- CopyToBuffer(buffer, &offsetAttributeIndex);
+ InsertToBuffer(buffer, &offsetPGIndex);
+ InsertToBuffer(buffer, &offsetVarsIndex);
+ InsertToBuffer(buffer, &offsetAttributeIndex);
// version
if (IsLittleEndian())
{
const std::uint8_t endian = 0;
- CopyToBuffer(buffer, &endian);
+ InsertToBuffer(buffer, &endian);
buffer.insert(buffer.end(), 2, 0);
- CopyToBuffer(buffer, &m_Version);
+ InsertToBuffer(buffer, &m_Version);
}
else
{
--
GitLab