Unverified Commit dac5e428 authored by pnorbert's avatar pnorbert Committed by GitHub
Browse files

Merge pull request #1579 from pnorbert/bp4-localvalues

- Handle LocalValueDim (local values turned into 1D global array) in …
parents c307debb f2ce93ee
...@@ -349,8 +349,10 @@ void VariableBase::InitShapeType() ...@@ -349,8 +349,10 @@ void VariableBase::InitShapeType()
if (m_Shape.size() == 1 && m_Shape.front() == LocalValueDim) if (m_Shape.size() == 1 && m_Shape.front() == LocalValueDim)
{ {
m_ShapeID = ShapeID::LocalValue; m_ShapeID = ShapeID::LocalValue;
m_Start.resize(1); m_Start.resize(1, 0); // start[0] == 0
m_Count.resize(1); // real count = 1 m_Count.resize(1, 1); // count[0] == 1
// Count[0] == 1 makes sure the value will be written
// into the data file (PutVariablePayload())
m_SingleValue = true; m_SingleValue = true;
} }
else else
......
...@@ -1134,6 +1134,7 @@ std::vector<typename core::Variable<T>::Info> BP4Deserializer::BlocksInfoCommon( ...@@ -1134,6 +1134,7 @@ std::vector<typename core::Variable<T>::Info> BP4Deserializer::BlocksInfoCommon(
std::vector<typename core::Variable<T>::Info> blocksInfo; std::vector<typename core::Variable<T>::Info> blocksInfo;
blocksInfo.reserve(blocksIndexOffsets.size()); blocksInfo.reserve(blocksIndexOffsets.size());
size_t n = 0;
for (const size_t blockIndexOffset : blocksIndexOffsets) for (const size_t blockIndexOffset : blocksIndexOffsets)
{ {
size_t position = blockIndexOffset; size_t position = blockIndexOffset;
...@@ -1166,7 +1167,23 @@ std::vector<typename core::Variable<T>::Info> BP4Deserializer::BlocksInfoCommon( ...@@ -1166,7 +1167,23 @@ std::vector<typename core::Variable<T>::Info> BP4Deserializer::BlocksInfoCommon(
blockInfo.Min = blockCharacteristics.Statistics.Min; blockInfo.Min = blockCharacteristics.Statistics.Min;
blockInfo.Max = blockCharacteristics.Statistics.Max; blockInfo.Max = blockCharacteristics.Statistics.Max;
} }
if (blockInfo.Shape.size() == 1 &&
blockInfo.Shape.front() == LocalValueDim)
{
blockInfo.Shape = Dims{blocksIndexOffsets.size()};
blockInfo.Count = Dims{1};
blockInfo.Start = Dims{n};
blockInfo.Min = blockCharacteristics.Statistics.Value;
blockInfo.Max = blockCharacteristics.Statistics.Value;
}
// bp index starts at 1
blockInfo.Step =
static_cast<size_t>(blockCharacteristics.Statistics.Step - 1);
blockInfo.BlockID = n;
blocksInfo.push_back(blockInfo); blocksInfo.push_back(blockInfo);
++n;
} }
return blocksInfo; return blocksInfo;
} }
......
...@@ -254,11 +254,11 @@ private: ...@@ -254,11 +254,11 @@ private:
const Stats<T> &stats, std::vector<char> &buffer) noexcept; const Stats<T> &stats, std::vector<char> &buffer) noexcept;
template <class T> template <class T>
void PutVariableCharacteristics( void PutVariableCharacteristicsInData(
const core::Variable<T> &variable, const core::Variable<T> &variable,
const typename core::Variable<T>::Info &blockInfo, const typename core::Variable<T>::Info &blockInfo,
const Stats<T> &stats, std::vector<char> &buffer, size_t &position, const Stats<T> &stats, std::vector<char> &buffer,
const bool putDimensions = true) noexcept; size_t &position) noexcept;
/** /**
* Writes from &buffer[position]: [2 * Writes from &buffer[position]: [2
......
...@@ -493,8 +493,8 @@ void BP4Serializer::PutVariableMetadataInData( ...@@ -493,8 +493,8 @@ void BP4Serializer::PutVariableMetadataInData(
buffer, position); buffer, position);
// CHARACTERISTICS // CHARACTERISTICS
PutVariableCharacteristics(variable, blockInfo, stats, buffer, position, PutVariableCharacteristicsInData(variable, blockInfo, stats, buffer,
false); position);
// pad metadata so that data will fall on aligned position in memory // pad metadata so that data will fall on aligned position in memory
// write a padding plus block identifier VMD] // write a padding plus block identifier VMD]
...@@ -662,13 +662,8 @@ void BP4Serializer::PutBoundsRecord(const bool singleValue, ...@@ -662,13 +662,8 @@ void BP4Serializer::PutBoundsRecord(const bool singleValue,
{ {
if (singleValue) if (singleValue)
{ {
const uint8_t id = characteristic_value; PutCharacteristicRecord(characteristic_value, characteristicsCounter,
helper::CopyToBuffer(buffer, position, &id); stats.Min, buffer, position);
// special case required by bpdump
const uint16_t length = sizeof(T);
helper::CopyToBuffer(buffer, position, &length);
helper::CopyToBuffer(buffer, position, &stats.Min);
++characteristicsCounter;
} }
else else
{ {
...@@ -836,11 +831,10 @@ void BP4Serializer::PutVariableCharacteristics( ...@@ -836,11 +831,10 @@ void BP4Serializer::PutVariableCharacteristics(
} }
template <class T> template <class T>
void BP4Serializer::PutVariableCharacteristics( void BP4Serializer::PutVariableCharacteristicsInData(
const core::Variable<T> &variable, const core::Variable<T> &variable,
const typename core::Variable<T>::Info &blockInfo, const Stats<T> &stats, const typename core::Variable<T>::Info &blockInfo, const Stats<T> &stats,
std::vector<char> &buffer, size_t &position, std::vector<char> &buffer, size_t &position) noexcept
const bool putDimensions) noexcept
{ {
// going back at the end // going back at the end
const size_t characteristicsCountPosition = position; const size_t characteristicsCountPosition = position;
...@@ -848,24 +842,10 @@ void BP4Serializer::PutVariableCharacteristics( ...@@ -848,24 +842,10 @@ void BP4Serializer::PutVariableCharacteristics(
position += 5; position += 5;
uint8_t characteristicsCounter = 0; uint8_t characteristicsCounter = 0;
if (putDimensions) // Write STAT min, max characteristics for an ARRAY
{ // VALUE variable data is not written into characteristics
// DIMENSIONS // in the data file (only in metadata file in other function)
uint8_t characteristicID = characteristic_dimensions; if (blockInfo.Data != nullptr && !variable.m_SingleValue)
helper::CopyToBuffer(buffer, position, &characteristicID);
const uint8_t dimensions = static_cast<uint8_t>(blockInfo.Count.size());
helper::CopyToBuffer(buffer, position, &dimensions); // count
const uint16_t dimensionsLength =
static_cast<uint16_t>(24 * dimensions);
helper::CopyToBuffer(buffer, position, &dimensionsLength); // length
PutDimensionsRecord(blockInfo.Count, blockInfo.Shape, blockInfo.Start,
buffer, position, true);
++characteristicsCounter;
}
// VALUE for SCALAR or STAT min, max for ARRAY
if (blockInfo.Data != nullptr)
{ {
PutBoundsRecord(variable.m_SingleValue, stats, characteristicsCounter, PutBoundsRecord(variable.m_SingleValue, stats, characteristicsCounter,
buffer, position); buffer, position);
......
...@@ -22,8 +22,8 @@ def SetupArgs(): ...@@ -22,8 +22,8 @@ def SetupArgs():
action="store_true") action="store_true")
parser.add_argument("--no-metadata", "-m", parser.add_argument("--no-metadata", "-m",
help="Do not print metadata md.0", action="store_true") help="Do not print metadata md.0", action="store_true")
parser.add_argument( parser.add_argument("--no-data", "-d",
"--no-data", "-d", help="Do not print data data.*", action="store_true") help="Do not print data data.*", action="store_true")
args = parser.parse_args() args = parser.parse_args()
# default values # default values
......
...@@ -154,7 +154,7 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize, ...@@ -154,7 +154,7 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize,
currentPosition = f.tell() currentPosition = f.tell()
print(" Payload offset : {0}".format(currentPosition)) print(" Payload offset : {0}".format(currentPosition))
nBytes = np.ones(1, dtype=np.uint64) nBytes = np.ones(1, dtype=np.uint64)
nBytes[0] = nElements[0] * typeSize nBytes[0] = nElements * typeSize
if (currentPosition + nBytes[0] > varsStartPosition + varsTotalLength): if (currentPosition + nBytes[0] > varsStartPosition + varsTotalLength):
print("ERROR: Variable data block of size would reach beyond all " print("ERROR: Variable data block of size would reach beyond all "
"variable blocks") "variable blocks")
...@@ -168,12 +168,21 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize, ...@@ -168,12 +168,21 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize,
"calculated from var block length") "calculated from var block length")
print("Expected size = {0} calculated size from dimensions = {1}". print("Expected size = {0} calculated size from dimensions = {1}".
format(expectedSize, nBytes[0])) format(expectedSize, nBytes[0]))
print(" Variable Data : {0} bytes".format(nBytes[0]))
# seek instead of reading for now if nElements == 1:
f.read(nBytes[0]) # single value. read and print
# f.seek(nBytes[0], 1) value = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID),
# data = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID), nElements)
# nElements) print(" Payload (value) : {0} ({1} bytes)".format(
value[0], nBytes[0]))
else:
# seek instead of reading for now
f.read(nBytes[0])
# f.seek(nBytes[0], 1)
# data = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID),
# nElements)
print(" Payload (array) : {0} bytes".format(nBytes[0]))
return True return True
# Read a variable's metadata # Read a variable's metadata
...@@ -248,8 +257,9 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength): ...@@ -248,8 +257,9 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
print(" Dims Length : {0}".format( print(" Dims Length : {0}".format(
dimsLen)) dimsLen))
nElements = np.ones(1, dtype=np.uint64) nElements = np.uint64(1)
ldims = np.zeros(ndims, dtype=np.uint64) ldims = np.zeros(ndims, dtype=np.uint64)
isLocalValueArray = False
for i in range(ndims): for i in range(ndims):
print(" Dim[{0}]".format(i)) print(" Dim[{0}]".format(i))
# Read Local Dimensions (1 byte flag + 8 byte value) # Read Local Dimensions (1 byte flag + 8 byte value)
...@@ -278,7 +288,11 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength): ...@@ -278,7 +288,11 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
if (isDimensionVarID == b'\0'): if (isDimensionVarID == b'\0'):
isDimensionVarID = b'n' isDimensionVarID = b'n'
gdim = np.fromfile(f, dtype=np.uint64, count=1)[0] gdim = np.fromfile(f, dtype=np.uint64, count=1)[0]
print(" global dim : {0}".format(gdim)) if i == 0 and ldims[i] == 0 and gdim == bp4dbg_utils.LocalValueDim:
print(" global dim : LocalValueDim ({0})".format(gdim))
isLocalValueArray = True
else:
print(" global dim : {0}".format(gdim))
# Read Offset Dimensions (1 byte flag + 8 byte value) # Read Offset Dimensions (1 byte flag + 8 byte value)
# Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0' # Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0'
...@@ -310,9 +324,16 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength): ...@@ -310,9 +324,16 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
print(" Tag (pad {0:2d}) : {1}".format( print(" Tag (pad {0:2d}) : {1}".format(
endTagLen - 4, tag.decode('ascii'))) endTagLen - 4, tag.decode('ascii')))
expectedVarDataSize = expectedVarBlockLength - (f.tell() - startPosition) # special case: LocalValueDim: local values turned into 1D global array
status = ReadVarData(f, nElements, typeID, ldims, expectedVarDataSize, # but it seems there is no data block at all for these variables
varsStartPosition, varsTotalLength) if isLocalValueArray:
ldims[0] = 1
nElements = np.uint64(1)
else:
expectedVarDataSize = expectedVarBlockLength - \
(f.tell() - startPosition)
status = ReadVarData(f, nElements, typeID, ldims, expectedVarDataSize,
varsStartPosition, varsTotalLength)
if not status: if not status:
return False return False
......
...@@ -209,8 +209,10 @@ def ReadPGMD(buf, idx, pos, limit, pgStartOffset): ...@@ -209,8 +209,10 @@ def ReadPGMD(buf, idx, pos, limit, pgStartOffset):
# 2 bytes PG Length + Name length # 2 bytes PG Length + Name length
pgLength = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0] pgLength = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0]
pos = pos + 2 pos = pos + 2
print(" PG length : {0} bytes".format(pgLength)) print(
if (pgStartPosition + pgLength > limit): " PG length : {0} bytes (+2 for length)".format(
pgLength))
if (pgStartPosition + pgLength + 2 > limit):
print("ERROR: There is not enough bytes {0} left in PG index block " print("ERROR: There is not enough bytes {0} left in PG index block "
"to read this single PG index ({1} bytes)").format( "to read this single PG index ({1} bytes)").format(
limit - pgStartPosition, pgLength) limit - pgStartPosition, pgLength)
...@@ -277,8 +279,9 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset): ...@@ -277,8 +279,9 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset):
# 4 bytes VAR Index Length # 4 bytes VAR Index Length
varLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0] varLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0]
pos = pos + 4 pos = pos + 4
print(" Var idx length : {0} bytes".format(varLength)) print(" Var idx length : {0} bytes (+4 for idx length)".format(
if (varStartPosition + varLength > limit): varLength))
if (varStartPosition + varLength + 4 > limit):
print("ERROR: There is not enough bytes in Var index block " print("ERROR: There is not enough bytes in Var index block "
"to read this single Var index") "to read this single Var index")
return False, pos return False, pos
...@@ -344,8 +347,9 @@ def ReadAttrMD(buf, idx, pos, limit, attrStartOffset): ...@@ -344,8 +347,9 @@ def ReadAttrMD(buf, idx, pos, limit, attrStartOffset):
# 4 bytes ATTR Index Length # 4 bytes ATTR Index Length
attrLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0] attrLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0]
pos = pos + 4 pos = pos + 4
print(" Attr idx length : {0} bytes".format(attrLength)) print(" Attr idx length : {0} bytes (+4 for idx length)".format(
if (attrStartPosition + attrLength > limit): attrLength))
if (attrStartPosition + attrLength + 4 > limit):
print("ERROR: There is not enough bytes in Attr index block " print("ERROR: There is not enough bytes in Attr index block "
"to read this single Attr index") "to read this single Attr index")
return False, pos return False, pos
......
from __future__ import (absolute_import, division, print_function, from __future__ import (absolute_import, division, print_function,
unicode_literals) unicode_literals)
LocalValueDim = 18446744073709551613
dataTypes = { dataTypes = {
-1: 'unknown', -1: 'unknown',
0: 'byte', 0: 'byte',
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment