Unverified Commit dac5e428 authored by pnorbert's avatar pnorbert Committed by GitHub
Browse files

Merge pull request #1579 from pnorbert/bp4-localvalues

- Handle LocalValueDim (local values turned into 1D global array) in …
parents c307debb f2ce93ee
......@@ -349,8 +349,10 @@ void VariableBase::InitShapeType()
if (m_Shape.size() == 1 && m_Shape.front() == LocalValueDim)
{
m_ShapeID = ShapeID::LocalValue;
m_Start.resize(1);
m_Count.resize(1); // real count = 1
m_Start.resize(1, 0); // start[0] == 0
m_Count.resize(1, 1); // count[0] == 1
// Count[0] == 1 makes sure the value will be written
// into the data file (PutVariablePayload())
m_SingleValue = true;
}
else
......
......@@ -1134,6 +1134,7 @@ std::vector<typename core::Variable<T>::Info> BP4Deserializer::BlocksInfoCommon(
std::vector<typename core::Variable<T>::Info> blocksInfo;
blocksInfo.reserve(blocksIndexOffsets.size());
size_t n = 0;
for (const size_t blockIndexOffset : blocksIndexOffsets)
{
size_t position = blockIndexOffset;
......@@ -1166,7 +1167,23 @@ std::vector<typename core::Variable<T>::Info> BP4Deserializer::BlocksInfoCommon(
blockInfo.Min = blockCharacteristics.Statistics.Min;
blockInfo.Max = blockCharacteristics.Statistics.Max;
}
if (blockInfo.Shape.size() == 1 &&
blockInfo.Shape.front() == LocalValueDim)
{
blockInfo.Shape = Dims{blocksIndexOffsets.size()};
blockInfo.Count = Dims{1};
blockInfo.Start = Dims{n};
blockInfo.Min = blockCharacteristics.Statistics.Value;
blockInfo.Max = blockCharacteristics.Statistics.Value;
}
// bp index starts at 1
blockInfo.Step =
static_cast<size_t>(blockCharacteristics.Statistics.Step - 1);
blockInfo.BlockID = n;
blocksInfo.push_back(blockInfo);
++n;
}
return blocksInfo;
}
......
......@@ -254,11 +254,11 @@ private:
const Stats<T> &stats, std::vector<char> &buffer) noexcept;
template <class T>
void PutVariableCharacteristics(
void PutVariableCharacteristicsInData(
const core::Variable<T> &variable,
const typename core::Variable<T>::Info &blockInfo,
const Stats<T> &stats, std::vector<char> &buffer, size_t &position,
const bool putDimensions = true) noexcept;
const Stats<T> &stats, std::vector<char> &buffer,
size_t &position) noexcept;
/**
* Writes from &buffer[position]: [2
......
......@@ -493,8 +493,8 @@ void BP4Serializer::PutVariableMetadataInData(
buffer, position);
// CHARACTERISTICS
PutVariableCharacteristics(variable, blockInfo, stats, buffer, position,
false);
PutVariableCharacteristicsInData(variable, blockInfo, stats, buffer,
position);
// pad metadata so that data will fall on aligned position in memory
// write a padding plus block identifier VMD]
......@@ -662,13 +662,8 @@ void BP4Serializer::PutBoundsRecord(const bool singleValue,
{
if (singleValue)
{
const uint8_t id = characteristic_value;
helper::CopyToBuffer(buffer, position, &id);
// special case required by bpdump
const uint16_t length = sizeof(T);
helper::CopyToBuffer(buffer, position, &length);
helper::CopyToBuffer(buffer, position, &stats.Min);
++characteristicsCounter;
PutCharacteristicRecord(characteristic_value, characteristicsCounter,
stats.Min, buffer, position);
}
else
{
......@@ -836,11 +831,10 @@ void BP4Serializer::PutVariableCharacteristics(
}
template <class T>
void BP4Serializer::PutVariableCharacteristics(
void BP4Serializer::PutVariableCharacteristicsInData(
const core::Variable<T> &variable,
const typename core::Variable<T>::Info &blockInfo, const Stats<T> &stats,
std::vector<char> &buffer, size_t &position,
const bool putDimensions) noexcept
std::vector<char> &buffer, size_t &position) noexcept
{
// going back at the end
const size_t characteristicsCountPosition = position;
......@@ -848,24 +842,10 @@ void BP4Serializer::PutVariableCharacteristics(
position += 5;
uint8_t characteristicsCounter = 0;
if (putDimensions)
{
// DIMENSIONS
uint8_t characteristicID = characteristic_dimensions;
helper::CopyToBuffer(buffer, position, &characteristicID);
const uint8_t dimensions = static_cast<uint8_t>(blockInfo.Count.size());
helper::CopyToBuffer(buffer, position, &dimensions); // count
const uint16_t dimensionsLength =
static_cast<uint16_t>(24 * dimensions);
helper::CopyToBuffer(buffer, position, &dimensionsLength); // length
PutDimensionsRecord(blockInfo.Count, blockInfo.Shape, blockInfo.Start,
buffer, position, true);
++characteristicsCounter;
}
// VALUE for SCALAR or STAT min, max for ARRAY
if (blockInfo.Data != nullptr)
// Write STAT min, max characteristics for an ARRAY
// VALUE variable data is not written into characteristics
// in the data file (only in metadata file in other function)
if (blockInfo.Data != nullptr && !variable.m_SingleValue)
{
PutBoundsRecord(variable.m_SingleValue, stats, characteristicsCounter,
buffer, position);
......
......@@ -22,8 +22,8 @@ def SetupArgs():
action="store_true")
parser.add_argument("--no-metadata", "-m",
help="Do not print metadata md.0", action="store_true")
parser.add_argument(
"--no-data", "-d", help="Do not print data data.*", action="store_true")
parser.add_argument("--no-data", "-d",
help="Do not print data data.*", action="store_true")
args = parser.parse_args()
# default values
......
......@@ -154,7 +154,7 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize,
currentPosition = f.tell()
print(" Payload offset : {0}".format(currentPosition))
nBytes = np.ones(1, dtype=np.uint64)
nBytes[0] = nElements[0] * typeSize
nBytes[0] = nElements * typeSize
if (currentPosition + nBytes[0] > varsStartPosition + varsTotalLength):
print("ERROR: Variable data block of size would reach beyond all "
"variable blocks")
......@@ -168,12 +168,21 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize,
"calculated from var block length")
print("Expected size = {0} calculated size from dimensions = {1}".
format(expectedSize, nBytes[0]))
print(" Variable Data : {0} bytes".format(nBytes[0]))
# seek instead of reading for now
f.read(nBytes[0])
# f.seek(nBytes[0], 1)
# data = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID),
# nElements)
if nElements == 1:
# single value. read and print
value = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID),
nElements)
print(" Payload (value) : {0} ({1} bytes)".format(
value[0], nBytes[0]))
else:
# seek instead of reading for now
f.read(nBytes[0])
# f.seek(nBytes[0], 1)
# data = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID),
# nElements)
print(" Payload (array) : {0} bytes".format(nBytes[0]))
return True
# Read a variable's metadata
......@@ -248,8 +257,9 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
print(" Dims Length : {0}".format(
dimsLen))
nElements = np.ones(1, dtype=np.uint64)
nElements = np.uint64(1)
ldims = np.zeros(ndims, dtype=np.uint64)
isLocalValueArray = False
for i in range(ndims):
print(" Dim[{0}]".format(i))
# Read Local Dimensions (1 byte flag + 8 byte value)
......@@ -278,7 +288,11 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
if (isDimensionVarID == b'\0'):
isDimensionVarID = b'n'
gdim = np.fromfile(f, dtype=np.uint64, count=1)[0]
print(" global dim : {0}".format(gdim))
if i == 0 and ldims[i] == 0 and gdim == bp4dbg_utils.LocalValueDim:
print(" global dim : LocalValueDim ({0})".format(gdim))
isLocalValueArray = True
else:
print(" global dim : {0}".format(gdim))
# Read Offset Dimensions (1 byte flag + 8 byte value)
# Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0'
......@@ -310,9 +324,16 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
print(" Tag (pad {0:2d}) : {1}".format(
endTagLen - 4, tag.decode('ascii')))
expectedVarDataSize = expectedVarBlockLength - (f.tell() - startPosition)
status = ReadVarData(f, nElements, typeID, ldims, expectedVarDataSize,
varsStartPosition, varsTotalLength)
# special case: LocalValueDim: local values turned into 1D global array
# but it seems there is no data block at all for these variables
if isLocalValueArray:
ldims[0] = 1
nElements = np.uint64(1)
else:
expectedVarDataSize = expectedVarBlockLength - \
(f.tell() - startPosition)
status = ReadVarData(f, nElements, typeID, ldims, expectedVarDataSize,
varsStartPosition, varsTotalLength)
if not status:
return False
......
......@@ -209,8 +209,10 @@ def ReadPGMD(buf, idx, pos, limit, pgStartOffset):
# 2 bytes PG Length + Name length
pgLength = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0]
pos = pos + 2
print(" PG length : {0} bytes".format(pgLength))
if (pgStartPosition + pgLength > limit):
print(
" PG length : {0} bytes (+2 for length)".format(
pgLength))
if (pgStartPosition + pgLength + 2 > limit):
print("ERROR: There is not enough bytes {0} left in PG index block "
"to read this single PG index ({1} bytes)").format(
limit - pgStartPosition, pgLength)
......@@ -277,8 +279,9 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset):
# 4 bytes VAR Index Length
varLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0]
pos = pos + 4
print(" Var idx length : {0} bytes".format(varLength))
if (varStartPosition + varLength > limit):
print(" Var idx length : {0} bytes (+4 for idx length)".format(
varLength))
if (varStartPosition + varLength + 4 > limit):
print("ERROR: There is not enough bytes in Var index block "
"to read this single Var index")
return False, pos
......@@ -344,8 +347,9 @@ def ReadAttrMD(buf, idx, pos, limit, attrStartOffset):
# 4 bytes ATTR Index Length
attrLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0]
pos = pos + 4
print(" Attr idx length : {0} bytes".format(attrLength))
if (attrStartPosition + attrLength > limit):
print(" Attr idx length : {0} bytes (+4 for idx length)".format(
attrLength))
if (attrStartPosition + attrLength + 4 > limit):
print("ERROR: There is not enough bytes in Attr index block "
"to read this single Attr index")
return False, pos
......
from __future__ import (absolute_import, division, print_function,
unicode_literals)
LocalValueDim = 18446744073709551613
dataTypes = {
-1: 'unknown',
0: 'byte',
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment