Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Podhorszki, Norbert
ADIOS2
Commits
dac5e428
Unverified
Commit
dac5e428
authored
Jun 29, 2019
by
pnorbert
Committed by
GitHub
Jun 29, 2019
Browse files
Merge pull request #1579 from pnorbert/bp4-localvalues
- Handle LocalValueDim (local values turned into 1D global array) in …
parents
c307debb
f2ce93ee
Changes
8
Hide whitespace changes
Inline
Side-by-side
source/adios2/core/VariableBase.cpp
View file @
dac5e428
...
...
@@ -349,8 +349,10 @@ void VariableBase::InitShapeType()
if
(
m_Shape
.
size
()
==
1
&&
m_Shape
.
front
()
==
LocalValueDim
)
{
m_ShapeID
=
ShapeID
::
LocalValue
;
m_Start
.
resize
(
1
);
m_Count
.
resize
(
1
);
// real count = 1
m_Start
.
resize
(
1
,
0
);
// start[0] == 0
m_Count
.
resize
(
1
,
1
);
// count[0] == 1
// Count[0] == 1 makes sure the value will be written
// into the data file (PutVariablePayload())
m_SingleValue
=
true
;
}
else
...
...
source/adios2/toolkit/format/bp4/BP4Deserializer.tcc
View file @
dac5e428
...
...
@@ -1134,6 +1134,7 @@ std::vector<typename core::Variable<T>::Info> BP4Deserializer::BlocksInfoCommon(
std::vector<typename core::Variable<T>::Info> blocksInfo;
blocksInfo.reserve(blocksIndexOffsets.size());
size_t n = 0;
for (const size_t blockIndexOffset : blocksIndexOffsets)
{
size_t position = blockIndexOffset;
...
...
@@ -1166,7 +1167,23 @@ std::vector<typename core::Variable<T>::Info> BP4Deserializer::BlocksInfoCommon(
blockInfo.Min = blockCharacteristics.Statistics.Min;
blockInfo.Max = blockCharacteristics.Statistics.Max;
}
if (blockInfo.Shape.size() == 1 &&
blockInfo.Shape.front() == LocalValueDim)
{
blockInfo.Shape = Dims{blocksIndexOffsets.size()};
blockInfo.Count = Dims{1};
blockInfo.Start = Dims{n};
blockInfo.Min = blockCharacteristics.Statistics.Value;
blockInfo.Max = blockCharacteristics.Statistics.Value;
}
// bp index starts at 1
blockInfo.Step =
static_cast<size_t>(blockCharacteristics.Statistics.Step - 1);
blockInfo.BlockID = n;
blocksInfo.push_back(blockInfo);
++n;
}
return blocksInfo;
}
...
...
source/adios2/toolkit/format/bp4/BP4Serializer.h
View file @
dac5e428
...
...
@@ -254,11 +254,11 @@ private:
const
Stats
<
T
>
&
stats
,
std
::
vector
<
char
>
&
buffer
)
noexcept
;
template
<
class
T
>
void
PutVariableCharacteristics
(
void
PutVariableCharacteristics
InData
(
const
core
::
Variable
<
T
>
&
variable
,
const
typename
core
::
Variable
<
T
>::
Info
&
blockInfo
,
const
Stats
<
T
>
&
stats
,
std
::
vector
<
char
>
&
buffer
,
size_t
&
position
,
const
bool
putDimensions
=
true
)
noexcept
;
const
Stats
<
T
>
&
stats
,
std
::
vector
<
char
>
&
buffer
,
size_t
&
position
)
noexcept
;
/**
* Writes from &buffer[position]: [2
...
...
source/adios2/toolkit/format/bp4/BP4Serializer.tcc
View file @
dac5e428
...
...
@@ -493,8 +493,8 @@ void BP4Serializer::PutVariableMetadataInData(
buffer, position);
// CHARACTERISTICS
PutVariableCharacteristics(variable, blockInfo, stats, buffer,
position,
false
);
PutVariableCharacteristics
InData
(variable, blockInfo, stats, buffer,
position
);
// pad metadata so that data will fall on aligned position in memory
// write a padding plus block identifier VMD]
...
...
@@ -662,13 +662,8 @@ void BP4Serializer::PutBoundsRecord(const bool singleValue,
{
if (singleValue)
{
const uint8_t id = characteristic_value;
helper::CopyToBuffer(buffer, position, &id);
// special case required by bpdump
const uint16_t length = sizeof(T);
helper::CopyToBuffer(buffer, position, &length);
helper::CopyToBuffer(buffer, position, &stats.Min);
++characteristicsCounter;
PutCharacteristicRecord(characteristic_value, characteristicsCounter,
stats.Min, buffer, position);
}
else
{
...
...
@@ -836,11 +831,10 @@ void BP4Serializer::PutVariableCharacteristics(
}
template <class T>
void BP4Serializer::PutVariableCharacteristics(
void BP4Serializer::PutVariableCharacteristics
InData
(
const core::Variable<T> &variable,
const typename core::Variable<T>::Info &blockInfo, const Stats<T> &stats,
std::vector<char> &buffer, size_t &position,
const bool putDimensions) noexcept
std::vector<char> &buffer, size_t &position) noexcept
{
// going back at the end
const size_t characteristicsCountPosition = position;
...
...
@@ -848,24 +842,10 @@ void BP4Serializer::PutVariableCharacteristics(
position += 5;
uint8_t characteristicsCounter = 0;
if (putDimensions)
{
// DIMENSIONS
uint8_t characteristicID = characteristic_dimensions;
helper::CopyToBuffer(buffer, position, &characteristicID);
const uint8_t dimensions = static_cast<uint8_t>(blockInfo.Count.size());
helper::CopyToBuffer(buffer, position, &dimensions); // count
const uint16_t dimensionsLength =
static_cast<uint16_t>(24 * dimensions);
helper::CopyToBuffer(buffer, position, &dimensionsLength); // length
PutDimensionsRecord(blockInfo.Count, blockInfo.Shape, blockInfo.Start,
buffer, position, true);
++characteristicsCounter;
}
// VALUE for SCALAR or STAT min, max for ARRAY
if (blockInfo.Data != nullptr)
// Write STAT min, max characteristics for an ARRAY
// VALUE variable data is not written into characteristics
// in the data file (only in metadata file in other function)
if (blockInfo.Data != nullptr && !variable.m_SingleValue)
{
PutBoundsRecord(variable.m_SingleValue, stats, characteristicsCounter,
buffer, position);
...
...
source/utils/bp4dbg/bp4dbg.py
View file @
dac5e428
...
...
@@ -22,8 +22,8 @@ def SetupArgs():
action
=
"store_true"
)
parser
.
add_argument
(
"--no-metadata"
,
"-m"
,
help
=
"Do not print metadata md.0"
,
action
=
"store_true"
)
parser
.
add_argument
(
"--no-data"
,
"-d"
,
help
=
"Do not print data data.*"
,
action
=
"store_true"
)
parser
.
add_argument
(
"--no-data"
,
"-d"
,
help
=
"Do not print data data.*"
,
action
=
"store_true"
)
args
=
parser
.
parse_args
()
# default values
...
...
source/utils/bp4dbg/bp4dbg_data.py
View file @
dac5e428
...
...
@@ -154,7 +154,7 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize,
currentPosition
=
f
.
tell
()
print
(
" Payload offset : {0}"
.
format
(
currentPosition
))
nBytes
=
np
.
ones
(
1
,
dtype
=
np
.
uint64
)
nBytes
[
0
]
=
nElements
[
0
]
*
typeSize
nBytes
[
0
]
=
nElements
*
typeSize
if
(
currentPosition
+
nBytes
[
0
]
>
varsStartPosition
+
varsTotalLength
):
print
(
"ERROR: Variable data block of size would reach beyond all "
"variable blocks"
)
...
...
@@ -168,12 +168,21 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize,
"calculated from var block length"
)
print
(
"Expected size = {0} calculated size from dimensions = {1}"
.
format
(
expectedSize
,
nBytes
[
0
]))
print
(
" Variable Data : {0} bytes"
.
format
(
nBytes
[
0
]))
# seek instead of reading for now
f
.
read
(
nBytes
[
0
])
# f.seek(nBytes[0], 1)
# data = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID),
# nElements)
if
nElements
==
1
:
# single value. read and print
value
=
readDataToNumpyArray
(
f
,
bp4dbg_utils
.
GetTypeName
(
typeID
),
nElements
)
print
(
" Payload (value) : {0} ({1} bytes)"
.
format
(
value
[
0
],
nBytes
[
0
]))
else
:
# seek instead of reading for now
f
.
read
(
nBytes
[
0
])
# f.seek(nBytes[0], 1)
# data = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID),
# nElements)
print
(
" Payload (array) : {0} bytes"
.
format
(
nBytes
[
0
]))
return
True
# Read a variable's metadata
...
...
@@ -248,8 +257,9 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
print
(
" Dims Length : {0}"
.
format
(
dimsLen
))
nElements
=
np
.
ones
(
1
,
dtype
=
np
.
uint64
)
nElements
=
np
.
uint64
(
1
)
ldims
=
np
.
zeros
(
ndims
,
dtype
=
np
.
uint64
)
isLocalValueArray
=
False
for
i
in
range
(
ndims
):
print
(
" Dim[{0}]"
.
format
(
i
))
# Read Local Dimensions (1 byte flag + 8 byte value)
...
...
@@ -278,7 +288,11 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
if
(
isDimensionVarID
==
b
'
\0
'
):
isDimensionVarID
=
b
'n'
gdim
=
np
.
fromfile
(
f
,
dtype
=
np
.
uint64
,
count
=
1
)[
0
]
print
(
" global dim : {0}"
.
format
(
gdim
))
if
i
==
0
and
ldims
[
i
]
==
0
and
gdim
==
bp4dbg_utils
.
LocalValueDim
:
print
(
" global dim : LocalValueDim ({0})"
.
format
(
gdim
))
isLocalValueArray
=
True
else
:
print
(
" global dim : {0}"
.
format
(
gdim
))
# Read Offset Dimensions (1 byte flag + 8 byte value)
# Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0'
...
...
@@ -310,9 +324,16 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
print
(
" Tag (pad {0:2d}) : {1}"
.
format
(
endTagLen
-
4
,
tag
.
decode
(
'ascii'
)))
expectedVarDataSize
=
expectedVarBlockLength
-
(
f
.
tell
()
-
startPosition
)
status
=
ReadVarData
(
f
,
nElements
,
typeID
,
ldims
,
expectedVarDataSize
,
varsStartPosition
,
varsTotalLength
)
# special case: LocalValueDim: local values turned into 1D global array
# but it seems there is no data block at all for these variables
if
isLocalValueArray
:
ldims
[
0
]
=
1
nElements
=
np
.
uint64
(
1
)
else
:
expectedVarDataSize
=
expectedVarBlockLength
-
\
(
f
.
tell
()
-
startPosition
)
status
=
ReadVarData
(
f
,
nElements
,
typeID
,
ldims
,
expectedVarDataSize
,
varsStartPosition
,
varsTotalLength
)
if
not
status
:
return
False
...
...
source/utils/bp4dbg/bp4dbg_metadata.py
View file @
dac5e428
...
...
@@ -209,8 +209,10 @@ def ReadPGMD(buf, idx, pos, limit, pgStartOffset):
# 2 bytes PG Length + Name length
pgLength
=
np
.
frombuffer
(
buf
,
dtype
=
np
.
uint16
,
count
=
1
,
offset
=
pos
)[
0
]
pos
=
pos
+
2
print
(
" PG length : {0} bytes"
.
format
(
pgLength
))
if
(
pgStartPosition
+
pgLength
>
limit
):
print
(
" PG length : {0} bytes (+2 for length)"
.
format
(
pgLength
))
if
(
pgStartPosition
+
pgLength
+
2
>
limit
):
print
(
"ERROR: There is not enough bytes {0} left in PG index block "
"to read this single PG index ({1} bytes)"
).
format
(
limit
-
pgStartPosition
,
pgLength
)
...
...
@@ -277,8 +279,9 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset):
# 4 bytes VAR Index Length
varLength
=
np
.
frombuffer
(
buf
,
dtype
=
np
.
uint32
,
count
=
1
,
offset
=
pos
)[
0
]
pos
=
pos
+
4
print
(
" Var idx length : {0} bytes"
.
format
(
varLength
))
if
(
varStartPosition
+
varLength
>
limit
):
print
(
" Var idx length : {0} bytes (+4 for idx length)"
.
format
(
varLength
))
if
(
varStartPosition
+
varLength
+
4
>
limit
):
print
(
"ERROR: There is not enough bytes in Var index block "
"to read this single Var index"
)
return
False
,
pos
...
...
@@ -344,8 +347,9 @@ def ReadAttrMD(buf, idx, pos, limit, attrStartOffset):
# 4 bytes ATTR Index Length
attrLength
=
np
.
frombuffer
(
buf
,
dtype
=
np
.
uint32
,
count
=
1
,
offset
=
pos
)[
0
]
pos
=
pos
+
4
print
(
" Attr idx length : {0} bytes"
.
format
(
attrLength
))
if
(
attrStartPosition
+
attrLength
>
limit
):
print
(
" Attr idx length : {0} bytes (+4 for idx length)"
.
format
(
attrLength
))
if
(
attrStartPosition
+
attrLength
+
4
>
limit
):
print
(
"ERROR: There is not enough bytes in Attr index block "
"to read this single Attr index"
)
return
False
,
pos
...
...
source/utils/bp4dbg/bp4dbg_utils.py
View file @
dac5e428
from
__future__
import
(
absolute_import
,
division
,
print_function
,
unicode_literals
)
LocalValueDim
=
18446744073709551613
dataTypes
=
{
-
1
:
'unknown'
,
0
:
'byte'
,
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment