Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
LEFEBVREJP email
radix
Commits
846f92da
Commit
846f92da
authored
Oct 24, 2018
by
Purves, Murray
Browse files
WIP ARLDataStrean write methods
parent
ce22f586
Pipeline
#16653
failed with stages
in 6 minutes and 17 seconds
Changes
5
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
radixio/arldatastream.cc
View file @
846f92da
...
...
@@ -74,12 +74,13 @@ void ARLDataStream::expand(const std::string& val, const ARLIndexHeader& index,
}
}
ARLDataStream
::
ARLDataStream
(
const
std
::
string
&
file
)
ARLDataStream
::
ARLDataStream
(
const
std
::
string
&
file
,
std
::
ios_base
::
openmode
mode
)
:
p
(
new
PImpl
(),
[](
PImpl
*
impl
)
{
delete
impl
;
})
{
p
->
file
=
file
;
p
->
stream
=
std
::
make_shared
<
radix
::
eafstream
>
(
p
->
file
.
c_str
(),
std
::
ios
::
in
|
std
::
ios
::
binary
);
p
->
stream
=
std
::
make_shared
<
radix
::
eafstream
>
(
p
->
file
.
c_str
(),
mode
|
std
::
ios
::
binary
);
if
(
!
p
->
stream
->
is_open
())
{
radix_line
(
"Failed to open ARL file: "
<<
file
);
...
...
@@ -98,11 +99,33 @@ bool ARLDataStream::read_index_header(ARLIndexHeader& iheader)
radix_line
(
" variable = "
<<
iheader
.
kvar
);
radix_line
(
" scaling exponent = "
<<
iheader
.
nexp
);
radix_line
(
" initial value = "
<<
iheader
.
var1
);
radix_line
(
headerString
);
radix_line
(
" header as string:
\n
"
<<
headerString
);
return
true
;
}
bool
ARLDataStream
::
write_index_header
(
const
ARLIndexHeader
&
iheader
)
{}
bool
ARLDataStream
::
write_index_header
(
const
ARLIndexHeader
&
iheader
)
{
radix_line
(
"Write index header: year = "
<<
iheader
.
year
);
radix_line
(
" month = "
<<
iheader
.
month
);
radix_line
(
" day = "
<<
iheader
.
day
);
radix_line
(
" hour = "
<<
iheader
.
hour
);
radix_line
(
" variable = "
<<
iheader
.
kvar
);
radix_line
(
" scaling exponent = "
<<
iheader
.
nexp
);
radix_line
(
" initial value = "
<<
iheader
.
var1
);
// Construct index header string
char
indexHeader
[
50
];
sprintf
(
indexHeader
,
"%2d%2d%2d%2d%2d%2d%2d%4s%4d%14.7E%14.7E"
,
iheader
.
year
,
iheader
.
month
,
iheader
.
day
,
iheader
.
hour
,
iheader
.
ic
,
iheader
.
il
,
std
::
stoi
(
iheader
.
cgrid
.
c_str
()),
iheader
.
kvar
.
c_str
(),
iheader
.
nexp
,
iheader
.
prec
,
iheader
.
var1
);
p
->
stream
->
writeString
(
std
::
string
(
indexHeader
),
50
);
radix_line
(
" header as string:
\n
"
<<
indexHeader
);
return
true
;
}
bool
ARLDataStream
::
read_next_index_header
(
ARLIndexHeader
&
iheader
)
{
...
...
@@ -129,13 +152,45 @@ bool ARLDataStream::read_record_header(const ARLIndexHeader& iheader,
radix_line
(
" ny = "
<<
rheader
.
ny
);
radix_line
(
" nz = "
<<
rheader
.
nz
);
radix_line
(
" Size of each record = "
<<
p
->
recordSize
);
radix_line
(
headerString
);
radix_line
(
" header as string:
\n
"
<<
headerString
);
return
true
;
}
bool
ARLDataStream
::
write_record_header
(
const
ARLIndexHeader
&
iheader
,
const
ARLRecordHeader
&
rheader
)
{
// Calculate and save record size
p
->
recordSize
=
(
rheader
.
nx
*
rheader
.
ny
)
+
indexHeaderLength
;
radix_line
(
" Write record header: model = "
<<
rheader
.
model_id
);
radix_line
(
" nx = "
<<
rheader
.
nx
);
radix_line
(
" ny = "
<<
rheader
.
ny
);
radix_line
(
" nz = "
<<
rheader
.
nz
);
radix_line
(
" Size of each record = "
<<
p
->
recordSize
);
char
recordHeader
[
108
];
sprintf
(
recordHeader
,
"%4s%3d%2d%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7."
"2f%3d%3d%3d%2d%4d"
,
rheader
.
model_id
.
c_str
(),
rheader
.
icx
,
rheader
.
mn
,
rheader
.
pole_lat
,
rheader
.
pole_lon
,
rheader
.
ref_lat
,
rheader
.
ref_lon
,
rheader
.
size
,
rheader
.
orient
,
rheader
.
tang_lat
,
rheader
.
sync_xp
,
rheader
.
sync_yp
,
rheader
.
sync_lat
,
rheader
.
sync_lon
,
rheader
.
dummy
,
rheader
.
nx
,
rheader
.
ny
,
rheader
.
nz
,
rheader
.
z_flag
,
rheader
.
lenh
);
p
->
stream
->
writeString
(
std
::
string
(
recordHeader
),
108
);
// Skip enough bytes to get to the next index header
int
bytesToSkip
=
roundUpInt
(
p
->
stream
->
bytesWritten
(),
p
->
recordSize
)
-
p
->
stream
->
bytesWritten
();
radix_line
(
" Write complete: skipping "
<<
bytesToSkip
<<
" bytes to the next index header"
);
p
->
stream
->
writeString
(
""
,
bytesToSkip
);
radix_line
(
" header as string:
\n
"
<<
recordHeader
);
return
true
;
}
bool
ARLDataStream
::
read_record
(
const
ARLIndexHeader
&
iheader
,
...
...
@@ -161,7 +216,6 @@ bool ARLDataStream::read_record(const ARLIndexHeader& iheader,
// Read the raw value
unsigned
char
ch
=
(
unsigned
char
)
p
->
stream
->
readChar
();
int
packedValue
=
ch
-
127
;
// int packedValue = p->stream->readChar() - 127;
// Calculate the unpacked value
double
unpackedValue
=
0.0
;
...
...
@@ -197,7 +251,54 @@ bool ARLDataStream::write_record(
const
ARLIndexHeader
&
iheader
,
const
ARLRecordHeader
&
rheader
,
const
std
::
vector
<
std
::
vector
<
double
>
>&
record
)
{
return
false
;
if
(
record
.
size
()
==
0
)
{
radix_line
(
"No data here - returning false"
);
return
false
;
}
// Calculate the scaling factor
double
scaleFactor
=
pow
(
2.0
,
7.0
-
(
double
)
iheader
.
nexp
),
lastValue
=
0.0
;
radix_line
(
" Writing record data:"
);
for
(
int
y
=
0
;
y
<
rheader
.
ny
;
++
y
)
{
for
(
int
x
=
0
;
x
<
rheader
.
nx
;
++
x
)
{
// Get the correct 'last value' if we are at a 0 index
if
(
x
==
0
)
{
if
(
y
==
0
)
{
lastValue
=
iheader
.
var1
;
}
else
{
lastValue
=
record
[
x
][
y
-
1
];
}
}
// Calculate the packed value
double
unpackedValue
=
record
[
x
][
y
];
int
packedInt
=
(
int
)((
unpackedValue
-
lastValue
)
*
scaleFactor
);
unsigned
char
packedValue
=
(
unsigned
char
)(
packedInt
+
127
);
p
->
stream
->
writeChar
(
packedValue
);
lastValue
=
unpackedValue
;
if
((
x
<
2
&&
y
<
2
)
||
((
rheader
.
nx
-
x
<
3
)
&&
(
rheader
.
ny
-
y
<
3
)))
{
radix_line
(
" ["
<<
x
<<
","
<<
y
<<
"]: packed = "
<<
packedInt
<<
", unpacked = "
<<
unpackedValue
);
}
}
}
return
true
;
}
void
ARLDataStream
::
close_stream
()
{
p
->
stream
->
close
();
}
}
// namespace radix
radixio/arldatastream.hh
View file @
846f92da
...
...
@@ -47,7 +47,7 @@ class RADIX_PUBLIC ARLDataStream
public:
ARLDataStream
()
=
delete
;
ARLDataStream
(
const
std
::
string
&
file
);
ARLDataStream
(
const
std
::
string
&
file
,
std
::
ios_base
::
openmode
mode
);
/**
* @brief read_index_header Reads a single index header from the stream
...
...
@@ -84,6 +84,8 @@ class RADIX_PUBLIC ARLDataStream
const
ARLRecordHeader
&
rheader
,
const
std
::
vector
<
std
::
vector
<
double
>
>&
record
);
void
close_stream
();
};
// class ARLDataStream
struct
RADIX_PUBLIC
ARLRecordHeader
...
...
radixio/eafstream.cc
View file @
846f92da
...
...
@@ -639,6 +639,21 @@ char eafstream::readChar()
return
ch
;
}
void
eafstream
::
writeChar
(
const
char
&
var
)
{
this
->
write
(
&
var
,
1
);
if
(
this
->
bad
()
||
this
->
eof
())
{
std
::
stringstream
ss
;
ss
<<
"Error, cannot write the char to file"
<<
((
this
->
bad
()
!=
false
)
?
"- this->bad()"
:
"- this->eof()"
);
std
::
cerr
<<
ss
.
str
()
<<
std
::
endl
;
throw
std
::
logic_error
(
ss
.
str
());
}
mBytesWritten
++
;
}
void
eafstream
::
writeString
(
const
std
::
string
&
var
,
size_t
length
,
char
filler
)
{
std
::
string
copy
(
var
);
...
...
radixio/eafstream.hh
View file @
846f92da
...
...
@@ -103,6 +103,7 @@ class RADIX_PUBLIC eafstream : public std::fstream
void
writeString
(
const
std
::
string
&
var
,
size_t
length
,
char
filler
=
' '
);
char
readChar
();
void
writeChar
(
const
char
&
var
);
/**
* \brief Returns the number of bytes read since the last header marker
...
...
radixio/tests/tstArlDataStream.cc
View file @
846f92da
...
...
@@ -11,7 +11,7 @@ TEST(RadixIO, ReadArlElements)
double
tolerance
=
0.001
;
std
::
string
testFile
=
"/home/ohp/data/met/RP195111.gbl"
;
ARLDataStream
testStream
(
testFile
);
ARLDataStream
testStream
(
testFile
,
std
::
ios
::
in
);
// Read the initial index header
ARLIndexHeader
testTopIndexHeader
;
...
...
@@ -83,35 +83,76 @@ TEST(RadixIO, ReadArlElements)
EXPECT_NEAR
(
236.1
,
testRecord
[
143
][
71
],
236.1
*
tolerance
);
EXPECT_NEAR
(
240.6
,
testRecord
[
142
][
72
],
240.6
*
tolerance
);
EXPECT_NEAR
(
240.6
,
testRecord
[
143
][
72
],
240.6
*
tolerance
);
testStream
.
close_stream
();
}
TEST
(
RadixIO
,
WriteArlElements
)
{
double
tolerance
=
0.001
;
std
::
string
testFile
=
"/home/ohp/data/met/RP195111.gbl"
;
ARLDataStream
testStream
(
testFile
);
std
::
string
testReadFile
=
"/home/ohp/data/met/RP195111.gbl"
;
ARLDataStream
setupStream
(
testReadFile
,
std
::
ios
::
in
);
// Read the initial data
ARLIndexHeader
setupTopIndexHeader
;
bool
success
=
setupStream
.
read_index_header
(
setupTopIndexHeader
);
ASSERT_TRUE
(
success
);
ARLRecordHeader
setupRecordHeader
;
success
=
setupStream
.
read_record_header
(
setupTopIndexHeader
,
setupRecordHeader
);
ASSERT_TRUE
(
success
);
ARLIndexHeader
setupIndexHeader
;
success
=
setupStream
.
read_next_index_header
(
setupIndexHeader
);
ASSERT_TRUE
(
success
);
std
::
vector
<
std
::
vector
<
double
>>
setupRecord
;
success
=
setupStream
.
read_record
(
setupIndexHeader
,
setupRecordHeader
,
setupRecord
);
ASSERT_TRUE
(
success
);
setupStream
.
close_stream
();
// Read the initial index header
ARLIndexHeader
testTopIndexHeader
;
bool
success
=
testStream
.
read_index_header
(
testTopIndexHeader
);
// Write this data back out to a file
std
::
string
testWriteFile
=
"test-out.gbl"
;
ARLDataStream
testWriteStream
(
testWriteFile
,
std
::
ios
::
out
);
success
=
testWriteStream
.
write_index_header
(
setupTopIndexHeader
);
ASSERT_TRUE
(
success
);
success
=
testWriteStream
.
write_record_header
(
setupTopIndexHeader
,
setupRecordHeader
);
ASSERT_TRUE
(
success
);
success
=
testWriteStream
.
write_index_header
(
setupIndexHeader
);
ASSERT_TRUE
(
success
);
success
=
testWriteStream
.
write_record
(
setupIndexHeader
,
setupRecordHeader
,
setupRecord
);
ASSERT_TRUE
(
success
);
testWriteStream
.
close_stream
();
// Read the file again and compare with original
ARLDataStream
testReadStream
(
testWriteFile
,
std
::
ios
::
in
);
ARLIndexHeader
testTopIndexHeader
;
success
=
testReadStream
.
read_index_header
(
testTopIndexHeader
);
ASSERT_TRUE
(
success
);
ARLRecordHeader
testRecordHeader
;
success
=
testStream
.
read_record_header
(
testTopIndexHeader
,
testRecordHeader
);
success
=
testReadStream
.
read_record_header
(
testTopIndexHeader
,
testRecordHeader
);
ASSERT_TRUE
(
success
);
// Read the data starting from the next index header
ARLIndexHeader
testIndexHeader
;
success
=
testStream
.
read_next_index_header
(
testIndexHeader
);
success
=
test
Read
Stream
.
read_next_index_header
(
testIndexHeader
);
ASSERT_TRUE
(
success
);
// Read data
std
::
vector
<
std
::
vector
<
double
>>
testRecord
;
success
=
testStream
.
read_record
(
testIndexHeader
,
testRecordHeader
,
testRecord
);
test
Read
Stream
.
read_record
(
testIndexHeader
,
testRecordHeader
,
testRecord
);
ASSERT_TRUE
(
success
);
// Write this data back out to a file
ASSERT_EQ
(
setupRecord
.
size
(),
testRecord
.
size
());
ASSERT_EQ
(
setupRecord
[
0
].
size
(),
testRecord
[
0
].
size
());
for
(
int
x
=
0
;
x
<
testRecord
.
size
();
++
x
)
{
for
(
int
y
=
0
;
y
<
testRecord
[
0
].
size
();
++
y
)
{
// EXPECT_NEAR(setupRecord[x][y], testRecord[x][y],
// setupRecord[x][y] * tolerance);
}
}
testReadStream
.
close_stream
();
}
// TEST(RadixIO, ReadArlWholeFile)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment