Commit 846f92da authored by Purves, Murray's avatar Purves, Murray
Browse files

WIP ARLDataStrean write methods

parent ce22f586
Pipeline #16653 failed with stages
in 6 minutes and 17 seconds
......@@ -74,12 +74,13 @@ void ARLDataStream::expand(const std::string& val, const ARLIndexHeader& index,
}
}
ARLDataStream::ARLDataStream(const std::string& file)
ARLDataStream::ARLDataStream(const std::string& file,
std::ios_base::openmode mode)
: p(new PImpl(), [](PImpl* impl) { delete impl; })
{
p->file = file;
p->stream = std::make_shared<radix::eafstream>(
p->file.c_str(), std::ios::in | std::ios::binary);
p->stream = std::make_shared<radix::eafstream>(p->file.c_str(),
mode | std::ios::binary);
if (!p->stream->is_open())
{
radix_line("Failed to open ARL file: " << file);
......@@ -98,11 +99,33 @@ bool ARLDataStream::read_index_header(ARLIndexHeader& iheader)
radix_line(" variable = " << iheader.kvar);
radix_line(" scaling exponent = " << iheader.nexp);
radix_line(" initial value = " << iheader.var1);
radix_line(headerString);
radix_line(" header as string:\n " << headerString);
return true;
}
bool ARLDataStream::write_index_header(const ARLIndexHeader& iheader) {}
bool ARLDataStream::write_index_header(const ARLIndexHeader& iheader)
{
radix_line("Write index header: year = " << iheader.year);
radix_line(" month = " << iheader.month);
radix_line(" day = " << iheader.day);
radix_line(" hour = " << iheader.hour);
radix_line(" variable = " << iheader.kvar);
radix_line(" scaling exponent = " << iheader.nexp);
radix_line(" initial value = " << iheader.var1);
// Construct index header string
char indexHeader[50];
sprintf(indexHeader, "%2d%2d%2d%2d%2d%2d%2d%4s%4d%14.7E%14.7E", iheader.year,
iheader.month, iheader.day, iheader.hour, iheader.ic, iheader.il,
std::stoi(iheader.cgrid.c_str()), iheader.kvar.c_str(), iheader.nexp,
iheader.prec, iheader.var1);
p->stream->writeString(std::string(indexHeader), 50);
radix_line(" header as string:\n " << indexHeader);
return true;
}
bool ARLDataStream::read_next_index_header(ARLIndexHeader& iheader)
{
......@@ -129,13 +152,45 @@ bool ARLDataStream::read_record_header(const ARLIndexHeader& iheader,
radix_line(" ny = " << rheader.ny);
radix_line(" nz = " << rheader.nz);
radix_line(" Size of each record = " << p->recordSize);
radix_line(headerString);
radix_line(" header as string:\n " << headerString);
return true;
}
bool ARLDataStream::write_record_header(const ARLIndexHeader& iheader,
const ARLRecordHeader& rheader)
{
// Calculate and save record size
p->recordSize = (rheader.nx * rheader.ny) + indexHeaderLength;
radix_line(" Write record header: model = " << rheader.model_id);
radix_line(" nx = " << rheader.nx);
radix_line(" ny = " << rheader.ny);
radix_line(" nz = " << rheader.nz);
radix_line(" Size of each record = " << p->recordSize);
char recordHeader[108];
sprintf(recordHeader,
"%4s%3d%2d%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7.2f%7."
"2f%3d%3d%3d%2d%4d",
rheader.model_id.c_str(), rheader.icx, rheader.mn, rheader.pole_lat,
rheader.pole_lon, rheader.ref_lat, rheader.ref_lon, rheader.size,
rheader.orient, rheader.tang_lat, rheader.sync_xp, rheader.sync_yp,
rheader.sync_lat, rheader.sync_lon, rheader.dummy, rheader.nx,
rheader.ny, rheader.nz, rheader.z_flag, rheader.lenh);
p->stream->writeString(std::string(recordHeader), 108);
// Skip enough bytes to get to the next index header
int bytesToSkip = roundUpInt(p->stream->bytesWritten(), p->recordSize) -
p->stream->bytesWritten();
radix_line(" Write complete: skipping "
<< bytesToSkip << " bytes to the next index header");
p->stream->writeString("", bytesToSkip);
radix_line(" header as string:\n " << recordHeader);
return true;
}
bool ARLDataStream::read_record(const ARLIndexHeader& iheader,
......@@ -161,7 +216,6 @@ bool ARLDataStream::read_record(const ARLIndexHeader& iheader,
// Read the raw value
unsigned char ch = (unsigned char)p->stream->readChar();
int packedValue = ch - 127;
// int packedValue = p->stream->readChar() - 127;
// Calculate the unpacked value
double unpackedValue = 0.0;
......@@ -197,7 +251,54 @@ bool ARLDataStream::write_record(
const ARLIndexHeader& iheader, const ARLRecordHeader& rheader,
const std::vector<std::vector<double> >& record)
{
return false;
if (record.size() == 0)
{
radix_line("No data here - returning false");
return false;
}
// Calculate the scaling factor
double scaleFactor = pow(2.0, 7.0 - (double)iheader.nexp), lastValue = 0.0;
radix_line(" Writing record data:");
for (int y = 0; y < rheader.ny; ++y)
{
for (int x = 0; x < rheader.nx; ++x)
{
// Get the correct 'last value' if we are at a 0 index
if (x == 0)
{
if (y == 0)
{
lastValue = iheader.var1;
}
else
{
lastValue = record[x][y - 1];
}
}
// Calculate the packed value
double unpackedValue = record[x][y];
int packedInt = (int)((unpackedValue - lastValue) * scaleFactor);
unsigned char packedValue = (unsigned char)(packedInt + 127);
p->stream->writeChar(packedValue);
lastValue = unpackedValue;
if ((x < 2 && y < 2) || ((rheader.nx - x < 3) && (rheader.ny - y < 3)))
{
radix_line(" [" << x << "," << y << "]: packed = " << packedInt
<< ", unpacked = " << unpackedValue);
}
}
}
return true;
}
void ARLDataStream::close_stream() { p->stream->close(); }
} // namespace radix
......@@ -47,7 +47,7 @@ class RADIX_PUBLIC ARLDataStream
public:
ARLDataStream() = delete;
ARLDataStream(const std::string& file);
ARLDataStream(const std::string& file, std::ios_base::openmode mode);
/**
* @brief read_index_header Reads a single index header from the stream
......@@ -84,6 +84,8 @@ class RADIX_PUBLIC ARLDataStream
const ARLRecordHeader& rheader,
const std::vector<std::vector<double> >& record);
void close_stream();
}; // class ARLDataStream
struct RADIX_PUBLIC ARLRecordHeader
......
......@@ -639,6 +639,21 @@ char eafstream::readChar()
return ch;
}
void eafstream::writeChar(const char &var)
{
this->write(&var, 1);
if (this->bad() || this->eof())
{
std::stringstream ss;
ss << "Error, cannot write the char to file"
<< ((this->bad() != false) ? "- this->bad()" : "- this->eof()");
std::cerr << ss.str() << std::endl;
throw std::logic_error(ss.str());
}
mBytesWritten++;
}
void eafstream::writeString(const std::string &var, size_t length, char filler)
{
std::string copy(var);
......
......@@ -103,6 +103,7 @@ class RADIX_PUBLIC eafstream : public std::fstream
void writeString(const std::string &var, size_t length, char filler = ' ');
char readChar();
void writeChar(const char &var);
/**
* \brief Returns the number of bytes read since the last header marker
......
......@@ -11,7 +11,7 @@ TEST(RadixIO, ReadArlElements)
double tolerance = 0.001;
std::string testFile = "/home/ohp/data/met/RP195111.gbl";
ARLDataStream testStream(testFile);
ARLDataStream testStream(testFile, std::ios::in);
// Read the initial index header
ARLIndexHeader testTopIndexHeader;
......@@ -83,35 +83,76 @@ TEST(RadixIO, ReadArlElements)
EXPECT_NEAR(236.1, testRecord[143][71], 236.1 * tolerance);
EXPECT_NEAR(240.6, testRecord[142][72], 240.6 * tolerance);
EXPECT_NEAR(240.6, testRecord[143][72], 240.6 * tolerance);
testStream.close_stream();
}
TEST(RadixIO, WriteArlElements)
{
double tolerance = 0.001;
std::string testFile = "/home/ohp/data/met/RP195111.gbl";
ARLDataStream testStream(testFile);
std::string testReadFile = "/home/ohp/data/met/RP195111.gbl";
ARLDataStream setupStream(testReadFile, std::ios::in);
// Read the initial data
ARLIndexHeader setupTopIndexHeader;
bool success = setupStream.read_index_header(setupTopIndexHeader);
ASSERT_TRUE(success);
ARLRecordHeader setupRecordHeader;
success =
setupStream.read_record_header(setupTopIndexHeader, setupRecordHeader);
ASSERT_TRUE(success);
ARLIndexHeader setupIndexHeader;
success = setupStream.read_next_index_header(setupIndexHeader);
ASSERT_TRUE(success);
std::vector<std::vector<double>> setupRecord;
success =
setupStream.read_record(setupIndexHeader, setupRecordHeader, setupRecord);
ASSERT_TRUE(success);
setupStream.close_stream();
// Read the initial index header
ARLIndexHeader testTopIndexHeader;
bool success = testStream.read_index_header(testTopIndexHeader);
// Write this data back out to a file
std::string testWriteFile = "test-out.gbl";
ARLDataStream testWriteStream(testWriteFile, std::ios::out);
success = testWriteStream.write_index_header(setupTopIndexHeader);
ASSERT_TRUE(success);
success = testWriteStream.write_record_header(setupTopIndexHeader,
setupRecordHeader);
ASSERT_TRUE(success);
success = testWriteStream.write_index_header(setupIndexHeader);
ASSERT_TRUE(success);
success = testWriteStream.write_record(setupIndexHeader, setupRecordHeader,
setupRecord);
ASSERT_TRUE(success);
testWriteStream.close_stream();
// Read the file again and compare with original
ARLDataStream testReadStream(testWriteFile, std::ios::in);
ARLIndexHeader testTopIndexHeader;
success = testReadStream.read_index_header(testTopIndexHeader);
ASSERT_TRUE(success);
ARLRecordHeader testRecordHeader;
success = testStream.read_record_header(testTopIndexHeader, testRecordHeader);
success =
testReadStream.read_record_header(testTopIndexHeader, testRecordHeader);
ASSERT_TRUE(success);
// Read the data starting from the next index header
ARLIndexHeader testIndexHeader;
success = testStream.read_next_index_header(testIndexHeader);
success = testReadStream.read_next_index_header(testIndexHeader);
ASSERT_TRUE(success);
// Read data
std::vector<std::vector<double>> testRecord;
success =
testStream.read_record(testIndexHeader, testRecordHeader, testRecord);
testReadStream.read_record(testIndexHeader, testRecordHeader, testRecord);
ASSERT_TRUE(success);
// Write this data back out to a file
ASSERT_EQ(setupRecord.size(), testRecord.size());
ASSERT_EQ(setupRecord[0].size(), testRecord[0].size());
for (int x = 0; x < testRecord.size(); ++x)
{
for (int y = 0; y < testRecord[0].size(); ++y)
{
// EXPECT_NEAR(setupRecord[x][y], testRecord[x][y],
// setupRecord[x][y] * tolerance);
}
}
testReadStream.close_stream();
}
// TEST(RadixIO, ReadArlWholeFile)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment