LoadANSTOHelper.cpp 15.1 KB
Newer Older
1
2
3
// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
4
5
//   NScD Oak Ridge National Laboratory, European Spallation Source,
//   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
6
// SPDX - License - Identifier: GPL - 3.0 +
7
#include "MantidDataHandling/LoadANSTOHelper.h"
8
9
#include "MantidAPI/FileProperty.h"
#include "MantidAPI/RegisterFileLoader.h"
LamarMoore's avatar
LamarMoore committed
10
#include "MantidDataObjects/EventWorkspace.h"
11
12
13
#include "MantidGeometry/Instrument.h"
#include "MantidGeometry/Instrument/RectangularDetector.h"
#include "MantidGeometry/Objects/ShapeFactory.h"
LamarMoore's avatar
LamarMoore committed
14
#include "MantidKernel/UnitFactory.h"
15
16
#include "MantidNexus/NexusClasses.h"

17
18
#include <numeric>

19
namespace Mantid::DataHandling::ANSTO {
Owen Arnold's avatar
Owen Arnold committed
20

21
// ProgressTracker
Samuel Jones's avatar
Samuel Jones committed
22
23
ProgressTracker::ProgressTracker(API::Progress &progBar, const char *msg, int64_t target, size_t count)
    : m_msg(msg), m_count(count), m_step(target / count), m_next(m_step), m_progBar(progBar) {
24
25
26

  m_progBar.doReport(m_msg);
}
27
ProgressTracker::~ProgressTracker() { complete(); }
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
void ProgressTracker::update(int64_t position) {
  while (m_next <= position) {
    m_progBar.report(m_msg);

    switch (m_count) {
    case 0:
      return;

    case 1:
      m_count = 0;
      m_next = std::numeric_limits<int64_t>::max();
      return;

    default:
      m_count--;
      m_next += m_step;
    }
  }
}
void ProgressTracker::complete() {
  if (m_count != 0) {
    m_progBar.reportIncrement(m_count, m_msg);
    m_count = 0;
  }
}

Owen Arnold's avatar
Owen Arnold committed
54
// EventProcessor
Samuel Jones's avatar
Samuel Jones committed
55
56
57
58
59
EventProcessor::EventProcessor(const std::vector<bool> &roi, const size_t stride, const double period,
                               const double phase, const int64_t startTime, const double tofMinBoundary,
                               const double tofMaxBoundary, const double timeMinBoundary, const double timeMaxBoundary)
    : m_roi(roi), m_stride(stride), m_frames(0), m_framesValid(0), m_startTime(startTime), m_period(period),
      m_phase(phase), m_tofMinBoundary(tofMinBoundary), m_tofMaxBoundary(tofMaxBoundary),
60
      m_timeMinBoundary(timeMinBoundary), m_timeMaxBoundary(timeMaxBoundary) {}
61
62
bool EventProcessor::validFrame() const {
  // frame boundary
Samuel Jones's avatar
Samuel Jones committed
63
  double frameTime = (static_cast<double>(m_frames) * m_period) * 1e-6; // in seconds
64

mantid-builder's avatar
mantid-builder committed
65
  return (frameTime >= m_timeMinBoundary) && (frameTime <= m_timeMaxBoundary);
66
67
68
69
70
71
}
void EventProcessor::newFrame() {
  m_frames++;
  if (validFrame())
    m_framesValid++;
}
Owen Arnold's avatar
Owen Arnold committed
72
73
void EventProcessor::addEvent(size_t x, size_t y, double tof) {
  // tof correction
74
75
76
77
78
79
80
81
  if (m_period > 0.0) {
    tof += m_phase;
    while (tof > m_period)
      tof -= m_period;
    while (tof < 0)
      tof += m_period;
  }

Owen Arnold's avatar
Owen Arnold committed
82
  // check if event is in valid range
83
  if (!validFrame())
84
    return;
85

86
87
88
89
90
91
92
93
94
95
96
97
  // ToF boundary
  if ((tof < m_tofMinBoundary) && (tof > m_tofMaxBoundary))
    return;

  // detector id
  size_t id = m_stride * x + y;

  // image size
  if ((y >= m_stride) || (id >= m_roi.size()))
    return;

  // check if neutron is in region of intreset
98
99
  if (m_roi[id]) {
    // absolute pulse time in nanoseconds
100
101
    auto frames = static_cast<double>(m_frames);
    auto frameTime = static_cast<int64_t>(m_period * frames * 1.0e3);
102
103
104
105
    int64_t pulse = m_startTime + frameTime;

    addEventImpl(id, pulse, tof);
  }
106
107
}

Owen Arnold's avatar
Owen Arnold committed
108
// EventCounter
Samuel Jones's avatar
Samuel Jones committed
109
110
111
112
113
EventCounter::EventCounter(const std::vector<bool> &roi, const size_t stride, const double period, const double phase,
                           const int64_t startTime, const double tofMinBoundary, const double tofMaxBoundary,
                           const double timeMinBoundary, const double timeMaxBoundary, std::vector<size_t> &eventCounts)
    : EventProcessor(roi, stride, period, phase, startTime, tofMinBoundary, tofMaxBoundary, timeMinBoundary,
                     timeMaxBoundary),
114
115
      m_eventCounts(eventCounts), m_tofMin(std::numeric_limits<double>::max()),
      m_tofMax(std::numeric_limits<double>::min()) {}
116
size_t EventCounter::numFrames() const { return m_framesValid; }
Samuel Jones's avatar
Samuel Jones committed
117
118
double EventCounter::tofMin() const { return m_tofMin <= m_tofMax ? m_tofMin : 0.0; }
double EventCounter::tofMax() const { return m_tofMin <= m_tofMax ? m_tofMax : 0.0; }
119
120
void EventCounter::addEventImpl(size_t id, int64_t pulse, double tof) {
  UNUSED_ARG(pulse);
Owen Arnold's avatar
Owen Arnold committed
121
122
123
124
125
126
127
  if (m_tofMin > tof)
    m_tofMin = tof;
  if (m_tofMax < tof)
    m_tofMax = tof;

  m_eventCounts[id]++;
}
128

Owen Arnold's avatar
Owen Arnold committed
129
// EventAssigner
Samuel Jones's avatar
Samuel Jones committed
130
131
132
133
134
135
EventAssigner::EventAssigner(const std::vector<bool> &roi, const size_t stride, const double period, const double phase,
                             const int64_t startTime, const double tofMinBoundary, const double tofMaxBoundary,
                             const double timeMinBoundary, const double timeMaxBoundary,
                             std::vector<EventVector_pt> &eventVectors)
    : EventProcessor(roi, stride, period, phase, startTime, tofMinBoundary, tofMaxBoundary, timeMinBoundary,
                     timeMaxBoundary),
136
      m_eventVectors(eventVectors) {}
137
138
void EventAssigner::addEventImpl(size_t id, int64_t pulse, double tof) {
  m_eventVectors[id]->emplace_back(tof, Types::Core::DateAndTime(pulse));
139
140
}

141
// EventAssignerFixedWavelength
Samuel Jones's avatar
Samuel Jones committed
142
143
144
145
146
147
148
149
EventAssignerFixedWavelength::EventAssignerFixedWavelength(const std::vector<bool> &roi, const size_t stride,
                                                           const double wavelength, const double period,
                                                           const double phase, const int64_t startTime,
                                                           const double tofMinBoundary, const double tofMaxBoundary,
                                                           const double timeMinBoundary, const double timeMaxBoundary,
                                                           std::vector<EventVector_pt> &eventVectors)
    : EventAssigner(roi, stride, period, phase, startTime, tofMinBoundary, tofMaxBoundary, timeMinBoundary,
                    timeMaxBoundary, eventVectors),
mantid-builder's avatar
mantid-builder committed
150
      m_wavelength(wavelength) {}
Samuel Jones's avatar
Samuel Jones committed
151
void EventAssignerFixedWavelength::addEventImpl(size_t id, int64_t pulse, double tof) {
152
  UNUSED_ARG(pulse);
153
  UNUSED_ARG(tof);
154
  m_eventVectors[id]->emplace_back(m_wavelength);
155
156
}

157
// ISISRawOnlyFile
Hahn, Steven's avatar
Hahn, Steven committed
158
#ifdef _WIN32
159
FastReadOnlyFile::FastReadOnlyFile(const char *filename) {
Samuel Jones's avatar
Samuel Jones committed
160
  m_handle = CreateFileA(filename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
161
}
162
163
FastReadOnlyFile::~FastReadOnlyFile() { close(); }
void *FastReadOnlyFile::handle() const { return m_handle; }
164
void FastReadOnlyFile::close() {
165
166
167
168
169
  CloseHandle(m_handle);
  m_handle = NULL;
}
bool FastReadOnlyFile::read(void *buffer, uint32_t size) {
  DWORD bytesRead;
Samuel Jones's avatar
Samuel Jones committed
170
  return (FALSE != ReadFile(m_handle, buffer, size, &bytesRead, NULL)) && (bytesRead == size);
171
172
}
bool FastReadOnlyFile::seek(int64_t offset, int whence, int64_t *newPosition) {
Samuel Jones's avatar
Samuel Jones committed
173
  return FALSE != SetFilePointerEx(m_handle, *(LARGE_INTEGER *)&offset, (LARGE_INTEGER *)newPosition, whence);
174
}
175
#else
Samuel Jones's avatar
Samuel Jones committed
176
FastReadOnlyFile::FastReadOnlyFile(const char *filename) { m_handle = fopen(filename, "rb"); }
177
178
FastReadOnlyFile::~FastReadOnlyFile() { close(); }
void *FastReadOnlyFile::handle() const { return m_handle; }
179
void FastReadOnlyFile::close() {
180
  fclose(m_handle);
181
  m_handle = nullptr;
182
183
}
bool FastReadOnlyFile::read(void *buffer, uint32_t size) {
184
  return 1 == fread(buffer, static_cast<size_t>(size), 1, m_handle);
185
186
187
}
bool FastReadOnlyFile::seek(int64_t offset, int whence, int64_t *newPosition) {
  return (0 == fseek(m_handle, offset, whence)) &&
Samuel Jones's avatar
Samuel Jones committed
188
         ((newPosition == nullptr) || (0 <= (*newPosition = static_cast<int64_t>(ftell(m_handle)))));
189
}
190
#endif
191
192

namespace Tar {
193

194
195
void EntryHeader::writeChecksum() {
  memset(Checksum, ' ', sizeof(Checksum));
Samuel Jones's avatar
Samuel Jones committed
196
  size_t value = std::accumulate((const char *)this, (const char *)this + sizeof(EntryHeader), (size_t)0);
197

198
199
  std::ostringstream buffer;

Samuel Jones's avatar
Samuel Jones committed
200
  buffer << std::oct << std::setfill('0') << std::setw(static_cast<int>(sizeof(Checksum)) - 1) << value;
201
202
203
204
205
206
207
208
  std::string string = buffer.str();

  std::copy(string.cbegin(), string.cend(), Checksum);
  Checksum[string.size()] = 0;
}
void EntryHeader::writeFileSize(int64_t value) {
  std::ostringstream buffer;

Samuel Jones's avatar
Samuel Jones committed
209
  buffer << std::oct << std::setfill('0') << std::setw(static_cast<int>(sizeof(FileSize)) - 1) << value;
210
  std::string string = buffer.str();
211

212
213
214
215
  std::copy(string.cbegin(), string.cend(), FileSize);
  FileSize[string.size()] = 0;
}
int64_t EntryHeader::readFileSize() {
216
  int64_t result = 0;
217
218
  const char *p = FileSize;
  for (size_t n = sizeof(FileSize) - 1; n != 0; --n) { // last character is '\0'
219
220
221
222
223
224
225
226
227
    char c = *p++;
    if (('0' <= c) && (c <= '9'))
      result = result * 8 + (c - '0');
  }
  return result;
}

// construction
File::File(const std::string &path)
Samuel Jones's avatar
Samuel Jones committed
228
229
    : m_good(true), m_file(path.c_str()), m_selected(static_cast<size_t>(-1)), m_position(0), m_size(0),
      m_bufferPosition(0), m_bufferAvailable(0) {
230

231
  m_good = m_file.handle() != nullptr;
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
  while (m_good) {
    EntryHeader header;
    int64_t position;

    m_good &= m_file.read(&header, sizeof(EntryHeader));
    m_good &= m_file.seek(512 - sizeof(EntryHeader), SEEK_CUR, &position);
    if (!m_good)
      break;

    std::string fileName(header.FileName);
    if (fileName.length() == 0)
      return;

    FileInfo fileInfo;
    fileInfo.Offset = position;
247
    fileInfo.Size = header.readFileSize();
248
249

    if (header.TypeFlag == TarTypeFlag_NormalFile) {
250
251
      m_fileNames.emplace_back(fileName);
      m_fileInfos.emplace_back(fileInfo);
252
    }
253

254
    auto offset = static_cast<size_t>(fileInfo.Size % 512);
255
256
257
258
259
260
    if (offset != 0)
      offset = 512 - offset;

    m_good &= m_file.seek(fileInfo.Size + offset, SEEK_CUR);
  }
}
261
262
263
264
265
void File::close() {
  m_good = false;
  m_file.close();
  m_fileNames.clear();
  m_fileInfos.clear();
266
  m_selected = static_cast<size_t>(-1);
267
268
269
270
271
  m_position = 0;
  m_size = 0;
  m_bufferPosition = 0;
  m_bufferAvailable = 0;
}
272
273

// properties
274
275
bool File::good() const { return m_good; }
const std::vector<std::string> &File::files() const { return m_fileNames; }
Samuel Jones's avatar
Samuel Jones committed
276
const std::string &File::selected_name() const { return m_fileNames[m_selected]; }
277
278
int64_t File::selected_position() const { return m_position; }
int64_t File::selected_size() const { return m_size; }
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299

// methods
bool File::select(const char *file) {
  if (!m_good)
    return false;

  // reset buffer
  m_bufferPosition = 0;
  m_bufferAvailable = 0;

  for (size_t i = 0; i != m_fileNames.size(); i++)
    if (m_fileNames[i] == file) {
      const FileInfo &info = m_fileInfos[i];

      m_selected = i;
      m_position = 0;
      m_size = info.Size;

      return m_good &= m_file.seek(info.Offset, SEEK_SET);
    }

300
  m_selected = static_cast<size_t>(-1);
301
302
303
304
305
  m_position = 0;
  m_size = 0;
  return false;
}
bool File::skip(uint64_t offset) {
306
  if (!m_good || (m_selected == static_cast<size_t>(-1)))
307
308
    return false;

309
  bool overrun = offset > static_cast<uint64_t>(m_size - m_position);
310
311
312
313
314
  if (overrun)
    offset = m_size - m_position;

  m_position += offset;

315
  uint64_t bufferPosition = static_cast<uint64_t>(m_bufferPosition) + offset;
316
  if (bufferPosition <= m_bufferAvailable)
317
    m_bufferPosition = static_cast<size_t>(bufferPosition);
318
319
320
321
322
323
324
325
326
327
  else {
    m_good &= m_file.seek(bufferPosition - m_bufferAvailable, SEEK_CUR);

    m_bufferPosition = 0;
    m_bufferAvailable = 0;
  }

  return m_good && !overrun;
}
size_t File::read(void *dst, size_t size) {
328
  if (!m_good || (m_selected == static_cast<size_t>(-1)))
329
330
    return 0;

331
332
  if (static_cast<int64_t>(size) > (m_size - m_position))
    size = static_cast<size_t>(m_size - m_position);
333

334
  auto ptr = reinterpret_cast<uint8_t *>(dst);
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
  size_t result = 0;

  if (m_bufferPosition != m_bufferAvailable) {
    result = m_bufferAvailable - m_bufferPosition;
    if (result > size)
      result = size;

    memcpy(ptr, m_buffer, result);
    ptr += result;

    size -= result;
    m_position += result;
    m_bufferPosition += result;
  }

  while (size != 0) {
Samuel Jones's avatar
Samuel Jones committed
351
    auto bytesToRead = static_cast<uint32_t>(std::min<size_t>(size, std::numeric_limits<uint32_t>::max()));
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366

    m_good &= m_file.read(ptr, bytesToRead);
    if (!m_good)
      break;

    ptr += bytesToRead;

    size -= bytesToRead;
    result += bytesToRead;
    m_position += bytesToRead;
  }

  return result;
}
int File::read_byte() {
367
  if (!m_good || (m_selected == static_cast<size_t>(-1)))
368
369
370
371
372
373
374
375
376
    return -1;

  if (m_bufferPosition == m_bufferAvailable) {
    if (m_position >= m_size)
      return -1;

    m_bufferPosition = 0;
    m_bufferAvailable = 0;

Samuel Jones's avatar
Samuel Jones committed
377
    uint32_t size = static_cast<uint32_t>(std::min<int64_t>(sizeof(m_buffer), m_size - m_position));
378
379
380
381
382
383
384
385
386
387
388
    m_good &= m_file.read(m_buffer, size);

    if (m_good)
      m_bufferAvailable = size;
    else
      return -1;
  }

  m_position++;
  return m_buffer[m_bufferPosition++];
}
Samuel Jones's avatar
Samuel Jones committed
389
390
bool File::append(const std::string &path, const std::string &name, const void *buffer, size_t size) {
  std::unique_ptr<FILE, decltype(&fclose)> handle(fopen(path.c_str(), "rb+"), fclose);
391

392
  bool good = handle != nullptr;
393
394
395
396
397
398
399
  int64_t lastHeaderPosition = 0;
  int64_t targetPosition = -1;

  while (good) {
    EntryHeader header;
    int64_t position;

400
    lastHeaderPosition = static_cast<int64_t>(ftell(handle.get()));
401

402
403
    good &= 1 == fread(&header, sizeof(EntryHeader), 1, handle.get());
    good &= 0 == fseek(handle.get(), 512 - sizeof(EntryHeader), SEEK_CUR);
404
    good &= 0 <= (position = static_cast<int64_t>(ftell(handle.get())));
405
406
407
408
409
410
411

    if (!good)
      return false;

    std::string fileName(header.FileName);
    if (fileName.length() == 0)
      break;
412

413
    if (fileName == name)
414
415
      targetPosition = lastHeaderPosition;
    else if (targetPosition != -1)
Samuel Jones's avatar
Samuel Jones committed
416
      throw std::runtime_error("format exception"); // it has to be the last file in the archive
417
418
419
420
421

    FileInfo fileInfo;
    fileInfo.Offset = position;
    fileInfo.Size = header.readFileSize();

422
    auto offset = static_cast<size_t>(fileInfo.Size % 512);
423
424
425
    if (offset != 0)
      offset = 512 - offset;

Samuel Jones's avatar
Samuel Jones committed
426
    good &= 0 == fseek(handle.get(), static_cast<long>(fileInfo.Size + offset), SEEK_CUR);
427
428
429
430
431
  }

  if (!good)
    return false;

432
  if (targetPosition < 0)
433
    targetPosition = lastHeaderPosition;
434

435
436
437
438
439
440
441
442
443
444
445
446
  // empty buffer
  char padding[512];
  memset(padding, 0, 512);

  // prepare new header
  EntryHeader header;
  memset(&header, 0, sizeof(EntryHeader));
  memcpy(header.FileName, name.c_str(), name.size());
  memset(header.FileMode, '0', sizeof(header.FileMode) - 1);
  memset(header.OwnerUserID, '0', sizeof(header.OwnerUserID) - 1);
  memset(header.OwnerGroupID, '0', sizeof(header.OwnerGroupID) - 1);
  memset(header.LastModification, '0', sizeof(header.LastModification) - 1);
447

448
449
450
451
452
  header.TypeFlag = TarTypeFlag_NormalFile;
  header.writeFileSize(size);
  header.writeChecksum();

  // write header
453
  good &= 0 == fseek(handle.get(), static_cast<long>(targetPosition), SEEK_SET);
454
455
456
457
458
  good &= 1 == fwrite(&header, sizeof(EntryHeader), 1, handle.get());
  good &= 1 == fwrite(padding, 512 - sizeof(EntryHeader), 1, handle.get());

  // write content
  good &= 1 == fwrite(buffer, size, 1, handle.get());
459

460
  // write padding
461
  auto offset = static_cast<size_t>(size % 512);
462
463
464
465
466
467
468
469
470
471
472
  if (offset != 0) {
    offset = 512 - offset;

    good &= 1 == fwrite(padding, offset, 1, handle.get());
  }

  // write final
  good &= 1 == fwrite(padding, 512, 1, handle.get());

  return good;
}
Owen Arnold's avatar
Owen Arnold committed
473

LamarMoore's avatar
LamarMoore committed
474
} // namespace Tar
475
} // namespace Mantid::DataHandling::ANSTO