LoadANSTOHelper.cpp

#include "MantidDataHandling/LoadANSTOHelper.h"
#include "MantidDataObjects/EventWorkspace.h"
#include "MantidAPI/FileProperty.h"
#include "MantidAPI/RegisterFileLoader.h"
#include "MantidKernel/UnitFactory.h"
#include "MantidGeometry/Instrument.h"
#include "MantidGeometry/Instrument/RectangularDetector.h"
#include "MantidGeometry/Objects/ShapeFactory.h"
#include "MantidNexus/NexusClasses.h"

#include <numeric>

namespace Mantid {
namespace DataHandling {
namespace ANSTO {

// ProgressTracker
ProgressTracker::ProgressTracker(API::Progress &progBar, const char *msg,
                                 int64_t target, size_t count)
    : m_msg(msg), m_count(count), m_step(target / count), m_next(m_step),
      m_progBar(progBar) {

  m_progBar.doReport(m_msg);
}
ProgressTracker::~ProgressTracker() { complete(); }
void ProgressTracker::update(int64_t position) {
  while (m_next <= position) {
    m_progBar.report(m_msg);

    switch (m_count) {
    case 0:
      return;

    case 1:
      m_count = 0;
      m_next = std::numeric_limits<int64_t>::max();
      return;

    default:
      m_count--;
      m_next += m_step;
    }
  }
}
void ProgressTracker::complete() {
  if (m_count != 0) {
    m_progBar.reportIncrement(m_count, m_msg);
    m_count = 0;
  }
}

// EventProcessor
EventProcessor::EventProcessor(const std::vector<bool> &roi,
                               const size_t stride, const double period,
                               const double phase, const double tofMinBoundary,
                               const double tofMaxBoundary,
                               const double timeMinBoundary,
                               const double timeMaxBoundary)
    : m_roi(roi), m_stride(stride), m_frames(0), m_framesValid(0),
      m_period(period), m_phase(phase), m_tofMinBoundary(tofMinBoundary),
      m_tofMaxBoundary(tofMaxBoundary), m_timeMinBoundary(timeMinBoundary),
      m_timeMaxBoundary(timeMaxBoundary) {}
bool EventProcessor::validFrame() const {
  // frame boundary
  double frameTime =
      (static_cast<double>(m_frames) * m_period) * 1e-6; // in seconds

  return (frameTime >= m_timeMinBoundary) && (frameTime <= m_timeMaxBoundary);
}
void EventProcessor::newFrame() {
  m_frames++;
  if (validFrame())
    m_framesValid++;
}
void EventProcessor::addEvent(size_t x, size_t y, double tof) {
  // tof correction
  if (m_period > 0.0) {
    tof += m_phase;
    while (tof > m_period)
      tof -= m_period;
    while (tof < 0)
      tof += m_period;
  }

  // check if event is in valid range
  if (!validFrame())
    return;

  // ToF boundary
  if ((tof < m_tofMinBoundary) && (tof > m_tofMaxBoundary))
    return;

  // detector id
  size_t id = m_stride * x + y;

  // image size
  if ((y >= m_stride) || (id >= m_roi.size()))
    return;

  // check if neutron is in region of intreset
  if (m_roi[id])
    addEventImpl(id, tof);
}

// EventCounter
EventCounter::EventCounter(const std::vector<bool> &roi, const size_t stride,
                           const double period, const double phase,
                           const double tofMinBoundary,
                           const double tofMaxBoundary,
                           const double timeMinBoundary,
                           const double timeMaxBoundary,
                           std::vector<size_t> &eventCounts)
    : EventProcessor(roi, stride, period, phase, tofMinBoundary, tofMaxBoundary,
                     timeMinBoundary, timeMaxBoundary),
      m_eventCounts(eventCounts), m_tofMin(std::numeric_limits<double>::max()),
      m_tofMax(std::numeric_limits<double>::min()) {}
size_t EventCounter::numFrames() const { return m_framesValid; }
double EventCounter::tofMin() const {
  return m_tofMin <= m_tofMax ? m_tofMin : 0.0;
}
double EventCounter::tofMax() const {
  return m_tofMin <= m_tofMax ? m_tofMax : 0.0;
}
void EventCounter::addEventImpl(size_t id, double tof) {
  if (m_tofMin > tof)
    m_tofMin = tof;
  if (m_tofMax < tof)
    m_tofMax = tof;

  m_eventCounts[id]++;
}

// EventAssigner
EventAssigner::EventAssigner(const std::vector<bool> &roi, const size_t stride,
                             const double period, const double phase,
                             const double tofMinBoundary,
                             const double tofMaxBoundary,
                             const double timeMinBoundary,
                             const double timeMaxBoundary,
                             std::vector<EventVector_pt> &eventVectors)
    : EventProcessor(roi, stride, period, phase, tofMinBoundary, tofMaxBoundary,
                     timeMinBoundary, timeMaxBoundary),
      m_eventVectors(eventVectors) {}
void EventAssigner::addEventImpl(size_t id, double tof) {
  m_eventVectors[id]->push_back(tof);
}

// EventAssignerFixedWavelength
EventAssignerFixedWavelength::EventAssignerFixedWavelength(
    const std::vector<bool> &roi, const size_t stride, const double wavelength,
    const double period, const double phase, const double tofMinBoundary,
    const double tofMaxBoundary, const double timeMinBoundary,
    const double timeMaxBoundary, std::vector<EventVector_pt> &eventVectors)
    : EventAssigner(roi, stride, period, phase, tofMinBoundary, tofMaxBoundary,
                    timeMinBoundary, timeMaxBoundary, eventVectors),
      m_wavelength(wavelength) {}
void EventAssignerFixedWavelength::addEventImpl(size_t id, double tof) {
  UNUSED_ARG(tof);
  m_eventVectors[id]->push_back(m_wavelength);
}

// FastReadOnlyFile
#ifdef _WIN32
FastReadOnlyFile::FastReadOnlyFile(const char *filename) {
  m_handle = CreateFileA(filename, GENERIC_READ, FILE_SHARE_READ, NULL,
                         OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
}
FastReadOnlyFile::~FastReadOnlyFile() { close(); }
void *FastReadOnlyFile::handle() const { return m_handle; }
void FastReadOnlyFile::close() {
  CloseHandle(m_handle);
  m_handle = NULL;
}
bool FastReadOnlyFile::read(void *buffer, uint32_t size) {
  DWORD bytesRead;
  return (FALSE != ReadFile(m_handle, buffer, size, &bytesRead, NULL)) &&
         (bytesRead == size);
}
bool FastReadOnlyFile::seek(int64_t offset, int whence, int64_t *newPosition) {
  return FALSE != SetFilePointerEx(m_handle, *(LARGE_INTEGER *)&offset,
                                   (LARGE_INTEGER *)newPosition, whence);
}
#else
FastReadOnlyFile::FastReadOnlyFile(const char *filename) {
  m_handle = fopen(filename, "rb");
}
FastReadOnlyFile::~FastReadOnlyFile() { close(); }
void *FastReadOnlyFile::handle() const { return m_handle; }
void FastReadOnlyFile::close() {
  fclose(m_handle);
  m_handle = nullptr;
}
bool FastReadOnlyFile::read(void *buffer, uint32_t size) {
  return 1 == fread(buffer, static_cast<size_t>(size), 1, m_handle);
}
bool FastReadOnlyFile::seek(int64_t offset, int whence, int64_t *newPosition) {
  return (0 == fseek(m_handle, offset, whence)) &&
         ((newPosition == nullptr) ||
          (0 <= (*newPosition = static_cast<int64_t>(ftell(m_handle)))));
}
#endif

namespace Tar {

void EntryHeader::writeChecksum() {
  memset(Checksum, ' ', sizeof(Checksum));
  size_t value = std::accumulate(
      (const char *)this, (const char *)this + sizeof(EntryHeader), (size_t)0);

  std::ostringstream buffer;

  buffer << std::oct << std::setfill('0')
         << std::setw(static_cast<int>(sizeof(Checksum)) - 1) << value;
  std::string string = buffer.str();

  std::copy(string.cbegin(), string.cend(), Checksum);
  Checksum[string.size()] = 0;
}
void EntryHeader::writeFileSize(int64_t value) {
  std::ostringstream buffer;

  buffer << std::oct << std::setfill('0')
         << std::setw(static_cast<int>(sizeof(FileSize)) - 1) << value;
  std::string string = buffer.str();

  std::copy(string.cbegin(), string.cend(), FileSize);
  FileSize[string.size()] = 0;
}
int64_t EntryHeader::readFileSize() {
  int64_t result = 0;
  const char *p = FileSize;
  for (size_t n = sizeof(FileSize) - 1; n != 0; --n) { // last character is '\0'
    char c = *p++;
    if (('0' <= c) && (c <= '9'))
      result = result * 8 + (c - '0');
  }
  return result;
}

// construction
File::File(const std::string &path)
    : m_good(true), m_file(path.c_str()), m_selected(static_cast<size_t>(-1)),
      m_position(0), m_size(0), m_bufferPosition(0), m_bufferAvailable(0) {

  m_good = m_file.handle() != nullptr;
  while (m_good) {
    EntryHeader header;
    int64_t position;

    m_good &= m_file.read(&header, sizeof(EntryHeader));
    m_good &= m_file.seek(512 - sizeof(EntryHeader), SEEK_CUR, &position);
    if (!m_good)
      break;

    std::string fileName(header.FileName);
    if (fileName.length() == 0)
      return;

    FileInfo fileInfo;
    fileInfo.Offset = position;
    fileInfo.Size = header.readFileSize();

    if (header.TypeFlag == TarTypeFlag_NormalFile) {
      m_fileNames.push_back(fileName);
      m_fileInfos.push_back(fileInfo);
    }

    size_t offset = static_cast<size_t>(fileInfo.Size % 512);
    if (offset != 0)
      offset = 512 - offset;

    m_good &= m_file.seek(fileInfo.Size + offset, SEEK_CUR);
  }
}
void File::close() {
  m_good = false;
  m_file.close();
  m_fileNames.clear();
  m_fileInfos.clear();
  m_selected = static_cast<size_t>(-1);
  m_position = 0;
  m_size = 0;
  m_bufferPosition = 0;
  m_bufferAvailable = 0;
}

// properties
bool File::good() const { return m_good; }
const std::vector<std::string> &File::files() const { return m_fileNames; }
const std::string &File::selected_name() const {
  return m_fileNames[m_selected];
}
int64_t File::selected_position() const { return m_position; }
int64_t File::selected_size() const { return m_size; }

// methods
bool File::select(const char *file) {
  if (!m_good)
    return false;

  // reset buffer
  m_bufferPosition = 0;
  m_bufferAvailable = 0;

  for (size_t i = 0; i != m_fileNames.size(); i++)
    if (m_fileNames[i] == file) {
      const FileInfo &info = m_fileInfos[i];

      m_selected = i;
      m_position = 0;
      m_size = info.Size;

      return m_good &= m_file.seek(info.Offset, SEEK_SET);
    }

  m_selected = static_cast<size_t>(-1);
  m_position = 0;
  m_size = 0;
  return false;
}
bool File::skip(uint64_t offset) {
  if (!m_good || (m_selected == static_cast<size_t>(-1)))
    return false;

  bool overrun = offset > static_cast<uint64_t>(m_size - m_position);
  if (overrun)
    offset = m_size - m_position;

  m_position += offset;

  uint64_t bufferPosition = static_cast<uint64_t>(m_bufferPosition) + offset;
  if (bufferPosition <= m_bufferAvailable)
    m_bufferPosition = static_cast<size_t>(bufferPosition);
  else {
    m_good &= m_file.seek(bufferPosition - m_bufferAvailable, SEEK_CUR);

    m_bufferPosition = 0;
    m_bufferAvailable = 0;
  }

  return m_good && !overrun;
}
size_t File::read(void *dst, size_t size) {
  if (!m_good || (m_selected == static_cast<size_t>(-1)))
    return 0;

  if (static_cast<int64_t>(size) > (m_size - m_position))
    size = static_cast<size_t>(m_size - m_position);

  auto ptr = reinterpret_cast<uint8_t *>(dst);
  size_t result = 0;

  if (m_bufferPosition != m_bufferAvailable) {
    result = m_bufferAvailable - m_bufferPosition;
    if (result > size)
      result = size;

    memcpy(ptr, m_buffer, result);
    ptr += result;

    size -= result;
    m_position += result;
    m_bufferPosition += result;
  }

  while (size != 0) {
    auto bytesToRead = static_cast<uint32_t>(
        std::min<size_t>(size, std::numeric_limits<uint32_t>::max()));

    m_good &= m_file.read(ptr, bytesToRead);
    if (!m_good)
      break;

    ptr += bytesToRead;

    size -= bytesToRead;
    result += bytesToRead;
    m_position += bytesToRead;
  }

  return result;
}
int File::read_byte() {
  if (!m_good || (m_selected == static_cast<size_t>(-1)))
    return -1;

  if (m_bufferPosition == m_bufferAvailable) {
    if (m_position >= m_size)
      return -1;

    m_bufferPosition = 0;
    m_bufferAvailable = 0;

    uint32_t size = static_cast<uint32_t>(
        std::min<int64_t>(sizeof(m_buffer), m_size - m_position));
    m_good &= m_file.read(m_buffer, size);

    if (m_good)
      m_bufferAvailable = size;
    else
      return -1;
  }

  m_position++;
  return m_buffer[m_bufferPosition++];
}
bool File::append(const std::string &path, const std::string &name,
                  const void *buffer, size_t size) {
  std::unique_ptr<FILE, decltype(&fclose)> handle(fopen(path.c_str(), "rb+"),
                                                  fclose);

  bool good = handle != nullptr;
  int64_t lastHeaderPosition = 0;
  int64_t targetPosition = -1;

  while (good) {
    EntryHeader header;
    int64_t position;

    lastHeaderPosition = static_cast<int64_t>(ftell(handle.get()));

    good &= 1 == fread(&header, sizeof(EntryHeader), 1, handle.get());
    good &= 0 == fseek(handle.get(), 512 - sizeof(EntryHeader), SEEK_CUR);
    good &= 0 <= (position = static_cast<int64_t>(ftell(handle.get())));

    if (!good)
      return false;

    std::string fileName(header.FileName);
    if (fileName.length() == 0)
      break;

    if (fileName.compare(name) == 0)
      targetPosition = lastHeaderPosition;
    else if (targetPosition != -1)
      throw std::runtime_error(
          "format exception"); // it has to be the last file in the archive

    FileInfo fileInfo;
    fileInfo.Offset = position;
    fileInfo.Size = header.readFileSize();

    size_t offset = static_cast<size_t>(fileInfo.Size % 512);
    if (offset != 0)
      offset = 512 - offset;

    good &= 0 == fseek(handle.get(), static_cast<long>(fileInfo.Size + offset),
                       SEEK_CUR);
  }

  if (!good)
    return false;

  if (targetPosition < 0)
    targetPosition = lastHeaderPosition;

  // empty buffer
  char padding[512];
  memset(padding, 0, 512);

  // prepare new header
  EntryHeader header;
  memset(&header, 0, sizeof(EntryHeader));
  memcpy(header.FileName, name.c_str(), name.size());
  memset(header.FileMode, '0', sizeof(header.FileMode) - 1);
  memset(header.OwnerUserID, '0', sizeof(header.OwnerUserID) - 1);
  memset(header.OwnerGroupID, '0', sizeof(header.OwnerGroupID) - 1);
  memset(header.LastModification, '0', sizeof(header.LastModification) - 1);

  header.TypeFlag = TarTypeFlag_NormalFile;
  header.writeFileSize(size);
  header.writeChecksum();

  // write header
  good &= 0 == fseek(handle.get(), static_cast<long>(targetPosition), SEEK_SET);
  good &= 1 == fwrite(&header, sizeof(EntryHeader), 1, handle.get());
  good &= 1 == fwrite(padding, 512 - sizeof(EntryHeader), 1, handle.get());

  // write content
  good &= 1 == fwrite(buffer, size, 1, handle.get());

  // write padding
  size_t offset = static_cast<size_t>(size % 512);
  if (offset != 0) {
    offset = 512 - offset;

    good &= 1 == fwrite(padding, offset, 1, handle.get());
  }

  // write final
  good &= 1 == fwrite(padding, 512, 1, handle.get());

  return good;
}

} // Tar
} // ANSTO
} // DataHandling
} // Mantid