LoadEventNexus.cpp 66.6 KB
Newer Older
1
2
3
// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
4
5
//   NScD Oak Ridge National Laboratory, European Spallation Source,
//   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
6
// SPDX - License - Identifier: GPL - 3.0 +
7
#include "MantidDataHandling/LoadEventNexus.h"
8
9
10
#include "MantidAPI/Axis.h"
#include "MantidAPI/FileProperty.h"
#include "MantidAPI/RegisterFileLoader.h"
11
12
#include "MantidAPI/Run.h"
#include "MantidAPI/Sample.h"
LamarMoore's avatar
LamarMoore committed
13
14
15
#include "MantidDataHandling/DefaultEventLoader.h"
#include "MantidDataHandling/EventWorkspaceCollection.h"
#include "MantidDataHandling/LoadEventNexusIndexSetup.h"
Mathieu Tillet's avatar
Mathieu Tillet committed
16
#include "MantidDataHandling/LoadHelper.h"
LamarMoore's avatar
LamarMoore committed
17
#include "MantidDataHandling/ParallelEventLoader.h"
18
#include "MantidGeometry/Instrument.h"
19
#include "MantidGeometry/Instrument/Goniometer.h"
20
#include "MantidGeometry/Instrument/RectangularDetector.h"
LamarMoore's avatar
LamarMoore committed
21
#include "MantidIndexing/IndexInfo.h"
22
23
#include "MantidKernel/ArrayProperty.h"
#include "MantidKernel/BoundedValidator.h"
24
#include "MantidKernel/DateAndTimeHelpers.h"
25
#include "MantidKernel/ListValidator.h"
26
#include "MantidKernel/MultiThreaded.h"
27
#include "MantidKernel/TimeSeriesProperty.h"
28
#include "MantidKernel/Timer.h"
29
30
#include "MantidKernel/UnitFactory.h"
#include "MantidKernel/VisibleWhenProperty.h"
Neil Vaytet's avatar
Neil Vaytet committed
31
#include "MantidNexus/NexusIOHelper.h"
32

33
#include <H5Cpp.h>
34
#include <memory>
Russell Taylor's avatar
Russell Taylor committed
35

36
37
#include <regex>

38
using Mantid::Types::Core::DateAndTime;
39
40
41
42
43
using std::map;
using std::string;
using std::vector;
using namespace ::NeXus;

44
namespace Mantid::DataHandling {
45

William F Godoy's avatar
William F Godoy committed
46
DECLARE_NEXUS_HDF5_FILELOADER_ALGORITHM(LoadEventNexus)
47

48
using namespace Kernel;
49
using namespace DateAndTimeHelpers;
50
51
52
using namespace Geometry;
using namespace API;
using namespace DataObjects;
53
using Types::Core::DateAndTime;
54

55
56
57
58
59
namespace {
// detnotes the end of iteration for NeXus::getNextEntry
const std::string NULL_STR("NULL");
} // namespace

60
61
62
63
64
65
66
/**
 * Based on the current group in the file, does the named sub-entry exist?
 * @param file : File handle. This is not modified, but cannot be const
 * @param name : sub entry name to look for
 * @return true only if it exists
 */
bool exists(::NeXus::File &file, const std::string &name) {
67
68
69
70
  const auto entries = file.getEntries();
  return exists(entries, name);
}

Samuel Jones's avatar
Samuel Jones committed
71
bool exists(const std::map<std::string, std::string> &entries, const std::string &name) {
72
73
74
  return entries.find(name) != entries.end();
}

75
76
//----------------------------------------------------------------------------------------------
/** Empty default constructor
LamarMoore's avatar
LamarMoore committed
77
 */
78
LoadEventNexus::LoadEventNexus()
Samuel Jones's avatar
Samuel Jones committed
79
80
81
    : filter_tof_min(0), filter_tof_max(0), m_specMin(0), m_specMax(0), longest_tof(0), shortest_tof(0), bad_tofs(0),
      discarded_events(0), compressTolerance(0), m_instrument_loaded_correctly(false), loadlogs(false),
      event_id_is_spec(false) {}
82

William F Godoy's avatar
William F Godoy committed
83
84
85
86
87
88
89
90
91
92
//----------------------------------------------------------------------------------------------
/**
 * Return the confidence with with this algorithm can load the file
 * @param descriptor A descriptor for the file
 * @returns An integer specifying the confidence level. 0 indicates it will not
 * be used
 */
int LoadEventNexus::confidence(Kernel::NexusHDF5Descriptor &descriptor) const {

  int confidence = 0;
Samuel Jones's avatar
Samuel Jones committed
93
  const std::map<std::string, std::set<std::string>> &allEntries = descriptor.getAllEntries();
William F Godoy's avatar
William F Godoy committed
94
  if (allEntries.count("NXevent_data") == 1) {
Samuel Jones's avatar
Samuel Jones committed
95
    if (descriptor.isEntry("/entry", "NXentry") || descriptor.isEntry("/raw_data_1", "NXentry")) {
William F Godoy's avatar
William F Godoy committed
96
97
98
99
100
101
102
      confidence = 80;
    }
  }

  return confidence;
}

103
104
//----------------------------------------------------------------------------------------------
/** Initialisation method.
LamarMoore's avatar
LamarMoore committed
105
 */
106
void LoadEventNexus::init() {
Savici, Andrei T's avatar
Savici, Andrei T committed
107
  const std::vector<std::string> exts{".nxs.h5", ".nxs", "_event.nxs"};
Samuel Jones's avatar
Samuel Jones committed
108
109
110
111
112
  this->declareProperty(std::make_unique<FileProperty>("Filename", "", FileProperty::Load, exts),
                        "The name of the Event NeXus file to read, including its full or "
                        "relative path. "
                        "The file name is typically of the form INST_####_event.nxs (N.B. case "
                        "sensitive if running on Linux).");
113

Samuel Jones's avatar
Samuel Jones committed
114
115
116
117
118
119
120
121
  this->declareProperty(std::make_unique<WorkspaceProperty<Workspace>>("OutputWorkspace", "", Direction::Output),
                        "The name of the output EventWorkspace or WorkspaceGroup in which to "
                        "load the EventNexus file.");

  declareProperty(std::make_unique<PropertyWithValue<string>>("NXentryName", "", Direction::Input),
                  "Optional: Name of the NXentry to load if it's not the default.");

  declareProperty(std::make_unique<PropertyWithValue<double>>("FilterByTofMin", EMPTY_DBL(), Direction::Input),
122
123
124
125
126
                  "Optional: To exclude events that do not fall within a range "
                  "of times-of-flight. "
                  "This is the minimum accepted value in microseconds. Keep "
                  "blank to load all events.");

Samuel Jones's avatar
Samuel Jones committed
127
  declareProperty(std::make_unique<PropertyWithValue<double>>("FilterByTofMax", EMPTY_DBL(), Direction::Input),
128
129
130
131
132
                  "Optional: To exclude events that do not fall within a range "
                  "of times-of-flight. "
                  "This is the maximum accepted value in microseconds. Keep "
                  "blank to load all events.");

Samuel Jones's avatar
Samuel Jones committed
133
  declareProperty(std::make_unique<PropertyWithValue<double>>("FilterByTimeStart", EMPTY_DBL(), Direction::Input),
134
135
136
                  "Optional: To only include events after the provided start "
                  "time, in seconds (relative to the start of the run).");

Samuel Jones's avatar
Samuel Jones committed
137
  declareProperty(std::make_unique<PropertyWithValue<double>>("FilterByTimeStop", EMPTY_DBL(), Direction::Input),
138
139
140
141
142
143
144
145
146
                  "Optional: To only include events before the provided stop "
                  "time, in seconds (relative to the start of the run).");

  std::string grp1 = "Filter Events";
  setPropertyGroup("FilterByTofMin", grp1);
  setPropertyGroup("FilterByTofMax", grp1);
  setPropertyGroup("FilterByTimeStart", grp1);
  setPropertyGroup("FilterByTimeStop", grp1);

Samuel Jones's avatar
Samuel Jones committed
147
148
149
150
  declareProperty(std::make_unique<ArrayProperty<string>>("BankName", Direction::Input),
                  "Optional: To only include events from one bank. Any bank "
                  "whose name does not match the given string will have no "
                  "events.");
151

Samuel Jones's avatar
Samuel Jones committed
152
  declareProperty(std::make_unique<PropertyWithValue<bool>>("SingleBankPixelsOnly", true, Direction::Input),
153
154
155
156
                  "Optional: Only applies if you specified a single bank to "
                  "load with BankName. "
                  "Only pixels in the specified bank will be created if true; "
                  "all of the instrument's pixels will be created otherwise.");
Samuel Jones's avatar
Samuel Jones committed
157
  setPropertySettings("SingleBankPixelsOnly", std::make_unique<VisibleWhenProperty>("BankName", IS_NOT_DEFAULT));
158
159
160
161
162

  std::string grp2 = "Loading a Single Bank";
  setPropertyGroup("BankName", grp2);
  setPropertyGroup("SingleBankPixelsOnly", grp2);

Samuel Jones's avatar
Samuel Jones committed
163
164
165
166
167
  declareProperty(std::make_unique<PropertyWithValue<bool>>("Precount", true, Direction::Input),
                  "Pre-count the number of events in each pixel before allocating memory "
                  "(optional, default True). "
                  "This can significantly reduce memory use and memory fragmentation; it "
                  "may also speed up loading.");
168

Samuel Jones's avatar
Samuel Jones committed
169
  declareProperty(std::make_unique<PropertyWithValue<double>>("CompressTolerance", -1.0, Direction::Input),
170
171
172
173
174
                  "Run CompressEvents while loading (optional, leave blank or "
                  "negative to not do). "
                  "This specified the tolerance to use (in microseconds) when "
                  "compressing.");

175
  auto mustBePositive = std::make_shared<BoundedValidator<int>>();
176
177
178
179
180
181
182
183
184
185
  mustBePositive->setLower(1);
  declareProperty("ChunkNumber", EMPTY_INT(), mustBePositive,
                  "If loading the file by sections ('chunks'), this is the "
                  "section number of this execution of the algorithm.");
  declareProperty("TotalChunks", EMPTY_INT(), mustBePositive,
                  "If loading the file by sections ('chunks'), this is the "
                  "total number of sections.");
  // TotalChunks is only meaningful if ChunkNumber is set
  // Would be nice to be able to restrict ChunkNumber to be <= TotalChunks at
  // validation
Samuel Jones's avatar
Samuel Jones committed
186
  setPropertySettings("TotalChunks", std::make_unique<VisibleWhenProperty>("ChunkNumber", IS_NOT_DEFAULT));
187
188
189
190
191
192
193

  std::string grp3 = "Reduce Memory Use";
  setPropertyGroup("Precount", grp3);
  setPropertyGroup("CompressTolerance", grp3);
  setPropertyGroup("ChunkNumber", grp3);
  setPropertyGroup("TotalChunks", grp3);

Samuel Jones's avatar
Samuel Jones committed
194
  declareProperty(std::make_unique<PropertyWithValue<bool>>("LoadMonitors", false, Direction::Input),
195
                  "Load the monitors from the file (optional, default False).");
196

mantid-builder's avatar
mantid-builder committed
197
  std::vector<std::string> options{"", "Events", "Histogram"};
Samuel Jones's avatar
Samuel Jones committed
198
  declareProperty("MonitorsLoadOnly", "", std::make_shared<Kernel::StringListValidator>(options),
199
200
                  "If multiple repesentations exist, which one to load. "
                  "Default is to load the one that is present.");
201

Samuel Jones's avatar
Samuel Jones committed
202
  declareProperty(std::make_unique<PropertyWithValue<double>>("FilterMonByTofMin", EMPTY_DBL(), Direction::Input),
203
204
205
206
                  "Optional: To exclude events from monitors that do not fall "
                  "within a range of times-of-flight. "
                  "This is the minimum accepted value in microseconds.");

Samuel Jones's avatar
Samuel Jones committed
207
  declareProperty(std::make_unique<PropertyWithValue<double>>("FilterMonByTofMax", EMPTY_DBL(), Direction::Input),
208
209
210
211
                  "Optional: To exclude events from monitors that do not fall "
                  "within a range of times-of-flight. "
                  "This is the maximum accepted value in microseconds.");

Samuel Jones's avatar
Samuel Jones committed
212
  declareProperty(std::make_unique<PropertyWithValue<double>>("FilterMonByTimeStart", EMPTY_DBL(), Direction::Input),
213
214
215
216
                  "Optional: To only include events from monitors after the "
                  "provided start time, in seconds (relative to the start of "
                  "the run).");

Samuel Jones's avatar
Samuel Jones committed
217
  declareProperty(std::make_unique<PropertyWithValue<double>>("FilterMonByTimeStop", EMPTY_DBL(), Direction::Input),
218
219
220
221
                  "Optional: To only include events from monitors before the "
                  "provided stop time, in seconds (relative to the start of "
                  "the run).");

Samuel Jones's avatar
Samuel Jones committed
222
  setPropertySettings("MonitorsLoadOnly", std::make_unique<VisibleWhenProperty>("LoadMonitors", IS_EQUAL_TO, "1"));
223
  auto asEventsIsOn = [] {
Samuel Jones's avatar
Samuel Jones committed
224
    std::unique_ptr<IPropertySettings> prop = std::make_unique<VisibleWhenProperty>("LoadMonitors", IS_EQUAL_TO, "1");
225
226
227
228
229
230
    return prop;
  };
  setPropertySettings("FilterMonByTofMin", asEventsIsOn());
  setPropertySettings("FilterMonByTofMax", asEventsIsOn());
  setPropertySettings("FilterMonByTimeStart", asEventsIsOn());
  setPropertySettings("FilterMonByTimeStop", asEventsIsOn());
231
232
233

  std::string grp4 = "Monitors";
  setPropertyGroup("LoadMonitors", grp4);
234
  setPropertyGroup("MonitorsLoadOnly", grp4);
235
236
237
238
239
  setPropertyGroup("FilterMonByTofMin", grp4);
  setPropertyGroup("FilterMonByTofMax", grp4);
  setPropertyGroup("FilterMonByTimeStart", grp4);
  setPropertyGroup("FilterMonByTimeStop", grp4);

Samuel Jones's avatar
Samuel Jones committed
240
241
  declareProperty("SpectrumMin", EMPTY_INT(), mustBePositive, "The number of the first spectrum to read.");
  declareProperty("SpectrumMax", EMPTY_INT(), mustBePositive, "The number of the last spectrum to read.");
242
  declareProperty(std::make_unique<ArrayProperty<int32_t>>("SpectrumList"),
243
244
                  "A comma-separated list of individual spectra to read.");

Samuel Jones's avatar
Samuel Jones committed
245
246
  declareProperty(std::make_unique<PropertyWithValue<bool>>("MetaDataOnly", false, Direction::Input),
                  "If true, only the meta data and sample logs will be loaded.");
247

Samuel Jones's avatar
Samuel Jones committed
248
  declareProperty(std::make_unique<PropertyWithValue<bool>>("LoadLogs", true, Direction::Input),
249
                  "Load only the Sample/DAS logs from the file (default True).");
Mathieu Tillet's avatar
Mathieu Tillet committed
250
251

  declareProperty(std::make_unique<PropertyWithValue<bool>>("LoadAllLogs", false, Direction::Input),
252
253
                  "Load all the logs from the nxs, without checking or processing them; if checked, LoadLogs will be "
                  "ignored; use with caution");
Mathieu Tillet's avatar
Mathieu Tillet committed
254

255
  std::vector<std::string> loadType{"Default"};
256
257

#ifndef _WIN32
258
  loadType.emplace_back("Multiprocess (experimental)");
259
#endif // _WIN32
260
261

#ifdef MPI_EXPERIMENTAL
Igor Gudich's avatar
Igor Gudich committed
262
  loadType.emplace_back("MPI");
263
264
#endif // MPI_EXPERIMENTAL

265
  auto loadTypeValidator = std::make_shared<StringListValidator>(loadType);
266
  declareProperty("LoadType", "Default", loadTypeValidator,
267
                  "Set type of loader. 2 options {Default, Multiproceess},"
268
                  "'Multiprocess' should work faster for big files and it is "
269
                  "experimental, available only in Linux");
270

Samuel Jones's avatar
Samuel Jones committed
271
  declareProperty(std::make_unique<PropertyWithValue<bool>>("LoadNexusInstrumentXML", true, Direction::Input),
272
273
                  "Reads the embedded Instrument XML from the NeXus file "
                  "(optional, default True). ");
274
275
276

  declareProperty("NumberOfBins", 500, mustBePositive,
                  "The number of bins intially defined. Use Rebin to change "
Nick Draper's avatar
Nick Draper committed
277
278
                  "the binning later.  If there is no data loaded, or you "
                  "select meta data only you will only get 1 bin.");
279
280

  // Flexible log loading
Samuel Jones's avatar
Samuel Jones committed
281
282
283
284
285
286
287
288
  declareProperty(std::make_unique<PropertyWithValue<std::vector<std::string>>>("AllowList", std::vector<std::string>(),
                                                                                Direction::Input),
                  "If specified, only these logs will be loaded from the file (each "
                  "separated by a space).");
  declareProperty(std::make_unique<PropertyWithValue<std::vector<std::string>>>("BlockList", std::vector<std::string>(),
                                                                                Direction::Input),
                  "If specified, these logs will NOT be loaded from the file (each "
                  "separated by a space).");
289
}
290

291
292
//----------------------------------------------------------------------------------------------
/** set the name of the top level NXentry m_top_entry_name
LamarMoore's avatar
LamarMoore committed
293
 */
294
295
void LoadEventNexus::setTopEntryName() {
  std::string nxentryProperty = getProperty("NXentryName");
296
  if (!nxentryProperty.empty()) {
297
298
299
300
    m_top_entry_name = nxentryProperty;
    return;
  }

301
302
303
304
305
306
307
308
309
  try {
    while (true) {
      const auto entry = m_file->getNextEntry();
      if (entry.second == "NXentry") {
        if ((entry.first == "entry") || (entry.first == "raw_data_1")) {
          m_top_entry_name = entry.first;
          break;
        }
      } else if (entry.first == NULL_STR && entry.second == NULL_STR) {
Samuel Jones's avatar
Samuel Jones committed
310
311
        g_log.error() << "Unable to determine name of top level NXentry - assuming "
                         "\"entry\".\n";
312
        m_top_entry_name = "entry";
313
        break;
314
      }
315
316
    }
  } catch (const std::exception &) {
Hahn, Steven's avatar
Hahn, Steven committed
317
318
    g_log.error() << "Unable to determine name of top level NXentry - assuming "
                     "\"entry\".\n";
319
320
321
    m_top_entry_name = "entry";
  }
}
322

323
template <typename T> void LoadEventNexus::filterDuringPause(T workspace) {
324
  try {
Samuel Jones's avatar
Samuel Jones committed
325
    if ((!ConfigService::Instance().hasProperty("loadeventnexus.keeppausedevents")) &&
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
        (m_ws->run().getLogData("pause")->size() > 1)) {
      g_log.notice("Filtering out events when the run was marked as paused. "
                   "Set the loadeventnexus.keeppausedevents configuration "
                   "property to override this.");

      auto filter = createChildAlgorithm("FilterByLogValue");
      filter->setProperty("InputWorkspace", workspace);
      filter->setProperty("OutputWorkspace", workspace);
      filter->setProperty("LogName", "pause");
      // The log value is set to 1 when the run is paused, 0 otherwise.
      filter->setProperty("MinimumValue", 0.0);
      filter->setProperty("MaximumValue", 0.0);
      filter->setProperty("LogBoundary", "Left");
      filter->execute();
    }
  } catch (Exception::NotFoundError &) {
    // No "pause" log, just carry on
  }
}

346
template <>
Samuel Jones's avatar
Samuel Jones committed
347
void LoadEventNexus::filterDuringPause<EventWorkspaceCollection_sptr>(EventWorkspaceCollection_sptr workspace) {
348
  // We provide a function pointer to the filter method of the object
349
  using std::placeholders::_1;
Samuel Jones's avatar
Samuel Jones committed
350
  auto func = std::bind(&LoadEventNexus::filterDuringPause<MatrixWorkspace_sptr>, this, _1);
351
352
353
  workspace->applyFilter(func);
}

354
355
//------------------------------------------------------------------------------------------------
/** Executes the algorithm. Reading in the file and creating and populating
LamarMoore's avatar
LamarMoore committed
356
357
 *  the output workspace
 */
William F Godoy's avatar
William F Godoy committed
358
void LoadEventNexus::execLoader() {
359
360
361
362
363
364
365
366
367
  // Retrieve the filename from the properties
  m_filename = getPropertyValue("Filename");

  compressTolerance = getProperty("CompressTolerance");

  loadlogs = getProperty("LoadLogs");

  // Check to see if the monitors need to be loaded later
  bool load_monitors = this->getProperty("LoadMonitors");
368
369
370
371
372

  // this must make absolutely sure that m_file is a valid (and open)
  // NeXus::File object
  safeOpenFile(m_filename);

373
374
375
376
377
378
379
380
381
  setTopEntryName();

  // Initialize progress reporting.
  int reports = 3;
  if (load_monitors)
    reports++;
  Progress prog(this, 0.0, 0.3, reports);

  // Load the detector events
382
383
  m_ws = std::make_shared<EventWorkspaceCollection>(); // Algorithm currently
                                                       // relies on an
384
385
  // object-level workspace ptr
  loadEvents(&prog, false); // Do not load monitor blocks
386
387
388
389
390
391
392
393
394
395

  if (discarded_events > 0) {
    g_log.information() << discarded_events
                        << " events were encountered coming from pixels which "
                           "are not in the Instrument Definition File."
                           "These events were discarded.\n";
  }

  // If the run was paused at any point, filter out those events (SNS only, I
  // think)
396
  filterDuringPause(m_ws->getSingleHeldWorkspace());
397
398

  // add filename
Owen Arnold's avatar
Owen Arnold committed
399
  m_ws->mutableRun().addProperty("Filename", m_filename);
400
  // Save output
401
  this->setProperty("OutputWorkspace", m_ws->combinedWorkspace());
Peterson, Peter's avatar
Peterson, Peter committed
402
403
404
405
406
407

  // close the file since LoadNexusMonitors will take care of its own file
  // handle
  m_file->close();

  // Load the monitors with child algorithm 'LoadNexusMonitors'
408
409
  if (load_monitors) {
    prog.report("Loading monitors");
Peterson, Peter's avatar
Peterson, Peter committed
410
    this->runLoadMonitors();
411
412
  }
}
413

Samuel Jones's avatar
Samuel Jones committed
414
std::pair<DateAndTime, DateAndTime> firstLastPulseTimes(::NeXus::File &file, Kernel::Logger &logger) {
415
  file.openData("event_time_zero");
416
  std::string isooffset; // ISO8601 offset
Neil Vaytet's avatar
Neil Vaytet committed
417
418
419
420
421
422
  DateAndTime offset;
  // According to the Nexus standard, if the offset is not present, it implies
  // the offset is and absolute timestamp, which is relative to the start of
  // Unix epoch (https://manual.nexusformat.org/classes/base_classes/NXlog.html)
  if (!file.hasAttr("offset")) {
    offset = DateAndTime("1970-01-01T00:00:00Z");
Neil Vaytet's avatar
Neil Vaytet committed
423
424
    logger.warning("In firstLastPulseTimes: no ISO8601 offset attribute "
                   "provided for event_time_zero, using UNIX epoch instead");
Neil Vaytet's avatar
Neil Vaytet committed
425
426
427
428
  } else {
    file.getAttr("offset", isooffset);
    offset = DateAndTime(isooffset);
  }
429
430
431
  std::string units; // time units
  if (file.hasAttr("units"))
    file.getAttr("units", units);
432
  // Read in the pulse times
Samuel Jones's avatar
Samuel Jones committed
433
  auto pulse_times = NeXus::NeXusIOHelper::readNexusVector<double>(file, "event_time_zero");
434
  // Remember to close the entry
435
436
  file.closeData();
  // Convert to seconds
Neil Vaytet's avatar
Neil Vaytet committed
437
  auto conv = Kernel::Units::timeConversionValue(units, "s");
Samuel Jones's avatar
Samuel Jones committed
438
439
  return std::make_pair(DateAndTime(pulse_times.front() * conv, 0.0) + offset.totalNanoseconds(),
                        DateAndTime(pulse_times.back() * conv, 0.0) + offset.totalNanoseconds());
440
} // namespace DataHandling
441

442
/**
LamarMoore's avatar
LamarMoore committed
443
444
445
 * Get the number of events in the currently opened group.
 *
 * @param file The handle to the nexus file opened to the group to look at.
446
447
 * @param hasTotalCounts Whether to try looking at the total_counts field.
 * This variable will be changed if the field is not there.
LamarMoore's avatar
LamarMoore committed
448
449
 * @param oldNeXusFileNames Whether to try using old names. This variable will
 * be changed if it is determined that old names are being used.
William F Godoy's avatar
William F Godoy committed
450
 * @param prefix current entry name prefix (e.g. /entry)
William F Godoy's avatar
William F Godoy committed
451
 * @param descriptor input containing metadata information
LamarMoore's avatar
LamarMoore committed
452
453
 * @return The number of events.
 */
Samuel Jones's avatar
Samuel Jones committed
454
std::size_t numEvents(::NeXus::File &file, bool &hasTotalCounts, bool &oldNeXusFileNames, const std::string &prefix,
455
                      const NexusHDF5Descriptor &descriptor) {
456
457
  // try getting the value of total_counts
  if (hasTotalCounts) {
458
    hasTotalCounts = false;
459
    if (descriptor.isEntry(prefix + "/total_counts")) {
460
461
462
463
      try {
        file.openData("total_counts");
        auto info = file.getInfo();
        file.closeData();
Neil Vaytet's avatar
Neil Vaytet committed
464
        if (info.type == ::NeXus::UINT64) {
465
466
          uint64_t eventCount;
          file.readData("total_counts", eventCount);
467
          hasTotalCounts = true;
468
          return eventCount;
469
470
471
        }
      } catch (::NeXus::Exception &) {
      }
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
    }
  }

  // just get the length of the event pixel ids
  try {
    if (oldNeXusFileNames)
      file.openData("event_pixel_id");
    else
      file.openData("event_id");
  } catch (::NeXus::Exception &) {
    // Older files (before Nov 5, 2010) used this field.
    try {
      file.openData("event_pixel_id");
      oldNeXusFileNames = true;
    } catch (::NeXus::Exception &) {
      // Some groups have neither indicating there are not events here
      return 0;
    }
  }
491

492
493
494
495
  size_t numEvents = static_cast<std::size_t>(file.getInfo().dims[0]);
  file.closeData();
  return numEvents;
}
496

Zhang, Chen's avatar
Zhang, Chen committed
497
/** Load the log from the nexus file
LamarMoore's avatar
LamarMoore committed
498
499
500
501
502
 *
 * @param nexusfilename :: The name of the nexus file being loaded
 * @param localWorkspace :: Templated workspace in which to put the instrument
 *geometry
 * @param alg :: Handle of the algorithm
503
504
 * @param returnpulsetimes :: flag to return shared pointer for
 *BankPulseTimes, otherwise NULL.
LamarMoore's avatar
LamarMoore committed
505
506
507
508
509
 * @param nPeriods : Number of periods (write to)
 * @param periodLog : Period logs DateAndTime to int map.
 *
 * @return Pulse times given in the DAS logs
 */
510
template <typename T>
Samuel Jones's avatar
Samuel Jones committed
511
512
513
514
std::shared_ptr<BankPulseTimes>
LoadEventNexus::runLoadNexusLogs(const std::string &nexusfilename, T localWorkspace, API::Algorithm &alg,
                                 bool returnpulsetimes, int &nPeriods,
                                 std::unique_ptr<const TimeSeriesProperty<int>> &periodLog) {
515
516
517
  // --------------------- Load DAS Logs -----------------
  // The pulse times will be empty if not specified in the DAS logs.
  // BankPulseTimes * out = NULL;
518
  std::shared_ptr<BankPulseTimes> out;
519
  auto loadLogs = alg.createChildAlgorithm("LoadNexusLogs");
520
521
522
523
524
525

  // Now execute the Child Algorithm. Catch and log any error, but don't stop.
  try {
    alg.getLogger().information() << "Loading logs from NeXus file..."
                                  << "\n";
    loadLogs->setPropertyValue("Filename", nexusfilename);
Samuel Jones's avatar
Samuel Jones committed
526
    loadLogs->setProperty<API::MatrixWorkspace_sptr>("Workspace", localWorkspace);
527
    try {
Samuel Jones's avatar
Samuel Jones committed
528
      loadLogs->setPropertyValue("NXentryName", alg.getPropertyValue("NXentryName"));
529
    } catch (...) {
530
531
    }

532
533
    loadLogs->execute();

534
    const Run &run = localWorkspace->run();
535
    // Get the number of periods
536
537
    if (run.hasProperty("nperiods")) {
      nPeriods = run.getPropertyValueAsType<int>("nperiods");
538
539
    }
    // Get the period log. Map of DateAndTime to Period int values.
540
541
    if (run.hasProperty("period_log")) {
      auto *temp = run.getProperty("period_log");
Tom Titcombe's avatar
Tom Titcombe committed
542
      // Check for corrupted period logs
Samuel Jones's avatar
Samuel Jones committed
543
544
      std::unique_ptr<TimeSeriesProperty<int>> tempPeriodLog(dynamic_cast<TimeSeriesProperty<int> *>(temp->clone()));
      checkForCorruptedPeriods(std::move(tempPeriodLog), periodLog, nPeriods, nexusfilename);
545
546
547
    }

    // If successful, we can try to load the pulse times
548
    std::vector<Types::Core::DateAndTime> temp;
549
    if (localWorkspace->run().hasProperty("proton_charge")) {
Samuel Jones's avatar
Samuel Jones committed
550
551
      auto *log =
          dynamic_cast<Kernel::TimeSeriesProperty<double> *>(localWorkspace->mutableRun().getProperty("proton_charge"));
552
553
554
      if (log)
        temp = log->timesAsVector();
    }
555
    if (returnpulsetimes)
556
      out = std::make_shared<BankPulseTimes>(temp);
557
558
559

    // Use the first pulse as the run_start time.
    if (!temp.empty()) {
560
      if (temp[0] < Types::Core::DateAndTime("1991-01-01T00:00:00"))
561
562
563
        alg.getLogger().warning() << "Found entries in the proton_charge "
                                     "sample log with invalid pulse time!\n";

564
      Types::Core::DateAndTime run_start = localWorkspace->getFirstPulseTime();
565
566
567
568
      // add the start of the run as a ISO8601 date/time string. The start =
      // first non-zero time.
      // (this is used in LoadInstrument to find the right instrument file to
      // use).
Samuel Jones's avatar
Samuel Jones committed
569
      localWorkspace->mutableRun().addProperty("run_start", run_start.toISO8601String(), true);
Mathieu Tillet's avatar
Mathieu Tillet committed
570
571
    } else if (run.hasProperty("start_time")) {
      localWorkspace->mutableRun().addProperty("run_start", run.getProperty("start_time")->value(), true);
572
573
574
575
576
577
578
579
580
581
582
    } else {
      alg.getLogger().warning() << "Empty proton_charge sample log. You will "
                                   "not be able to filter by time.\n";
    }
    /// Attempt to make a gonoimeter from the logs
    try {
      Geometry::Goniometer gm;
      gm.makeUniversalGoniometer();
      localWorkspace->mutableRun().setGoniometer(gm, true);
    } catch (std::runtime_error &) {
    }
583
  } catch (const InvalidLogPeriods &) {
584
585
    // Rethrow so LoadEventNexus fails.
    // If we don't, Mantid will crash.
586
    throw;
587
588
589
590
591
592
593
594
595
  } catch (...) {
    alg.getLogger().error() << "Error while loading Logs from SNS Nexus. Some "
                               "sample logs may be missing."
                            << "\n";
    return out;
  }
  return out;
}

Zhang, Chen's avatar
Zhang, Chen committed
596
/** Load the log from the nexus file
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
 *
 * @param nexusfilename :: The name of the nexus file being loaded
 * @param localWorkspace :: Templated workspace in which to put the instrument
 *geometry
 * @param alg :: Handle of the algorithm
 * @param returnpulsetimes :: flag to return shared pointer for
 *BankPulseTimes, otherwise NULL.
 * @param nPeriods : Number of periods (write to)
 * @param periodLog : Period logs DateAndTime to int map.
 * @param allow_list: list of properties that will be loaded
 * @param block_list: list of properties that will be excluded from loading
 *
 * @return Pulse times given in the DAS logs
 */
template <typename T>
std::shared_ptr<BankPulseTimes> LoadEventNexus::runLoadNexusLogs(
Samuel Jones's avatar
Samuel Jones committed
613
614
    const std::string &nexusfilename, T localWorkspace, API::Algorithm &alg, bool returnpulsetimes, int &nPeriods,
    std::unique_ptr<const TimeSeriesProperty<int>> &periodLog, const std::vector<std::string> &allow_list,
615
616
617
618
619
    const std::vector<std::string> &block_list) {
  // --------------------- Load DAS Logs -----------------
  // The pulse times will be empty if not specified in the DAS logs.
  // BankPulseTimes * out = NULL;
  std::shared_ptr<BankPulseTimes> out;
620
  auto loadLogs = alg.createChildAlgorithm("LoadNexusLogs");
621
622
623
624
625
626

  // Now execute the Child Algorithm. Catch and log any error, but don't stop.
  try {
    alg.getLogger().information() << "Loading logs from NeXus file..."
                                  << "\n";
    loadLogs->setPropertyValue("Filename", nexusfilename);
Samuel Jones's avatar
Samuel Jones committed
627
    loadLogs->setProperty<API::MatrixWorkspace_sptr>("Workspace", localWorkspace);
628
629
630
631
    loadLogs->setProperty<std::vector<std::string>>("AllowList", allow_list);
    loadLogs->setProperty<std::vector<std::string>>("BlockList", block_list);

    try {
Samuel Jones's avatar
Samuel Jones committed
632
      loadLogs->setPropertyValue("NXentryName", alg.getPropertyValue("NXentryName"));
633
634
635
636
637
638
639
640
641
642
643
644
645
646
    } catch (...) {
    }

    loadLogs->execute();

    const Run &run = localWorkspace->run();
    // Get the number of periods
    if (run.hasProperty("nperiods")) {
      nPeriods = run.getPropertyValueAsType<int>("nperiods");
    }
    // Get the period log. Map of DateAndTime to Period int values.
    if (run.hasProperty("period_log")) {
      auto *temp = run.getProperty("period_log");
      // Check for corrupted period logs
Samuel Jones's avatar
Samuel Jones committed
647
648
      std::unique_ptr<TimeSeriesProperty<int>> tempPeriodLog(dynamic_cast<TimeSeriesProperty<int> *>(temp->clone()));
      checkForCorruptedPeriods(std::move(tempPeriodLog), periodLog, nPeriods, nexusfilename);
649
650
651
652
653
    }

    // If successful, we can try to load the pulse times
    std::vector<Types::Core::DateAndTime> temp;
    if (localWorkspace->run().hasProperty("proton_charge")) {
Samuel Jones's avatar
Samuel Jones committed
654
655
      auto *log =
          dynamic_cast<Kernel::TimeSeriesProperty<double> *>(localWorkspace->mutableRun().getProperty("proton_charge"));
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
      if (log)
        temp = log->timesAsVector();
    }
    if (returnpulsetimes)
      out = std::make_shared<BankPulseTimes>(temp);

    // Use the first pulse as the run_start time.
    if (!temp.empty()) {
      if (temp[0] < Types::Core::DateAndTime("1991-01-01T00:00:00"))
        alg.getLogger().warning() << "Found entries in the proton_charge "
                                     "sample log with invalid pulse time!\n";

      Types::Core::DateAndTime run_start = localWorkspace->getFirstPulseTime();
      // add the start of the run as a ISO8601 date/time string. The start =
      // first non-zero time.
      // (this is used in LoadInstrument to find the right instrument file to
      // use).
Samuel Jones's avatar
Samuel Jones committed
673
      localWorkspace->mutableRun().addProperty("run_start", run_start.toISO8601String(), true);
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
    } else {
      alg.getLogger().warning() << "Empty proton_charge sample log. You will "
                                   "not be able to filter by time.\n";
    }
    /// Attempt to make a gonoimeter from the logs
    try {
      Geometry::Goniometer gm;
      gm.makeUniversalGoniometer();
      localWorkspace->mutableRun().setGoniometer(gm, true);
    } catch (std::runtime_error &) {
    }
  } catch (const InvalidLogPeriods &) {
    // Rethrow so LoadEventNexus fails.
    // If we don't, Mantid will crash.
    throw;
  } catch (...) {
    alg.getLogger().error() << "Error while loading Logs from SNS Nexus. Some "
                               "sample logs may be missing."
                            << "\n";
    return out;
  }
  return out;
}

Tom Titcombe's avatar
Tom Titcombe committed
698
699
700
701
702
703
704
705
/** Check for corrupted period logs
 * If data is historical (1 periods, period is labelled 0) then change period
 * labels to 1 If number of periods does not match expected number of periods
 * then throw an error
 * @param tempPeriodLog :: a temporary local copy of period logs, which will
 * change
 * @param periodLog :: unique pointer which will point to period logs once they
 * have been changed
706
707
708
 * @param nPeriods :: the value in the nperiods log of the run. Number of
 * expected periods
 * @param nexusfilename :: the filename of the run to load
Tom Titcombe's avatar
Tom Titcombe committed
709
 */
Samuel Jones's avatar
Samuel Jones committed
710
711
712
void LoadEventNexus::checkForCorruptedPeriods(std::unique_ptr<TimeSeriesProperty<int>> tempPeriodLog,
                                              std::unique_ptr<const TimeSeriesProperty<int>> &periodLog,
                                              const int &nPeriods, const std::string &nexusfilename) {
Tom Titcombe's avatar
Tom Titcombe committed
713
  const auto valuesAsVector = tempPeriodLog->valuesAsVector();
Samuel Jones's avatar
Samuel Jones committed
714
  const auto nPeriodsInLog = *std::max_element(valuesAsVector.begin(), valuesAsVector.end());
Tom Titcombe's avatar
Tom Titcombe committed
715
716
717
718
719
720
721

  // Check for historic files
  if (nPeriodsInLog == 0 && nPeriods == 1) {
    // "modernize" the local copy here by making period_log
    // a vector of 1s
    const std::vector<int> newValues(tempPeriodLog->realSize(), 1);
    const auto times = tempPeriodLog->timesAsVector();
Samuel Jones's avatar
Samuel Jones committed
722
    periodLog.reset(new const TimeSeriesProperty<int>("period_log", times, newValues));
Tom Titcombe's avatar
Tom Titcombe committed
723
724
725
726
727
728
729
730
731
  } else if (nPeriodsInLog != nPeriods) {
    // Sanity check here that period_log only contains period numbers up to
    // nperiods. These values can be different due to instrument noise, and
    // cause undescriptive crashes if not caught.
    // We throw here to make it clear
    // that the file is corrupted and must be manually assessed.
    const auto msg = "File " + nexusfilename +
                     " has been corrupted. The log framelog/period_log/value "
                     "contains " +
Samuel Jones's avatar
Samuel Jones committed
732
733
                     std::to_string(nPeriodsInLog) + " periods, but periods/number contains " +
                     std::to_string(nPeriods) + ". This file should be manually inspected and corrected.";
Tom Titcombe's avatar
Tom Titcombe committed
734
735
736
737
738
739
740
741
    throw InvalidLogPeriods(msg);
  } else {
    // periodLog should point to a copy of the period logs
    periodLog = std::make_unique<const TimeSeriesProperty<int>>(*tempPeriodLog);
    tempPeriodLog.reset();
  }
}

742
/** Load the instrument from the nexus file
LamarMoore's avatar
LamarMoore committed
743
744
745
746
747
748
 *
 * @param nexusfilename :: The name of the nexus file being loaded
 * @param localWorkspace :: EventWorkspaceCollection in which to put the
 *instrument
 *geometry
 * @param alg :: Handle of the algorithm
749
750
 * @param returnpulsetimes :: flag to return shared pointer for
 *BankPulseTimes, otherwise NULL.
LamarMoore's avatar
LamarMoore committed
751
752
753
754
755
 * @param nPeriods : Number of periods (write to)
 * @param periodLog : Period logs DateAndTime to int map.
 *
 * @return Pulse times given in the DAS logs
 */
756
template <>
Samuel Jones's avatar
Samuel Jones committed
757
758
759
std::shared_ptr<BankPulseTimes> LoadEventNexus::runLoadNexusLogs<EventWorkspaceCollection_sptr>(
    const std::string &nexusfilename, EventWorkspaceCollection_sptr localWorkspace, API::Algorithm &alg,
    bool returnpulsetimes, int &nPeriods, std::unique_ptr<const TimeSeriesProperty<int>> &periodLog) {
760
  auto ws = localWorkspace->getSingleHeldWorkspace();
Samuel Jones's avatar
Samuel Jones committed
761
  auto ret = runLoadNexusLogs<MatrixWorkspace_sptr>(nexusfilename, ws, alg, returnpulsetimes, nPeriods, periodLog);
762
763
764
  return ret;
}

765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
/** Load the instrument from the nexus file
 *
 * @param nexusfilename :: The name of the nexus file being loaded
 * @param localWorkspace :: EventWorkspaceCollection in which to put the
 *instrument
 *geometry
 * @param alg :: Handle of the algorithm
 * @param returnpulsetimes :: flag to return shared pointer for
 *BankPulseTimes, otherwise NULL.
 * @param nPeriods : Number of periods (write to)
 * @param periodLog : Period logs DateAndTime to int map.
 * @param allow_list: log entry that will be loaded
 * @param block_list: log entry that will be excluded
 *
 * @return Pulse times given in the DAS logs
 */
template <>
Samuel Jones's avatar
Samuel Jones committed
782
783
784
785
std::shared_ptr<BankPulseTimes> LoadEventNexus::runLoadNexusLogs<EventWorkspaceCollection_sptr>(
    const std::string &nexusfilename, EventWorkspaceCollection_sptr localWorkspace, API::Algorithm &alg,
    bool returnpulsetimes, int &nPeriods, std::unique_ptr<const TimeSeriesProperty<int>> &periodLog,
    const std::vector<std::string> &allow_list, const std::vector<std::string> &block_list) {
786
  auto ws = localWorkspace->getSingleHeldWorkspace();
Samuel Jones's avatar
Samuel Jones committed
787
788
  auto ret = runLoadNexusLogs<MatrixWorkspace_sptr>(nexusfilename, ws, alg, returnpulsetimes, nPeriods, periodLog,
                                                    allow_list, block_list);
789
790
791
  return ret;
}

792
enum class LoadEventNexus::LoaderType { MPI, MULTIPROCESS, DEFAULT };
793

794
795
//-----------------------------------------------------------------------------
/**
LamarMoore's avatar
LamarMoore committed
796
797
798
799
800
801
802
803
804
 * Load events from the file.
 * @param prog :: A pointer to the progress reporting object
 * @param monitors :: If true the events from the monitors are loaded and not
 *the main banks
 *
 * This also loads the instrument, but only if it has not been set in the
 *workspace
 * being used as input (m_ws data member). Same applies to the logs.
 */
Samuel Jones's avatar
Samuel Jones committed
805
void LoadEventNexus::loadEvents(API::Progress *const prog, const bool monitors) {
806
807
808
809
810
  bool metaDataOnly = getProperty("MetaDataOnly");

  // Get the time filters
  setTimeFilters(monitors);

811
812
813
814
  // Get the log filter if provided
  std::vector<std::string> allow_list = getProperty("AllowList");
  std::vector<std::string> block_list = getProperty("BlockList");

815
816
  // The run_start will be loaded from the pulse times.
  DateAndTime run_start(0, 0);
817
  bool takeTimesFromEvents = false;
818
  // Initialize the counter of bad TOFs
819
  bad_tofs = 0;
820
  int nPeriods = 1;
Samuel Jones's avatar
Samuel Jones committed
821
  auto periodLog = std::make_unique<const TimeSeriesProperty<int>>("period_log");
Mathieu Tillet's avatar
Mathieu Tillet committed
822
823
824

  bool loadAllLogs = getProperty("LoadAllLogs");

825
  if (loadlogs) {
Mathieu Tillet's avatar
Mathieu Tillet committed
826
827
828
829
830
831
832
833
834
835
    if (!loadAllLogs) {
      prog->doReport("Loading DAS logs");

      if (allow_list.empty() && block_list.empty()) {
        m_allBanksPulseTimes =
            runLoadNexusLogs<EventWorkspaceCollection_sptr>(m_filename, m_ws, *this, true, nPeriods, periodLog);
      } else {
        m_allBanksPulseTimes = runLoadNexusLogs<EventWorkspaceCollection_sptr>(m_filename, m_ws, *this, true, nPeriods,
                                                                               periodLog, allow_list, block_list);
      }
836

Mathieu Tillet's avatar
Mathieu Tillet committed
837
838
839
840
841
842
843
844
845
846
847
      try {
        run_start = m_ws->getFirstPulseTime();
      } catch (Kernel::Exception::NotFoundError &) {
        /*
          This is added to (a) support legacy behaviour of continuing to take
          times from the proto_charge log, but (b) allowing a fall back of
          getting run start and end from actual pulse times within the
          NXevent_data group. Note that the latter is better Nexus compliant.
        */
        takeTimesFromEvents = true;
      }
848
    } else {
849
      prog->doReport("Loading all logs");
Mathieu Tillet's avatar
Mathieu Tillet committed
850
851
852
853
854
855
856
857
858
      // Open NeXus file
      NXhandle nxHandle;
      NXstatus nxStat = NXopen(m_filename.c_str(), NXACC_READ, &nxHandle);

      if (nxStat != NX_ERROR) {
        LoadHelper loadHelper;
        loadHelper.addNexusFieldsToWsRun(nxHandle, m_ws->mutableRun());
        NXclose(&nxHandle);
      }
859
    }
860
861
  } else {
    g_log.information() << "Skipping the loading of sample logs!\n"
Samuel Jones's avatar
Samuel Jones committed
862
                        << "Reading the start time directly from /" << m_top_entry_name << "/start_time\n";
863
864
865
866
867
868
869
    // start_time is read and set
    m_file->openPath("/");
    m_file->openGroup(m_top_entry_name, "NXentry");
    std::string tmp;
    m_file->readData("start_time", tmp);
    m_file->closeGroup();
    run_start = createFromSanitizedISO8601(tmp);
Samuel Jones's avatar
Samuel Jones committed
870
    m_ws->mutableRun().addProperty("run_start", run_start.toISO8601String(), true);
871
  }
872
  // set more properties on the workspace
873
874
  const std::shared_ptr<NexusHDF5Descriptor> descriptor = getFileInfo();

875
876
877
  try {
    // this is a static method that is why it is passing the
    // file object and the file path
878

Samuel Jones's avatar
Samuel Jones committed
879
    loadEntryMetadata<EventWorkspaceCollection_sptr>(m_filename, m_ws, m_top_entry_name, *descriptor);
880
881
882
883
  } catch (std::runtime_error &e) {
    // Missing metadata is not a fatal error. Log and go on with your life
    g_log.error() << "Error loading metadata: " << e.what() << '\n';
  }
Tom Titcombe's avatar
Tom Titcombe committed
884

Samuel Jones's avatar
Samuel Jones committed
885
886
  m_ws->setNPeriods(static_cast<size_t>(nPeriods),
                    periodLog); // This is how many workspaces we are going to make.
887

888
  // Make sure you have a non-NULL m_allBanksPulseTimes
889
  if (m_allBanksPulseTimes == nullptr) {
890
    std::vector<DateAndTime> temp;
891
    m_allBanksPulseTimes = std::make_shared<BankPulseTimes>(temp);
892
893
  }

Owen Arnold's avatar
Owen Arnold committed
894
  if (!m_ws->getInstrument() || !m_instrument_loaded_correctly) {
895
    // Load the instrument (if not loaded before)
896
    prog->report("Loading instrument");
897
898
899
900
901
    // Note that closing an re-opening the file is needed here for loading
    // instruments directly from the nexus file containing the event data.
    // This may not be needed in the future if both LoadEventNexus and
    // LoadInstrument are made to use the same Nexus/HDF5 library
    m_file->close();
Samuel Jones's avatar
Samuel Jones committed
902
    m_instrument_loaded_correctly = loadInstrument(m_filename, m_ws, m_top_entry_name, this, descriptor.get());
903

904
    if (!m_instrument_loaded_correctly)
905
906
      throw std::runtime_error("Instrument was not initialized correctly! "
                               "Loading cannot continue.");
907
908
    // reopen file
    safeOpenFile(m_filename);
909
  }
910
911

  // top level file information
912
  m_file->openPath("/");
913
  // Start with the base entry
914
  m_file->openGroup(m_top_entry_name, "NXentry");
915
916
917
918
919
920
921

  // Now we want to go through all the bankN_event entries
  vector<string> bankNames;
  vector<std::size_t> bankNumEvents;
  std::string classType = monitors ? "NXmonitor" : "NXevent_data";
  ::NeXus::Info info;
  bool oldNeXusFileNames(false);
922
  bool haveWeights = false;
923
  auto firstPulseT = DateAndTime::maximum();
924

Samuel Jones's avatar
Samuel Jones committed
925
  const std::map<std::string, std::set<std::string>> &allEntries = descriptor->getAllEntries();
926
927
928
929

  auto itClassEntries = allEntries.find(classType);

  if (itClassEntries != allEntries.end()) {
930

931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
    const std::set<std::string> &classEntries = itClassEntries->second;
    const std::regex classRegex("(/" + m_top_entry_name + "/)([^/]*)");
    std::smatch groups;

    for (const std::string &classEntry : classEntries) {

      if (std::regex_match(classEntry, groups, classRegex)) {
        const std::string entry_name(groups[2].str());
        m_file->openGroup(entry_name, classType);

        if (takeTimesFromEvents) {
          /* If we are here, we are loading logs, but have failed to establish
           * the run_start from the proton_charge log. We are going to get this
           * from our event_time_zero instead
           */
          auto localFirstLast = firstLastPulseTimes(*m_file, this->g_log);
          firstPulseT = std::min(firstPulseT, localFirstLast.first);
        }
        // get the number of events
        const std::string prefix = "/" + m_top_entry_name + "/" + entry_name;
951
        bool hasTotalCounts = true;
Samuel Jones's avatar
Samuel Jones committed
952
        std::size_t num = numEvents(*m_file, hasTotalCounts, oldNeXusFileNames, prefix, *descriptor);
953
954
955
956
957
958
959
960
        bankNames.emplace_back(entry_name);
        bankNumEvents.emplace_back(num);

        // Look for weights in simulated file
        const std::string absoluteEventWeightName = prefix + "/event_weight";
        haveWeights = descriptor->isEntry(absoluteEventWeightName);
        m_file->closeGroup();
      }
961
962
    }
  }
963

964
965
  if (takeTimesFromEvents)
    run_start = firstPulseT;
966

967
  loadSampleDataISIScompatibility(*m_file, *m_ws);
968

969
970
  // Close the 'top entry' group (raw_data_1 for NexusProcessed, etc.)
  m_file->closeGroup();
971
972
973
974
975
976
977
978
979
980
981
982
983
984

  // Delete the output workspace name if it existed
  std::string outName = getPropertyValue("OutputWorkspace");
  if (AnalysisDataService::Instance().doesExist(outName))
    AnalysisDataService::Instance().remove(outName);

  // --------------------------- Time filtering
  // ------------------------------------
  double filter_time_start_sec, filter_time_stop_sec;
  filter_time_start_sec = getProperty("FilterByTimeStart");
  filter_time_stop_sec = getProperty("FilterByTimeStop");

  // Default to ALL pulse times
  bool is_time_filtered = false;
985
986
  filter_time_start = Types::Core::DateAndTime::minimum();
  filter_time_stop = Types::Core::DateAndTime::maximum();
987

988
  if (m_allBanksPulseTimes->pulseTimes.size() > 0) {
989
990
991
992
993
    // If not specified, use the limits of doubles. Otherwise, convert from
    // seconds to absolute PulseTime
    if (filter_time_start_sec != EMPTY_DBL()) {
      filter_time_start = run_start + filter_time_start_sec;
      is_time_filtered = true;
994
995
    }

996
997
998
    if (filter_time_stop_sec != EMPTY_DBL()) {
      filter_time_stop = run_start + filter_time_stop_sec;
      is_time_filtered = true;
999
    }
1000

1001
1002
1003
1004
1005
1006
1007
    // Silly values?
    if (filter_time_stop < filter_time_start) {
      std::string msg = "Your ";
      if (monitors)
        msg += "monitor ";
      msg += "filter for time's Stop value is smaller than the Start value.";
      throw std::invalid_argument(msg);
1008
    }
1009
1010
1011
1012
  }

  if (is_time_filtered) {
    // Now filter out the run, using the DateAndTime type.
Owen Arnold's avatar
Owen Arnold committed
1013
    m_ws->mutableRun().filterByTime(filter_time_start, filter_time_stop);
1014
1015
1016
1017
  }

  if (metaDataOnly) {
    // Now, create a default X-vector for histogramming, with just 2 bins.
Samuel Jones's avatar
Samuel Jones committed
1018
    auto axis = HistogramData::BinEdges{1, static_cast<double>(std::numeric_limits<uint32_t>::max()) * 0.1 - 1};
1019
    // Set the binning axis using this.
Owen Arnold's avatar
Owen Arnold committed
1020
    m_ws->setAllX(axis);
1021

1022
    createSpectraMapping(m_filename, monitors, std::vector<std::string>());
1023
    return;
1024
1025
1026
1027
  }

  // --------- Loading only one bank ? ----------------------------------
  std::vector<std::string> someBanks = getProperty("BankName");
1028
  const bool SingleBankPixelsOnly = getProperty("SingleBankPixelsOnly");
1029
  if ((!someBanks.empty()) && (!monitors)) {
1030
1031
1032
1033
1034
    std::vector<std::string> eventedBanks;
    eventedBanks.reserve(someBanks.size());
    for (const auto &bank : someBanks) {
      eventedBanks.emplace_back(bank + "_events");
    }
1035
    // check that all of the requested banks are in the file
1036
    const auto invalidBank =
Samuel Jones's avatar
Samuel Jones committed
1037
1038
1039
1040
        std::find_if(eventedBanks.cbegin(), eventedBanks.cend(), [&bankNames](const auto &someBank) {
          return std::none_of(bankNames.cbegin(), bankNames.cend(),
                              [&someBank](const auto &name) { return name == someBank; });
        });
1041
    if (invalidBank != eventedBanks.cend()) {
Samuel Jones's avatar
Samuel Jones committed
1042
      throw std::invalid_argument("No entry named '" + *invalidBank + "' was found in the .NXS file.");