Unverified Commit 3da2a5c6 authored by Antti Soininen's avatar Antti Soininen Committed by GitHub
Browse files

Merge pull request #24798 from mantidproject/24797_resolve_date_stamp

Helper function to retrieve date-sorted filenames for IDF's or Parameter files
parents 439cf10c 3fb4868f
......@@ -164,10 +164,13 @@ public:
static void getValidFromTo(const std::string &IDFfilename,
std::string &outValidFrom,
std::string &outValidTo);
/// Utility to retrieve a resource file (IDF, Parameters, ..)
static std::vector<std::string> getResourceFilenames(
const std::string &prefix, const std::vector<std::string> &fileFormats,
const std::vector<std::string> &directoryNames, const std::string &date);
/// Get the IDF using the instrument name and date
static std::string getInstrumentFilename(const std::string &instrumentName,
const std::string &date = "");
const Geometry::DetectorInfo &detectorInfo() const;
Geometry::DetectorInfo &mutableDetectorInfo();
......
......@@ -80,7 +80,7 @@ public:
class myContentHandler : public Poco::XML::ContentHandler {
void startElement(const XMLString &, const XMLString &localName,
const XMLString &, const Attributes &attrList) override {
if (localName == "instrument") {
if (localName == "instrument" || localName == "parameter-file") {
throw DummyException(
static_cast<std::string>(attrList.getValue("", "valid-from")),
static_cast<std::string>(attrList.getValue("", "valid-to")));
......@@ -934,69 +934,65 @@ void ExperimentInfo::getValidFromTo(const std::string &IDFfilename,
}
}
/** A given instrument may have multiple definition files associated with it.
*This method returns a file name which identifies a given instrument definition
*for a given instrument.
*The instrument geometry can be loaded from either a ".xml" file (old-style
*IDF) or a ".hdf5/.nxs" file (new-style nexus).
*The filename is required to be of the form InstrumentName + _Definition +
*Identifier + extension. The identifier then is the part of a filename that
*identifies the instrument definition valid at a given date.
*
* If several instrument files files are valid at the given date the file with
*the most recent from date is selected. If no such files are found the file
*with the latest from date is selected.
/** Compile a list of files in compliance with name pattern-matching, file
* format, and date-stamp constraints
*
* If no file is found for the given instrument, an empty string is returned.
*
* @param instrumentName :: Instrument name e.g. GEM, TOPAS or BIOSANS
* @param date :: ISO 8601 date
* @return full path of instrument geometry file
* Ideally, the valid-from and valid-to of any valid file should encapsulate
* argument date. If this is not possible, then the file with the most recent
* valid-from stamp is selected.
*
* @throws Exception::NotFoundError If no valid instrument definition filename
*is found
* @param prefix :: the name of a valid file must begin with this pattern
* @param fileFormats :: the extension of a valid file must be one of these
* formats
* @param directoryNames :: search only in these directories
* @param date :: the valid-from and valid-to of a valid file should encapsulate
* this date
* @return list of absolute paths for each valid file
*/
std::string
ExperimentInfo::getInstrumentFilename(const std::string &instrumentName,
const std::string &date) {
std::vector<std::string> ExperimentInfo::getResourceFilenames(
const std::string &prefix, const std::vector<std::string> &fileFormats,
const std::vector<std::string> &directoryNames, const std::string &date) {
if (date.empty()) {
// Just use the current date
g_log.debug() << "No date specified, using current date and time.\n";
const std::string now =
Types::Core::DateAndTime::getCurrentTime().toISO8601String();
// Recursively call this method, but with both parameters.
return ExperimentInfo::getInstrumentFilename(instrumentName, now);
// Recursively call this method, but with all parameters.
return ExperimentInfo::getResourceFilenames(prefix, fileFormats,
directoryNames, now);
}
g_log.debug() << "Looking for instrument file for " << instrumentName
<< " that is valid on '" << date << "'\n";
// Lookup the instrument (long) name
std::string instrument(
Kernel::ConfigService::Instance().getInstrument(instrumentName).name());
// Get the instrument directories for instrument file search
const std::vector<std::string> &directoryNames =
Kernel::ConfigService::Instance().getInstrumentDirectories();
// Join all the file formats into a single string
std::stringstream ss;
ss << "(";
for (size_t i = 0; i < fileFormats.size(); ++i) {
if (i != 0)
ss << "|";
ss << fileFormats[i];
}
ss << ")";
const std::string allFileFormats = ss.str();
const boost::regex regex(instrument + "_Definition.*\\.(xml|nxs|hdf5)",
const boost::regex regex(prefix + ".*\\." + allFileFormats,
boost::regex_constants::icase);
Poco::DirectoryIterator end_iter;
DateAndTime d(date);
bool foundGoodFile =
false; // True if we have found a matching file (valid at the given date)
std::string mostRecentInstFile; // store most recently starting matching
// instrument file if found, else most
// recently starting instrument file.
DateAndTime refDate("1900-01-31 23:59:00"); // used to help determine the most
// recently starting instrument
// file, if none match
DateAndTime refDateGoodFile("1900-01-31 23:59:00"); // used to help determine
// the most recently
// starting matching
// instrument file
// recently starting file, if none match
DateAndTime refDateGoodFile(
"1900-01-31 23:59:00"); // used to help determine the most recently
// Two files could have the same `from` date so multimap is required.
// Sort with newer dates placed at the beginning
std::multimap<DateAndTime, std::string, std::greater<DateAndTime>>
matchingFiles;
bool foundFile = false;
std::string mostRecentFile; // path to the file with most recent "valid-from"
for (const auto &directoryName : directoryNames) {
// This will iterate around the directories from user ->etc ->install, and
// find the first beat file
// Iterate over the directories from user ->etc ->install, and find the
// first beat file
for (Poco::DirectoryIterator dir_itr(directoryName); dir_itr != end_iter;
++dir_itr) {
......@@ -1008,11 +1004,11 @@ ExperimentInfo::getInstrumentFilename(const std::string &instrumentName,
if (regex_match(l_filenamePart, regex)) {
const auto &pathName = filePath.toString();
g_log.debug() << "Found file: '" << pathName << "'\n";
std::string validFrom, validTo;
getValidFromTo(pathName, validFrom, validTo);
g_log.debug() << "File '" << pathName << " valid dates: from '"
<< validFrom << "' to '" << validTo << "'\n";
// Use default valid "from" and "to" dates if none were found.
DateAndTime to, from;
if (validFrom.length() > 0)
......@@ -1025,25 +1021,79 @@ ExperimentInfo::getInstrumentFilename(const std::string &instrumentName,
to.setFromISO8601("2100-01-01T00:00:00");
if (from <= d && d <= to) {
if (from > refDateGoodFile) { // We'd found a matching file more
// recently starting than any other
// matching file found
foundGoodFile = true;
refDateGoodFile = from;
mostRecentInstFile = pathName;
}
foundFile = true;
matchingFiles.insert(
std::pair<DateAndTime, std::string>(from, pathName));
}
if (!foundGoodFile && (from >= refDate)) { // Use most recently starting
// file, in case we don't
// find a matching file.
// Consider the most recent file in the absence of matching files
if (!foundFile && (from >= refDate)) {
refDate = from;
mostRecentInstFile = pathName;
mostRecentFile = pathName;
}
}
}
}
g_log.debug() << "Instrument file selected is " << mostRecentInstFile << '\n';
return mostRecentInstFile;
// Retrieve the file names only
std::vector<std::string> pathNames;
if (!matchingFiles.empty()) {
pathNames.reserve(matchingFiles.size());
for (auto elem : matchingFiles)
pathNames.emplace_back(std::move(elem.second));
} else {
pathNames.emplace_back(std::move(mostRecentFile));
}
return pathNames;
}
/** A given instrument may have multiple definition files associated with it.
*This method returns a file name which identifies a given instrument definition
*for a given instrument.
*The instrument geometry can be loaded from either a ".xml" file (old-style
*IDF) or a ".hdf5/.nxs" file (new-style nexus).
*The filename is required to be of the form InstrumentName + _Definition +
*Identifier + extension. The identifier then is the part of a filename that
*identifies the instrument definition valid at a given date.
*
* If several instrument files files are valid at the given date the file with
*the most recent from date is selected. If no such files are found the file
*with the latest from date is selected.
*
* If no file is found for the given instrument, an empty string is returned.
*
* @param instrumentName :: Instrument name e.g. GEM, TOPAS or BIOSANS
* @param date :: ISO 8601 date
* @return full path of instrument geometry file
*
* @throws Exception::NotFoundError If no valid instrument definition filename
*is found
*/
std::string
ExperimentInfo::getInstrumentFilename(const std::string &instrumentName,
const std::string &date) {
const std::vector<std::string> validFormats = {"xml", "nxs", "hdf5"};
g_log.debug() << "Looking for instrument file for " << instrumentName
<< " that is valid on '" << date << "'\n";
// Lookup the instrument (long) name
const std::string instrument(
Kernel::ConfigService::Instance().getInstrument(instrumentName).name());
// Get the instrument directories for instrument file search
const std::vector<std::string> &directoryNames =
Kernel::ConfigService::Instance().getInstrumentDirectories();
// matching files sorted with newest files coming first
const std::vector<std::string> matchingFiles = getResourceFilenames(
instrument + "_Definition", validFormats, directoryNames, date);
std::string instFile;
if (!matchingFiles.empty()) {
instFile = matchingFiles[0];
g_log.debug() << "Instrument file selected is " << instFile << '\n';
} else {
g_log.debug() << "No instrument file found\n";
}
return instFile;
}
/** Return a const reference to the DetectorInfo object.
......
......@@ -586,6 +586,26 @@ public:
TS_ASSERT_DIFFERS(boevs.find("TEST1_ValidDateOverlap"), std::string::npos);
ConfigService::Instance().setString("instrumentDefinition.directory",
instDir);
std::vector<std::string> formats = {"xml"};
std::vector<std::string> dirs;
dirs.push_back(testDir);
std::vector<std::string> fnames = helper.getResourceFilenames(
"ARGUS", formats, dirs, "1909-01-31 22:59:59");
TS_ASSERT_DIFFERS(fnames[0].find("TEST1_ValidDateOverlap"),
std::string::npos);
TS_ASSERT_EQUALS(fnames.size(), 1);
fnames = helper.getResourceFilenames("ARGUS", formats, dirs,
"1909-03-31 22:59:59");
TS_ASSERT_DIFFERS(fnames[0].find("TEST2_ValidDateOverlap"),
std::string::npos);
TS_ASSERT_DIFFERS(fnames[1].find("TEST1_ValidDateOverlap"),
std::string::npos);
fnames = helper.getResourceFilenames("ARGUS", formats, dirs,
"1909-05-31 22:59:59");
TS_ASSERT_DIFFERS(fnames[0].find("TEST1_ValidDateOverlap"),
std::string::npos);
TS_ASSERT_EQUALS(fnames.size(), 1);
}
void test_nexus_geometry_getInstrumentFilename() {
......
......@@ -12,19 +12,32 @@
#include "MantidGeometry/Instrument/ComponentInfo.h"
#include "MantidGeometry/Instrument/DetectorInfo.h"
#include "MantidKernel/WarningSuppressions.h"
#include "MantidPythonInterface/kernel/Converters/PySequenceToVector.h"
#include "MantidPythonInterface/kernel/Converters/ToPyList.h"
#include "MantidPythonInterface/kernel/GetPointer.h"
#include "MantidPythonInterface/kernel/Policies/RemoveConst.h"
#include <boost/python/class.hpp>
#include <boost/python/copy_const_reference.hpp>
#include <boost/python/list.hpp>
#include <boost/python/overloads.hpp>
#include <boost/python/register_ptr_to_python.hpp>
using Mantid::API::ExperimentInfo;
using Mantid::PythonInterface::Policies::RemoveConstSharedPtr;
using namespace Mantid::PythonInterface;
using namespace boost::python;
GET_POINTER_SPECIALIZATION(ExperimentInfo)
/// Converter from C++ signature to python signature
list getResourceFilenames(const std::string &prefix, const list &fileFormats,
const list &directoryNames, const std::string &date) {
return Converters::ToPyList<std::string>()(
ExperimentInfo::getResourceFilenames(
prefix, Converters::PySequenceToVector<std::string>(fileFormats)(),
Converters::PySequenceToVector<std::string>(directoryNames)(), date));
}
GNU_DIAG_OFF("unused-local-typedef")
// Ignore -Wconversion warnings coming from boost::python
// Seen with GCC 7.1.1 and Boost 1.63.0
......@@ -42,7 +55,22 @@ void export_ExperimentInfo() {
.def("getInstrument", &ExperimentInfo::getInstrument,
return_value_policy<RemoveConstSharedPtr>(), args("self"),
"Returns the :class:`~mantid.geometry.Instrument` for this run.")
.def("getResourceFilenames", &getResourceFilenames,
(arg("prefix"), arg("fileFormats"), arg("directoryNames"),
arg("date")),
"Compile a list of files in compliance with name pattern-matching,\n"
"file format, and date-stamp constraints\n\n"
"Ideally, the valid-from and valid-to of any valid file should\n"
"encapsulate the argument date. If this is not possible, then\n"
"the file with the most recent valid-from stamp is selected\n\n"
"prefix: the name of a valid file must begin with this "
"pattern\n"
"fileFormats: list of valid file extensions\n"
"directoryNames: list of directories to be searched\n"
"date : the 'valid-from' and 'valid-to 'dates of a valid\n"
"file will encapsulate this date (e.g '1900-01-31 23:59:00')\n"
"\nreturns : list of absolute paths for each valid file\n")
.staticmethod("getResourceFilenames")
.def("getInstrumentFilename", &ExperimentInfo::getInstrumentFilename,
getInstrumentFilename_Overload(
"Returns IDF filename", (arg("instrument"), arg("date") = "")))
......
......@@ -27,10 +27,9 @@ Logging
Nexus Geometry Loading
----------------------
:ref:`LoadEmptyInstrument <algm-LoadEmptyInstrument>` will now load instrument geometry from hdf5 `NeXus <https://www.nexusformat.org/>`_ format files. Files consistent with the standard following the introduction of `NXoff_geometry <http://download.nexusformat.org/sphinx/classes/base_classes/NXoff_geometry.html>`_ and `NXcylindrical_geometry <http://download.nexusformat.org/sphinx/classes/base_classes/NXcylindrical_geometry.html>`_ will be used to build the entire in-memory instrument geometry within Mantid. This IDF-free route is primarily envisioned for the ESS. While dependent on the instrument, we are overall seeing significant improvements in instrument load times over loading from equivalent IDF based implementations. :ref:`LoadInstrument <algm-LoadInstrument>` also supports the nexus geometry format in the same was as LoadEmptyInstrument.
The changes above have also been encorporated directly into :ref:`LoadEventNexus <algm-LoadEventNexus>`, so it is possible to store data and geometry together for the first time in a NeXus compliant format for Mantid. These changes have made it possible to load experimental ESS event data files directly into Mantid.
- :ref:`LoadEmptyInstrument <algm-LoadEmptyInstrument>` will now load instrument geometry from hdf5 `NeXus <https://www.nexusformat.org/>`_ format files. Files consistent with the standard following the introduction of `NXoff_geometry <http://download.nexusformat.org/sphinx/classes/base_classes/NXoff_geometry.html>`_ and `NXcylindrical_geometry <http://download.nexusformat.org/sphinx/classes/base_classes/NXcylindrical_geometry.html>`_ will be used to build the entire in-memory instrument geometry within Mantid. This IDF-free route is primarily envisioned for the ESS. While dependent on the instrument, we are overall seeing significant improvements in instrument load times over loading from equivalent IDF based implementations. :ref:`LoadInstrument <algm-LoadInstrument>` also supports the nexus geometry format in the same was as LoadEmptyInstrument.
- The changes above have also been encorporated directly into :ref:`LoadEventNexus <algm-LoadEventNexus>`, so it is possible to store data and geometry together for the first time in a NeXus compliant format for Mantid. These changes have made it possible to load experimental ESS event data files directly into Mantid.
- New helper function `ExperimentInfo.getResourceFilenames` returns a list of instrument definition files and/or parameter files in accordance to a query time stamp.
Archive Searching
-----------------
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment