Commit bc3283b4 authored by David Fairbrother's avatar David Fairbrother
Browse files

Move static file handling into own class

Moves multiple static file handling methods, such as FindFileX out of
ExperimentInfo into their own class
parent 622fa5dd
......@@ -39,6 +39,7 @@ set(SRC_FILES
src/Expression.cpp
src/FileBackedExperimentInfo.cpp
src/FileFinder.cpp
src/FileFinderUtils.cpp
src/FileLoaderRegistry.cpp
src/FileProperty.cpp
src/FrameworkManager.cpp
......@@ -68,7 +69,7 @@ set(SRC_FILES
src/IFunctionGeneral.cpp
src/IFunctionMD.cpp
src/IFunctionMW.cpp
src/IJournal.cpp
src/IJournal.cpp
src/ILatticeFunction.cpp
src/IMDEventWorkspace.cpp
src/IMDHistoWorkspace.cpp
......@@ -152,7 +153,8 @@ set(SRC_FILES
src/WorkspaceNearestNeighbours.cpp
src/WorkspaceOpOverloads.cpp
src/WorkspaceProperty.cpp
src/WorkspaceUnitValidator.cpp)
src/WorkspaceUnitValidator.cpp
)
set(SRC_UNITY_IGNORE_FILES
src/CompositeFunction.cpp
......@@ -161,7 +163,8 @@ set(SRC_UNITY_IGNORE_FILES
src/IEventWorkspace.cpp
src/IFunctionMD.cpp
src/IPeakFunction.cpp
src/IPowderDiffPeakFunction.cpp)
src/IPowderDiffPeakFunction.cpp
)
set(INC_FILES
inc/MantidAPI/ADSValidator.h
......@@ -208,6 +211,7 @@ set(INC_FILES
inc/MantidAPI/Expression.h
inc/MantidAPI/FileBackedExperimentInfo.h
inc/MantidAPI/FileFinder.h
inc/MantidAPI/FileFinderUtils.h
inc/MantidAPI/FileLoaderRegistry.h
inc/MantidAPI/FileProperty.h
inc/MantidAPI/FrameworkManager.h
......@@ -360,14 +364,14 @@ set(INC_FILES
inc/MantidAPI/WorkspaceProperty.h
inc/MantidAPI/WorkspaceProperty.tcc
inc/MantidAPI/WorkspaceUnitValidator.h
inc/MantidAPI/Workspace_fwd.h)
inc/MantidAPI/Workspace_fwd.h
)
option(PROFILE_ALGORITHM_LINUX "Profile algorithm execution on Linux" OFF)
if(PROFILE_ALGORITHM_LINUX)
set(SRC_FILES
"${SRC_FILES}"
"src/AlgorithmExecuteProfile.cpp"
"src/AlgoTimeRegister.cpp")
set(SRC_FILES "${SRC_FILES}" "src/AlgorithmExecuteProfile.cpp"
"src/AlgoTimeRegister.cpp"
)
set(INC_FILES "${INC_FILES}" "inc/MantidAPI/AlgoTimeRegister.h")
else()
set(SRC_FILES "${SRC_FILES}" "src/AlgorithmExecute.cpp")
......@@ -403,6 +407,7 @@ set(TEST_FILES
ExpressionTest.h
FileBackedExperimentInfoTest.h
FileFinderTest.h
FileFinderUtilsTest.h
FilePropertyTest.h
FrameworkManagerTest.h
FuncMinimizerFactoryTest.h
......@@ -484,18 +489,20 @@ set(TEST_FILES
WorkspaceNearestNeighboursTest.h
WorkspaceOpOverloadsTest.h
WorkspacePropertyTest.h
WorkspaceUnitValidatorTest.h)
WorkspaceUnitValidatorTest.h
)
set(GMOCK_TEST_FILES
ImplicitFunctionFactoryTest.h
ImplicitFunctionParameterParserFactoryTest.h
MatrixWorkspaceTest.h)
ImplicitFunctionFactoryTest.h ImplicitFunctionParameterParserFactoryTest.h
MatrixWorkspaceTest.h
)
if(COVERALLS)
foreach(loop_var ${SRC_FILES} ${INC_FILES})
set_property(GLOBAL APPEND
PROPERTY COVERAGE_SRCS
"${CMAKE_CURRENT_SOURCE_DIR}/${loop_var}")
set_property(
GLOBAL APPEND PROPERTY COVERAGE_SRCS
"${CMAKE_CURRENT_SOURCE_DIR}/${loop_var}"
)
endforeach(loop_var)
endif()
......@@ -515,14 +522,15 @@ enable_precompiled_headers(inc/MantidAPI/PrecompiledHeader.h SRC_FILES)
# Add the target for this directory
add_library(API ${SRC_FILES} ${INC_FILES})
# Set the name of the generated library
set_target_properties(API
PROPERTIES OUTPUT_NAME
MantidAPI
COMPILE_DEFINITIONS
IN_MANTID_API)
set_target_properties(
API PROPERTIES OUTPUT_NAME MantidAPI COMPILE_DEFINITIONS IN_MANTID_API
)
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set_target_properties(API PROPERTIES INSTALL_RPATH "@loader_path/../MacOS;@loader_path/../Frameworks")
set_target_properties(
API PROPERTIES INSTALL_RPATH
"@loader_path/../MacOS;@loader_path/../Frameworks"
)
elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
set_target_properties(API PROPERTIES INSTALL_RPATH "\$ORIGIN/../${LIB_DIR}")
endif()
......@@ -530,17 +538,20 @@ endif()
# Add to the 'Framework' group in VS
set_property(TARGET API PROPERTY FOLDER "MantidFramework")
target_include_directories (API SYSTEM PUBLIC ${NEXUS_INCLUDE_DIR} ${Boost_INCLUDE_DIRS})
target_link_libraries(API
LINK_PRIVATE
${TCMALLOC_LIBRARIES_LINKTIME}
${JSONCPP_LIBRARIES}
${MANTIDLIBS}
${GSL_LIBRARIES}
${NEXUS_LIBRARIES}
${WINSOCK}
${BCRYPT})
target_include_directories(
API SYSTEM PUBLIC ${NEXUS_INCLUDE_DIR} ${Boost_INCLUDE_DIRS}
)
target_link_libraries(
API
LINK_PRIVATE
${TCMALLOC_LIBRARIES_LINKTIME}
${JSONCPP_LIBRARIES}
${MANTIDLIBS}
${GSL_LIBRARIES}
${NEXUS_LIBRARIES}
${WINSOCK}
${BCRYPT}
)
# Add the unit tests directory
add_subdirectory(test)
......
......@@ -147,23 +147,6 @@ public:
// run end time if available, empty otherwise
std::string getAvailableWorkspaceEndDate() const;
/// Utility to retrieve the validity dates for the given IDF
static void getValidFromTo(const std::string &IDFfilename,
std::string &outValidFrom,
std::string &outValidTo);
/// Utility to retrieve a resource file (IDF, Parameters, ..)
static std::vector<std::string> getResourceFilenames(
const std::string &prefix, const std::vector<std::string> &fileFormats,
const std::vector<std::string> &directoryNames, const std::string &date);
/// Get the IDF using the instrument name and date
static std::string getInstrumentFilename(const std::string &instrumentName,
const std::string &date = "");
/// Search instrument directories for Parameter file,
/// return full path name if found, else "".
static std::string getFullPathParamIDF(std::string instName,
const std::string &dirHint = "");
const Geometry::DetectorInfo &detectorInfo() const;
Geometry::DetectorInfo &mutableDetectorInfo();
......
// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright &copy; 2020 ISIS Rutherford Appleton Laboratory UKRI,
// NScD Oak Ridge National Laboratory, European Spallation Source,
// Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#pragma once
#include "MantidAPI/DllConfig.h"
#include <string>
#include <vector>
namespace Mantid::API {
class MANTID_API_DLL FileFinderUtils {
public:
/// Search instrument directories for Parameter file,
/// return full path name if found, else "".
static std::string getFullPathParamIDF(std::string instName,
const std::string &dirHint = "");
/// Get the IDF using the instrument name and date
static std::string getInstrumentFilename(const std::string &instrumentName,
const std::string &date = "");
/// Utility to retrieve a resource file (IDF, Parameters, ..)
static std::vector<std::string> getResourceFilenames(
const std::string &prefix, const std::vector<std::string> &fileFormats,
const std::vector<std::string> &directoryNames, const std::string &date);
/// Utility to retrieve the validity dates for the given IDF
static void getValidFromTo(const std::string &IDFfilename,
std::string &outValidFrom,
std::string &outValidTo);
private:
};
} // Namespace Mantid::API
\ No newline at end of file
......@@ -5,6 +5,7 @@
// Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#include "MantidAPI/ExperimentInfo.h"
#include "MantidAPI/FileFinderUtils.h"
#include "MantidAPI/InstrumentDataService.h"
#include "MantidAPI/ResizeRectangularDetectorHelper.h"
#include "MantidAPI/Run.h"
......@@ -44,11 +45,7 @@
#include <boost/lexical_cast.hpp>
#include <boost/regex.hpp>
#include <Poco/DirectoryIterator.h>
#include <Poco/Path.h>
#include <Poco/SAX/Attributes.h>
#include <Poco/SAX/ContentHandler.h>
#include <Poco/SAX/SAXParser.h>
#include <nexus/NeXusException.hpp>
#include <algorithm>
......@@ -58,7 +55,6 @@
using namespace Mantid::Geometry;
using namespace Mantid::Kernel;
using namespace Mantid::Types::Core;
using namespace Poco::XML;
namespace Mantid {
......@@ -67,42 +63,6 @@ namespace {
/// static logger object
Kernel::Logger g_log("ExperimentInfo");
// used to terminate SAX process
class DummyException {
public:
std::string m_validFrom;
std::string m_validTo;
DummyException(const std::string &validFrom, const std::string &validTo)
: m_validFrom(validFrom), m_validTo(validTo) {}
};
// SAX content handler for grapping stuff quickly from IDF
class myContentHandler : public Poco::XML::ContentHandler {
void startElement(const XMLString & /*uri*/, const XMLString &localName,
const XMLString & /*qname*/,
const Attributes &attrList) override {
if (localName == "instrument" || localName == "parameter-file") {
throw DummyException(
static_cast<std::string>(attrList.getValue("", "valid-from")),
static_cast<std::string>(attrList.getValue("", "valid-to")));
}
}
void endElement(const XMLString & /*uri*/, const XMLString & /*localName*/,
const XMLString & /*qname*/) override {}
void startDocument() override {}
void endDocument() override {}
void characters(const XMLChar /*ch*/[], int /*start*/,
int /*length*/) override {}
void endPrefixMapping(const XMLString & /*prefix*/) override {}
void ignorableWhitespace(const XMLChar /*ch*/[], int /*start*/,
int /*length*/) override {}
void processingInstruction(const XMLString & /*target*/,
const XMLString & /*data*/) override {}
void setDocumentLocator(const Locator * /*loc*/) override {}
void skippedEntity(const XMLString & /*name*/) override {}
void startPrefixMapping(const XMLString & /*prefix*/,
const XMLString & /*uri*/) override {}
};
} // namespace
/** Constructor
......@@ -817,273 +777,7 @@ std::string ExperimentInfo::getAvailableWorkspaceEndDate() const {
return date;
}
/** Return from an IDF the values of the valid-from and valid-to attributes
*
* @param IDFfilename :: Full path of an IDF
* @param[out] outValidFrom :: Used to return valid-from date
* @param[out] outValidTo :: Used to return valid-to date
*/
void ExperimentInfo::getValidFromTo(const std::string &IDFfilename,
std::string &outValidFrom,
std::string &outValidTo) {
SAXParser pParser;
// Create on stack to ensure deletion. Relies on pParser also being local
// variable.
myContentHandler conHand;
pParser.setContentHandler(&conHand);
try {
pParser.parse(IDFfilename);
} catch (DummyException &e) {
outValidFrom = e.m_validFrom;
outValidTo = e.m_validTo;
} catch (...) {
// should throw some sensible here
}
}
/** Compile a list of files in compliance with name pattern-matching, file
* format, and date-stamp constraints
*
* Ideally, the valid-from and valid-to of any valid file should encapsulate
* argument date. If this is not possible, then the file with the most recent
* valid-from stamp is selected.
*
* @param prefix :: the name of a valid file must begin with this pattern
* @param fileFormats :: the extension of a valid file must be one of these
* formats
* @param directoryNames :: search only in these directories
* @param date :: the valid-from and valid-to of a valid file should encapsulate
* this date
* @return list of absolute paths for each valid file
*/
std::vector<std::string> ExperimentInfo::getResourceFilenames(
const std::string &prefix, const std::vector<std::string> &fileFormats,
const std::vector<std::string> &directoryNames, const std::string &date) {
if (date.empty()) {
// Just use the current date
g_log.debug() << "No date specified, using current date and time.\n";
const std::string now =
Types::Core::DateAndTime::getCurrentTime().toISO8601String();
// Recursively call this method, but with all parameters.
return ExperimentInfo::getResourceFilenames(prefix, fileFormats,
directoryNames, now);
}
// Join all the file formats into a single string
std::stringstream ss;
ss << "(";
for (size_t i = 0; i < fileFormats.size(); ++i) {
if (i != 0)
ss << "|";
ss << fileFormats[i];
}
ss << ")";
const std::string allFileFormats = ss.str();
const boost::regex regex(prefix + ".*\\." + allFileFormats,
boost::regex_constants::icase);
Poco::DirectoryIterator end_iter;
DateAndTime d(date);
DateAndTime refDate("1900-01-31 23:59:00"); // used to help determine the most
// recently starting file, if none match
DateAndTime refDateGoodFile(
"1900-01-31 23:59:00"); // used to help determine the most recently
// Two files could have the same `from` date so multimap is required.
// Sort with newer dates placed at the beginning
std::multimap<DateAndTime, std::string, std::greater<DateAndTime>>
matchingFiles;
bool foundFile = false;
std::string mostRecentFile; // path to the file with most recent "valid-from"
for (const auto &directoryName : directoryNames) {
// Iterate over the directories from user ->etc ->install, and find the
// first beat file
for (Poco::DirectoryIterator dir_itr(directoryName); dir_itr != end_iter;
++dir_itr) {
const auto &filePath = dir_itr.path();
if (!filePath.isFile())
continue;
const std::string &l_filenamePart = filePath.getFileName();
if (regex_match(l_filenamePart, regex)) {
const auto &pathName = filePath.toString();
g_log.debug() << "Found file: '" << pathName << "'\n";
std::string validFrom, validTo;
getValidFromTo(pathName, validFrom, validTo);
g_log.debug() << "File '" << pathName << " valid dates: from '"
<< validFrom << "' to '" << validTo << "'\n";
// Use default valid "from" and "to" dates if none were found.
DateAndTime to, from;
if (validFrom.length() > 0)
from.setFromISO8601(validFrom);
else
from = refDate;
if (validTo.length() > 0)
to.setFromISO8601(validTo);
else
to.setFromISO8601("2100-01-01T00:00:00");
if (from <= d && d <= to) {
foundFile = true;
matchingFiles.insert(
std::pair<DateAndTime, std::string>(from, pathName));
}
// Consider the most recent file in the absence of matching files
if (!foundFile && (from >= refDate)) {
refDate = from;
mostRecentFile = pathName;
}
}
}
}
// Retrieve the file names only
std::vector<std::string> pathNames;
if (!matchingFiles.empty()) {
pathNames.reserve(matchingFiles.size());
std::transform(matchingFiles.begin(), matchingFiles.end(),
std::back_inserter(pathNames),
[](const auto &elem) { return elem.second; });
} else {
pathNames.emplace_back(std::move(mostRecentFile));
}
return pathNames;
}
/** A given instrument may have multiple definition files associated with it.
*This method returns a file name which identifies a given instrument definition
*for a given instrument.
*The instrument geometry can be loaded from either a ".xml" file (old-style
*IDF) or a ".hdf5/.nxs" file (new-style nexus).
*The filename is required to be of the form InstrumentName + _Definition +
*Identifier + extension. The identifier then is the part of a filename that
*identifies the instrument definition valid at a given date.
*
* If several instrument files files are valid at the given date the file with
*the most recent from date is selected. If no such files are found the file
*with the latest from date is selected.
*
* If no file is found for the given instrument, an empty string is returned.
*
* @param instrumentName :: Instrument name e.g. GEM, TOPAS or BIOSANS
* @param date :: ISO 8601 date
* @return full path of instrument geometry file
*
* @throws Exception::NotFoundError If no valid instrument definition filename
*is found
*/
std::string
ExperimentInfo::getInstrumentFilename(const std::string &instrumentName,
const std::string &date) {
const std::vector<std::string> validFormats = {"xml", "nxs", "hdf5"};
g_log.debug() << "Looking for instrument file for " << instrumentName
<< " that is valid on '" << date << "'\n";
// Lookup the instrument (long) name
const std::string instrument(
Kernel::ConfigService::Instance().getInstrument(instrumentName).name());
// Get the instrument directories for instrument file search
const std::vector<std::string> &directoryNames =
Kernel::ConfigService::Instance().getInstrumentDirectories();
// matching files sorted with newest files coming first
const std::vector<std::string> matchingFiles = getResourceFilenames(
instrument + "_Definition", validFormats, directoryNames, date);
std::string instFile;
if (!matchingFiles.empty()) {
instFile = matchingFiles[0];
g_log.debug() << "Instrument file selected is " << instFile << '\n';
} else {
g_log.debug() << "No instrument file found\n";
}
return instFile;
}
//-----------------------------------------------------------------------------------------------------------------------
/// Search the directory for the Parameter IDF file and return full path name if
/// found, else return "".
// directoryName must include a final '/'.
std::string ExperimentInfo::getFullPathParamIDF(std::string instName,
const std::string &dirHint) {
constexpr auto lookupFile = [](const std::string &dir,
const std::string &filename) {
Poco::Path directoryPath(dir);
directoryPath.makeDirectory();
// Remove the path from the filename
Poco::Path filePath(filename);
const std::string &instrumentFile = filePath.getFileName();
// First check whether there is a parameter file whose name is the same as
// the IDF file, but with 'Parameters' instead of 'Definition'.
std::string definitionPart("_Definition");
const std::string::size_type prefix_end(
instrumentFile.find(definitionPart));
const std::string::size_type suffix_start =
prefix_end + definitionPart.length();
// Get prefix and leave case sensitive
std::string prefix = instrumentFile.substr(0, prefix_end);
// Make suffix ensuring it has positive length
std::string suffix = ".xml";
if (suffix_start < instrumentFile.length()) {
suffix = instrumentFile.substr(suffix_start, std::string::npos);
}
// Assemble parameter file name
std::string fullPathParamIDF =
directoryPath.setFileName(prefix + "_Parameters" + suffix).toString();
if (!Poco::File(fullPathParamIDF)
.exists()) { // No such file exists, so look
// for file based on instrument
// ID
// given by the prefix
fullPathParamIDF =
directoryPath.setFileName(prefix + "_Parameters.xml").toString();
}
if (!Poco::File(fullPathParamIDF)
.exists()) { // No such file exists, indicate
// none found in this directory.
fullPathParamIDF = "";
}
return fullPathParamIDF;
};
// All inst names are stored as capitals currently,
// so we need to also do this for case-sensitive filesystems
std::transform(instName.begin(), instName.end(), instName.begin(), ::toupper);
// Try the hinted dir first
if (!dirHint.empty()) {
const std::string result = lookupFile(dirHint, instName);
if (!result.empty()) {
return result;
}
}
Kernel::ConfigServiceImpl &configService = Kernel::ConfigService::Instance();
std::vector<std::string> directoryNames =
configService.getInstrumentDirectories();
for (const auto &dirName : directoryNames) {
// This will iterate around the directories from user ->etc ->install, and
// find the first beat file
const std::string result = lookupFile(dirName, instName);
if (!result.empty()) {
return result;
}
}
return "";
}
/** Return a const reference to the DetectorInfo object.
*
......@@ -1405,8 +1099,8 @@ void ExperimentInfo::setInstumentFromXML(const std::string &nxFilename,
} else {
// XML was not included or was empty
// Use the instrument name to find the file
instrumentFilename =
getInstrumentFilename(instrumentName, getWorkspaceStartDate());
instrumentFilename = FileFinderUtils::getInstrumentFilename(
instrumentName, getWorkspaceStartDate());
// And now load the contents
instrumentXml = loadInstrumentXML(instrumentFilename);
}
......
#include "MantidAPI/FileFinderUtils.h"
#include "MantidKernel/ConfigService.h"
#include "MantidKernel/DateAndTime.h"
#include "MantidKernel/InstrumentInfo.h"
#include "MantidKernel/Logger.h"
#include <Poco/DirectoryIterator.h>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Poco/SAX/Attributes.h>
#include <Poco/SAX/ContentHandler.h>
#include <Poco/SAX/SAXParser.h>
#include <boost/regex.hpp>
#include <string>
#include <utility>
#include <vector>
using namespace Mantid::Kernel;
using namespace Mantid::Types::Core;
using namespace Poco::XML;
namespace {
/// static logger object
Mantid::Kernel::Logger g_log("FileFinderUtils");
// used to terminate SAX process
class DummyException {
public:
std::string m_validFrom;
std::string m_validTo;
DummyException(const std::string &validFrom, const std::string &validTo)
: m_validFrom(validFrom), m_validTo(validTo) {}
};
// SAX content handler for grapping stuff quickly from IDF
class myContentHandler : public Poco::XML::ContentHandler {
void startElement(const XMLString & /*uri*/, const XMLString &localName,
const XMLString & /*qname*/,
const Attributes &attrList) override {
if (localName == "instrument" || localName == "parameter-file") {