From d08e2d206a9013a62def9f97ae256b17b7e831fe Mon Sep 17 00:00:00 2001
From: Martyn Gigg <martyn.gigg@stfc.ac.uk>
Date: Thu, 3 Dec 2009 14:07:42 +0000
Subject: [PATCH] Changes to FileValidator so that extensions are no longer
mandated, they are a suggestion. A warning is issued if the current extension
does not match those stored but the string is still marked as valid. This
required changes to LoadRaw and LoadLog so that they don't rely on extensions
any longer. Re #1053
---
.../inc/MantidDataHandling/LoadLog.h | 5 +-
.../inc/MantidDataHandling/LoadRaw.h | 1 +
.../inc/MantidDataHandling/LoadRaw2.h | 2 +
.../inc/MantidDataHandling/LoadRaw3.h | 2 +
Code/Mantid/DataHandling/src/LoadLog.cpp | 321 +++++++++---------
Code/Mantid/DataHandling/src/LoadRaw.cpp | 37 +-
Code/Mantid/DataHandling/src/LoadRaw2.cpp | 37 +-
Code/Mantid/DataHandling/src/LoadRaw3.cpp | 37 ++
.../Kernel/inc/MantidKernel/FileValidator.h | 5 +-
Code/Mantid/Kernel/src/FileValidator.cpp | 124 +++----
Code/Mantid/Kernel/test/FilePropertyTest.h | 7 +-
Code/Mantid/Kernel/test/FileValidatorTest.h | 78 ++---
12 files changed, 364 insertions(+), 292 deletions(-)
diff --git a/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadLog.h b/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadLog.h
index 279c319facc..b92202754b0 100644
--- a/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadLog.h
+++ b/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadLog.h
@@ -86,7 +86,6 @@ namespace Mantid
/// Overwrites Algorithm method
void exec();
-
/// The name and path of an input file. This may be the filename of a
/// raw datafile or the name of a specific log file.
std::string m_filename;
@@ -100,8 +99,8 @@ namespace Mantid
/// convert string to lower case
std::string stringToLower(std::string strToConvert);
- /// look at whether filename has the .txt extension and contain a '_'
- bool isLogFile(const std::string& filenamePart);
+ /// Checks if the file is an ASCII file
+ bool isAscii(const std::string& filenamePart);
/// check if first 19 characters of a string is data-time string according to yyyy-mm-ddThh:mm:ss
bool isDateTimeString(const std::string& str);
diff --git a/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw.h b/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw.h
index ef8a7994e3c..59fdb2ae3af 100644
--- a/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw.h
+++ b/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw.h
@@ -80,6 +80,7 @@ namespace Mantid
/// Overwrites Algorithm method
void exec();
+ bool isAscii(const std::string & filename) const;
void checkOptionalProperties();
void loadData(const DataObjects::Histogram1D::RCtype::ptr_type&,int, int&, ISISRAW& , const int& , int*, DataObjects::Workspace2D_sptr );
void runLoadInstrument(DataObjects::Workspace2D_sptr);
diff --git a/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw2.h b/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw2.h
index 1a2fdc727b8..73e7685a8fd 100644
--- a/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw2.h
+++ b/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw2.h
@@ -82,6 +82,8 @@ namespace Mantid
/// Overwrites Algorithm method
void exec();
+ /// Check if this actually is a binary file
+ bool isAscii(const std::string& filename) const;
void checkOptionalProperties();
void runLoadInstrument(DataObjects::Workspace2D_sptr);
void runLoadInstrumentFromRaw(DataObjects::Workspace2D_sptr);
diff --git a/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw3.h b/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw3.h
index fbd4e96136f..c5a3473a051 100644
--- a/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw3.h
+++ b/Code/Mantid/DataHandling/inc/MantidDataHandling/LoadRaw3.h
@@ -81,6 +81,8 @@ namespace Mantid
/// Overwrites Algorithm method
void exec();
+ /// Check if this actually is a binary file
+ bool isAscii(const std::string& filename) const;
void checkOptionalProperties();
int calculateWorkspaceSize();
void goManagedRaw(bool bincludeMonitors,bool bexcludeMonitors,bool bseparateMonitors );
diff --git a/Code/Mantid/DataHandling/src/LoadLog.cpp b/Code/Mantid/DataHandling/src/LoadLog.cpp
index 83139138cc5..0c35b9ecc12 100644
--- a/Code/Mantid/DataHandling/src/LoadLog.cpp
+++ b/Code/Mantid/DataHandling/src/LoadLog.cpp
@@ -8,10 +8,14 @@
#include "MantidDataObjects/Workspace2D.h"
#include "MantidKernel/Glob.h"
#include "MantidKernel/FileProperty.h"
+#include "LoadRaw/isisraw2.h"
#include "Poco/File.h"
#include "Poco/Path.h"
#include "Poco/DirectoryIterator.h"
+#include "Poco/DateTimeParser.h"
+#include "Poco/DateTimeFormat.h"
+#include "Poco/RegularExpression.h"
#include <fstream> // used to get ifstream
#include <sstream>
@@ -43,7 +47,12 @@ void LoadLog::init()
new WorkspaceProperty<MatrixWorkspace>("Workspace","Anonymous",Direction::InOut),
"The name of the workspace to which the log data will be added");
- declareProperty(new FileProperty("Filename", "", FileProperty::Load),
+ std::vector<std::string> exts(4, "");
+ exts[0] = "txt";
+ exts[1] = "raw";
+ exts[2] = "s*";
+ exts[3] = "add";
+ declareProperty(new FileProperty("Filename", "", FileProperty::Load, exts),
"The filename (including its full or relative path) of either an ISIS log file\n"
"or an ISIS raw file. If a raw file is specified all log files associated with\n"
"that raw file are loaded into the specified workspace. The file extension must\n"
@@ -52,6 +61,26 @@ void LoadLog::init()
declareProperty("Period",1);
}
+ //@cond NODOC
+ namespace
+ {
+ struct FileMatcher
+ {
+ FileMatcher(const std::string & expression) : m_expression(expression) {}
+
+ bool operator()(const std::string & test) const
+ {
+ Poco::RegularExpression regex(m_expression, Poco::RegularExpression::RE_CASELESS);
+ return regex.match(test);
+ }
+
+ private:
+ FileMatcher();
+ const std::string m_expression;
+ };
+ }
+ //@endcond
+
/** Executes the algorithm. Reading in ISIS log file(s)
*
* @throw Mantid::Kernel::Exception::FileError Thrown if file is not recognised to be a raw datafile or log file
@@ -63,14 +92,8 @@ void LoadLog::exec()
m_filename = getPropertyValue("Filename");
+ // File property checks whether the given path exists, just check that is actually a file
Poco::File l_path( m_filename );
-
- if ( !l_path.exists() )
- {
- g_log.error("In LoadLog: " + m_filename + " does not exist.");
- throw Exception::FileError("File does not exist:" , m_filename);
- }
-
if ( l_path.isDirectory() )
{
g_log.error("In LoadLog: " + m_filename + " must be a filename not a directory.");
@@ -83,187 +106,161 @@ void LoadLog::exec()
const MatrixWorkspace_sptr localWorkspace = getProperty("Workspace");
boost::shared_ptr<API::Sample> sample = localWorkspace->getSample();
- int Period = getProperty("Period");
-
// If m_filename is the filename of a raw datafile then search for potential log files
// in the directory of this raw datafile. Otherwise check if m_filename is a potential
// log file. Add the filename of these potential log files to: potentialLogFiles.
-
- std::vector<std::string> potentialLogFiles;
-
+ std::set<std::string> potentialLogFiles;
// start the process or populating potential log files into the container: potentialLogFiles
-
std::string l_filenamePart = Poco::Path(l_path.path()).getFileName();// get filename part only
bool rawFile = false;// Will be true if Filename property is a name of a RAW file
- if ( isLogFile(l_filenamePart) )
+ if ( isAscii(m_filename) && l_filenamePart.find("_") != std::string::npos )
{
// then we will assume that m_filename is an ISIS log file
- potentialLogFiles.push_back(m_filename);
+ potentialLogFiles.insert(m_filename);
}
- else if ( ( stringToLower(l_filenamePart).find(".raw") != std::string::npos ||
- stringToLower(l_filenamePart).find(".s") != std::string::npos ) && l_filenamePart.size() >= 10 )
+ else
{
- // then we will assume that m_filename is an ISIS raw file
- rawFile = true;
-
+ // then we will assume that m_filename is an ISIS raw file. The file validator will have warned the user if the
+ // extension is not one of the suggested ones
+ rawFile = true;
// strip out the raw data file identifier
-
- size_t l_pos = l_filenamePart.find(".");
-
- std::string l_rawID = stringToLower(l_filenamePart.substr(0,l_pos));
-
+ std::string l_rawID("");
+ size_t idx = l_filenamePart.rfind('.');
+ if( idx != std::string::npos )
+ {
+ l_rawID = l_filenamePart.substr(0, l_filenamePart.rfind('.'));
+ }
+ else
+ {
+ l_rawID = l_filenamePart;
+ }
+
// look for log files in the directory of the raw datafile
- size_t idot = m_filename.find('.');
- std::set<std::string> logNames;
- Poco::Path pattern(m_filename.substr(0,idot) + "_*.txt");
-
- Poco::Path base(pattern);
- base.makeParent();
- Kernel::Glob::glob(base,pattern,logNames);
+ std::string pattern(l_rawID + "_*.txt");
+ Poco::Path dir(m_filename);
+ dir.makeParent();
+ try
+ {
+ Kernel::Glob::glob(Poco::Path(dir).resolve(pattern),potentialLogFiles);
+ }
+ catch(std::exception &)
+ {
+ }
- if (logNames.size() > 0)
- potentialLogFiles.assign(logNames.begin(),logNames.end());
- else
+ if( potentialLogFiles.empty() )
{
+ Poco::RegularExpression regex(l_rawID + "_.*\\.txt", Poco::RegularExpression::RE_CASELESS );
+ Poco::DirectoryIterator end_iter;
+ for ( Poco::DirectoryIterator dir_itr(Poco::Path(m_filename).parent()); dir_itr != end_iter; ++dir_itr )
+ {
+ if ( !Poco::File(dir_itr->path() ).isFile() ) continue;
+
+ l_filenamePart = Poco::Path(dir_itr->path()).getFileName();
+
+ if ( regex.match(l_filenamePart) )
+ {
+ potentialLogFiles.insert( dir_itr->path() );
+ }
+ }
- Poco::DirectoryIterator end_iter;
- for ( Poco::DirectoryIterator dir_itr( Poco::Path(l_path.path()).makeAbsolute().parent() ); dir_itr != end_iter; ++dir_itr )
- {
- if ( !Poco::File(dir_itr->path() ).isFile() ) continue;
+ }
+ }
- l_filenamePart = Poco::Path(dir_itr->path()).getFileName();
- if ( !isLogFile(l_filenamePart) ) continue;
+ //If there are no log files by now, we have nothing else to do
+ if( potentialLogFiles.empty() ) return;
- if ( stringToLower(l_filenamePart).find(l_rawID) != std::string::npos )
- {
- potentialLogFiles.push_back( dir_itr->path() );
- }
- }
- }
+ //Do a quick search for the icpevent file
+ std::string icpevent_file_name("");
+ std::set<std::string>::const_iterator icpfile = find_if(potentialLogFiles.begin(), potentialLogFiles.end(), FileMatcher(std::string(".*icpevent.*")));
+ if( icpfile != potentialLogFiles.end() )
+ {
+ icpevent_file_name = *icpfile;
}
- else
+
+ API::LogParser parser(icpevent_file_name);
+ // Add mantid-created logs
+ int period = getProperty("Period");
+ Property* log = parser.createPeriodLog(period);
+ if (log)
{
- g_log.error("In LoadLog: " + m_filename + " found to be neither a raw datafile nor a log file.");
- throw Exception::FileError("Filename found to be neither a raw datafile nor a log file." , m_filename);
+ sample->addLogData(log);
}
+ sample->addLogData(parser.createAllPeriodsLog());
+ sample->addLogData(parser.createRunningLog());
- // Attempt to load the content of each potential log file into the Sample object
+ // Extract the common part of log file names (the workspace name)
+ std::string ws_name = Poco::Path(m_filename).getFileName();
+ ws_name.erase(ws_name.find_last_of('.'));
+ ws_name += '_';
+ size_t n_common_chars = ws_name.size();
- std::vector<std::string>::iterator file = potentialLogFiles.begin();
- for (; file != potentialLogFiles.end(); file++)
+ // Attempt to load the content of each potential log file into the Sample object
+ std::set<std::string>::const_iterator logs_end = potentialLogFiles.end();
+ for(std::set<std::string>::const_iterator logs_itr = potentialLogFiles.begin(); logs_itr != logs_end; ++logs_itr)
{
+ std::string filename = *logs_itr;
// open log file
-
- std::ifstream inLogFile(file->c_str());
-
- //std::cerr<<"processing "<<*file<<'\n';
+ std::ifstream inLogFile(filename.c_str());
if (!inLogFile)
{
// Unable to open file
- g_log.error("Unable to open file " + (*file));
- throw Exception::FileError("Unable to open file:" , (*file));
+ g_log.error("Unable to open file " + filename);
+ throw Exception::FileError("Unable to open file:" , filename);
}
-
-
// figure out if second column is a number or a string
-
std::string aLine;
- std::string dateAndTime;
-
- kind l_kind(LoadLog::empty);
-
if( std::getline(inLogFile, aLine, '\n') )
{
if ( !isDateTimeString(aLine) )
{
- g_log.warning("File" + (*file) + " is not a standard ISIS log file. Expected to be a two column file.");
+ g_log.warning("File" + filename + " is not a standard ISIS log file. Expected to be a two column file.");
inLogFile.close();
- file = potentialLogFiles.erase(file);
- if (file == potentialLogFiles.end()) break;
continue;
}
-
+ std::string dateAndTime;
std::stringstream ins(aLine);
-
ins >> dateAndTime;
// read in what follows the date-time string in the log file and figure out
// what type it is
-
std::string whatType;
-
ins >> whatType;
-
- l_kind = classify(whatType);
-
+ kind l_kind = classify(whatType);
if ( LoadLog::string != l_kind && LoadLog::number != l_kind )
{
- g_log.warning("ISIS log file contains unrecognised second column entries: " + (*file));
+ g_log.warning("ISIS log file contains unrecognised second column entries: " + filename);
inLogFile.close();
- file = potentialLogFiles.erase(file);
- if (file == potentialLogFiles.end()) break;
continue;
}
- }
-
- inLogFile.close();
- } // end for
-
- // Extract the common part of log file names (the workspace name)
- std::string ws_name = Poco::Path(m_filename).getFileName();
- ws_name.erase(ws_name.find_last_of('.'));
- ws_name += '_';
-
-
- // Find the icpevent filename
- std::string icpevent_file_name;
- for(size_t i=0;i<potentialLogFiles.size();i++)
- {
- if (stringToLower(potentialLogFiles[i]).find("icpevent") != std::string::npos)
- {
- icpevent_file_name = potentialLogFiles[i];
- break;
- }
- }
-
- API::LogParser parser(icpevent_file_name);
-
- // Add mantid-created logs
- Property* log = parser.createPeriodLog(Period);
- if (log)
- {
- sample->addLogData(log);
- }
- sample->addLogData(parser.createAllPeriodsLog());
- sample->addLogData(parser.createRunningLog());
-
- // Add log data from the files
- for(size_t i=0;i<potentialLogFiles.size();i++)
- {
+
try
{
- // Make the property name by removing the workspce name and file extension from the log filename
- std::string log_name = Poco::Path(potentialLogFiles[i]).getFileName();
-
- if (rawFile)
- log_name.erase(0,ws_name.size());
-
- size_t j = log_name.find_last_of('.');
- if (j != std::string::npos)
- log_name.erase(j);
-
- Property* log = parser.createLogProperty(potentialLogFiles[i],stringToLower(log_name));
- if (log)
- sample->addLogData(log);
+ // Make the property name by removing the workspce name and file extension from the log filename
+ std::string log_name = Poco::Path(Poco::Path(filename).getFileName()).getBaseName();
+
+ if (rawFile)
+ {
+ log_name.erase(0, n_common_chars);
+ }
+
+ Property* log = parser.createLogProperty(*logs_itr,stringToLower(log_name));
+ if (log)
+ {
+ sample->addLogData(log);
+ }
}
- catch(...)
+ catch(std::exception&)
{
- continue;
+ continue;
}
- }
+
+ }
+ inLogFile.close();
+ } // end for
+
// operation was a success and ended normally
return;
@@ -276,18 +273,24 @@ void LoadLog::exec()
*/
LoadLog::kind LoadLog::classify(const std::string& s)
{
+ if( s.empty() )
+ {
+ return LoadLog::empty;
+ }
+
using std::string;
const string lower("abcdefghijklmnopqrstuvwxyz");
const string upper("ABCDEFGHIJKLMNOPQRSTUVWXYZ");
const string letters = lower + upper + '_';
- if (s.empty())
- return LoadLog::empty;
-
if (letters.find_first_of(s) != string::npos)
- return LoadLog::string;
+ {
+ return LoadLog::string;
+ }
else
- return LoadLog::number;
+ {
+ return LoadLog::number;
+ }
}
/** change each element of the string to lower case
@@ -296,37 +299,45 @@ LoadLog::kind LoadLog::classify(const std::string& s)
*/
std::string LoadLog::stringToLower(std::string strToConvert)
{
- for(unsigned int i=0;i<strToConvert.length();i++)
- {
- strToConvert[i] = tolower(strToConvert[i]);
- }
- return strToConvert; //return the converted string
+ std::transform(strToConvert.begin(), strToConvert.end(), strToConvert.begin(), tolower);
+ return strToConvert;
}
-/** looks whether filename has the .txt extension and contain a '_'
+/** Checks whether filename is a simple text file
* @param filenamePart The filename to inspect
-* @returns true if the filename has the .txt extension and contain a '_'
+* @returns true if the filename has the .txt extension
*/
-bool LoadLog::isLogFile(const std::string& filenamePart)
+bool LoadLog::isAscii(const std::string& filename)
{
- if ( stringToLower(filenamePart).find(".txt") != std::string::npos && filenamePart.find("_") != std::string::npos )
+ FILE* file = fopen(filename.c_str(), "rb");
+ char data[256];
+ int n = fread(data, 1, sizeof(data), file);
+ char *pend = &data[n];
+ /*
+ * Call it a binary file if we find a non-ascii character in the
+ * first 256 bytes of the file.
+ */
+ for( char *p = data; p < pend; ++p )
+ {
+ unsigned long ch = (unsigned long)*p;
+ if( !(ch <= 0x7F) )
+ {
+ return false;
+ }
+
+ }
return true;
- else
- return false;
}
-/** check if first 19 characters of a string is data-time string according to yyyy-mm-ddThh:mm:ss
+/** check if first 19 characters of a string is date-time string according to yyyy-mm-ddThh:mm:ss
* @param str The string to test
* @returns true if the strings format matched the expected date format
*/
bool LoadLog::isDateTimeString(const std::string& str)
{
- if ( str.size() >= 19 )
- if ( str.compare(4,1,"-") == 0 && str.compare(7,1,"-") == 0 && str.compare(13,1,":") == 0
- && str.compare(16,1,":") == 0 && str.compare(10,1,"T") == 0 )
- return true;
-
- return false;
+ Poco::DateTime dt;
+ int tz_diff;
+ return Poco::DateTimeParser::tryParse(Poco::DateTimeFormat::ISO8601_FORMAT, str.substr(0,19), dt, tz_diff);
}
} // namespace DataHandling
diff --git a/Code/Mantid/DataHandling/src/LoadRaw.cpp b/Code/Mantid/DataHandling/src/LoadRaw.cpp
index 5ca2ab2a693..58c75695f0b 100644
--- a/Code/Mantid/DataHandling/src/LoadRaw.cpp
+++ b/Code/Mantid/DataHandling/src/LoadRaw.cpp
@@ -39,8 +39,7 @@ namespace Mantid
std::vector<std::string> exts;
exts.push_back("raw");
exts.push_back("s*");
- //exts.push_back("sav");
- //exts.push_back("s[0-9][0-9]");
+ exts.push_back("add");
declareProperty(new FileProperty("Filename", "", FileProperty::Load, exts),
"The name of the RAW file to read, including its full or relative\n"
@@ -75,7 +74,12 @@ namespace Mantid
{
// Retrieve the filename from the properties
m_filename = getPropertyValue("Filename");
-
+ if( isAscii(m_filename) )
+ {
+ g_log.error() << "File \"" << m_filename << "\" is not a valid RAW file.\n";
+ throw std::invalid_argument("Incorrect file type encountered.");
+ }
+
ISISRAW iraw(NULL);
if (iraw.readFromFile(m_filename.c_str()) != 0)
{
@@ -197,6 +201,33 @@ namespace Mantid
delete[] spectrum;
}
+ /**
+ * Check if a file is a text file
+ * @param filename The file path to check
+ * @returns true if the file an ascii text file, false otherwise
+ */
+ bool LoadRaw::isAscii(const std::string & filename) const
+ {
+ FILE* file = fopen(filename.c_str(), "rb");
+ char data[256];
+ int n = fread(data, 1, sizeof(data), file);
+ char *pend = &data[n];
+ /*
+ * Call it a binary file if we find a non-ascii character in the
+ * first 256 bytes of the file.
+ */
+ for( char *p = data; p < pend; ++p )
+ {
+ unsigned long ch = (unsigned long)*p;
+ if( !(ch <= 0x7F) )
+ {
+ return false;
+ }
+
+ }
+ return true;
+ }
+
/// Validates the optional 'spectra to read' properties, if they have been set
void LoadRaw::checkOptionalProperties()
{
diff --git a/Code/Mantid/DataHandling/src/LoadRaw2.cpp b/Code/Mantid/DataHandling/src/LoadRaw2.cpp
index 4bf14eaf96a..6a98bbb7134 100644
--- a/Code/Mantid/DataHandling/src/LoadRaw2.cpp
+++ b/Code/Mantid/DataHandling/src/LoadRaw2.cpp
@@ -48,8 +48,8 @@ namespace Mantid
std::vector<std::string> exts;
exts.push_back("raw");
exts.push_back("s*");
- //exts.push_back("sav");
- //exts.push_back("s[0-9][0-9]");
+ exts.push_back("add");
+
declareProperty(new FileProperty("Filename", "", FileProperty::Load, exts),
"The name of the RAW file to read, including its full or relative\n"
"path. (N.B. case sensitive if running on Linux).");
@@ -86,6 +86,12 @@ namespace Mantid
// Retrieve the filename from the properties
m_filename = getPropertyValue("Filename");
+ if( isAscii(m_filename) )
+ {
+ g_log.error() << "File \"" << m_filename << "\" is not a valid RAW file.\n";
+ throw std::invalid_argument("Incorrect file type encountered.");
+ }
+
//ISISRAW iraw(NULL);
FILE* file = fopen(m_filename.c_str(),"rb");
if (file == NULL)
@@ -268,6 +274,33 @@ namespace Mantid
fclose(file);
}
+ /**
+ * Check if a file is a text file
+ * @param filename The file path to check
+ * @returns true if the file an ascii text file, false otherwise
+ */
+ bool LoadRaw2::isAscii(const std::string & filename) const
+ {
+ FILE* file = fopen(filename.c_str(), "rb");
+ char data[256];
+ int n = fread(data, 1, sizeof(data), file);
+ char *pend = &data[n];
+ /*
+ * Call it a binary file if we find a non-ascii character in the
+ * first 256 bytes of the file.
+ */
+ for( char *p = data; p < pend; ++p )
+ {
+ unsigned long ch = (unsigned long)*p;
+ if( !(ch <= 0x7F) )
+ {
+ return false;
+ }
+
+ }
+ return true;
+ }
+
/// Validates the optional 'spectra to read' properties, if they have been set
void LoadRaw2::checkOptionalProperties()
{
diff --git a/Code/Mantid/DataHandling/src/LoadRaw3.cpp b/Code/Mantid/DataHandling/src/LoadRaw3.cpp
index 0efe66d67ee..2adf16fa0f2 100644
--- a/Code/Mantid/DataHandling/src/LoadRaw3.cpp
+++ b/Code/Mantid/DataHandling/src/LoadRaw3.cpp
@@ -19,6 +19,7 @@
#include "Poco/Path.h"
#include <cmath>
#include <cstdio> //Required for gcc 4.4
+
namespace Mantid
{
namespace DataHandling
@@ -50,6 +51,7 @@ void LoadRaw3::init()
std::vector<std::string> exts;
exts.push_back("raw");
exts.push_back("s*");
+ exts.push_back("add");
declareProperty(new FileProperty("Filename", "", FileProperty::Load, exts),
"The name of the RAW file to read, including its full or relative\n"
@@ -102,6 +104,14 @@ void LoadRaw3::exec()
{
// Retrieve the filename from the properties
m_filename = getPropertyValue("Filename");
+ // Need to check that the file is not a text file as the ISISRAW routines don't deal with these very well, i.e
+ // reading continues until a bad_alloc is encountered.
+ if( isAscii(m_filename) )
+ {
+ g_log.error() << "File \"" << m_filename << "\" is not a valid RAW file.\n";
+ throw std::invalid_argument("Incorrect file type encountered.");
+ }
+
bool bLoadlogFiles = getProperty("LoadLogFiles");
bool bincludeMonitors = true;
bool bseparateMonitors = false;
@@ -638,6 +648,33 @@ void LoadRaw3::setWorkspaceProperty(const std::string & propertyName, const std:
}
}
+/**
+ * Check if a file is a text file
+ * @param filename The file path to check
+ * @returns true if the file an ascii text file, false otherwise
+ */
+bool LoadRaw3::isAscii(const std::string & filename) const
+{
+ FILE* file = fopen(filename.c_str(), "rb");
+ char data[256];
+ int n = fread(data, 1, sizeof(data), file);
+ char *pend = &data[n];
+ /*
+ * Call it a binary file if we find a non-ascii character in the
+ * first 256 bytes of the file.
+ */
+ for( char *p = data; p < pend; ++p )
+ {
+ unsigned long ch = (unsigned long)*p;
+ if( !(ch <= 0x7F) )
+ {
+ return false;
+ }
+
+ }
+ return true;
+}
+
/// Validates the optional 'spectra to read' properties, if they have been set
void LoadRaw3::checkOptionalProperties()
{
diff --git a/Code/Mantid/Kernel/inc/MantidKernel/FileValidator.h b/Code/Mantid/Kernel/inc/MantidKernel/FileValidator.h
index 6d1c99395b3..b5c1346c02d 100644
--- a/Code/Mantid/Kernel/inc/MantidKernel/FileValidator.h
+++ b/Code/Mantid/Kernel/inc/MantidKernel/FileValidator.h
@@ -48,12 +48,13 @@ public:
private:
/// The list of permitted extensions
const std::set<std::string> m_extensions;
- /// An internal list of extensions that, if necessary, have been transformed into regular expressions
- std::set<std::string> m_regex_exts;
/// Flag indicating whether to test for existence of filename
const bool m_fullTest;
std::string checkValidity(const std::string &value) const;
+
+ /// A reference to the logger
+ static Logger & g_log;
};
} // namespace Kernel
diff --git a/Code/Mantid/Kernel/src/FileValidator.cpp b/Code/Mantid/Kernel/src/FileValidator.cpp
index 2ec2706e245..2048688c8ab 100644
--- a/Code/Mantid/Kernel/src/FileValidator.cpp
+++ b/Code/Mantid/Kernel/src/FileValidator.cpp
@@ -1,57 +1,32 @@
#include "MantidKernel/FileValidator.h"
#include <algorithm>
#include "Poco/File.h"
-#include "Poco/RegularExpression.h"
+#include "Poco/Path.h"
#include <iostream>
-/// Functors for checking the file extensions against a regular expression
-namespace {
- /// A struct holding a string to be tested against a regular expression
- struct RegExMatcher
- {
- /// Constructor
- RegExMatcher(std::string ext) : m_ext(ext) {}
-
- /// Operator matches expressions
- bool operator()(const std::string & test)
- {
- Poco::RegularExpression regex(test, Poco::RegularExpression::RE_CASELESS);
- bool matched = regex.match(m_ext);
- return matched;
- }
-
- private:
- /// Private default constructor
- RegExMatcher();
- /// The extension to test
- std::string m_ext;
- };
- /// Converts a shell-like pattern to a regular expression pattern
- struct RegExConverter
+namespace
+{
+ /// Functor object to supply to for_each
+ struct lowercase
{
- /// Operator to convert from shell-like patterns to regular expression syntax
- std::string operator()(const std::string & pattern)
+ void operator()(std::string s)
{
- std::string replacement;
- for( std::string::const_iterator itr = pattern.begin(); itr != pattern.end(); ++itr )
- {
- char ch = *itr;
- if( ch == '?' ) replacement.append(".");
- else if( ch == '*' ) replacement.append(".*");
- else replacement.push_back(ch);
- }
- return replacement;
+ std::transform(s.begin(), s.end(), s.begin(), tolower);
}
- };
+ };
}
namespace Mantid
{
namespace Kernel
{
+
+/// Initialize the logger
+Logger& FileValidator::g_log = Logger::get("FileValidator");
+
/// Default constructor.
-FileValidator::FileValidator() : IValidator<std::string>(), m_extensions(), m_regex_exts(), m_fullTest(true)
+FileValidator::FileValidator() : IValidator<std::string>(), m_extensions(), m_fullTest(true)
{}
/** Constructor
@@ -61,17 +36,9 @@ FileValidator::FileValidator() : IValidator<std::string>(), m_extensions(), m_re
FileValidator::FileValidator(const std::vector<std::string>& extensions, bool testFileExists) :
IValidator<std::string>(),
m_extensions(extensions.begin(),extensions.end()),
- m_regex_exts(),
m_fullTest(testFileExists)
{
- // Transform the file extensions to regular expression syntax for matching. This could be done every time
- // checkValidity() is called but that would create unnecessary work
- RegExConverter conv;
- std::set<std::string>::const_iterator it;
- for (it = m_extensions.begin(); it != m_extensions.end(); ++it)
- {
- m_regex_exts.insert(conv(*it));
- }
+ for_each(m_extensions.begin(), m_extensions.end(), lowercase());
}
/// Destructor
@@ -92,50 +59,39 @@ IValidator<std::string>* FileValidator::clone()
return new FileValidator(*this);
}
-/** If m_fullTest=true if checks that the files exists, otherwise just that the extension is good
+/** If m_fullTest=true if checks that the files exists, otherwise just that path syntax looks valid
* @param value file name
* @returns An error message to display to users or an empty string on no error
*/
std::string FileValidator::checkValidity(const std::string &value) const
-{
- //check the file has a good extension
- if( !m_regex_exts.empty() )
- {
- //Find extension of value
- const std::string ext = value.substr(value.rfind(".") + 1);
- //Use a functor to test each allowed extension in turn
- RegExMatcher matcher(ext);
- std::set<std::string>::const_iterator itr =
- std::find_if(m_regex_exts.begin(), m_regex_exts.end(), matcher);
-
- // If not found, return the valid extensions
- if ( itr == m_regex_exts.end() )
+{
+ // Check if the path is syntactically valid
+ if( !Poco::Path().tryParse(value) )
+ {
+ return "Error in path syntax: \"" + value + "\".";
+ }
+
+ //Check the extension but just issue a warning if it is not one of the suggested values
+ std::string::size_type idx = value.rfind(".");
+ if( idx != std::string::npos )
+ {
+ std::string ext = value.substr(idx + 1);
+ std::transform(ext.begin(), ext.end(), ext.begin(), tolower);
+ if( !m_extensions.empty() && m_extensions.find(ext) == m_extensions.end() )
{
- if (m_extensions.size() == 1)
- {
- return "The file must have extension " + *(m_extensions.begin());
- }
- else
- {
- std::string error("The file must have one of these extensions: ");
- std::set<std::string>::const_iterator itr = m_extensions.begin();
- error += *itr;
- for ( ++itr; itr != m_extensions.end(); ++itr )
- {
- error += ", " + *itr;
- }
- return error;
- }
- }
+ g_log.warning() << "Unrecognised extension in file \"" << value << "\"." << std::endl;
+ }
}
-
- //If the file is required to exist check it is there
+
+ //If the file is required to exist check it is there
if ( m_fullTest && ( value.empty() || !Poco::File(value).exists() ) )
- {
- return "File \"" + value + "\" not found";
- }
- return "";
-}
+ {
+ return "File \"" + value + "\" not found";
+ }
+
+ //Otherwise we are okay, file extensions are just a suggestion so no validation on them is necessary
+ return "";
+}
} // namespace Kernel
diff --git a/Code/Mantid/Kernel/test/FilePropertyTest.h b/Code/Mantid/Kernel/test/FilePropertyTest.h
index 1c88bda6d06..cc284411572 100644
--- a/Code/Mantid/Kernel/test/FilePropertyTest.h
+++ b/Code/Mantid/Kernel/test/FilePropertyTest.h
@@ -12,6 +12,9 @@ public:
void testSearchDirs()
{
+ // It wasn't happy having this in the constructor as I think all of the objects in the test
+ //get created first and then all of the tests run
+ Mantid::Kernel::ConfigService::Instance().loadConfig("Mantid.properties");
TS_ASSERT_DIFFERS(Mantid::Kernel::ConfigService::Instance().getDataSearchDirs().size(), 0);
}
@@ -45,9 +48,9 @@ public:
msg = fp->setValue("ALF15739.RAW");
TS_ASSERT_EQUALS(msg, "")
- //Check incorrect extension
+ //Check different extension
msg = fp->setValue("48098.Q");
- TS_ASSERT_EQUALS(msg, "The file must have extension raw");
+ TS_ASSERT_EQUALS(msg, "");
delete fp;
}
diff --git a/Code/Mantid/Kernel/test/FileValidatorTest.h b/Code/Mantid/Kernel/test/FileValidatorTest.h
index 51337a2b1bf..8ffa725826f 100644
--- a/Code/Mantid/Kernel/test/FileValidatorTest.h
+++ b/Code/Mantid/Kernel/test/FileValidatorTest.h
@@ -4,6 +4,7 @@
#include <cxxtest/TestSuite.h>
#include "MantidKernel/FileValidator.h"
+#include "Poco/File.h"
using namespace Mantid::Kernel;
@@ -20,32 +21,37 @@ void testVectorConstructor()
TS_ASSERT_EQUALS ( v.allowedValues().size(), 2 )
}
-void testFailsOnWrongExtension()
+void testPassesOnExistentFile()
{
- //You can't find this file? It just has to be a file that is in the test directory of all build servers, with the same, case. Any ideas? If you change the extension then also change the file extension test below ... Steve Williams
- std::string testFile("RunTests.bat");
-
- std::vector<std::string> vec;
- vec.push_back("raw");
-
+ //Create two files, one with the extension within the validator and one without
+
+ const std::string file_stub = "scratch.";
+ const std::string ext1 = "txt";
+ const std::string ext2 = "raw";
+ Poco::File txt_file(file_stub + ext1);
+ Poco::File raw_file(file_stub + ext2);
+
+ try
+ {
+ txt_file.createFile();
+ raw_file.createFile();
+ }
+ catch(std::exception & )
+ {
+ TS_FAIL("Error creating test file for \"testPassesOnExistentFile\" test.");
+ }
+
+ //FileValidator will suggest txt files as correct extension
+ std::vector<std::string> vec(1, "txt");
FileValidator v1(vec);
- TS_ASSERT_EQUALS( v1.isValid(testFile),
- "The file must have extension raw" )
- vec.push_back("RAW");
- FileValidator v2(vec);
- TS_ASSERT_EQUALS( v2.isValid(testFile),
- "The file must have one of these extensions: RAW, raw" )
-}
-
-void testPassesOnRightExtension()
-{
- std::string testFile("runTests.bat");
+
+ TS_ASSERT_EQUALS( v1.isValid(txt_file.path()), "" );
+ // Not correct extension but the file exists so we allow it
+ TS_ASSERT_EQUALS( v1.isValid(raw_file.path()), "" );
- std::vector<std::string> vec;
- vec.push_back("bat");
- FileValidator v(vec);
- TS_ASSERT_EQUALS( v.isValid(testFile), "" )
+ txt_file.remove();
+ raw_file.remove();
}
void testFailsOnNonexistentFile()
@@ -58,29 +64,19 @@ void testFailsOnNonexistentFile()
"File \"" + NoFile + "\" not found" )
}
-void testFailsOnEmptyFileString()
+void testPassesOnNonexistentFile()
{
- FileValidator file_val;
- TS_ASSERT_EQUALS( file_val.isValid(""),
- "File \"\" not found" )
+ std::string NoFile("myJunkFile_hgfvj.cpp");
+ std::vector<std::string> vec;
+ vec.push_back("cpp");
+ FileValidator v(vec, false);
+ TS_ASSERT_EQUALS( v.isValid(NoFile), "" );
}
-void testPassesOnWildcardExtensions()
+void testFailsOnEmptyFileString()
{
- std::vector<std::string> exts;
- exts.push_back("c[a-z][a-z]");
- exts.push_back("h??");
- exts.push_back("h*");
- FileValidator validator(exts, false);
- TS_ASSERT_EQUALS( validator.isValid("fli.cpp"), "" )
- const std::string outputString("The file must have one of these extensions: c[a-z][a-z], h*, h??");
- TS_ASSERT_EQUALS( validator.isValid("fli.cp"), outputString )
- TS_ASSERT_EQUALS( validator.isValid("fli.c01"), outputString )
- TS_ASSERT_EQUALS( validator.isValid("fli.cxx"), "" )
- TS_ASSERT_EQUALS( validator.isValid("fli.hxx"), "" )
- TS_ASSERT_EQUALS( validator.isValid("fli.habc"), "" )
- TS_ASSERT_EQUALS( validator.isValid("fli.z"), outputString )
- TS_ASSERT_EQUALS( validator.isValid("fli.bpp"), outputString )
+ FileValidator file_val;
+ TS_ASSERT_EQUALS( file_val.isValid(""), "File \"\" not found" )
}
};
--
GitLab