Commit 47336152 authored by Nick Draper's avatar Nick Draper
Browse files

Add filtering by log status values

applied automatically for ISIS data

Unfiltered data can be accessed using the Unfiltered() method
Although this will also remove the filtering for perio and running status
parent 43a7ec91
......@@ -42,6 +42,10 @@ namespace API {
*/
class MANTID_API_DLL LogManager {
public:
// Gets the correct log name for the matching invalid values log for a given
// log name
static std::string getInvalidValuesFilterLogName(const std::string logName);
LogManager();
LogManager(const LogManager &other);
/// Destructor. Doesn't need to be virtual as long as nothing inherits from
......@@ -184,6 +188,13 @@ public:
/// Clear the logs
void clearLogs();
// returns true if the log has a matching invalid values log filter
bool hasInvalidValuesFilter(const std::string logName) const;
// returns the invalid values log if the log has a matching invalid values log filter
Kernel::TimeSeriesProperty<bool> *
getInvalidValuesFilter(const std::string logName) const;
bool operator==(const LogManager &other) const;
bool operator!=(const LogManager &other) const;
......
......@@ -95,7 +95,6 @@ bool convertPropertyToDouble(const Property *property, double &value,
/// Name of the log entry containing the proton charge when retrieved using
/// getProtonCharge
const char *LogManager::PROTON_CHARGE_LOG_NAME = "gd_prtn_chrg";
//----------------------------------------------------------------------
// Public member functions
//----------------------------------------------------------------------
......@@ -553,6 +552,34 @@ void LogManager::loadNexus(::NeXus::File *file,
*/
void LogManager::clearLogs() { m_manager->clear(); }
/// Gets the correct log name for the matching invalid values log for a given log
/// name
std::string
LogManager::getInvalidValuesFilterLogName(const std::string logName) {
return PropertyManager::getInvalidValuesFilterLogName(logName);
}
/// returns true if the log has a matching invalid values log filter
bool LogManager::hasInvalidValuesFilter(const std::string logName) const {
return hasProperty(getInvalidValuesFilterLogName(logName));
}
/// returns the invalid values log if the log has a matching invalid values log
/// filter
Kernel::TimeSeriesProperty<bool> *
LogManager::getInvalidValuesFilter(const std::string logName) const {
try {
auto log = getLogData(getInvalidValuesFilterLogName(logName));
if (auto tsp = dynamic_cast<TimeSeriesProperty<bool> *>(log)) {
return tsp;
}
}
catch(Exception::NotFoundError &) {
//do nothing, just drop through tto the return line below
}
return nullptr;
}
bool LogManager::operator==(const LogManager &other) const {
return *m_manager == *(other.m_manager);
}
......
......@@ -9,6 +9,7 @@
#include "MantidAPI/LogManager.h"
#include "MantidGeometry/Instrument/Goniometer.h"
#include "MantidKernel/Exception.h"
#include "MantidKernel/FilteredTimeSeriesProperty.h"
#include "MantidKernel/Matrix.h"
#include "MantidKernel/Property.h"
#include "MantidKernel/TimeSeriesProperty.h"
......@@ -53,6 +54,22 @@ private:
std::string m_value = "Nothing";
};
void addTestTimeSeriesFilter(LogManager &run, const std::string &name) {
auto timeSeries = new TimeSeriesProperty<bool>(name);
timeSeries->addValue("2012-07-19T16:17:00", true);
timeSeries->addValue("2012-07-19T16:17:10", true);
timeSeries->addValue("2012-07-19T16:17:20", true);
timeSeries->addValue("2012-07-19T16:17:30", true);
timeSeries->addValue("2012-07-19T16:17:40", false);
timeSeries->addValue("2012-07-19T16:17:50", false);
timeSeries->addValue("2012-07-19T16:18:00", true);
timeSeries->addValue("2012-07-19T16:18:10", true);
timeSeries->addValue("2012-07-19T16:19:20", true);
timeSeries->addValue("2012-07-19T16:19:20", true);
run.addProperty(timeSeries);
}
template <typename T>
void addTestTimeSeries(LogManager &run, const std::string &name) {
auto timeSeries = new TimeSeriesProperty<T>(name);
......@@ -579,6 +596,50 @@ public:
TS_ASSERT(!(a == b));
}
void test_has_invalid_values_filter() {
LogManager runInfo;
const std::string name = "test_has_invalid_values_filter";
const std::string filterName = runInfo.getInvalidValuesFilterLogName(name);
TSM_ASSERT("The filter name should start with the log name",filterName.rfind(name, 0) == 0);
addTestTimeSeries<double>(runInfo, name);
TS_ASSERT_EQUALS(runInfo.hasInvalidValuesFilter(name), false);
addTestTimeSeriesFilter(runInfo, filterName);
TS_ASSERT_EQUALS(runInfo.hasInvalidValuesFilter(name), true);
}
void test_get_invalid_values_filter() {
LogManager runInfo;
const std::string name = "test_get_invalid_values_filter";
const std::string filterName = runInfo.getInvalidValuesFilterLogName(name);
addTestTimeSeries<double>(runInfo, name);
auto *filterfail = runInfo.getInvalidValuesFilter(name);
TSM_ASSERT("The filter was returned correrctly as NULL",
filterfail == nullptr);
addTestTimeSeriesFilter(runInfo, filterName);
auto * filter = runInfo.getInvalidValuesFilter(name);
TSM_ASSERT("The filter was returned incorrectly as NULL", filter);
TS_ASSERT_THROWS_NOTHING(filter->firstTime());
//check it can be used to filter the log
auto *log = runInfo.getProperty(name);
auto *tsLog = dynamic_cast<TimeSeriesProperty<double>*>(log);
TS_ASSERT(tsLog);
if (tsLog) {
auto filtered =
std::make_unique<FilteredTimeSeriesProperty<double>>(tsLog,
*filter);
TS_ASSERT_DELTA(filtered->nthValue(0), 2, 1e-5);
TS_ASSERT_DELTA(filtered->nthValue(1), 3, 1e-5);
TS_ASSERT_DELTA(filtered->nthValue(2), 4, 1e-5);
TS_ASSERT_DELTA(filtered->nthValue(3), 5, 1e-5);
TS_ASSERT_DELTA(filtered->nthValue(4), 21, 1e-5);
TS_ASSERT_DELTA(filtered->nthValue(5), 22, 1e-5);
}
}
private:
template <typename T>
void doTest_GetPropertyAsSingleValue_SingleType(const T value) {
......
......@@ -7,6 +7,7 @@
#include "MantidDataHandling/LoadNexusLogs.h"
#include "MantidAPI/FileProperty.h"
#include "MantidAPI/Run.h"
#include "MantidAPI/LogManager.h"
#include "MantidKernel/ArrayProperty.h"
#include "MantidKernel/TimeSeriesProperty.h"
#include <locale>
......@@ -196,46 +197,6 @@ std::unique_ptr<Kernel::Property> createTimeSeries(::NeXus::File &file,
std::bind(std::multiplies<double>(), _1, 60.0));
}
// Now the the validity of the values
// this should be a match int array to the data values (or times)
// If not present assume all data is valid
try {
file.openData("value_valid");
// Now the validity data
::NeXus::Info info = file.getInfo();
// Check the size
if (size_t(info.dims[0]) != time_double.size()) {
throw ::NeXus::Exception("Invalid value entry for validity data");
}
if (file.isDataInt()) // Int type
{
std::vector<int> values;
try {
file.getDataCoerce(values);
for (auto validity : values) {
if (validity != 0) {
log.warning() << "Some \"" << propName
<< "\" log data is invalid!\n";
break;
}
}
} catch (::NeXus::Exception &) {
throw;
}
} else {
throw ::NeXus::Exception(
"Invalid value type for validity data. Only int is supported");
}
} catch (::NeXus::Exception &ex) {
std::string error_msg = ex.what();
if (error_msg != "NXopendata(value_valid) failed") {
log.warning() << error_msg << "\n";
file.closeData();
}
}
// Now the values: Could be a string, int or double
file.openData("value");
// Get the units of the property
......@@ -322,6 +283,84 @@ std::unique_ptr<Kernel::Property> createTimeSeries(::NeXus::File &file,
}
}
/**
* Creates a time series validity filter property from the currently opened log entry. It is
* assumed to
* have been checked to have a time field and the value entry's name is given
* as an argument
* @param file :: A reference to the file handle
* @param prop :: The property to check for a validity array
* @param log :: Reference to logger to print out to
* @returns A pointer to a new property containing the time series filter or null
*/
std::unique_ptr<Kernel::Property>
createTimeSeriesValidityFilter(::NeXus::File &file,
const std::unique_ptr<Kernel::Property> &prop,
Kernel::Logger &log) {
;
auto tsProp = dynamic_cast<Kernel::ITimeSeriesProperty *>(prop.get());
auto times = tsProp->timesAsVector();
std::vector<int> values;
std::vector<bool> boolValues;
// Now the the validity of the values
// this should be a match int array to the data values (or times)
// If not present assume all data is valid
try {
file.openData("value_valid");
// Now the validity data
::NeXus::Info info = file.getInfo();
// Check the size
if (size_t(info.dims[0]) != times.size()) {
throw ::NeXus::Exception("Invalid value entry for validity data");
}
if (file.isDataInt()) // Int type
{
try {
file.getDataCoerce(values);
file.closeData();
} catch (::NeXus::Exception &) {
throw;
}
} else {
throw ::NeXus::Exception(
"Invalid value type for validity data. Only int is supported");
}
} catch (::NeXus::Exception &ex) {
std::string error_msg = ex.what();
if (error_msg != "NXopendata(value_valid) failed") {
log.warning() << error_msg << "\n";
file.closeData();
// no data found
return std::unique_ptr<Kernel::Property>(nullptr);
}
}
bool invalidDataFound = false;
boolValues.reserve(values.size());
//convert the integer values to boolean with 0=valid data
for (size_t i = 0; i < values.size(); i++) {
bool boolValue = (values[i] != 0);
boolValues.emplace_back(boolValue);
if ((boolValue) && (!invalidDataFound)) {
invalidDataFound = true;
}
}
if (invalidDataFound) {
log.warning() << "Some \"" << prop->name() << "\" log data is invalid!\n";
auto tspName =
API::LogManager::getInvalidValuesFilterLogName(prop->name());
auto tsp = std::make_unique<TimeSeriesProperty<bool>>(tspName);
tsp->create(times, boolValues);
log.debug() << " done reading \"value_valid\" array\n";
return tsp;
} else {
// no data found
return std::unique_ptr<Kernel::Property>(nullptr);
}
}
/**
* Appends an additional entry to a TimeSeriesProperty which is at the end
* time of the run and contains the last value of the property recorded before
......@@ -811,6 +850,13 @@ void LoadNexusLogs::loadNXLog(
try {
if (overwritelogs || !(workspace->run().hasProperty(entry_name))) {
auto logValue = createTimeSeries(file, entry_name, freqStart, g_log);
auto validityLogValue =
createTimeSeriesValidityFilter(file, logValue, g_log);
if (validityLogValue) {
appendEndTimeLog(validityLogValue.get(), workspace->run());
workspace->mutableRun().addProperty(std::move(validityLogValue),
overwritelogs);
}
appendEndTimeLog(logValue.get(), workspace->run());
workspace->mutableRun().addProperty(std::move(logValue), overwritelogs);
}
......@@ -869,6 +915,12 @@ void LoadNexusLogs::loadSELog(
}
logValue = createTimeSeries(file, propName, freqStart, g_log);
auto validityLogValue =
createTimeSeriesValidityFilter(file, logValue, g_log);
if (validityLogValue) {
appendEndTimeLog(validityLogValue.get(), workspace->run());
workspace->mutableRun().addProperty(std::move(validityLogValue));
}
appendEndTimeLog(logValue.get(), workspace->run());
file.closeGroup();
......
......@@ -8,6 +8,7 @@
#include "MantidAPI/AnalysisDataService.h"
#include "MantidAPI/FrameworkManager.h"
#include "MantidAPI/LogManager.h"
#include "MantidAPI/MatrixWorkspace.h"
#include "MantidAPI/Run.h"
#include "MantidAPI/Workspace.h"
......@@ -301,6 +302,43 @@ public:
ld.setProperty("Workspace", ws);
ld.execute();
TS_ASSERT(ld.isExecuted());
auto run = ws->run();
// This one should not be present as there is no invalid data
TS_ASSERT_THROWS_ANYTHING(run.getLogData(
LogManager::getInvalidValuesFilterLogName("slitpos")));
//This one should not be present as there is no invalid data
TS_ASSERT_THROWS_ANYTHING(run.getLogData(
LogManager::getInvalidValuesFilterLogName("cryo_Sample")));
//these two both contain invalid data
auto pclog1 =
dynamic_cast<TimeSeriesProperty<bool> *>(run.getLogData(
LogManager::getInvalidValuesFilterLogName("cryo_temp1")));
std::vector<bool> correct{true, false, true};
TS_ASSERT_EQUALS(pclog1->valuesAsVector(), correct);
auto pclog2 =
dynamic_cast<TimeSeriesProperty<bool> *>(run.getLogData(
LogManager::getInvalidValuesFilterLogName("cryo_temp2")));
std::vector<bool> correct2{true, true, true};
TS_ASSERT_EQUALS(pclog2->valuesAsVector(), correct2);
// force the filtering by passing in an empty log
auto emptyProperty = new TimeSeriesProperty<bool>("empty");
run.filterByLog(*emptyProperty);
auto pclogFiltered1 = dynamic_cast<TimeSeriesProperty<double> *>(run.getLogData("cryo_temp1"));
std::vector<double> correctFiltered1{0., 0., 0.};
TS_ASSERT_EQUALS(pclogFiltered1->valuesAsVector(), correctFiltered1);
auto pclogFiltered2 = dynamic_cast<TimeSeriesProperty<double> *>(
run.getLogData("cryo_temp2"));
std::vector<double> correctFiltered2{0., 0., 0.};
TS_ASSERT_EQUALS(pclogFiltered2->valuesAsVector(), correctFiltered2);
}
private:
......
......@@ -349,6 +349,7 @@ public:
virtual void
splitByTime(std::vector<SplittingInterval> & /*splitter*/,
std::vector<PropertyManager *> /* outputs*/) const = 0;
virtual void filterByProperty(const TimeSeriesProperty<bool> & /*filter*/,
const std::vector<std::string> &
/* excludedFromFiltering */) = 0;
......
......@@ -41,6 +41,12 @@ template <typename T> class TimeSeriesProperty;
*/
class MANTID_KERNEL_DLL PropertyManager : virtual public IPropertyManager {
public:
static const std::string INVALID_VALUES_SUFFIX;
// Gets the correct log name for the matching invalid values log for a given
// log name
static std::string getInvalidValuesFilterLogName(const std::string logName);
static bool isAnInvalidValuesFilterLog(const std::string logName);
PropertyManager();
PropertyManager(const PropertyManager &);
PropertyManager &operator=(const PropertyManager &);
......
......@@ -10,6 +10,7 @@
#include "MantidKernel/PropertyManager.h"
#include "MantidKernel/FilteredTimeSeriesProperty.h"
#include "MantidKernel/IPropertySettings.h"
#include "MantidKernel/LogFilter.h"
#include "MantidKernel/PropertyWithValueJSON.h"
#include "MantidKernel/StringTokenizer.h"
......@@ -36,6 +37,23 @@ const std::string createKey(const std::string &name) {
}
} // namespace
const std::string PropertyManager::INVALID_VALUES_SUFFIX = "_invalid_values";
/// Gets the correct log name for the matching invalid values log for a given
/// log name
std::string
PropertyManager::getInvalidValuesFilterLogName(const std::string logName) {
return logName + PropertyManager::INVALID_VALUES_SUFFIX;
}
bool PropertyManager::isAnInvalidValuesFilterLog(const std::string logName) {
const auto fullString = logName;
const auto ending = PropertyManager::INVALID_VALUES_SUFFIX;
if (fullString.length() >= ending.length()) {
return (0 == fullString.compare(fullString.length() - ending.length(),
ending.length(), ending));
} else {
return false;
}
}
//-----------------------------------------------------------------------------------------------
/// Default constructor
PropertyManager::PropertyManager() : m_properties(), m_orderedProperties() {}
......@@ -172,6 +190,8 @@ void PropertyManager::splitByTime(
* @param filter :: A boolean time series to filter each property on
* @param excludedFromFiltering :: A string list of properties that
* will be excluded from filtering
* @param applyInvalidDataFilters :: Wether to also apply any matching invalid
* data filter logs (default:true)
*/
void PropertyManager::filterByProperty(
const Kernel::TimeSeriesProperty<bool> &filter,
......@@ -186,9 +206,30 @@ void PropertyManager::filterByProperty(
Property *currentProp = orderedProperty;
if (auto doubleSeries =
dynamic_cast<TimeSeriesProperty<double> *>(currentProp)) {
std::unique_ptr<Property> filtered =
std::make_unique<FilteredTimeSeriesProperty<double>>(doubleSeries,
filter);
// don't filter the invalid values filters
if (PropertyManager::isAnInvalidValuesFilterLog(currentProp->name()))
break;
std::unique_ptr<Property> filtered(nullptr);
if (this->existsProperty(PropertyManager::getInvalidValuesFilterLogName(
currentProp->name()))) {
// add the filter to the passed in filters
std::unique_ptr<LogFilter> logFilter =
std::make_unique<LogFilter>(filter);
auto filterProp =
getPointerToProperty(PropertyManager::getInvalidValuesFilterLogName(
currentProp->name()));
auto tspFilterProp =
dynamic_cast<TimeSeriesProperty<bool> *>(filterProp);
if (!tspFilterProp)
break;
logFilter->addFilter(*tspFilterProp);
filtered = std::make_unique<FilteredTimeSeriesProperty<double>>(
doubleSeries, *logFilter->filter());
} else {
filtered = std::make_unique<FilteredTimeSeriesProperty<double>>(
doubleSeries, filter);
}
orderedProperty = filtered.get();
// Now replace in the map
this->m_properties[createKey(currentProp->name())] = std::move(filtered);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment