GenericDataProcessorPresenter.cpp

#include "MantidQtWidgets/Common/DataProcessorUI/GenericDataProcessorPresenter.h"
#include "MantidAPI/AlgorithmManager.h"
#include "MantidAPI/IEventWorkspace.h"
#include "MantidAPI/ITableWorkspace.h"
#include "MantidAPI/MatrixWorkspace.h"
#include "MantidAPI/NotebookWriter.h"
#include "MantidAPI/TableRow.h"
#include "MantidAPI/WorkspaceFactory.h"
#include "MantidGeometry/Instrument.h"
#include "MantidKernel/Strings.h"
#include "MantidKernel/TimeSeriesProperty.h"
#include "MantidKernel/Utils.h"
#include "MantidKernel/make_unique.h"
#include "MantidQtWidgets/Common/AlgorithmHintStrategy.h"
#include "MantidQtWidgets/Common/DataProcessorUI/DataProcessorView.h"
#include "MantidQtWidgets/Common/DataProcessorUI/GenerateNotebook.h"
#include "MantidQtWidgets/Common/DataProcessorUI/GenericDataProcessorPresenterGroupReducerWorker.h"
#include "MantidQtWidgets/Common/DataProcessorUI/GenericDataProcessorPresenterRowReducerWorker.h"
#include "MantidQtWidgets/Common/DataProcessorUI/GenericDataProcessorPresenterThread.h"
#include "MantidQtWidgets/Common/DataProcessorUI/OptionsMap.h"
#include "MantidQtWidgets/Common/DataProcessorUI/QtDataProcessorOptionsDialog.h"
#include "MantidQtWidgets/Common/DataProcessorUI/WorkspaceCommand.h"
#include "MantidQtWidgets/Common/DataProcessorUI/WorkspaceNameUtils.h"
#include "MantidQtWidgets/Common/ParseKeyValueString.h"
#include "MantidQtWidgets/Common/ProgressableView.h"

#include <QHash>
#include <QStringList>
#include <algorithm>
#include <boost/tokenizer.hpp>
#include <fstream>
#include <iterator>
#include <sstream>

using namespace Mantid::API;
using namespace Mantid::Geometry;
using namespace Mantid::Kernel;
using namespace MantidQt::MantidWidgets;

namespace {
void setAlgorithmProperty(IAlgorithm *const alg, std::string const &name,
                          std::string const &value) {
  if (!value.empty())
    alg->setProperty(name, value);
}

void setAlgorithmProperty(IAlgorithm *const alg, QString const &name,
                          std::string const &value) {
  if (!value.empty())
    setAlgorithmProperty(alg, name.toStdString(), value);
}

void setAlgorithmProperty(IAlgorithm *const alg, QString const &name,
                          QString const &value) {
  if (!value.isEmpty())
    setAlgorithmProperty(alg, name.toStdString(), value.toStdString());
}

bool workspaceExists(QString const &workspaceName) {
  return AnalysisDataService::Instance().doesExist(workspaceName.toStdString());
}

void removeWorkspace(QString const &workspaceName) {
  AnalysisDataService::Instance().remove(workspaceName.toStdString());
}
}

namespace MantidQt {
namespace MantidWidgets {
namespace DataProcessor {

/**
* Constructor
* @param whitelist : The set of properties we want to show as columns
* @param preprocessMap : A map containing instructions for pre-processing
* @param processor : A ProcessingAlgorithm
* @param postprocessor : A PostprocessingAlgorithm
* workspaces
* @param postprocessMap : A map containing instructions for post-processing.
* This map links column name to properties of the post-processing algorithm
* @param loader : The algorithm responsible for loading data
*/
GenericDataProcessorPresenter::GenericDataProcessorPresenter(
    WhiteList whitelist,
    std::map<QString, PreprocessingAlgorithm> preprocessMap,
    ProcessingAlgorithm processor, PostprocessingAlgorithm postprocessor,
    std::map<QString, QString> postprocessMap, QString loader)
    : WorkspaceObserver(), m_view(nullptr), m_progressView(nullptr),
      m_mainPresenter(), m_loader(std::move(loader)),
      m_postprocessing(postprocessor.name().isEmpty()
                           ? boost::optional<PostprocessingStep>()
                           : PostprocessingStep(QString(),
                                                std::move(postprocessor),
                                                std::move(postprocessMap))),
      m_preprocessing(ColumnOptionsMap(), std::move(preprocessMap)),
      m_whitelist(std::move(whitelist)), m_processor(std::move(processor)),
      m_progressReporter(nullptr), m_promptUser(true), m_tableDirty(false),
      m_pauseReduction(false), m_reductionPaused(true),
      m_nextActionFlag(ReductionFlag::StopReduceFlag) {

  // Column Options must be added to the whitelist
  m_whitelist.addElement("Options", "Options",
                         "<b>Override <samp>" + processor.name() +
                             "</samp> properties</b><br /><i>optional</i><br "
                             "/>This column allows you to "
                             "override the properties used when executing "
                             "the main reduction algorithm. "
                             "Options are given as "
                             "key=value pairs, separated by commas. Values "
                             "containing commas must be quoted. In case of "
                             "conflict between options "
                             "specified via this column and global options "
                             "specified externally, the former prevail.");

  // Column Hidden Options must be added to the whitelist
  m_whitelist.addElement("HiddenOptions", "HiddenOptions",
                         "<b>Override <samp>" + processor.name() +
                             "</samp> properties</b><br /><i>optional</i><br "
                             "/>This column allows you to "
                             "override the properties used when executing "
                             "the main reduction algorithm in the same way"
                             "as the Options column, but this column is hidden"
                             "from the user. "
                             "Hidden Options are given as "
                             "key=value pairs, separated by commas. Values "
                             "containing commas must be quoted. In case of "
                             "conflict between options "
                             "specified via this column and global options "
                             "specified externally, the former prevail.");

  if (hasPostprocessing()) {
    m_manager =
        Mantid::Kernel::make_unique<TwoLevelTreeManager>(this, m_whitelist);
  } else {
    m_manager =
        Mantid::Kernel::make_unique<OneLevelTreeManager>(this, m_whitelist);
  }
}

/**
* Delegating constructor (no pre-processing needed)
* @param whitelist : The set of properties we want to show as columns
* @param processor : A ProcessingAlgorithm
* @param postprocessor : A PostprocessingAlgorithm
* workspaces
*/
GenericDataProcessorPresenter::GenericDataProcessorPresenter(
    WhiteList whitelist, ProcessingAlgorithm processor,
    PostprocessingAlgorithm postprocessor)
    : GenericDataProcessorPresenter(
          std::move(whitelist), std::map<QString, PreprocessingAlgorithm>(),
          std::move(processor), std::move(postprocessor)) {}

/**
 * Delegating constructor (only whitelist specified)
 * @param whitelist : The set of properties we want to show as columns
 */
GenericDataProcessorPresenter::GenericDataProcessorPresenter(
    WhiteList whitelist)
    : GenericDataProcessorPresenter(
          std::move(whitelist), std::map<QString, PreprocessingAlgorithm>(),
          ProcessingAlgorithm(), PostprocessingAlgorithm()) {}

/**
* Delegating constructor (no post-processing needed)
* @param whitelist : The set of properties we want to show as columns
* @param preprocessMap : A map containing instructions for pre-processing
* @param processor : A ProcessingAlgorithm
* workspaces
*/
GenericDataProcessorPresenter::GenericDataProcessorPresenter(
    WhiteList whitelist,
    std::map<QString, PreprocessingAlgorithm> preprocessMap,
    ProcessingAlgorithm processor)
    : GenericDataProcessorPresenter(
          std::move(whitelist), std::move(preprocessMap), std::move(processor),
          PostprocessingAlgorithm()) {}

/**
* Delegating constructor (no pre-processing needed, no post-processing needed)
* @param whitelist : The set of properties we want to show as columns
* @param processor : A ProcessingAlgorithm
* workspaces
*/
GenericDataProcessorPresenter::GenericDataProcessorPresenter(
    WhiteList whitelist, ProcessingAlgorithm processor)
    : GenericDataProcessorPresenter(
          std::move(whitelist), std::map<QString, PreprocessingAlgorithm>(),
          std::move(processor), PostprocessingAlgorithm()) {}

/**
* Destructor
*/
GenericDataProcessorPresenter::~GenericDataProcessorPresenter() {}

namespace {
std::set<std::string> toStdStringSet(std::set<QString> in) {
  auto out = std::set<std::string>();
  std::transform(in.cbegin(), in.cend(), std::inserter(out, out.begin()),
                 [](QString const &inStr)
                     -> std::string { return inStr.toStdString(); });
  return out;
}
}
/**
* Sets the views this presenter is going to handle
* @param tableView : The table view
* @param progressView : The progress view
*/
void GenericDataProcessorPresenter::acceptViews(
    DataProcessorView *tableView, ProgressableView *progressView) {

  // As soon as we are given a view, initialize everything
  m_view = tableView;
  m_progressView = progressView;

  // Add actions to toolbar
  addCommands();

  // Initialise options
  // Load saved values from disk
  initOptions();

  // Populate an initial list of valid tables to open, and subscribe to the ADS
  // to keep it up to date
  Mantid::API::AnalysisDataServiceImpl &ads =
      Mantid::API::AnalysisDataService::Instance();

  auto items = ads.getObjectNames();
  for (auto const &name : items) {
    Workspace_sptr ws = ads.retrieve(name);

    if (m_manager->isValidModel(
            boost::dynamic_pointer_cast<ITableWorkspace>(ws),
            m_whitelist.size()))
      m_workspaceList.insert(QString::fromStdString(name));
  }
  observeAdd();
  observePostDelete();
  observeRename();
  observeADSClear();
  observeAfterReplace();

  // Provide autocompletion hints for the options column. We use the algorithm's
  // properties minus those we blacklist. We blacklist any useless properties or
  // ones we're handling that the user should'nt touch.
  IAlgorithm_sptr alg =
      AlgorithmManager::Instance().create(m_processor.name().toStdString());
  m_view->setOptionsHintStrategy(
      new AlgorithmHintStrategy(alg, toStdStringSet(m_processor.blacklist())),
      static_cast<int>(m_whitelist.size()) - 2);

  // Start with a blank table
  newTable();

  // Update enabled/disabled states on the view (processing is not yet in
  // progress)
  updateWidgetEnabledState(false);
}

/**
Returns the name of the reduced workspace for a given row
@param data :: [input] The data for this row
@param prefix : A prefix to be appended to the generated ws name
@throws std::runtime_error if the workspace could not be prepared
@returns : The name of the workspace
*/
QString GenericDataProcessorPresenter::getReducedWorkspaceName(
    const QStringList &data, const QString &prefix) const {
  return MantidQt::MantidWidgets::DataProcessor::getReducedWorkspaceName(
      data, m_whitelist, prefix);
}

void GenericDataProcessorPresenter::settingsChanged() {
  m_preprocessing.m_options =
      convertColumnOptionsFromQMap(m_mainPresenter->getPreprocessingOptions());
  m_processingOptions =
      convertOptionsFromQMap(m_mainPresenter->getProcessingOptions());

  if (hasPostprocessing())
    m_postprocessing->m_options =
        m_mainPresenter->getPostprocessingOptionsAsString();

  m_manager->invalidateAllProcessed();
}

bool GenericDataProcessorPresenter::rowOutputExists(RowItem const &row) const {
  for (auto i = 0u; i < m_processor.numberOfOutputProperties(); i++) {
    auto outputWorkspaceName =
        getReducedWorkspaceName(row.second, m_processor.prefix(i));
    if (!workspaceExists(outputWorkspaceName))
      return false;
  }
  return true;
}

/**
Process selected data
*/
void GenericDataProcessorPresenter::process() {
  // Emit a signal hat the process is starting
  m_view->emitProcessClicked();
  if (GenericDataProcessorPresenter::m_skipProcessing) {
    m_skipProcessing = false;
    return;
  }
  m_selectedData = m_manager->selectedData(m_promptUser);

  // Don't continue if there are no items selected
  if (m_selectedData.size() == 0)
    return;

  // Clear the group queue
  m_group_queue = GroupQueue();

  // Progress: each group and each row within count as a progress step.
  int maxProgress = 0;

  for (const auto &group : m_selectedData) {
    auto groupOutputNotFound =
        hasPostprocessing() &&
        !workspaceExists(getPostprocessedWorkspaceName(group.second));

    if (groupOutputNotFound)
      m_manager->setProcessed(false, group.first);

    // Groups that are already processed or cannot be post-processed (only 1
    // child row selected) do not count in progress
    if (!isProcessed(group.first) && group.second.size() > 1)
      maxProgress++;

    RowQueue rowQueue;

    for (const auto &row : group.second) {

      // Add all row items to queue
      rowQueue.push(row);

      // Set group as unprocessed if settings have changed or the expected
      // output workspaces cannot be found
      if (!rowOutputExists(row))
        m_manager->setProcessed(false, row.first, group.first);

      // Rows that are already processed do not count in progress
      if (!isProcessed(row.first, group.first))
        maxProgress++;
    }
    m_group_queue.emplace(group.first, rowQueue);
  }

  // Create progress reporter bar
  if (maxProgress > 0) {
    int progress = 0;
    m_progressReporter = new ProgressPresenter(progress, maxProgress,
                                               maxProgress, m_progressView);
  }
  // Start processing the first group
  m_nextActionFlag = ReductionFlag::ReduceGroupFlag;
  resume();
}

/**
Decide which processing action to take next
*/
void GenericDataProcessorPresenter::doNextAction() {

  switch (m_nextActionFlag) {
  case ReductionFlag::ReduceRowFlag:
    nextRow();
    break;
  case ReductionFlag::ReduceGroupFlag:
    nextGroup();
    break;
  case ReductionFlag::StopReduceFlag:
    endReduction();
    break;
  }
  // Not having a 'default' case is deliberate. gcc issues a warning if there's
  // a flag we aren't handling.
}

/**
Process a new row
*/
void GenericDataProcessorPresenter::nextRow() {

  if (m_pauseReduction) {
    // Notify presenter that reduction is paused
    m_mainPresenter->confirmReductionPaused();
    m_reductionPaused = true;
    return;
  }

  // Add processed row data to the group
  int rowIndex = m_rowItem.first;
  m_groupData[rowIndex] = m_rowItem.second;
  int groupIndex = m_group_queue.front().first;
  auto &rqueue = m_group_queue.front().second;

  if (!rqueue.empty()) {
    // Set next action flag
    m_nextActionFlag = ReductionFlag::ReduceRowFlag;
    // Reduce next row
    m_rowItem = rqueue.front();
    rqueue.pop();
    // Skip reducing rows that are already processed
    if (!isProcessed(m_rowItem.first, groupIndex)) {
      startAsyncRowReduceThread(&m_rowItem, groupIndex);
      return;
    }
  } else {
    m_group_queue.pop();
    // Set next action flag
    m_nextActionFlag = ReductionFlag::ReduceGroupFlag;

    // Skip post-processing groups that are already processed or only contain a
    // single row
    if (!isProcessed(groupIndex) && m_groupData.size() > 1) {
      startAsyncGroupReduceThread(m_groupData, groupIndex);
      completedReductionSuccessfully(m_groupData);
      return;
    }
  }

  // Row / group skipped, perform next action
  doNextAction();
}

void GenericDataProcessorPresenter::completedReductionSuccessfully(
    GroupData const &) {}

/**
Process a new group
*/
void GenericDataProcessorPresenter::nextGroup() {

  if (m_pauseReduction) {
    // Notify presenter that reduction is paused
    m_mainPresenter->confirmReductionPaused();
    m_reductionPaused = true;
    return;
  }

  if (!m_group_queue.empty()) {
    // Set next action flag
    m_nextActionFlag = ReductionFlag::ReduceRowFlag;
    // Reduce first row
    auto &rqueue = m_group_queue.front().second;
    m_rowItem = rqueue.front();
    rqueue.pop();
    // Skip reducing rows that are already processed
    if (!isProcessed(m_rowItem.first, m_group_queue.front().first))
      startAsyncRowReduceThread(&m_rowItem, m_group_queue.front().first);
    else
      doNextAction();
  } else {
    // If "Output Notebook" checkbox is checked then create an ipython notebook
    if (m_view->getEnableNotebook())
      saveNotebook(m_selectedData);
    endReduction();
  }
  // Clear group data from any previously processed groups
  m_groupData.clear();
}

/*
Reduce the current row asynchronously
*/
void GenericDataProcessorPresenter::startAsyncRowReduceThread(RowItem *rowItem,
                                                              int groupIndex) {

  auto *worker = new GenericDataProcessorPresenterRowReducerWorker(
      this, rowItem, groupIndex);
  m_workerThread.reset(new GenericDataProcessorPresenterThread(this, worker));
  m_workerThread->start();
}

/*
Reduce the current group asynchronously
*/
void GenericDataProcessorPresenter::startAsyncGroupReduceThread(
    GroupData &groupData, int groupIndex) {

  auto *worker = new GenericDataProcessorPresenterGroupReducerWorker(
      this, groupData, groupIndex);
  m_workerThread.reset(new GenericDataProcessorPresenterThread(this, worker));
  m_workerThread->start();
}

/**
End reduction
*/
void GenericDataProcessorPresenter::endReduction() {

  pause();
  m_reductionPaused = true;
  m_mainPresenter->confirmReductionPaused();
}

/**
Handle reduction error
*/
void GenericDataProcessorPresenter::reductionError(QString ex) {
  m_view->giveUserCritical(ex, "Error");
}

/**
Handle thread completion
*/
void GenericDataProcessorPresenter::threadFinished(const int exitCode) {

  m_workerThread.release();

  if (exitCode == 0) { // Success
    m_progressReporter->report();
    doNextAction();
  } else { // Error
    m_progressReporter->clear();
    endReduction();
  }
}

/**
Display a dialog to choose save location for notebook, then save the notebook
there
@param data : the processed data
*/
void GenericDataProcessorPresenter::saveNotebook(const TreeData &data) {
  QString filename = m_view->requestNotebookPath();
  if (!filename.isEmpty()) {
    // Global pre-processing options as a map where keys are column
    // name and values are pre-processing options as a map of
    // property name to value
    const auto preprocessingOptionsMap = m_preprocessing.m_options;

    auto notebook = Mantid::Kernel::make_unique<GenerateNotebook>(
        m_wsName, m_view->getProcessInstrument(), m_whitelist,
        m_preprocessing.m_map, m_processor, m_postprocessing,
        preprocessingOptionsMap, m_processingOptions);
    auto generatedNotebook =
        std::string(notebook->generateNotebook(data).toStdString());

    std::ofstream file(filename.toStdString(), std::ofstream::trunc);
    file << generatedNotebook;
    file.flush();
    file.close();
  }
}

bool GenericDataProcessorPresenter::hasPostprocessing() const {
  return bool(m_postprocessing);
}

/**
Post-processes the workspaces created by the given rows together.
@param groupData : the data in a given group as received from the tree manager
*/
void GenericDataProcessorPresenter::postProcessGroup(
    const GroupData &groupData) {
  if (hasPostprocessing())
    m_postprocessing->postProcessGroup(m_processor.prefix(0), m_whitelist,
                                       groupData);
}

/**
Takes a user specified run, or list of runs, and returns a pointer to the
desired workspace
@param runStr : The run or list of runs (separated by '+')
@param preprocessor : The pre-processing algorithm acting on this column
@param optionsMap : User-specified options as a map
@throws std::runtime_error if the workspace could not be prepared
@returns a shared pointer to the workspace
*/
Workspace_sptr GenericDataProcessorPresenter::prepareRunWorkspace(
    const QString &runStr, const PreprocessingAlgorithm &preprocessor,
    const OptionsMap &optionsMap) {
  auto const instrument = m_view->getProcessInstrument();

  auto runs = preprocessingStringToList(runStr);

  if (runs.isEmpty())
    throw std::runtime_error("No runs given");

  // If we're only given one run, just return that
  if (runs.size() == 1)
    return getRun(runs[0], instrument, preprocessor.prefix());

  auto const outputName =
      preprocessingListToString(runs, preprocessor.prefix());

  /* Ideally, this should be executed as a child algorithm to keep the ADS tidy,
  * but that doesn't preserve history nicely, so we'll just take care of tidying
  * up in the event of failure.
  */
  IAlgorithm_sptr alg =
      AlgorithmManager::Instance().create(preprocessor.name().toStdString());
  alg->initialize();
  setAlgorithmProperty(
      alg.get(), preprocessor.lhsProperty(),
      getRun(runs[0], instrument, preprocessor.prefix())->getName());
  setAlgorithmProperty(alg.get(), preprocessor.outputProperty(), outputName);

  // Drop the first run from the runs list
  runs.erase(runs.begin());

  try {
    // Iterate through all the remaining runs, adding them to the first run
    for (auto runIt = runs.begin(); runIt != runs.end(); ++runIt) {

      // Loop all options and set them on the algorithm unless they
      // are the LHS or RHS property
      for (auto &kvp : optionsMap) {
        try {
          if (kvp.first != preprocessor.lhsProperty() &&
              kvp.first != preprocessor.rhsProperty())
            setAlgorithmProperty(alg.get(), kvp.first, kvp.second);
        } catch (Mantid::Kernel::Exception::NotFoundError &) {
          // We can't apply this option to this pre-processing alg
          throw;
        }
      }

      // Now set this run as the RHS property
      setAlgorithmProperty(
          alg.get(), preprocessor.rhsProperty(),
          getRun(*runIt, instrument, preprocessor.prefix())->getName());
      alg->execute();

      if (runIt != --runs.end()) {
        // After the first execution we replace the LHS with the previous output
        setAlgorithmProperty(alg.get(), preprocessor.lhsProperty(), outputName);
      }
    }
  } catch (...) {
    // If we're unable to create the full workspace, discard the partial version
    removeWorkspace(outputName);
    // We've tidied up, now re-throw.
    throw;
  }

  return AnalysisDataService::Instance().retrieveWS<Workspace>(
      outputName.toStdString());
}
/**
Returns the name of the reduced workspace for a given group
@param groupData : The data in a given group
@returns : The name of the workspace
*/
QString GenericDataProcessorPresenter::getPostprocessedWorkspaceName(
    const GroupData &groupData) {
  if (!hasPostprocessing())
    throw std::runtime_error("Attempted to get postprocessing workspace but no "
                             "postprocessing is specified.");
  return m_postprocessing->getPostprocessedWorkspaceName(m_whitelist,
                                                         groupData);
}

/** Loads a run found from disk or AnalysisDataService
 *
 * @param run : The name of the run
 * @param instrument : The instrument the run belongs to
 * @param prefix : The prefix to be prepended to the run number
 * @throws std::runtime_error if the run could not be loaded
 * @returns a shared pointer to the workspace
 */
Workspace_sptr GenericDataProcessorPresenter::getRun(const QString &run,
                                                     const QString &instrument,
                                                     const QString &prefix) {

  bool runFound;
  QString outName;
  QString fileName = instrument + run;

  outName = findRunInADS(run, prefix, runFound);
  if (!runFound) {
    outName = loadRun(run, instrument, prefix, m_loader, runFound);
    if (!runFound)
      throw std::runtime_error("Could not open " + fileName.toStdString());
  }

  return AnalysisDataService::Instance().retrieveWS<Workspace>(
      outName.toStdString());
}

bool isNumeric(QString const &numericCandidate) {
  return QRegExp("\\d+").exactMatch(numericCandidate);
}
/** Tries fetching a run from AnalysisDataService
 *
 * @param run : The name of the run
 * @param prefix : The prefix to be prepended to the run number
 * @param runFound : Whether or not the run was actually found
 * @returns string name of the run
 */
QString GenericDataProcessorPresenter::findRunInADS(const QString &run,
                                                    const QString &prefix,
                                                    bool &runFound) {

  runFound = true;

  // First, let's see if the run given is the name of a workspace in the ADS
  if (workspaceExists(run))
    return run;

  // Try with prefix
  if (workspaceExists(prefix + run))
    return prefix + run;

  // Is the run string is numeric?
  if (isNumeric(run)) {
    // Look for "<run_number>" in the ADS
    if (workspaceExists(run))
      return run;

    // Look for "<instrument><run_number>" in the ADS
    if (workspaceExists(prefix + run))
      return prefix + run;
  }

  // Run not found in ADS;
  runFound = false;
  return "";
}

/** Tries loading a run from disk
 *
 * @param run : The name of the run
 * @param instrument : The instrument the run belongs to
 * @param prefix : The prefix to be prepended to the run number
 * @param loader : The algorithm used for loading runs
 * @param runFound : Whether or not the run was actually found
 * @returns string name of the run
 */
QString GenericDataProcessorPresenter::loadRun(const QString &run,
                                               const QString &instrument,
                                               const QString &prefix,
                                               const QString &loader,
                                               bool &runFound) {

  runFound = true;
  auto const fileName = instrument + run;
  auto const outputName = prefix + run;

  IAlgorithm_sptr algLoadRun =
      AlgorithmManager::Instance().create(loader.toStdString());
  algLoadRun->initialize();
  algLoadRun->setProperty("Filename", fileName.toStdString());
  algLoadRun->setProperty("OutputWorkspace", outputName.toStdString());
  algLoadRun->execute();
  if (!algLoadRun->isExecuted()) {
    // Run not loaded from disk
    runFound = false;
    return "";
  }

  return outputName;
}

IAlgorithm_sptr
GenericDataProcessorPresenter::createProcessingAlgorithm() const {
  auto alg =
      AlgorithmManager::Instance().create(m_processor.name().toStdString());
  alg->initialize();
  return alg;
}

/** Preprocess the property value from the given column if
 * preprocessing is applicable for this column (does nothing
 * otherwise)
 * @param columnName [in] :: the name of the column
 * @param columnValue [inout] :: the original value in the column;
 * this gets updated to the new (preprocessed) value for the column
 * if preprocessing was performed
 * @param data [in] :: the data in the row
 */
void GenericDataProcessorPresenter::preprocessColumnValue(
    const QString &columnName, QString &columnValue, RowData *data) {
  // Check if preprocessing is required for this column
  if (!m_preprocessing.hasPreprocessing(columnName))
    return;

  // Get the options for the preprocessing algorithm
  auto preprocessor = m_preprocessing.m_map.at(columnName);
  OptionsMap options;
  if (m_preprocessing.hasOptions(columnName)) {
    auto globalOptions = m_preprocessing.m_options.at(columnName);
    options = getCanonicalOptions(data, globalOptions, m_whitelist, false);
  }

  // Run the preprocessing algorithm
  auto runWS = prepareRunWorkspace(columnValue, preprocessor, options);

  // Update the column value with the result of preprocessing
  columnValue = QString::fromStdString(runWS->getName());
}

/** Perform preprocessing on algorithm property values where applicable
 * @param options : the algorithm properties as a map of property name
 * to value
 * @param data : the data in the row
*/
void GenericDataProcessorPresenter::preprocessOptionValues(OptionsMap &options,
                                                           RowData *data) {
  // Loop through all columns (excluding the Options and Hidden options
  // columns)
  for (auto columnIt = m_whitelist.cbegin(); columnIt != m_whitelist.cend() - 2;
       ++columnIt) {
    auto column = *columnIt;
    auto &propertyName = column.algorithmProperty();

    // Check if the column has a value
    if (options.find(propertyName) != options.end()) {
      preprocessColumnValue(column.name(), options[propertyName], data);
    }
  }
}

/** Some columns in the model should be updated with outputs
 * from the algorithm if they were not set by the user. This is
 * so that the view can be updated show the user what values were used.
 */
void GenericDataProcessorPresenter::updateModelFromAlgorithm(
    IAlgorithm_sptr alg, RowData *data) {

  auto newData = data;

  if (alg->isExecuted()) {
    auto runNumbersIt2 = data->constBegin();
    auto newDataIt = newData->begin();
    auto columnIt2 = m_whitelist.cbegin();

    /* The reduction is complete, try to populate the columns */
    for (; columnIt2 != m_whitelist.cend() - 2;
         ++columnIt2, ++runNumbersIt2, ++newDataIt) {

      auto column = *columnIt2;
      auto runNumbers = *runNumbersIt2;

      if (runNumbers.isEmpty() && !m_preprocessing.m_map.count(column.name())) {

        QString propValue = QString::fromStdString(
            alg->getPropertyValue(column.algorithmProperty().toStdString()));

        if (m_options["Round"].toBool()) {
          QString exp = (propValue.indexOf("e") != -1)
                            ? propValue.right(propValue.indexOf("e"))
                            : "";
          propValue =
              propValue.mid(0, propValue.indexOf(".") +
                                   m_options["RoundPrecision"].toInt() + 1) +
              exp;
        }

        (*newDataIt) = propValue;
      }
    }
  }
}

/** Create an algorithm with the given properties and execute it
 * @param options : the options as a map of property name to value
 * @throws std::runtime_error if reduction fails
 * @returns : the algorithm
 */
IAlgorithm_sptr GenericDataProcessorPresenter::createAndRunAlgorithm(
    const OptionsMap &options) {
  auto alg = createProcessingAlgorithm();

  for (auto &kvp : options) {
    setAlgorithmProperty(alg.get(), kvp.first, kvp.second);
  }

  alg->execute();

  return alg;
}

/** Reduce a row
 *
 * @param data :: [input] The data in this row as a vector where elements
 * correspond to column contents
 * @throws std::runtime_error if reduction fails
 */
void GenericDataProcessorPresenter::reduceRow(RowData *data) {

  // Get the algorithm input properties as an options map
  OptionsMap options = getCanonicalOptions(
      data, m_processingOptions, m_whitelist, true,
      m_processor.outputProperties(), m_processor.prefixes());
  // Perform any preprocessing on the input properties
  preprocessOptionValues(options, data);
  // Run the algorithm
  const auto alg = createAndRunAlgorithm(options);
  // Populate any missing values in the model with output from the algorithm
  updateModelFromAlgorithm(alg, data);
}

/**
Insert a new row
*/
void GenericDataProcessorPresenter::appendRow() { m_manager->appendRow(); }

/**
Insert a new group
*/
void GenericDataProcessorPresenter::appendGroup() { m_manager->appendGroup(); }

/**
Delete row(s) from the model
*/
void GenericDataProcessorPresenter::deleteRow() { m_manager->deleteRow(); }

/**
Delete group(s) from the model
*/
void GenericDataProcessorPresenter::deleteGroup() { m_manager->deleteGroup(); }

/**
Group rows together
*/
void GenericDataProcessorPresenter::groupRows() { m_manager->groupRows(); }

/**
Expand all groups
*/
void GenericDataProcessorPresenter::expandAll() { m_view->expandAll(); }

/**
Collapse all groups
*/
void GenericDataProcessorPresenter::collapseAll() { m_view->collapseAll(); }

/**
Select all rows / groups
*/
void GenericDataProcessorPresenter::selectAll() { m_view->selectAll(); }

/**
Used by the view to tell the presenter something has changed
*/
void GenericDataProcessorPresenter::notify(DataProcessorPresenter::Flag flag) {

  switch (flag) {
  case DataProcessorPresenter::SaveAsFlag:
    saveTableAs();
    break;
  case DataProcessorPresenter::SaveFlag:
    saveTable();
    break;
  case DataProcessorPresenter::AppendRowFlag:
    appendRow();
    break;
  case DataProcessorPresenter::AppendGroupFlag:
    appendGroup();
    break;
  case DataProcessorPresenter::DeleteRowFlag:
    deleteRow();
    break;
  case DataProcessorPresenter::DeleteGroupFlag:
    deleteGroup();
    break;
  case DataProcessorPresenter::ProcessFlag:
    process();
    break;
  case DataProcessorPresenter::GroupRowsFlag:
    groupRows();
    break;
  case DataProcessorPresenter::NewTableFlag:
    newTable();
    break;
  case DataProcessorPresenter::TableUpdatedFlag:
    m_tableDirty = true;
    break;
  case DataProcessorPresenter::ExpandSelectionFlag:
    expandSelection();
    break;
  case DataProcessorPresenter::OptionsDialogFlag:
    showOptionsDialog();
    break;
  case DataProcessorPresenter::ClearSelectedFlag:
    clearSelected();
    break;
  case DataProcessorPresenter::CopySelectedFlag:
    copySelected();
    break;
  case DataProcessorPresenter::CutSelectedFlag:
    cutSelected();
    break;
  case DataProcessorPresenter::PasteSelectedFlag:
    pasteSelected();
    break;
  case DataProcessorPresenter::ImportTableFlag:
    importTable();
    break;
  case DataProcessorPresenter::OpenTableFlag:
    openTable();
    break;
  case DataProcessorPresenter::ExportTableFlag:
    exportTable();
    break;