GenericDataProcessorPresenter.cpp

#include "MantidQtMantidWidgets/DataProcessorUI/GenericDataProcessorPresenter.h"
#include "MantidAPI/AlgorithmManager.h"
#include "MantidAPI/IEventWorkspace.h"
#include "MantidAPI/ITableWorkspace.h"
#include "MantidAPI/MatrixWorkspace.h"
#include "MantidAPI/NotebookWriter.h"
#include "MantidAPI/TableRow.h"
#include "MantidAPI/WorkspaceFactory.h"
#include "MantidGeometry/Instrument.h"
#include "MantidKernel/Strings.h"
#include "MantidKernel/TimeSeriesProperty.h"
#include "MantidKernel/Utils.h"
#include "MantidKernel/make_unique.h"
#include "MantidQtMantidWidgets/AlgorithmHintStrategy.h"
#include "MantidQtMantidWidgets/DataProcessorUI/DataProcessorGenerateNotebook.h"
#include "MantidQtMantidWidgets/DataProcessorUI/DataProcessorView.h"
#include "MantidQtMantidWidgets/DataProcessorUI/DataProcessorWorkspaceCommand.h"
#include "MantidQtMantidWidgets/DataProcessorUI/GenericDataProcessorPresenterRowReducerWorker.h"
#include "MantidQtMantidWidgets/DataProcessorUI/GenericDataProcessorPresenterGroupReducerWorker.h"
#include "MantidQtMantidWidgets/DataProcessorUI/GenericDataProcessorPresenterThread.h"
#include "MantidQtMantidWidgets/DataProcessorUI/ParseKeyValueString.h"
#include "MantidQtMantidWidgets/DataProcessorUI/QtDataProcessorOptionsDialog.h"
#include "MantidQtMantidWidgets/ProgressableView.h"

#include <boost/algorithm/string/join.hpp>
#include <algorithm>
#include <unordered_map>
#include <boost/regex.hpp>
#include <boost/tokenizer.hpp>
#include <fstream>
#include <sstream>

using namespace Mantid::API;
using namespace Mantid::Geometry;
using namespace Mantid::Kernel;
using namespace MantidQt::MantidWidgets;

namespace {
std::map<std::string, std::string>
convertStringToMap(const std::string &options) {

  std::vector<std::string> optionsVec;
  std::map<std::string, std::string> optionsMap;

  boost::split(optionsVec, options, boost::is_any_of(";"));

  for (const auto &option : optionsVec) {

    std::vector<std::string> opt;
    boost::split(opt, option, boost::is_any_of(","));

    std::vector<std::string> temp(opt.begin() + 1, opt.end());

    optionsMap[opt[0]] = boost::algorithm::join(temp, ",");
  }
  return optionsMap;
}

std::unordered_map<std::string, std::set<std::string>>
convertStringToMapWithSet(const std::string &properties) {
  // The provided string has the form
  // key1: value11, value12; key2: value21;
  // The keys are keys in a map which maps to a set of values
  std::unordered_map<std::string, std::set<std::string>> props;

  if (properties.empty()) {
    return props;
  }

  // Split by each map pair
  std::vector<std::string> propVec;
  boost::split(propVec, properties, boost::is_any_of(";"));

  for (const auto &prop : propVec) {
    // Split the key and values
    std::vector<std::string> elements;
    boost::split(elements, prop, boost::is_any_of(":"));

    // Split values
    std::vector<std::string> vals;
    boost::split(vals, elements[1], boost::is_any_of(","));
    std::set<std::string> values(vals.begin(), vals.end());
    props[elements[0]] = values;
  }
  return props;
}
}

namespace MantidQt {
namespace MantidWidgets {

/**
* Constructor
* @param whitelist : The set of properties we want to show as columns
* @param preprocessMap : A map containing instructions for pre-processing
* @param processor : A DataProcessorProcessingAlgorithm
* @param postprocessor : A DataProcessorPostprocessingAlgorithm
* workspaces
* @param postprocessMap : A map containing instructions for post-processing.
* This map links column name to properties of the post-processing algorithm
* @param loader : The algorithm responsible for loading data
*/
GenericDataProcessorPresenter::GenericDataProcessorPresenter(
    const DataProcessorWhiteList &whitelist,
    const std::map<std::string, DataProcessorPreprocessingAlgorithm> &
        preprocessMap,
    const DataProcessorProcessingAlgorithm &processor,
    const DataProcessorPostprocessingAlgorithm &postprocessor,
    const std::map<std::string, std::string> &postprocessMap,
    const std::string &loader)
    : WorkspaceObserver(), m_view(nullptr), m_progressView(nullptr),
      m_mainPresenter(), m_loader(loader), m_whitelist(whitelist),
      m_preprocessMap(preprocessMap), m_processor(processor),
      m_postprocessor(postprocessor), m_postprocessMap(postprocessMap),
      m_progressReporter(nullptr), m_postprocess(true), m_promptUser(true),
      m_tableDirty(false), m_newSelection(true), m_reductionPaused(true),
      m_nextActionFlag(ReductionFlag::StopReduceFlag) {

  // Column Options must be added to the whitelist
  m_whitelist.addElement("Options", "Options",
                         "<b>Override <samp>" +
                             QString::fromStdString(processor.name()) +
                             "</samp> properties</b><br /><i>optional</i><br "
                             "/>This column allows you to "
                             "override the properties used when executing "
                             "the main reduction algorithm. "
                             "Options are given as "
                             "key=value pairs, separated by commas. Values "
                             "containing commas must be quoted. In case of "
                             "conflict between options "
                             "specified via this column and global options "
                             "specified externally, the former prevail.");
  m_columns = static_cast<int>(m_whitelist.size());

  if (m_postprocessor.name().empty()) {
    m_postprocess = false;
    m_manager = Mantid::Kernel::make_unique<DataProcessorOneLevelTreeManager>(
        this, m_whitelist);
  } else {
    m_manager = Mantid::Kernel::make_unique<DataProcessorTwoLevelTreeManager>(
        this, m_whitelist);
  }
}

/**
* Delegating constructor (no pre-processing needed)
* @param whitelist : The set of properties we want to show as columns
* @param processor : A DataProcessorProcessingAlgorithm
* @param postprocessor : A DataProcessorPostprocessingAlgorithm
* workspaces
*/
GenericDataProcessorPresenter::GenericDataProcessorPresenter(
    const DataProcessorWhiteList &whitelist,
    const DataProcessorProcessingAlgorithm &processor,
    const DataProcessorPostprocessingAlgorithm &postprocessor)
    : GenericDataProcessorPresenter(
          whitelist,
          std::map<std::string, DataProcessorPreprocessingAlgorithm>(),
          processor, postprocessor) {}

/**
* Delegating constructor (no post-processing needed)
* @param whitelist : The set of properties we want to show as columns
* @param preprocessMap : A map containing instructions for pre-processing
* @param processor : A DataProcessorProcessingAlgorithm
* workspaces
*/
GenericDataProcessorPresenter::GenericDataProcessorPresenter(
    const DataProcessorWhiteList &whitelist,
    const std::map<std::string, DataProcessorPreprocessingAlgorithm> &
        preprocessMap,
    const DataProcessorProcessingAlgorithm &processor)
    : GenericDataProcessorPresenter(whitelist, preprocessMap, processor,
                                    DataProcessorPostprocessingAlgorithm()) {}

/**
* Delegating constructor (no pre-processing needed, no post-processing needed)
* @param whitelist : The set of properties we want to show as columns
* @param processor : A DataProcessorProcessingAlgorithm
* workspaces
*/
GenericDataProcessorPresenter::GenericDataProcessorPresenter(
    const DataProcessorWhiteList &whitelist,
    const DataProcessorProcessingAlgorithm &processor)
    : GenericDataProcessorPresenter(
          whitelist,
          std::map<std::string, DataProcessorPreprocessingAlgorithm>(),
          processor, DataProcessorPostprocessingAlgorithm()) {}

/**
* Destructor
*/
GenericDataProcessorPresenter::~GenericDataProcessorPresenter() {}

/**
* Sets the views this presenter is going to handle
* @param tableView : The table view
* @param progressView : The progress view
*/
void GenericDataProcessorPresenter::acceptViews(
    DataProcessorView *tableView, ProgressableView *progressView) {

  // As soon as we are given a view, initialize everything
  m_view = tableView;
  m_progressView = progressView;

  // Add actions to toolbar
  addCommands();

  // Initialise options
  // Load saved values from disk
  initOptions();

  // Populate an initial list of valid tables to open, and subscribe to the ADS
  // to keep it up to date
  Mantid::API::AnalysisDataServiceImpl &ads =
      Mantid::API::AnalysisDataService::Instance();

  auto items = ads.getObjectNames();
  for (auto const &name : items) {
    Workspace_sptr ws = ads.retrieve(name);

    if (m_manager->isValidModel(
            boost::dynamic_pointer_cast<ITableWorkspace>(ws),
            m_whitelist.size()))
      m_workspaceList.insert(QString::fromStdString(name));
  }
  observeAdd();
  observePostDelete();
  observeRename();
  observeADSClear();
  observeAfterReplace();
  m_view->setTableList(m_workspaceList);

  // Provide autocompletion hints for the options column. We use the algorithm's
  // properties minus those we blacklist. We blacklist any useless properties or
  // ones we're handling that the user should'nt touch.
  IAlgorithm_sptr alg = AlgorithmManager::Instance().create(m_processor.name());
  m_view->setOptionsHintStrategy(
      new AlgorithmHintStrategy(alg, m_processor.blacklist()), m_columns - 1);

  // Start with a blank table
  newTable();

  // The view should currently be in the paused state
  m_view->pause();

  // Setup table selection model connections
  m_view->setSelectionModelConnections();
}

/**
Process selected data
*/
void GenericDataProcessorPresenter::process() {

  // If selection unchanged, resume processing the old selection
  if (!m_newSelection) {
    resume();
    return;
  }

  // Otherwise obtain new runs
  m_selectedData = m_manager->selectedData(m_promptUser);

  // Don't continue if there are no items to process
  if (m_selectedData.size() == 0)
    return;

  m_newSelection = false;

  // Set the global settings. If any have been changed, clear list of processed
  // group indexes
  std::string newPreprocessingOptions =
      m_mainPresenter->getPreprocessingOptionsAsString().toStdString();
  std::string newProcessingOptions =
      m_mainPresenter->getProcessingOptions().toStdString();
  std::string newPostprocessingOptions =
      m_mainPresenter->getPostprocessingOptions().toStdString();

  if (m_preprocessingOptions != newPreprocessingOptions ||
      m_processingOptions != newProcessingOptions ||
      m_postprocessingOptions != newPostprocessingOptions)
    m_processedGroupIndexes.clear();

  m_preprocessingOptions = newPreprocessingOptions;
  m_processingOptions = newProcessingOptions;
  m_postprocessingOptions = newPostprocessingOptions;

  // Clear any highlighted rows
  m_manager->clearHighlighted();

  // Clear the group queue
  m_gqueue = GroupQueue();

  for (const auto &item : m_selectedData) {
    // Loop over each group

    // Ignore any groups that are already processed
    if (m_processedGroupIndexes.find(item.first) !=
        m_processedGroupIndexes.end())
      continue;

    RowQueue rowQueue;

    for (const auto &data : item.second) {
      // Add all row items to queue
      rowQueue.push(data);
    }
    m_gqueue.emplace(item.first, rowQueue);
  }

  // Progress: each group and each row within count as a progress step.
  int progress = 0;
  int maxProgress = (int)(m_gqueue.size());
  for (const auto &subitem : m_gqueue._Get_container()) {
    maxProgress += (int)(subitem.second.size());
  }
  m_progressReporter =
      new ProgressPresenter(progress, maxProgress, maxProgress, m_progressView);

  // Start processing the first group
  m_nextActionFlag = ReductionFlag::ReduceGroupFlag;
  resume();
}

/**
Decide which processing action to take next
*/
void GenericDataProcessorPresenter::doNextAction() {

  switch (m_nextActionFlag) {
  case ReductionFlag::ReduceRowFlag:
    nextRow();
    break;
  case ReductionFlag::ReduceGroupFlag:
    nextGroup();
    break;
  case ReductionFlag::StopReduceFlag:
    break;
  }
  // Not having a 'default' case is deliberate. gcc issues a warning if there's
  // a flag we aren't handling.
}

/**
Process a new row
*/
void GenericDataProcessorPresenter::nextRow() {

  if (m_reductionPaused) {
    // Notify presenter that reduction is paused
    m_mainPresenter->confirmReductionPaused();
    return;
  }

  // Add processed row data to the group
  int rowIndex = m_rowItem.first;
  m_groupData[rowIndex] = m_rowItem.second;
  int groupIndex = m_gqueue.front().first;
  auto &rqueue = m_gqueue.front().second;

  if (!rqueue.empty()) {
    // Set next action flag
    m_nextActionFlag = ReductionFlag::ReduceRowFlag;
    // Reduce next row
    m_rowItem = rqueue.front();
    rqueue.pop();
    startAsyncRowReduceThread(&m_rowItem, groupIndex);
  } else {
    m_gqueue.pop();
    // Set next action flag
    m_nextActionFlag = ReductionFlag::ReduceGroupFlag;

    if (m_groupData.size() > 1) {
      // Multiple rows in containing group, do post-processing on the group
      startAsyncGroupReduceThread(m_groupData, groupIndex);
    } else {
      // Single row in containing group, skip to next group
      nextGroup();
    }
  }
}

/**
Process a new group
*/
void GenericDataProcessorPresenter::nextGroup() {

  if (m_reductionPaused) {
    // Notify presenter that reduction is paused
    m_mainPresenter->confirmReductionPaused();
    return;
  }

  if (!m_gqueue.empty()) {
    // Set next action flag
    m_nextActionFlag = ReductionFlag::ReduceRowFlag;
    // Reduce first row
    auto &rqueue = m_gqueue.front().second;
    m_rowItem = rqueue.front();
    rqueue.pop();
    startAsyncRowReduceThread(&m_rowItem, m_gqueue.front().first);
  } else {
    // If "Output Notebook" checkbox is checked then create an ipython notebook
    if (m_view->getEnableNotebook())
      saveNotebook(m_selectedData);
    endReduction();
  }
}

/*
Reduce the current row asynchronously
*/
void GenericDataProcessorPresenter::startAsyncRowReduceThread(RowItem *rowItem,
                                                              int groupIndex) {

  auto *worker = new GenericDataProcessorPresenterRowReducerWorker(
      this, rowItem, groupIndex);
  m_workerThread.reset(new GenericDataProcessorPresenterThread(this, worker));
  m_workerThread->start();
}

/*
Reduce the current group asynchronously
*/
void GenericDataProcessorPresenter::startAsyncGroupReduceThread(
    GroupData &groupData, int groupIndex) {

  auto *worker = new GenericDataProcessorPresenterGroupReducerWorker(
      this, groupData, groupIndex);
  m_workerThread.reset(new GenericDataProcessorPresenterThread(this, worker));
  m_workerThread->start();
}

/**
End reduction
*/
void GenericDataProcessorPresenter::endReduction() {

  pause();
  m_mainPresenter->confirmReductionPaused();
  m_newSelection = true; // Allow same selection to be processed again
}

/**
Handle reduction error
*/
void GenericDataProcessorPresenter::reductionError(std::exception ex) {

  m_view->giveUserCritical(ex.what(), "Error");
}

/**
Handle thread completion
*/
void GenericDataProcessorPresenter::threadFinished(const int exitCode) {

  m_workerThread.release();

  if (exitCode == 0) { // Success
    m_progressReporter->report();
    doNextAction();
  } else { // Error
    m_progressReporter->clear();
    endReduction();
  }
}

/**
Display a dialog to choose save location for notebook, then save the notebook
there
@param data : the processed data
*/
void GenericDataProcessorPresenter::saveNotebook(const TreeData &data) {

  std::string filename = m_view->requestNotebookPath();
  if (filename == "") {
    return;
  }

  // Global pre-processing options as a map where keys are column
  // name and values are pre-processing options as a string
  const auto preprocessingOptionsMap =
      convertStringToMap(m_preprocessingOptions);

  auto notebook = Mantid::Kernel::make_unique<DataProcessorGenerateNotebook>(
      m_wsName, m_view->getProcessInstrument(), m_whitelist, m_preprocessMap,
      m_processor, m_postprocessor, preprocessingOptionsMap,
      m_processingOptions, m_postprocessingOptions);
  std::string generatedNotebook = notebook->generateNotebook(data);

  std::ofstream file(filename.c_str(), std::ofstream::trunc);
  file << generatedNotebook;
  file.flush();
  file.close();
}

/**
Post-processes the workspaces created by the given rows together.
@param groupData : the data in a given group as received from the tree manager
*/
void GenericDataProcessorPresenter::postProcessGroup(
    const GroupData &groupData) {

  if (!m_postprocess)
    throw std::runtime_error("Cannot post-process workspaces");

  // The input workspace names
  std::vector<std::string> inputNames;

  // The name to call the post-processed ws
  const std::string outputWSName =
      getPostprocessedWorkspaceName(groupData, m_postprocessor.prefix());

  // Go through each row and get the input ws names
  for (const auto &row : groupData) {

    // The name of the reduced workspace for this row
    const std::string inputWSName =
        getReducedWorkspaceName(row.second, m_processor.prefix(0));

    if (AnalysisDataService::Instance().doesExist(inputWSName)) {

      inputNames.emplace_back(inputWSName);
    }
  }
  const std::string inputWSNames = boost::algorithm::join(inputNames, ", ");

  // If the previous result is in the ADS already, we'll need to remove it.
  // If it's a group, we'll get an error for trying to group into a used group
  // name
  if (AnalysisDataService::Instance().doesExist(outputWSName))
    AnalysisDataService::Instance().remove(outputWSName);

  IAlgorithm_sptr alg =
      AlgorithmManager::Instance().create(m_postprocessor.name());
  alg->initialize();
  alg->setProperty(m_postprocessor.inputProperty(), inputWSNames);
  alg->setProperty(m_postprocessor.outputProperty(), outputWSName);

  auto optionsMap = parseKeyValueString(m_postprocessingOptions);
  for (auto kvp = optionsMap.begin(); kvp != optionsMap.end(); ++kvp) {
    try {
      alg->setProperty(kvp->first, kvp->second);
    } catch (Mantid::Kernel::Exception::NotFoundError &) {
      throw std::runtime_error("Invalid property in options column: " +
                               kvp->first);
    }
  }

  // Options specified via post-process map
  for (const auto &prop : m_postprocessMap) {
    const std::string propName = prop.second;
    const std::string propValueStr =
        groupData.begin()->second[m_whitelist.colIndexFromColName(prop.first)];
    if (!propValueStr.empty()) {
      // Warning: we take minus the value of the properties because in
      // Reflectometry this property refers to the rebin step, and they want a
      // logarithmic binning. If other technique areas need to use a
      // post-process map we'll need to re-think how to do this.
      alg->setPropertyValue(propName, "-" + propValueStr);
    }
  }

  alg->execute();

  if (!alg->isExecuted())
    throw std::runtime_error("Failed to post-process workspaces.");
}

/**
Takes a user specified run, or list of runs, and returns a pointer to the
desired workspace
@param runStr : The run or list of runs (separated by '+')
@param preprocessor : The pre-processing algorithm acting on this column
@param optionsMap : User-specified options as a map
@throws std::runtime_error if the workspace could not be prepared
@returns a shared pointer to the workspace
*/
Workspace_sptr GenericDataProcessorPresenter::prepareRunWorkspace(
    const std::string &runStr,
    const DataProcessorPreprocessingAlgorithm &preprocessor,
    const std::map<std::string, std::string> &optionsMap) {
  const std::string instrument = m_view->getProcessInstrument();

  std::vector<std::string> runs;
  boost::split(runs, runStr, boost::is_any_of("+,"));

  if (runs.empty())
    throw std::runtime_error("No runs given");

  // Remove leading/trailing whitespace from each run
  for (auto runIt = runs.begin(); runIt != runs.end(); ++runIt)
    boost::trim(*runIt);

  // If we're only given one run, just return that
  if (runs.size() == 1)
    return getRun(runs[0], instrument, preprocessor.prefix());

  const std::string outputName =
      preprocessor.prefix() + boost::algorithm::join(runs, "_");

  /* Ideally, this should be executed as a child algorithm to keep the ADS tidy,
  * but that doesn't preserve history nicely, so we'll just take care of tidying
  * up in the event of failure.
  */
  IAlgorithm_sptr alg =
      AlgorithmManager::Instance().create(preprocessor.name());
  alg->initialize();
  alg->setProperty(
      preprocessor.lhsProperty(),
      getRun(runs[0], instrument, preprocessor.prefix())->getName());
  alg->setProperty(preprocessor.outputProperty(), outputName);

  // Drop the first run from the runs list
  runs.erase(runs.begin());

  try {
    // Iterate through all the remaining runs, adding them to the first run
    for (auto runIt = runs.begin(); runIt != runs.end(); ++runIt) {

      for (auto kvp = optionsMap.begin(); kvp != optionsMap.end(); ++kvp) {
        try {
          alg->setProperty(kvp->first, kvp->second);
        } catch (Mantid::Kernel::Exception::NotFoundError &) {
          // We can't apply this option to this pre-processing alg
          throw;
        }
      }

      alg->setProperty(
          preprocessor.rhsProperty(),
          getRun(*runIt, instrument, preprocessor.prefix())->getName());
      alg->execute();

      if (runIt != --runs.end()) {
        // After the first execution we replace the LHS with the previous output
        alg->setProperty(preprocessor.lhsProperty(), outputName);
      }
    }
  } catch (...) {
    // If we're unable to create the full workspace, discard the partial version
    AnalysisDataService::Instance().remove(outputName);

    // We've tidied up, now re-throw.
    throw;
  }

  return AnalysisDataService::Instance().retrieveWS<Workspace>(outputName);
}

/**
Returns the name of the reduced workspace for a given row
@param data :: [input] The data for this row
@param prefix : A prefix to be appended to the generated ws name
@throws std::runtime_error if the workspace could not be prepared
@returns : The name of the workspace
*/
std::string GenericDataProcessorPresenter::getReducedWorkspaceName(
    const std::vector<std::string> &data, const std::string &prefix) {

  if (static_cast<int>(data.size()) != m_columns)
    throw std::invalid_argument("Can't find reduced workspace name");

  /* This method calculates, for a given row, the name of the output (processed)
  * workspace. This is done using the white list, which contains information
  * about the columns that should be included to create the ws name. In
  * Reflectometry for example, we want to include values in the 'Run(s)' and
  * 'Transmission Run(s)' columns. We may also use a prefix associated with
  * the column when specified. Finally, to construct the ws name we may also
  * use a 'global' prefix associated with the processing algorithm (for
  * instance 'IvsQ_' in Reflectometry) this is given by the second argument to
  * this method */

  // Temporary vector of strings to construct the name
  std::vector<std::string> names;

  for (int col = 0; col < m_columns; col++) {

    // Do we want to use this column to generate the name of the output ws?
    if (m_whitelist.showValue(col)) {

      // Get what's in the column
      const std::string valueStr = data.at(col);

      // If it's not empty, use it
      if (!valueStr.empty()) {

        // But we may have things like '1+2' which we want to replace with '1_2'
        std::vector<std::string> value;
        boost::split(value, valueStr, boost::is_any_of("+"));

        names.push_back(m_whitelist.prefix(col) +
                        boost::algorithm::join(value, "_"));
      }
    }
  } // Columns

  std::string wsname = prefix;
  wsname += boost::algorithm::join(names, "_");

  return wsname;
}

/**
Returns the name of the reduced workspace for a given group
@param groupData : The data in a given group
@param prefix : A prefix to be appended to the generated ws name
@returns : The name of the workspace
*/
std::string GenericDataProcessorPresenter::getPostprocessedWorkspaceName(
    const GroupData &groupData, const std::string &prefix) {

  if (!m_postprocess)
    throw std::runtime_error("Cannot retrieve post-processed workspace name");

  /* This method calculates, for a given set of rows, the name of the output
  * (post-processed) workspace */

  std::vector<std::string> outputNames;

  for (const auto &data : groupData) {
    outputNames.push_back(getReducedWorkspaceName(data.second));
  }
  return prefix + boost::join(outputNames, "_");
}

/**
Sets the state of whether a new table selection has been made
@param newSelectionMade : Boolean on setting new table selection state
*/
void GenericDataProcessorPresenter::setNewSelectionState(
    bool newSelectionMade) {

  m_newSelection = newSelectionMade;
}

/** Loads a run found from disk or AnalysisDataService
 *
 * @param run : The name of the run
 * @param instrument : The instrument the run belongs to
 * @param prefix : The prefix to be prepended to the run number
 * @throws std::runtime_error if the run could not be loaded
 * @returns a shared pointer to the workspace
 */
Workspace_sptr
GenericDataProcessorPresenter::getRun(const std::string &run,
                                      const std::string &instrument,
                                      const std::string &prefix) {

  bool runFound;
  std::string outName;
  std::string fileName = instrument + run;

  outName = findRunInADS(run, prefix, runFound);
  if (!runFound) {
    outName = loadRun(run, instrument, prefix, m_loader, runFound);
    if (!runFound)
      throw std::runtime_error("Could not open " + fileName);
  }

  return AnalysisDataService::Instance().retrieveWS<Workspace>(outName);
}

/** Tries fetching a run from AnalysisDataService
 *
 * @param run : The name of the run
 * @param prefix : The prefix to be prepended to the run number
 * @param runFound : Whether or not the run was actually found
 * @returns string name of the run
 */
std::string GenericDataProcessorPresenter::findRunInADS(
    const std::string &run, const std::string &prefix, bool &runFound) {

  runFound = true;

  // First, let's see if the run given is the name of a workspace in the ADS
  if (AnalysisDataService::Instance().doesExist(run))
    return run;

  // Try with prefix
  if (AnalysisDataService::Instance().doesExist(prefix + run))
    return prefix + run;

  // Is the run string is numeric?
  if (boost::regex_match(run, boost::regex("\\d+"))) {

    // Look for "<run_number>" in the ADS
    if (AnalysisDataService::Instance().doesExist(run))
      return run;

    // Look for "<instrument><run_number>" in the ADS
    if (AnalysisDataService::Instance().doesExist(prefix + run))
      return prefix + run;
  }

  // Run not found in ADS;
  runFound = false;
  return "";
}

/** Tries loading a run from disk
 *
 * @param run : The name of the run
 * @param instrument : The instrument the run belongs to
 * @param prefix : The prefix to be prepended to the run number
 * @param loader : The algorithm used for loading runs
 * @param runFound : Whether or not the run was actually found
 * @returns string name of the run
 */
std::string GenericDataProcessorPresenter::loadRun(
    const std::string &run, const std::string &instrument,
    const std::string &prefix, const std::string &loader, bool &runFound) {

  runFound = true;
  const std::string fileName = instrument + run;
  const std::string outputName = prefix + run;

  IAlgorithm_sptr algLoadRun = AlgorithmManager::Instance().create(loader);
  algLoadRun->initialize();
  algLoadRun->setProperty("Filename", fileName);
  algLoadRun->setProperty("OutputWorkspace", outputName);
  algLoadRun->execute();
  if (!algLoadRun->isExecuted()) {
    // Run not loaded from disk
    runFound = false;
    return "";
  }

  return outputName;
}

/** Reduce a row
 *
 * @param data :: [input] The data in this row as a vector where elements
 * correspond to column contents
 * @throws std::runtime_error if reduction fails
 */
void GenericDataProcessorPresenter::reduceRow(RowData *data) {

  /* Create the processing algorithm */

  IAlgorithm_sptr alg = AlgorithmManager::Instance().create(m_processor.name());
  alg->initialize();

  /* Read input properties from the table */
  /* excluding 'Group' and 'Options' */

  // Global pre-processing options as a map
  std::map<std::string, std::string> globalOptions;
  if (!m_preprocessMap.empty())
    globalOptions = convertStringToMap(m_preprocessingOptions);

  // Pre-processing properties
  auto preProcessPropMap = convertStringToMapWithSet(
      m_mainPresenter->getPreprocessingProperties().toStdString());

  // Properties not to be used in processing
  std::set<std::string> restrictedProps;

  // Loop over all columns in the whitelist except 'Options'
  for (int i = 0; i < m_columns - 1; i++) {

    // The algorithm's property linked to this column
    auto propertyName = m_whitelist.algPropFromColIndex(i);
    // The column's name
    auto columnName = m_whitelist.colNameFromColIndex(i);

    // The value for which preprocessing can be conducted on
    std::string preProcessValue;

    if (globalOptions.count(columnName) && !globalOptions[columnName].empty()) {
      auto tmpOptionsMap = parseKeyValueString(globalOptions[columnName]);
      for (auto &optionMapEntry : tmpOptionsMap) {
        preProcessValue += optionMapEntry.second;
      }
    } else if (!data->at(i).empty()) {
      preProcessValue = data->at(i);
    } else {
      continue;
    }

    if (m_preprocessMap.count(columnName)) {
      // This column needs pre-processing

      // We do not want the associated properties to be set again in
      // processing
      if (preProcessPropMap.count(columnName) > 0) {
        for (auto &prop : preProcessPropMap[columnName]) {
          restrictedProps.insert(prop);
        }
      }

      auto preprocessor = m_preprocessMap.at(columnName);

      const std::string globalOptionsForColumn =
          globalOptions.count(columnName) > 0 ? globalOptions.at(columnName)
                                              : "";

      auto optionsMap = parseKeyValueString(globalOptionsForColumn);
      auto runWS =
          prepareRunWorkspace(preProcessValue, preprocessor, optionsMap);
      alg->setProperty(propertyName, runWS->getName());
    } else {
      // No pre-processing needed
      auto propertyValue = data->at(i);
      if (!propertyValue.empty())
        alg->setPropertyValue(propertyName, propertyValue);
    }
  }

  // Parse and set any user-specified options
  auto optionsMap = parseKeyValueString(m_processingOptions);
  for (auto kvp = optionsMap.begin(); kvp != optionsMap.end(); ++kvp) {
    try {
      if (restrictedProps.find(kvp->first) == restrictedProps.end())
        alg->setProperty(kvp->first, kvp->second);
    } catch (Mantid::Kernel::Exception::NotFoundError &) {
      throw std::runtime_error("Invalid property in options column: " +
                               kvp->first);
    }
  }

  /* Now deal with 'Options' column */
  const auto userOptions = data->back();

  // Parse and set any user-specified options
  optionsMap = parseKeyValueString(userOptions);
  for (auto kvp = optionsMap.begin(); kvp != optionsMap.end(); ++kvp) {
    try {
      alg->setProperty(kvp->first, kvp->second);
    } catch (Mantid::Kernel::Exception::NotFoundError &) {
      throw std::runtime_error("Invalid property in options column: " +
                               kvp->first);
    }
  }

  /* We need to give a name to the output workspaces */
  for (size_t i = 0; i < m_processor.numberOfOutputProperties(); i++) {
    alg->setProperty(m_processor.outputPropertyName(i),
                     getReducedWorkspaceName(*data, m_processor.prefix(i)));
  }

  /* Now run the processing algorithm */
  alg->execute();

  auto newData = data;
  if (alg->isExecuted()) {

    /* The reduction is complete, try to populate the columns */
    for (int i = 0; i < m_columns - 1; i++) {

      auto columnName = m_whitelist.colNameFromColIndex(i);

      if (data->at(i).empty() && !m_preprocessMap.count(columnName)) {

        std::string propValue =
            alg->getPropertyValue(m_whitelist.algPropFromColIndex(i));

        if (m_options["Round"].toBool()) {
          std::string exp = (propValue.find("e") != std::string::npos)
                                ? propValue.substr(propValue.find("e"))
                                : "";
          propValue =
              propValue.substr(0, propValue.find(".") +
                                      m_options["RoundPrecision"].toInt() + 1) +
              exp;
        }

        newData->at(i) = propValue;
      }
    }
  }
}

/**
Insert a new row
*/
void GenericDataProcessorPresenter::appendRow() {

  m_manager->appendRow();
  m_tableDirty = true;
}

/**
Insert a new group
*/
void GenericDataProcessorPresenter::appendGroup() {

  m_manager->appendGroup();
  m_tableDirty = true;
}

/**
Delete row(s) from the model
*/
void GenericDataProcessorPresenter::deleteRow() {

  m_manager->deleteRow();