Merge pull request #449 from mantidproject/9215_Speed_up_data_loading

Speed up data loading

Code/Mantid/Framework/Algorithms/inc/MantidAlgorithms/PlotAsymmetryByLogValue.h

@@ -79,7 +79,7 @@ private:
   void init();
   void exec();
   // Load run, apply dead time corrections and detector grouping
-  API::Workspace_sptr doLoad (int64_t runNumber );
+  void doLoad (int64_t runNumber, API::Workspace_sptr& loadedWs, API::Workspace_sptr& loadedDt, API::Workspace_sptr& loadedDg);
   // Analyse loaded run
   void doAnalysis (API::Workspace_sptr loadedWs, int64_t index);
   // Parse run names

Code/Mantid/Framework/Algorithms/src/PlotAsymmetryByLogValue.cpp

@@ -20,6 +20,7 @@
 #include "MantidKernel/ArrayProperty.h"
 #include "MantidKernel/ListValidator.h"
 #include "MantidKernel/MandatoryValidator.h"
+#include "MantidKernel/MultiThreaded.h"
 #include "MantidKernel/PropertyWithValue.h"
 #include "MantidKernel/TimeSeriesProperty.h"
 #include "Poco/File.h"
@@ -177,26 +178,63 @@ void PlotAsymmetryByLogValue::exec() {
 
   // Parse run names and get the number of runs
   parseRunNames( firstFN, lastFN, m_filenameBase, m_filenameExt, m_filenameZeros);
-  size_t is = atoi(firstFN.c_str()); // starting run number
-  size_t ie = atoi(lastFN.c_str());  // last run number
+  int64_t is = atoi(firstFN.c_str()); // starting run number
+  int64_t ie = atoi(lastFN.c_str());  // last run number
 
   // Resize vectors that will store results
   resizeVectors(ie-is+1);
 
   Progress progress(this, 0, 1, ie - is + 2);
 
-  // Loop through runs
-  for (size_t i = is; i <= ie; i++) {
+  // Load in serial
+  // We need to store the loaded ws, dead times and grouping in vectors
+  std::vector<Workspace_sptr> loadedWorkspace;
+  std::vector<Workspace_sptr> loadedDeadTimeTable;
+  std::vector<Workspace_sptr> loadedDetGroupingTable;
+  for (int64_t i = is; i <= ie; i++) {
 
-    // Load run, apply dead time corrections and detector grouping
-    Workspace_sptr loadedWs = doLoad(i);
+    Workspace_sptr loadedWs, loadedDt, loadedDg;
+    doLoad( i, loadedWs, loadedDt, loadedDg );
 
-    // Analyse loadedWs
-    doAnalysis (loadedWs, i-is);
+    loadedWorkspace.push_back(loadedWs);
+    loadedDeadTimeTable.push_back(loadedDt);
+    loadedDetGroupingTable.push_back(loadedDg);
 
     progress.report();
   }
 
+  // Analyse in parallel
+  // Use previously loaded ws, dead times and grouping
+  PARALLEL_FOR_NO_WSP_CHECK()
+  for (int64_t i = is; i <= ie; i++) {
+    PARALLEL_START_INTERUPT_REGION
+
+    int64_t runIndex = i - is;
+
+    Workspace_sptr loadedWs = loadedWorkspace.at(runIndex);
+    Workspace_sptr loadedDt = loadedDeadTimeTable.at(runIndex);
+    Workspace_sptr loadedDg = loadedDetGroupingTable.at(runIndex);
+
+    // Apply dead time corrections if requested
+    if ( m_dtcType != "None" ) {
+      applyDeadtimeCorr (loadedWs,loadedDt);
+    }
+    // Apply grouping if requested
+    if ( m_autogroup ) {
+      groupDetectors (loadedWs,loadedDg);
+    }
+
+    // Analyse loadedWs
+    doAnalysis (loadedWs, runIndex);
+
+    PARALLEL_END_INTERUPT_REGION
+  }
+  PARALLEL_CHECK_INTERUPT_REGION
+
+  // Clear vectors
+  loadedWorkspace.clear();
+  loadedDeadTimeTable.clear();
+  loadedDetGroupingTable.clear();
 
   // Create the 2D workspace for the output
   int nplots = (m_green!= EMPTY_INT()) ? 4 : 1;
@@ -215,8 +253,11 @@ void PlotAsymmetryByLogValue::exec() {
 
 /**  Loads one run and applies dead-time corrections and detector grouping if required
 *   @param runNumber :: [input] Run number specifying run to load
+*   @param loadedWs :: [output] Workspace loaded from nexus file
+*   @param loadedDt :: [output] Dead time table loaded from nexus file or specified file
+*   @param loadedDg :: [output] Detector grouping table loaded from nexus file
 */
-Workspace_sptr PlotAsymmetryByLogValue::doLoad (int64_t runNumber ) {
+void PlotAsymmetryByLogValue::doLoad (int64_t runNumber, Workspace_sptr& loadedWs, Workspace_sptr& loadedDt, Workspace_sptr& loadedDg) {
 
   // Get complete run name
   std::ostringstream fn, fnn;
@@ -227,33 +268,27 @@ Workspace_sptr PlotAsymmetryByLogValue::doLoad (int64_t runNumber ) {
   IAlgorithm_sptr load = createChildAlgorithm("LoadMuonNexus");
   load->setPropertyValue("Filename", fn.str());
   load->execute();
-  Workspace_sptr loadedWs = load->getProperty("OutputWorkspace");
+  loadedWs = load->getProperty("OutputWorkspace");
 
   // Check if dead-time corrections have to be applied
   if (m_dtcType != "None") {
     if (m_dtcType == "FromSpecifiedFile") {
-
-      // If user specifies a file, load corrections now
-      Workspace_sptr customDeadTimes;
-      loadCorrectionsFromFile (customDeadTimes, getPropertyValue("DeadTimeCorrFile"));
-      applyDeadtimeCorr (loadedWs, customDeadTimes);
+      // If user specifies a file, load corrections from file
+      loadCorrectionsFromFile (loadedDt, getPropertyValue("DeadTimeCorrFile"));
     } else {
       // Load corrections from run
-      Workspace_sptr deadTimes = load->getProperty("DeadTimeTable");
-      applyDeadtimeCorr (loadedWs, deadTimes);
+      loadedDt = load->getProperty("DeadTimeTable");
     }
   }
 
-  // If m_autogroup, group detectors
+  // If m_autogroup load detector grouping table
   if (m_autogroup) {
-    Workspace_sptr loadedDetGrouping = load->getProperty("DetectorGroupingTable");
-    if (!loadedDetGrouping)
+    loadedDg = load->getProperty("DetectorGroupingTable");
+    if (!loadedDg)
       throw std::runtime_error("No grouping info in the file.\n\nPlease "
       "specify grouping manually");
-    groupDetectors(loadedWs,loadedDetGrouping);
   }
 
-  return loadedWs;
 }
 
 /**  Load dead-time corrections from specified file
@@ -385,20 +420,36 @@ void PlotAsymmetryByLogValue::parseRunNames (std::string& firstFN, std::string&
 */
 void PlotAsymmetryByLogValue::applyDeadtimeCorr (Workspace_sptr &loadedWs, Workspace_sptr deadTimes)
 {
-  ScopedWorkspace ws(loadedWs);
-  ScopedWorkspace dt(deadTimes);
+  std::stringstream wsName, deadName;
+  wsName << "__input" << PARALLEL_THREAD_NUMBER;
+  deadName << "__dead" << PARALLEL_THREAD_NUMBER;
 
-  IAlgorithm_sptr applyCorr = AlgorithmManager::Instance().create("ApplyDeadTimeCorr");
+  // Could be groups of workspaces, so need to work with ADS
+  AnalysisDataService::Instance().add(wsName.str(),loadedWs);
+  AnalysisDataService::Instance().add(deadName.str(),deadTimes);
+
+  IAlgorithm_sptr applyCorr = createChildAlgorithm("ApplyDeadTimeCorr");
   applyCorr->setLogging(false);
   applyCorr->setRethrows(true);
-  applyCorr->setPropertyValue("InputWorkspace", ws.name());
-  applyCorr->setPropertyValue("OutputWorkspace", ws.name());
-  applyCorr->setProperty("DeadTimeTable", dt.name());
+  applyCorr->setPropertyValue("InputWorkspace", wsName.str());
+  applyCorr->setPropertyValue("OutputWorkspace", wsName.str());
+  applyCorr->setPropertyValue("DeadTimeTable", deadName.str());
   applyCorr->execute();
   // Workspace should've been replaced in the ADS by ApplyDeadTimeCorr, so
-  // need to
-  // re-assign it
-  loadedWs = ws.retrieve();
+  // need to re-assign it
+  loadedWs = AnalysisDataService::Instance().retrieve(wsName.str());
+
+  // Remove workspaces from ADS
+  if ( boost::dynamic_pointer_cast<WorkspaceGroup>(loadedWs) ) {
+    AnalysisDataService::Instance().deepRemoveGroup(wsName.str());
+  } else {
+    AnalysisDataService::Instance().remove(wsName.str());
+  }
+  if ( boost::dynamic_pointer_cast<WorkspaceGroup>(deadTimes) ) {
+    AnalysisDataService::Instance().deepRemoveGroup(deadName.str());
+  } else {
+    AnalysisDataService::Instance().remove(deadName.str());
+  }
 }
 
 /**  Group detectors from specified file
@@ -407,22 +458,35 @@ void PlotAsymmetryByLogValue::applyDeadtimeCorr (Workspace_sptr &loadedWs, Works
 */
 void PlotAsymmetryByLogValue::groupDetectors (Workspace_sptr &loadedWs, Workspace_sptr loadedDetGrouping)
 {
+  std::stringstream wsName, groupName;
+  wsName << "__input" << PARALLEL_THREAD_NUMBER;
+  groupName << "__group" << PARALLEL_THREAD_NUMBER;
 
   // Could be groups of workspaces, so need to work with ADS
-  ScopedWorkspace inWS(loadedWs);
-  ScopedWorkspace grouping(loadedDetGrouping);
-  ScopedWorkspace outWS;
+  AnalysisDataService::Instance().add(wsName.str(),loadedWs);
+  AnalysisDataService::Instance().add(groupName.str(),loadedDetGrouping);
 
-  IAlgorithm_sptr applyGrouping = AlgorithmManager::Instance().create("MuonGroupDetectors");
+  IAlgorithm_sptr applyGrouping = createChildAlgorithm("MuonGroupDetectors");
   applyGrouping->setLogging(false);
   applyGrouping->setRethrows(true);
-
-  applyGrouping->setPropertyValue("InputWorkspace", inWS.name());
-  applyGrouping->setPropertyValue("DetectorGroupingTable", grouping.name());
-  applyGrouping->setPropertyValue("OutputWorkspace", outWS.name());
+  applyGrouping->setPropertyValue("InputWorkspace", wsName.str());
+  applyGrouping->setPropertyValue("DetectorGroupingTable", groupName.str());
+  applyGrouping->setPropertyValue("OutputWorkspace", wsName.str());
   applyGrouping->execute();
 
-  loadedWs = outWS.retrieve();
+  loadedWs = AnalysisDataService::Instance().retrieve(wsName.str());
+
+  // Remove workspaces from ADS
+  if ( boost::dynamic_pointer_cast<WorkspaceGroup>(loadedWs) ) {
+    AnalysisDataService::Instance().deepRemoveGroup(wsName.str());
+  } else {
+    AnalysisDataService::Instance().remove(wsName.str());
+  }
+  if ( boost::dynamic_pointer_cast<WorkspaceGroup>(loadedDetGrouping) ) {
+    AnalysisDataService::Instance().deepRemoveGroup(groupName.str());
+  } else {
+    AnalysisDataService::Instance().remove(groupName.str());
+  }
 }
 
 /**  Performs asymmetry analysis on a loaded workspace