Reduce comparisons of logs in AlignAndFocus chunks

There are two big changes in this PR. The first is inspecting the what
specific Load algorithm was used and switching to that after the first
chunk is loaded (removes determining the specific loader for subsequent
chunks). The second is to remove the logs before accumulating the
chunks.

Framework/PythonInterface/plugins/algorithms/AlignAndFocusPowderFromFiles.py

@@ -9,9 +9,9 @@ from __future__ import (absolute_import, division, print_function)
 from mantid.api import mtd, AlgorithmFactory, DistributedDataProcessorAlgorithm, ITableWorkspaceProperty, \
     MatrixWorkspaceProperty, MultipleFileProperty, PropertyMode
 from mantid.kernel import ConfigService, Direction
-from mantid.simpleapi import AlignAndFocusPowder, CompressEvents, ConvertUnits, CreateCacheFilename, \
-    DeleteWorkspace, DetermineChunking, Divide, EditInstrumentGeometry, FilterBadPulses, Load, \
-    LoadNexusProcessed, PDDetermineCharacterizations, Plus, RenameWorkspace, SaveNexusProcessed
+from mantid.simpleapi import AlignAndFocusPowder, CompressEvents, ConvertUnits, CopyLogs, CreateCacheFilename, \
+    DeleteWorkspace, DetermineChunking, Divide, EditInstrumentGeometry, FilterBadPulses, LoadNexusProcessed, \
+    PDDetermineCharacterizations, Plus, RemoveLogs, RenameWorkspace, SaveNexusProcessed
 import os
 
 EXTENSIONS_NXS = ["_event.nxs", ".nxs.h5"]
@@ -19,7 +19,7 @@ PROPS_FOR_INSTR = ["PrimaryFlightPath", "SpectrumIDs", "L2", "Polar", "Azimuthal
 CAL_FILE, GROUP_FILE = "CalFileName", "GroupFilename"
 CAL_WKSP, GRP_WKSP, MASK_WKSP = "CalibrationWorkspace", "GroupingWorkspace", "MaskWorkspace"
 PROPS_FOR_ALIGN = [CAL_FILE, GROUP_FILE,
-                   GRP_WKSP,CAL_WKSP, "OffsetsWorkspace",
+                   GRP_WKSP, CAL_WKSP, "OffsetsWorkspace",
                    MASK_WKSP, "MaskBinTable",
                    "Params", "ResampleX", "Dspacing", "DMin", "DMax",
                    "TMin", "TMax", "PreserveEvents",
@@ -63,7 +63,7 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
         return "Diffraction\\Reduction"
 
     def seeAlso(self):
-        return [ "AlignAndFocusPowder" ]
+        return ["AlignAndFocusPowder"]
 
     def name(self):
         return "AlignAndFocusPowderFromFiles"
@@ -124,6 +124,21 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
                 linearizedRuns.append(item)
         return linearizedRuns
 
+    def __createLoader(self, filename, wkspname, progstart=None, progstop=None):
+        # load a chunk - this is a bit crazy long because we need to get an output property from `Load` when it
+        # is run and the algorithm history doesn't exist until the parent algorithm (this) has finished
+        if progstart is None or progstop is None:
+            loader = self.createChildAlgorithm(self.__loaderName)
+        else:
+            loader = self.createChildAlgorithm(self.__loaderName,
+                                               startProgress=progstart, endProgress=progstop)
+        loader.setAlwaysStoreInADS(True)
+        loader.setLogging(True)
+        loader.initialize()
+        loader.setPropertyValue('Filename', filename)
+        loader.setPropertyValue('OutputWorkspace', wkspname)
+        return loader
+
     def __getAlignAndFocusArgs(self):
         args = {}
         for name in PROPS_FOR_ALIGN:
@@ -161,18 +176,25 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
             if key not in self.kwargs:
                 self.kwargs[key] = instr + ext
 
-    def __determineCharacterizations(self, filename, wkspname, loadFile):
+    def __determineCharacterizations(self, filename, wkspname):
         useCharac = bool(self.charac is not None)
+        loadFile = not mtd.doesExist(wkspname)
 
         # input workspace is only needed to find a row in the characterizations table
         tempname = None
         if loadFile:
             if useCharac:
                 tempname = '__%s_temp' % wkspname
-                Load(Filename=filename, OutputWorkspace=tempname,
-                     MetaDataOnly=True)
+                # set the loader for this file
+                loader = self.__createLoader(filename, tempname)
+                loader.setProperty('MetaDataOnly', True)  # this is only supported by LoadEventNexus
+                loader.execute()
+
+                # get the underlying loader name if we used the generic one
+                if self.__loaderName == 'Load':
+                    self.__loaderName = loader.getPropertyValue('LoaderName')
         else:
-            tempname = wkspname # assume it is already loaded
+            tempname = wkspname  # assume it is already loaded
 
         # put together argument list
         args = dict(ReductionProperties=self.getProperty('ReductionProperties').valueAsStr)
@@ -181,7 +203,7 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
             if not prop.isDefault:
                 args[name] = prop.value
         if tempname is not None:
-            args['InputWorkspace']=tempname
+            args['InputWorkspace'] = tempname
         if useCharac:
             args['Characterizations'] = self.charac
 
@@ -211,12 +233,13 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
 
     def __processFile(self, filename, wkspname, unfocusname, file_prog_start, determineCharacterizations):
         chunks = determineChunking(filename, self.chunkSize)
-        numSteps = 6 # for better progress reporting - 6 steps per chunk
+        numSteps = 6  # for better progress reporting - 6 steps per chunk
         if unfocusname != '':
-            numSteps = 7 # one more for accumulating the unfocused workspace
+            numSteps = 7  # one more for accumulating the unfocused workspace
         self.log().information('Processing \'{}\' in {:d} chunks'.format(filename, len(chunks)))
         prog_per_chunk_step = self.prog_per_file * 1./(numSteps*float(len(chunks)))
         unfocusname_chunk = ''
+        canSkipLoadingLogs = False
 
         # inner loop is over chunks
         for (j, chunk) in enumerate(chunks):
@@ -225,11 +248,30 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
             if unfocusname != '':  # only create unfocus chunk if needed
                 unfocusname_chunk = '{}_c{:d}'.format(unfocusname, j)
 
-            Load(Filename=filename, OutputWorkspace=chunkname,
-                 startProgress=prog_start, endProgress=prog_start+prog_per_chunk_step,
-                 **chunk)
-            if determineCharacterizations:
-                self.__determineCharacterizations(filename, chunkname, False) # updates instance variable
+            # load a chunk - this is a bit crazy long because we need to get an output property from `Load` when it
+            # is run and the algorithm history doesn't exist until the parent algorithm (this) has finished
+            loader = self.__createLoader(filename, chunkname,
+                                         progstart=prog_start, progstop=prog_start + prog_per_chunk_step)
+            if canSkipLoadingLogs:
+                loader.setProperty('LoadLogs', False)
+            for key, value in chunk.items():
+                if isinstance(value, str):
+                    loader.setPropertyValue(key, value)
+                else:
+                    loader.setProperty(key, value)
+            loader.execute()
+
+            # copy the necessary logs onto the workspace
+            if canSkipLoadingLogs:
+                CopyLogs(InputWorkspace=wkspname, OutputWorkspace=chunkname, MergeStrategy='WipeExisting')
+
+            # get the underlying loader name if we used the generic one
+            if self.__loaderName == 'Load':
+                self.__loaderName = loader.getPropertyValue('LoaderName')
+            canSkipLoadingLogs = self.__loaderName == 'LoadEventNexus'
+
+            if determineCharacterizations and j == 0:
+                self.__determineCharacterizations(filename, chunkname)  # updates instance variable
                 determineCharacterizations = False
 
             prog_start += prog_per_chunk_step
@@ -252,7 +294,7 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
             AlignAndFocusPowder(InputWorkspace=chunkname, OutputWorkspace=chunkname, UnfocussedWorkspace=unfocusname_chunk,
                                 startProgress=prog_start, endProgress=prog_start+2.*prog_per_chunk_step,
                                 **self.kwargs)
-            prog_start += 2.*prog_per_chunk_step # AlignAndFocusPowder counts for two steps
+            prog_start += 2. * prog_per_chunk_step  # AlignAndFocusPowder counts for two steps
 
             if j == 0:
                 self.__updateAlignAndFocusArgs(chunkname)
@@ -260,12 +302,14 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
                 if unfocusname != '':
                     RenameWorkspace(InputWorkspace=unfocusname_chunk, OutputWorkspace=unfocusname)
             else:
+                RemoveLogs(Workspace=chunkname)  # accumulation has them already
                 Plus(LHSWorkspace=wkspname, RHSWorkspace=chunkname, OutputWorkspace=wkspname,
                      ClearRHSWorkspace=self.kwargs['PreserveEvents'],
                      startProgress=prog_start, endProgress=prog_start+prog_per_chunk_step)
                 DeleteWorkspace(Workspace=chunkname)
 
                 if unfocusname != '':
+                    RemoveLogs(Workspace=unfocusname_chunk)  # accumulation has them already
                     Plus(LHSWorkspace=unfocusname, RHSWorkspace=unfocusname_chunk, OutputWorkspace=unfocusname,
                          ClearRHSWorkspace=self.kwargs['PreserveEvents'],
                          startProgress=prog_start, endProgress=prog_start + prog_per_chunk_step)
@@ -274,7 +318,7 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
                 if self.kwargs['PreserveEvents'] and self.kwargs['CompressTolerance'] > 0.:
                     CompressEvents(InputWorkspace=wkspname, OutputWorkspace=wkspname,
                                    WallClockTolerance=self.kwargs['CompressWallClockTolerance'],
-                                   Tolerance= self.kwargs['CompressTolerance'],
+                                   Tolerance=self.kwargs['CompressTolerance'],
                                    StartTime=self.kwargs['CompressStartTime'])
         # end of inner loop
 
@@ -301,7 +345,7 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
         else:
             self.log().warning('CacheDir is not specified - functionality disabled')
 
-        self.prog_per_file = 1./float(len(filenames)) # for better progress reporting
+        self.prog_per_file = 1./float(len(filenames))  # for better progress reporting
 
         # these are also passed into the child-algorithms
         self.kwargs = self.__getAlignAndFocusArgs()
@@ -311,14 +355,14 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
             # default name is based off of filename
             wkspname = os.path.split(filename)[-1].split('.')[0]
 
+            self.__loaderName = 'Load'  # reset to generic load with each file
             if useCaching:
-                self.__determineCharacterizations(filename,
-                                                  wkspname, True) # updates instance variable
+                self.__determineCharacterizations(filename, wkspname)  # updates instance variable
                 cachefile = self.__getCacheName(wkspname)
             else:
                 cachefile = None
 
-            wkspname += '_f%d' % i # add file number to be unique
+            wkspname += '_f%d' % i  # add file number to be unique
 
             # if the unfocussed data is requested, don't read it from disk
             # because of the extra complication of the unfocussed workspace
@@ -335,7 +379,7 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
                 if editinstrargs:
                     EditInstrumentGeometry(Workspace=wkspname, **editinstrargs)
             else:
-                self.__processFile(filename, wkspname, unfocusname_file, self.prog_per_file*float(i), not useCaching)
+                self.__processFile(filename, wkspname, unfocusname_file, self.prog_per_file * float(i), not useCaching)
 
                 # write out the cachefile for the main reduced data independent of whether
                 # the unfocussed workspace was requested
@@ -361,7 +405,7 @@ class AlignAndFocusPowderFromFiles(DistributedDataProcessorAlgorithm):
                 if self.kwargs['PreserveEvents'] and self.kwargs['CompressTolerance'] > 0.:
                     CompressEvents(InputWorkspace=finalname, OutputWorkspace=finalname,
                                    WallClockTolerance=self.kwargs['CompressWallClockTolerance'],
-                                   Tolerance= self.kwargs['CompressTolerance'],
+                                   Tolerance=self.kwargs['CompressTolerance'],
                                    StartTime=self.kwargs['CompressStartTime'])
                     # not compressing unfocussed workspace because it is in d-spacing
                     # and is likely to be from a different part of the instrument

Framework/PythonInterface/plugins/algorithms/SNSPowderReduction.py

@@ -536,7 +536,8 @@ class SNSPowderReduction(DistributedDataProcessorAlgorithm):
                                               OutputWorkspace=self._charTable)
         # export the characterizations table
         charTable = results[0]
-        self.declareProperty(ITableWorkspaceProperty("CharacterizationsTable", self._charTable, Direction.Output))
+        if not self.existsProperty("CharacterizationsTable"):
+            self.declareProperty(ITableWorkspaceProperty("CharacterizationsTable", self._charTable, Direction.Output))
         self.setProperty("CharacterizationsTable", charTable)
 
         # get the focus positions from the properties
@@ -972,7 +973,8 @@ class SNSPowderReduction(DistributedDataProcessorAlgorithm):
                 self.log().warning(str(e))
 
             propertyName = "OutputWorkspace%s" % str(output_wksp_list[split_index])
-            self.declareProperty(WorkspaceProperty(propertyName, str(output_wksp_list[split_index]), Direction.Output))
+            if not self.existsProperty(propertyName):
+                self.declareProperty(WorkspaceProperty(propertyName, str(output_wksp_list[split_index]), Direction.Output))
             self.setProperty(propertyName, output_wksp_list[split_index])
             self._save(output_wksp_list[split_index], self._info, False, True)
             self.log().information("Done focussing data of %d." % (split_index))

docs/source/release/v3.14.0/diffraction.rst

@@ -20,6 +20,7 @@ Improvements
 - :ref:`SNAPReduce <algm-SNAPReduce>` now has progress bar and all output workspaces have history
 - :ref:`SNAPReduce <algm-SNAPReduce>` has been completely refactored. It now uses :ref:`AlignAndFocusPowderFromFiles <algm-AlignAndFocusPowderFromFiles>` for a large part of its functionality. It has progress bar and all output workspaces have history. It is also more memory efficient by reducing the number of temporary workspaces created.
 - :ref:`AlignAndFocusPowder <algm-AlignAndFocusPowder>` and :ref:`AlignAndFocusPowderFromFiles <algm-AlignAndFocusPowderFromFiles>` now support outputting the unfocussed data and weighted events (with time). This allows for event filtering **after** processing the data.
+- :ref:`AlignAndFocusPowderFromFiles <algm-AlignAndFocusPowderFromFiles>` has a significant performance improvement when used with chunking
 - :ref:`LoadWAND <algm-LoadWAND>` has grouping option added and loads faster
 - Mask workspace option added to :ref:`WANDPowderReduction <algm-WANDPowderReduction>`
 - :ref:`Le Bail concept page <Le Bail Fit>` moved from mediawiki