diff --git a/Framework/PythonInterface/mantid/plots/mantidaxes.py b/Framework/PythonInterface/mantid/plots/mantidaxes.py
index a9c7f4482fa1c3e75e2000739c3f3d1b48754cc2..8ad76112806bfcbfd13b60230126b159cc32a49f 100644
--- a/Framework/PythonInterface/mantid/plots/mantidaxes.py
+++ b/Framework/PythonInterface/mantid/plots/mantidaxes.py
@@ -285,7 +285,7 @@ class MantidAxes(Axes):
         Remove the artists reference by this workspace (if any) and return True
         if the axes is then empty
         :param workspace: A Workspace object
-        :return: True if the axes is empty, false if artists remain or this workspace is not associated here
+        :return: True is an artist was removed False if one was not
         """
         try:
             # pop to ensure we don't hold onto an artist reference
@@ -296,7 +296,7 @@ class MantidAxes(Axes):
         for workspace_artist in artist_info:
             workspace_artist.remove(self)
 
-        return self.is_empty(self)
+        return True
 
     def remove_artists_if(self, unary_predicate):
         """
diff --git a/Framework/PythonInterface/test/python/mantid/plots/plots__init__Test.py b/Framework/PythonInterface/test/python/mantid/plots/plots__init__Test.py
index 781ba2262e2c42f1ff9e52d726b1bfc7ec751dd7..6f4b1d2c4d4103fddb08368c1ac780c2a6efb49e 100644
--- a/Framework/PythonInterface/test/python/mantid/plots/plots__init__Test.py
+++ b/Framework/PythonInterface/test/python/mantid/plots/plots__init__Test.py
@@ -76,14 +76,15 @@ class Plots__init__Test(unittest.TestCase):
 
     def test_remove_workspace_artist_for_known_workspace_removes_plot(self):
         self.ax.plot(self.ws2d_histo, specNum=2, linewidth=6)
-        is_empty = self.ax.remove_workspace_artists(self.ws2d_histo)
-        self.assertEqual(True, is_empty)
+        workspace_removed = self.ax.remove_workspace_artists(self.ws2d_histo)
+        self.assertEqual(True, workspace_removed)
         self.assertEqual(0, len(self.ax.lines))
 
     def test_remove_workspace_artist_for_unknown_workspace_does_nothing(self):
         self.ax.plot(self.ws2d_histo, specNum=2, linewidth=6)
         unknown_ws = CreateSampleWorkspace()
-        self.ax.remove_workspace_artists(unknown_ws)
+        workspace_removed  = self.ax.remove_workspace_artists(unknown_ws)
+        self.assertEqual(False, workspace_removed)
         self.assertEqual(1, len(self.ax.lines))
 
     def test_remove_workspace_artist_for_removes_only_specified_workspace(self):
@@ -92,7 +93,8 @@ class Plots__init__Test(unittest.TestCase):
         line_second_ws = self.ax.plot(second_ws, specNum=5)[0]
         self.assertEqual(2, len(self.ax.lines))
 
-        self.ax.remove_workspace_artists(self.ws2d_histo)
+        workspace_removed = self.ax.remove_workspace_artists(self.ws2d_histo)
+        self.assertEqual(True, workspace_removed)
         self.assertEqual(1, len(self.ax.lines))
         self.assertTrue(line_ws2d_histo not in self.ax.lines)
         self.assertTrue(line_second_ws in self.ax.lines)
diff --git a/docs/source/release/v5.1.0/mantidworkbench.rst b/docs/source/release/v5.1.0/mantidworkbench.rst
index 040b994c8ec9e752a18ee7c0176f5008b34ed7c7..ab08d04905b1de6065bfdd90b59de522938e6735 100644
--- a/docs/source/release/v5.1.0/mantidworkbench.rst
+++ b/docs/source/release/v5.1.0/mantidworkbench.rst
@@ -9,6 +9,7 @@ Improvements
 ############
 
 - Tile plots are now reloaded correctly by project recovery.
+- Fixed an issue where some scripts were running slower if a  plot was open at the same time.
 
 
 Bugfixes
diff --git a/qt/applications/workbench/workbench/plotting/figuremanager.py b/qt/applications/workbench/workbench/plotting/figuremanager.py
index 7201eade9b7323e2bcafce57b9f529fa4716b849..adb612dd5fb18875ce33769f191b19de8fc903ee 100644
--- a/qt/applications/workbench/workbench/plotting/figuremanager.py
+++ b/qt/applications/workbench/workbench/plotting/figuremanager.py
@@ -92,9 +92,12 @@ class FigureManagerADSObserver(AnalysisDataServiceObserver):
         # managed via a workspace.
         # See https://github.com/mantidproject/mantid/issues/25135.
         empty_axes = []
+        redraw = False
         for ax in all_axes:
             if isinstance(ax, MantidAxes):
-                ax.remove_workspace_artists(workspace)
+                to_redraw = ax.remove_workspace_artists(workspace)
+            else:
+                to_redraw = False
             # We check for axes type below as a pseudo check for an axes being
             # a colorbar. Creating a colorfill plot creates 2 axes: one linked
             # to a workspace, the other a colorbar. Deleting the workspace
@@ -104,10 +107,11 @@ class FigureManagerADSObserver(AnalysisDataServiceObserver):
             # Axes object being closed.
             if type(ax) is not Axes:
                 empty_axes.append(MantidAxes.is_empty(ax))
+            redraw = redraw | to_redraw
 
         if all(empty_axes):
             self.window.emit_close()
-        else:
+        elif redraw:
             self.canvas.draw()
 
     @_catch_exceptions