From 70f86ad97bcb308ed31213469b073385f1010f79 Mon Sep 17 00:00:00 2001
From: Martyn Gigg <martyn.gigg@stfc.ac.uk>
Date: Fri, 23 Nov 2018 09:58:54 +0000
Subject: [PATCH] Use .pth files for updating python paths

This is more standard and removes the possibililty
of users to accidentally overriding the set of python
paths required by many interfaces to start.
The pythonscripts.directories config key is now empty by
default and only adds directories on starting the framework.
---
 Framework/Kernel/CMakeLists.txt               | 50 ---------------
 .../Properties/Mantid.properties.template     |  9 +--
 Framework/PythonInterface/CMakeLists.txt      |  1 +
 .../PythonInterface/mantid/api/__init__.py    |  1 -
 .../PythonInterface/mantid/api/_aliases.py    |  9 +--
 .../api/src/Exports/AlgorithmManager.cpp      | 20 +++++-
 .../api/src/Exports/AnalysisDataService.cpp   | 25 +++++++-
 .../api/src/Exports/FrameworkManager.cpp      | 63 ++++++++++++++++++-
 .../PythonInterface/mantid/api/src/api.cpp.in | 16 -----
 .../PythonInterface/mantid/kernel/__init__.py |  9 ---
 .../PythonInterface/mantid/kernel/_aliases.py | 30 +++------
 Framework/PythonInterface/mantid/simpleapi.py |  5 +-
 .../algorithms/SaveGEMMAUDParamFile.py        | 27 ++++----
 .../python/plugins/algorithms/CMakeLists.txt  |  1 +
 .../WorkflowAlgorithms/CMakeLists.txt         |  1 +
 buildconfig/CMake/FindPyUnitTest.cmake        | 12 ++--
 buildconfig/CMake/LinuxPackageScripts.cmake   |  8 ++-
 scripts/CMakeLists.txt                        | 53 ++++++++++++++--
 scripts/ExternalInterfaces/CMakeLists.txt     |  4 +-
 19 files changed, 195 insertions(+), 149 deletions(-)

diff --git a/Framework/Kernel/CMakeLists.txt b/Framework/Kernel/CMakeLists.txt
index 20123a0f948..8e05862d0ad 100644
--- a/Framework/Kernel/CMakeLists.txt
+++ b/Framework/Kernel/CMakeLists.txt
@@ -589,37 +589,6 @@ set ( COLORMAPS_FOLDER ${MANTID_ROOT}/installers/colormaps/ )
 set ( MANTIDPUBLISHER "http://upload.mantidproject.org/scriptrepository?debug=1" )
 set ( HTML_ROOT ${DOCS_BUILDDIR}/html )
 
-# Construct script paths.
-set ( MANTID_SCRIPTS ${MANTID_ROOT}/scripts )
-# First the required scripts variable...
-# Omitting the following (as of now) empty directories as CMake doesn't copy them on install and you then end up with a warning message....
-set ( REQUIREDSCRIPT_SUBDIRS Engineering Inelastic Reflectometry SANS)
-
-# If other external interfaces are added then we need a better approach here..
-set ( REQUIREDSCRIPT_DIRS ${MANTID_ROOT}/scripts;${CMAKE_BINARY_DIR}/scripts/ExternalInterfaces/mslice/src/mslice )
-foreach ( SUBDIR ${REQUIREDSCRIPT_SUBDIRS} )
-  set ( REQUIREDSCRIPT_DIRS "${REQUIREDSCRIPT_DIRS};${MANTID_SCRIPTS}/${SUBDIR}" )
-endforeach()
-
-# Second the standard scripts variable. The variable in the file is NOT recursive so add the top level directories from here
-set ( EXCLUDED_DIRS test )
-set ( PYTHONSCRIPT_DIRS "" )
-file ( GLOB SCRIPT_SUBDIRS RELATIVE ${MANTID_SCRIPTS} ${MANTID_SCRIPTS}/* )
-foreach ( SUBDIR ${SCRIPT_SUBDIRS} )
-  set ( DIR ${MANTID_SCRIPTS}/${SUBDIR} )
-  if ( IS_DIRECTORY ${DIR} AND NOT EXISTS ${DIR}/__init__.py )
-    list ( FIND REQUIREDSCRIPT_SUBDIRS ${SUBDIR} HAVE_DIR )
-    list ( FIND EXCLUDED_DIRS ${SUBDIR} EXCLUDE_DIR )
-    if ( HAVE_DIR EQUAL -1 AND EXCLUDE_DIR EQUAL -1 ) # If it is not in REQUIRED and not EXCLUDED
-      if ( PYTHONSCRIPT_DIRS )
-        set ( PYTHONSCRIPT_DIRS "${PYTHONSCRIPT_DIRS};${DIR}" )
-      else()
-        set ( PYTHONSCRIPT_DIRS "${DIR}" ) # Avoid first ;
-      endif()
-    endif()
-  endif()
-endforeach()
-
 # For an mpi-enabled build, do not log to file, format console output (which winds up in a file) differently
 # These settings carry down to the installation configuration below
 if ( MPI_BUILD )
@@ -694,25 +663,6 @@ set ( DATADIRS "" )
 set ( MANTIDPUBLISHER "http://upload.mantidproject.org/scriptrepository" )
 set ( HTML_ROOT ../share/doc/html )
 
-# script paths
-set ( MANTID_SCRIPTS ${MANTID_ROOT}/scripts )
-set ( REQUIREDSCRIPT_DIRS ${MANTID_SCRIPTS};${MANTID_SCRIPTS}/ExternalInterfaces )
-foreach ( SUBDIR ${REQUIREDSCRIPT_SUBDIRS} )
-  set ( REQUIREDSCRIPT_DIRS "${REQUIREDSCRIPT_DIRS};${MANTID_SCRIPTS}/${SUBDIR}" )
-endforeach()
-
-# PYTHONSCRIPT_DIRS
-set ( WITH_SEMICOLONS "" )
-foreach ( DIR ${PYTHONSCRIPT_DIRS} )
-  string ( REGEX REPLACE "${MANTID_ROOT_BUILD}" "${MANTID_ROOT}" DIR ${DIR} )
-  if ( WITH_SEMICOLONS )
-    set ( WITH_SEMICOLONS "${WITH_SEMICOLONS};${DIR}" )
-  else()
-    set ( WITH_SEMICOLONS "${DIR}" ) # Avoid first ;
-  endif()
-endforeach()
-set ( PYTHONSCRIPT_DIRS "${WITH_SEMICOLONS}" )
-
 # For a framework-only (e.g. MPI) build some of these are not relevant and should
 # be left empty to avoid warnings on starting Mantid
 if ( ${CMAKE_PROJECT_NAME} MATCHES "MantidFramework" )
diff --git a/Framework/Properties/Mantid.properties.template b/Framework/Properties/Mantid.properties.template
index 36893c2f6af..5e3ede53bb2 100644
--- a/Framework/Properties/Mantid.properties.template
+++ b/Framework/Properties/Mantid.properties.template
@@ -69,16 +69,9 @@ openclKernelFiles.directory = @MANTID_ROOT@/Framework/GPUAlgorithms/src
 # Where to load colormap files (.map)
 colormaps.directory = @COLORMAPS_FOLDER@
 
-# Location of Python scripts that are required for certain aspects of Mantid to function
-# correctly, i.e. customized interfaces
-#
-# WARNING: Altering this value will almost certainly break Mantid functionality
-#
-requiredpythonscript.directories = @REQUIREDSCRIPT_DIRS@
-
 # Additional directories for Python scripts. These are added to the Python path by the Python API.
 # This key is NOT recursive so sub-directories must be added in addition
-pythonscripts.directories = @PYTHONSCRIPT_DIRS@
+pythonscripts.directories =
 
 # Setting this to 1 will allow python algorithms to be reloaded before execution.
 pythonalgorithms.refresh.allowed = 0
diff --git a/Framework/PythonInterface/CMakeLists.txt b/Framework/PythonInterface/CMakeLists.txt
index 959fd20f4b0..ad78194c942 100644
--- a/Framework/PythonInterface/CMakeLists.txt
+++ b/Framework/PythonInterface/CMakeLists.txt
@@ -110,6 +110,7 @@ clean_orphaned_pyc_files ( ${CMAKE_CURRENT_SOURCE_DIR}/plugins )
 ###########################################################################
 # tests
 ###########################################################################
+set ( PYTHONINTERFACE_PLUGINS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/plugins )
 add_subdirectory( test )
 
 ###########################################################################
diff --git a/Framework/PythonInterface/mantid/api/__init__.py b/Framework/PythonInterface/mantid/api/__init__.py
index cef37ebd245..882f1a5d5bd 100644
--- a/Framework/PythonInterface/mantid/api/__init__.py
+++ b/Framework/PythonInterface/mantid/api/__init__.py
@@ -18,7 +18,6 @@ from __future__ import (absolute_import, division,
 from ..kernel import _shared_cextension
 with _shared_cextension():
     from ._api import *
-    from ._api import _declareCPPAlgorithms
 
 ###############################################################################
 # Make aliases accessible in this namespace
diff --git a/Framework/PythonInterface/mantid/api/_aliases.py b/Framework/PythonInterface/mantid/api/_aliases.py
index 4fcb30bbcb7..bc27830f565 100644
--- a/Framework/PythonInterface/mantid/api/_aliases.py
+++ b/Framework/PythonInterface/mantid/api/_aliases.py
@@ -27,15 +27,12 @@ from ..kernel._aliases import lazy_instance_access
 # delete the python objects.
 # If you see a segfault late in a python process related to the GIL
 # it is likely an exit handler is missing.
-AnalysisDataService = lazy_instance_access(AnalysisDataServiceImpl,
-                                           onexit=AnalysisDataServiceImpl.clear)
+AnalysisDataService = lazy_instance_access(AnalysisDataServiceImpl)
 AlgorithmFactory = lazy_instance_access(AlgorithmFactoryImpl)
-AlgorithmManager = lazy_instance_access(AlgorithmManagerImpl,
-                                        onexit=AlgorithmManagerImpl.clear)
+AlgorithmManager = lazy_instance_access(AlgorithmManagerImpl)
 FileFinder = lazy_instance_access(FileFinderImpl)
 FileLoaderRegistry = lazy_instance_access(FileLoaderRegistryImpl)
-FrameworkManager = lazy_instance_access(FrameworkManagerImpl,
-                                        onexit=FrameworkManagerImpl.shutdown)
+FrameworkManager = lazy_instance_access(FrameworkManagerImpl)
 FunctionFactory = lazy_instance_access(FunctionFactoryImpl)
 WorkspaceFactory = lazy_instance_access(WorkspaceFactoryImpl)
 CatalogManager = lazy_instance_access(CatalogManagerImpl)
diff --git a/Framework/PythonInterface/mantid/api/src/Exports/AlgorithmManager.cpp b/Framework/PythonInterface/mantid/api/src/Exports/AlgorithmManager.cpp
index cc0aee81a49..0bda0a3a804 100644
--- a/Framework/PythonInterface/mantid/api/src/Exports/AlgorithmManager.cpp
+++ b/Framework/PythonInterface/mantid/api/src/Exports/AlgorithmManager.cpp
@@ -18,6 +18,24 @@ using Mantid::PythonInterface::AlgorithmIDProxy;
 using namespace boost::python;
 
 namespace {
+
+std::once_flag INIT_FLAG;
+
+/**
+ * Returns a reference to the AlgorithmManager object, creating it
+ * if necessary. In addition to creating the object the first call also:
+ *   - register AlgorithmManager.shutdown as an atexit function
+ * @return A reference to the AlgorithmManager instance
+ */
+AlgorithmManagerImpl &instance() {
+  // start the framework (if necessary)
+  auto &ads = AlgorithmManager::Instance();
+  std::call_once(INIT_FLAG, []() {
+    Py_AtExit([]() { AlgorithmManager::Instance().clear(); });
+  });
+  return ads;
+}
+
 /**
  * Return the algorithm identified by the given ID. A wrapper version that takes
  * a
@@ -107,7 +125,7 @@ void export_AlgorithmManager() {
            "Clears the current list of managed algorithms")
       .def("cancelAll", &AlgorithmManagerImpl::cancelAll, arg("self"),
            "Requests that all currently running algorithms be cancelled")
-      .def("Instance", &AlgorithmManager::Instance,
+      .def("Instance", instance,
            return_value_policy<reference_existing_object>(),
            "Return a reference to the singleton instance")
       .staticmethod("Instance");
diff --git a/Framework/PythonInterface/mantid/api/src/Exports/AnalysisDataService.cpp b/Framework/PythonInterface/mantid/api/src/Exports/AnalysisDataService.cpp
index a778874d61e..705b9c23480 100644
--- a/Framework/PythonInterface/mantid/api/src/Exports/AnalysisDataService.cpp
+++ b/Framework/PythonInterface/mantid/api/src/Exports/AnalysisDataService.cpp
@@ -24,6 +24,29 @@ using namespace boost::python;
 GET_POINTER_SPECIALIZATION(AnalysisDataServiceImpl)
 
 namespace {
+std::once_flag INIT_FLAG;
+
+/**
+ * Returns a reference to the AnalysisDataService object, creating it
+ * if necessary. In addition to creating the object the first call also:
+ *   - register AnalysisDataService.clear as an atexit function
+ * @return A reference to the FrameworkManagerImpl instance
+ */
+AnalysisDataServiceImpl &instance() {
+  // start the framework (if necessary)
+  auto &ads = AnalysisDataService::Instance();
+  std::call_once(INIT_FLAG, []() {
+    Py_AtExit([]() { AnalysisDataService::Instance().clear(); });
+  });
+  return ads;
+}
+
+/**
+ * @param self A reference to the AnalysisDataServiceImpl
+ * @param names The list of names to extract
+ * @param unrollGroups If true unroll the workspace groups
+ * @return a python list of the workspaces in the ADS
+ */
 list retrieveWorkspaces(AnalysisDataServiceImpl &self, const list &names,
                         bool unrollGroups = false) {
   return Converters::ToPyList<Workspace_sptr>()(self.retrieveWorkspaces(
@@ -45,7 +68,7 @@ void export_AnalysisDataService() {
       DataServiceExporter<AnalysisDataServiceImpl, Workspace_sptr>;
   auto pythonClass = ADSExporter::define("AnalysisDataServiceImpl");
   pythonClass
-      .def("Instance", &AnalysisDataService::Instance,
+      .def("Instance", instance,
            return_value_policy<reference_existing_object>(),
            "Return a reference to the singleton instance")
       .staticmethod("Instance")
diff --git a/Framework/PythonInterface/mantid/api/src/Exports/FrameworkManager.cpp b/Framework/PythonInterface/mantid/api/src/Exports/FrameworkManager.cpp
index c87f29ca7f8..d8387b7c815 100644
--- a/Framework/PythonInterface/mantid/api/src/Exports/FrameworkManager.cpp
+++ b/Framework/PythonInterface/mantid/api/src/Exports/FrameworkManager.cpp
@@ -5,15 +5,76 @@
 //     & Institut Laue - Langevin
 // SPDX - License - Identifier: GPL - 3.0 +
 #include "MantidAPI/FrameworkManager.h"
+#include "MantidKernel/ConfigService.h"
+#include "MantidPythonInterface/api/Algorithms/RunPythonScript.h"
 
 #include <boost/python/class.hpp>
+#include <boost/python/import.hpp>
 #include <boost/python/reference_existing_object.hpp>
 #include <boost/python/return_value_policy.hpp>
 
+#include <iostream>
+#include <mutex>
+
+using Mantid::API::AlgorithmFactory;
 using Mantid::API::FrameworkManager;
 using Mantid::API::FrameworkManagerImpl;
+using Mantid::Kernel::ConfigService;
 using namespace boost::python;
 
+namespace {
+
+std::once_flag INIT_FLAG;
+constexpr auto PYTHONPATHS_KEY = "pythonscripts.directories";
+
+/**
+ * We don't want to register the C++ algorithms on loading the api python
+ * module since we want then be able to control when the various singletons
+ * are created if we are being imported from vanilla Python. This function
+ * registers the any C++ algorithms and should be called once.
+ */
+void declareCPPAlgorithms() {
+  AlgorithmFactory::Instance()
+      .subscribe<Mantid::PythonInterface::RunPythonScript>();
+}
+
+/**
+ * @brief Append to the sys.path any paths defined in the config key
+ * pythonscripts.directories
+ */
+void updatePythonPaths() {
+  auto packagesetup = import("mantid.kernel.packagesetup");
+  packagesetup.attr("update_sys_paths")(
+      ConfigService::Instance()
+          .getValue<std::string>(PYTHONPATHS_KEY)
+          .get_value_or(""));
+}
+
+/**
+ * Returns a reference to the FrameworkManager object, creating it
+ * if necessary. In addition to creating the object the first call also:
+ *   - registers the C++ algorithms declared in this library
+ *   - updates the Python paths with any user-defined directories
+ *     declared in the `pythonscripts.directories`
+ *   - register FrameworkManager.shutdown as an atexit function
+ * @return A reference to the FrameworkManagerImpl instance
+ */
+FrameworkManagerImpl &instance() {
+  // start the framework (if necessary)
+  auto &frameworkMgr = FrameworkManager::Instance();
+  std::call_once(INIT_FLAG, []() {
+    declareCPPAlgorithms();
+    updatePythonPaths();
+    // Without a python-based exit handler the singletons are only cleaned
+    // up after main() and this is too late to acquire the GIL to be able to
+    // delete any python objects still stored in other singletons like the
+    // ADS or AlgorithmManager.
+    Py_AtExit([]() { FrameworkManager::Instance().shutdown(); });
+  });
+  return frameworkMgr;
+}
+} // namespace
+
 void export_FrameworkManager() {
   class_<FrameworkManagerImpl, boost::noncopyable>("FrameworkManagerImpl",
                                                    no_init)
@@ -53,7 +114,7 @@ void export_FrameworkManager() {
       .def("shutdown", &FrameworkManagerImpl::shutdown, arg("self"),
            "Effectively shutdown this service")
 
-      .def("Instance", &FrameworkManager::Instance,
+      .def("Instance", instance,
            return_value_policy<reference_existing_object>(),
            "Return a reference to the singleton instance")
       .staticmethod("Instance");
diff --git a/Framework/PythonInterface/mantid/api/src/api.cpp.in b/Framework/PythonInterface/mantid/api/src/api.cpp.in
index 27d5554925e..33566e819bf 100644
--- a/Framework/PythonInterface/mantid/api/src/api.cpp.in
+++ b/Framework/PythonInterface/mantid/api/src/api.cpp.in
@@ -20,22 +20,9 @@
 
 #include "MantidAPI/AlgorithmFactory.h"
 #include "MantidAPI/Workspace.h"
-#include "MantidPythonInterface/api/Algorithms/RunPythonScript.h"
 
 namespace
 {
-  /**
-   * We don't want to register the C++ algorithms on loading the python module
-   * since we want then be able to control when the various singletons are started if
-   * we are being imported from vanilla Python.
-   * Here we export a single function that can be called after the FrameworkManager
-   * has been started to registered any hard-coded C++ algorithms that are
-   * contained within this module.
-   */
-  void _declareCPPAlgorithms()
-  {
-    Mantid::API::AlgorithmFactory::Instance().subscribe<Mantid::PythonInterface::RunPythonScript>();
-  }
 
   /**
    * Checks if two workspace shared pointers point to the same workspace
@@ -56,9 +43,6 @@ BOOST_PYTHON_MODULE(_api)
   boost::python::docstring_options docstrings(true, true, false);
   // Import numpy
   _import_array();
-  
-  // Internal function to declare C++ algorithms that are a part of this module
-  boost::python::def("_declareCPPAlgorithms", &_declareCPPAlgorithms);
 
   // Workspace address comparison
   boost::python::def("isSameWorkspaceObject",
diff --git a/Framework/PythonInterface/mantid/kernel/__init__.py b/Framework/PythonInterface/mantid/kernel/__init__.py
index 6f3bca4b269..8afb89228bd 100644
--- a/Framework/PythonInterface/mantid/kernel/__init__.py
+++ b/Framework/PythonInterface/mantid/kernel/__init__.py
@@ -53,7 +53,6 @@ from . import mpisetup
 with _shared_cextension():
     from ._kernel import *
 
-
 ###############################################################################
 # Make modules available in this namespace
 ###############################################################################
@@ -68,12 +67,4 @@ funcreturns = funcinspect
 # Do site-specific setup for packages
 ###############################################################################
 from . import packagesetup as _mantidsite
-
-_mantidsite.update_sys_paths(_os.environ.get('MANTIDPATH', ''))
 _mantidsite.set_NEXUSLIB_var()
-# Add directories to PYTHONPATH
-_path_keys = ['requiredpythonscript.directories','pythonscripts.directories']
-for key in _path_keys:
-    paths = config[key]
-    _mantidsite.update_sys_paths(paths)
-
diff --git a/Framework/PythonInterface/mantid/kernel/_aliases.py b/Framework/PythonInterface/mantid/kernel/_aliases.py
index fd9e71f45de..6e1c7338ef7 100644
--- a/Framework/PythonInterface/mantid/kernel/_aliases.py
+++ b/Framework/PythonInterface/mantid/kernel/_aliases.py
@@ -11,58 +11,42 @@
 from __future__ import (absolute_import, division,
                         print_function)
 
-import atexit
-
 from ._kernel import (ConfigServiceImpl, Logger, UnitFactoryImpl,
                       UsageServiceImpl, PropertyManagerDataServiceImpl)
 
 
-def lazy_instance_access(cls, onexit=None):
+def lazy_instance_access(cls):
     """
     Takes a singleton class and wraps it in an LazySingletonHolder
     that constructs the instance on first access.
     :param cls: The singleton class type
-    :param onexit: Unbound method accepting a cls instance
-    registered with atexit.register on instance construction
     :return: A new LazySingletonHolder wrapping cls
     """
-    if onexit is not None:
-        def instance(lazy_singleton):
-            if object.__getattribute__(lazy_singleton,
-                                       "atexit_not_registered"):
-                atexit.register(lambda: onexit(cls.Instance()))
-                object.__setattr__(lazy_singleton, "atexit_not_registered", False)
-            return cls.Instance()
-    else:
-        def instance(_):
-            return cls.Instance()
-
     class LazySingletonHolder(object):
         """
         Delays construction of a singleton instance until the
         first attribute access.
         """
-        atexit_not_registered = True
 
         def __getattribute__(self, item):
             # Called for each attribute access. cls.Instance() constructs
             # the singleton at first access
-            return cls.__getattribute__(instance(self), item)
+            return cls.__getattribute__(cls.Instance(), item)
 
         def __len__(self):
-            return cls.__getattribute__(instance(self), "__len__")()
+            return cls.__getattribute__(cls.Instance(), "__len__")()
 
         def __getitem__(self, item):
-            return cls.__getattribute__(instance(self), "__getitem__")(item)
+            return cls.__getattribute__(cls.Instance(), "__getitem__")(item)
 
         def __setitem__(self, item, value):
-            return cls.__getattribute__(instance(self), "__setitem__")(item, value)
+            return cls.__getattribute__(cls.Instance(), "__setitem__")(item, value)
 
         def __delitem__(self, item):
-            return cls.__getattribute__(instance(self), "__delitem__")(item)
+            return cls.__getattribute__(cls.Instance(), "__delitem__")(item)
 
         def __contains__(self, item):
-            return cls.__getattribute__(instance(self), "__contains__")(item)
+            return cls.__getattribute__(cls.Instance(), "__contains__")(item)
 
     return LazySingletonHolder()
 
diff --git a/Framework/PythonInterface/mantid/simpleapi.py b/Framework/PythonInterface/mantid/simpleapi.py
index 9187226f2e3..0c7be6dd570 100644
--- a/Framework/PythonInterface/mantid/simpleapi.py
+++ b/Framework/PythonInterface/mantid/simpleapi.py
@@ -1419,12 +1419,9 @@ def _attach_algorithm_func_as_method(method_name, algorithm_wrapper, algm_object
 
 # Initialization:
 #   - start FrameworkManager
-#   - create algorithm functions for C++ algorithms
 #   - loads the python plugins and create new algorithm functions
 
 _api.FrameworkManagerImpl.Instance()
-_api._declareCPPAlgorithms()
-_atexit.register(_api.FrameworkManagerImpl.Instance().shutdown)
 _translate()
 
 # Load the Python plugins
@@ -1479,4 +1476,4 @@ except Exception:
     # If an error gets raised remove the attribute to be consistent
     # with standard python behaviour and reraise the exception
     delattr(mantid, MODULE_NAME)
-    raise
\ No newline at end of file
+    raise
diff --git a/Framework/PythonInterface/plugins/algorithms/SaveGEMMAUDParamFile.py b/Framework/PythonInterface/plugins/algorithms/SaveGEMMAUDParamFile.py
index afb98fecc28..2fd7c1af90e 100644
--- a/Framework/PythonInterface/plugins/algorithms/SaveGEMMAUDParamFile.py
+++ b/Framework/PythonInterface/plugins/algorithms/SaveGEMMAUDParamFile.py
@@ -14,6 +14,17 @@ from string import Formatter
 
 from mantid.api import *
 from mantid.kernel import *
+import isis_powder.gem_routines
+
+_MAUD_TEMPLATE_PATH = None
+
+
+def _maud_template_path():
+    global _MAUD_TEMPLATE_PATH
+    if _MAUD_TEMPLATE_PATH is None:
+        _MAUD_TEMPLATE_PATH = os.path.join(os.path.dirname(isis_powder.gem_routines.__file__),
+                                           'maud_param_template.maud')
+    return _MAUD_TEMPLATE_PATH
 
 
 class SaveGEMMAUDParamFile(PythonAlgorithm):
@@ -52,7 +63,7 @@ class SaveGEMMAUDParamFile(PythonAlgorithm):
 
         self.declareProperty(FileProperty(name=self.PROP_TEMPLATE_FILE,
                                           action=FileAction.Load,
-                                          defaultValue=self._find_isis_powder_dir()),
+                                          defaultValue=_maud_template_path()),
                              doc="Template for the .maud file")
 
         self.declareProperty(IntArrayProperty(name=self.PROP_GROUPING_SCHEME),
@@ -135,20 +146,6 @@ class SaveGEMMAUDParamFile(PythonAlgorithm):
         """
         return (bank_param_list[grouping_scheme[spec_num] - 1] for spec_num in spectrum_numbers)
 
-    def _find_isis_powder_dir(self):
-        script_dirs = [directory for directory in config["pythonscripts.directories"].split(";")
-                       if "Diffraction" in directory]
-
-        for directory in script_dirs:
-            path_to_test = os.path.join(directory,
-                                        "isis_powder",
-                                        "gem_routines",
-                                        "maud_param_template.maud")
-            if os.path.exists(path_to_test):
-                return path_to_test
-
-        return ""
-
     def _format_param_list(self, param_list):
         return "\n".join(str(param) for param in param_list)
 
diff --git a/Framework/PythonInterface/test/python/plugins/algorithms/CMakeLists.txt b/Framework/PythonInterface/test/python/plugins/algorithms/CMakeLists.txt
index e43ec318686..b4bc9506769 100644
--- a/Framework/PythonInterface/test/python/plugins/algorithms/CMakeLists.txt
+++ b/Framework/PythonInterface/test/python/plugins/algorithms/CMakeLists.txt
@@ -118,4 +118,5 @@ set ( TEST_PY_FILES
 check_tests_valid ( ${CMAKE_CURRENT_SOURCE_DIR} ${TEST_PY_FILES} )
 
 # Prefix for test name=PythonAlgorithms
+set ( PYUNITTEST_PYTHONPATH_EXTRA ${PYTHONINTERFACE_PLUGINS_DIR}/algorithms )
 pyunittest_add_test ( ${CMAKE_CURRENT_SOURCE_DIR} python.algorithms ${TEST_PY_FILES} )
diff --git a/Framework/PythonInterface/test/python/plugins/algorithms/WorkflowAlgorithms/CMakeLists.txt b/Framework/PythonInterface/test/python/plugins/algorithms/WorkflowAlgorithms/CMakeLists.txt
index 6d10c935b18..4cc0bd75549 100644
--- a/Framework/PythonInterface/test/python/plugins/algorithms/WorkflowAlgorithms/CMakeLists.txt
+++ b/Framework/PythonInterface/test/python/plugins/algorithms/WorkflowAlgorithms/CMakeLists.txt
@@ -68,5 +68,6 @@ set ( TEST_PY_FILES
 check_tests_valid ( ${CMAKE_CURRENT_SOURCE_DIR} ${TEST_PY_FILES} )
 
 # Prefix for test name=PythonWorkflowAlgorithms
+set ( PYUNITTEST_PYTHONPATH_EXTRA ${PYTHONINTERFACE_PLUGINS_DIR}/algorithms/WorkflowAlgorithms )
 pyunittest_add_test ( ${CMAKE_CURRENT_SOURCE_DIR} python.WorkflowAlgorithms ${TEST_PY_FILES} )
 add_subdirectory( sans )
diff --git a/buildconfig/CMake/FindPyUnitTest.cmake b/buildconfig/CMake/FindPyUnitTest.cmake
index df9ad629319..5ac987e2e62 100644
--- a/buildconfig/CMake/FindPyUnitTest.cmake
+++ b/buildconfig/CMake/FindPyUnitTest.cmake
@@ -2,6 +2,8 @@
 # PYUNITTEST_ADD_TEST (public macro to add unit tests)
 #   Adds a set of python tests based upon the unittest module
 #
+#   The variable PYUNITTEST_PYTHONPATH_EXTRA can be defined with
+#   extra paths to add to PYTHONPATH during the tests
 #   Parameters:
 #       _test_src_dir_base :: A base directory when added to the relative test paths gives
 #                             an absolute path to that test. This directory is added to the
@@ -17,27 +19,27 @@ function ( PYUNITTEST_ADD_TEST _test_src_dir _testname_prefix )
   else()
     set ( _module_dir ${CMAKE_BINARY_DIR}/bin )
   endif()
+
   set ( _test_runner ${_module_dir}/mantidpython )
   if ( WIN32 )
     set ( _test_runner ${_test_runner}.bat )
   endif ()
   set ( _test_runner_module ${CMAKE_SOURCE_DIR}/Framework/PythonInterface/test/testhelpers/testrunner.py )
+
   # Environment
   if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
-    set ( _python_path ${PYTHON_XMLRUNNER_DIR};${_test_src_dir};$ENV{PYTHONPATH} )
+    set ( _python_path ${PYTHON_XMLRUNNER_DIR};${_test_src_dir};${PYUNITTEST_PYTHONPATH_EXTRA}$ENV{PYTHONPATH} )
     # cmake list separator and Windows environment separator are the same so escape the cmake one
     string ( REPLACE ";" "\\;" _python_path "${_python_path}" )
   else()
-    set ( _python_path ${PYTHON_XMLRUNNER_DIR}:${_test_src_dir}:$ENV{PYTHONPATH} )
+    string ( REPLACE ";" ":" _python_path "${PYUNITTEST_PYTHONPATH_EXTRA}" )
+    set ( _python_path ${PYTHON_XMLRUNNER_DIR}:${_test_src_dir}:${_python_path}:$ENV{PYTHONPATH} )
   endif()
   # Define the environment
   list ( APPEND _test_environment "PYTHONPATH=${_python_path}" )
   if ( PYUNITTEST_QT_API )
     list ( APPEND _test_environment "QT_API=${PYUNITTEST_QT_API}" )
   endif()
-  if ( PYUNITTEST_TESTRUNNER_IMPORT_MANTID )
-    list ( APPEND _test_environment "TESTRUNNER_IMPORT_MANTID=1" )
-  endif()
 
   # Add all of the individual tests so that they can be run in parallel
   foreach ( part ${ARGN} )
diff --git a/buildconfig/CMake/LinuxPackageScripts.cmake b/buildconfig/CMake/LinuxPackageScripts.cmake
index 8fb1be25b6b..e9f5f9fe8f4 100644
--- a/buildconfig/CMake/LinuxPackageScripts.cmake
+++ b/buildconfig/CMake/LinuxPackageScripts.cmake
@@ -53,7 +53,6 @@ file ( WRITE ${CMAKE_CURRENT_BINARY_DIR}/mantid.csh
 
 install ( PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/mantid.sh
   ${CMAKE_CURRENT_BINARY_DIR}/mantid.csh
-  ${CMAKE_CURRENT_BINARY_DIR}/mantid.pth
   DESTINATION ${ETC_DIR}
 )
 
@@ -66,10 +65,15 @@ print(sc.get_python_lib(plat_specific=True))"
   OUTPUT_VARIABLE PYTHON_SITE
   OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-file ( WRITE ${CMAKE_CURRENT_BINARY_DIR}/mantid.pth
+file ( WRITE ${CMAKE_CURRENT_BINARY_DIR}/mantid.pth.install
   "${CMAKE_INSTALL_PREFIX}/${BIN_DIR}\n"
 )
 
+install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/mantid.pth.install
+  DESTINATION ${ETC_DIR}
+  RENAME mantid.pth
+)
+
 ############################################################################
 # Setup file variables for pre/post installation
 # These are very different depending on the distribution so are contained
diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt
index b3cf15b9d79..84d6a96acab 100644
--- a/scripts/CMakeLists.txt
+++ b/scripts/CMakeLists.txt
@@ -19,13 +19,56 @@ set_property ( TARGET CompileUIErrorReporter PROPERTY FOLDER "CompilePyUI" )
 add_subdirectory ( ExternalInterfaces )
 
 # --------------------------------------------------------------------
-# Testing
+# .pth file
 # --------------------------------------------------------------------
-# All of the following tests (and those in subdirectories) require the
-# test format to import mantid to setup paths to the scripts
-# directory
-set ( PYUNITTEST_TESTRUNNER_IMPORT_MANTID 1 )
+set ( _pth_dirs .
+  Calibration
+  DiamondAttenuationCorrection
+  Diffraction
+  Engineering
+  GSAS-II
+  Inelastic
+  Interface
+  Reflectometry
+  SANS
+  SCD_Reduction
+  TemporaryREF_MScripts
+  Vates
+)
+
+
+set ( _pth_list_dev )
+set ( _pth_list_install )
+foreach ( _dir ${_pth_dirs} )
+  list ( APPEND _pth_list_dev "${CMAKE_SOURCE_DIR}/scripts/${_dir}" )
+  list ( APPEND _pth_list_install "../scripts/${_dir}" )
+endforeach()
+list ( APPEND _pth_list_dev ${MSLICE_DEV} )
+list ( APPEND _pth_list_install "../scripts/Externalinterfaces" )
+
+# development copy
+set ( _scripts_pth_src "${CMAKE_CURRENT_BINARY_DIR}/mantid-scripts.pth.src" )
+set ( _scripts_pth_dest "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_CFG_INTDIR}/mantid-scripts.pth" )
+string ( REPLACE ";" "\n" _pth_list_dev "${_pth_list_dev}" )
+file ( WRITE ${_scripts_pth_src} "${_pth_list_dev}\n" )
+add_custom_command ( OUTPUT ${_scripts_pth_dest}
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${_scripts_pth_src} ${_scripts_pth_dest}
+  DEPENDS ${_scripts_pth_src}
+  COMMENT "Generating scripts .pth file"
+)
+add_custom_target ( ScriptsDotPth ALL DEPENDS ${_scripts_pth_dest} )
+add_dependencies ( PythonInterface ScriptsDotPth )
+
+# install copy
+set ( _scripts_pth_install "${CMAKE_CURRENT_BINARY_DIR}/mantid-scripts.pth.install" )
+string ( REPLACE ";" "\n" _pth_list_install "${_pth_list_install}" )
+file ( WRITE ${_scripts_pth_install} "${_pth_list_install}\n" )
+install ( FILES ${_scripts_pth_install} DESTINATION ${BIN_DIR} )
 
+# --------------------------------------------------------------------
+# Testing
+# --------------------------------------------------------------------
 set ( TEST_PY_FILES
       test/AbinsAtomsDataTest.py
       test/AbinsCalculateDWSingleCrystalTest.py
diff --git a/scripts/ExternalInterfaces/CMakeLists.txt b/scripts/ExternalInterfaces/CMakeLists.txt
index b1156a8446f..675ac0a5052 100644
--- a/scripts/ExternalInterfaces/CMakeLists.txt
+++ b/scripts/ExternalInterfaces/CMakeLists.txt
@@ -14,7 +14,7 @@ ExternalProject_Add ( mslice
   TEST_COMMAND ""
   INSTALL_COMMAND ""
 )
-set ( _mslice_src ${_mslice_external_root}/src/mslice )
+set ( MSLICE_DEV ${_mslice_external_root}/src/mslice PARENT_SCOPE )
 message ( STATUS "Fetching/updating mslice" )
 execute_process ( COMMAND ${CMAKE_COMMAND} ARGS -P ${_mslice_external_root}/tmp/mslice-gitclone.cmake
                   RESULT_VARIABLE _exit_code )
@@ -29,5 +29,5 @@ else ()
 endif()
 
 # Installation
-install ( DIRECTORY ${_mslice_external_root}/src/mslice/mslice/
+install ( DIRECTORY ${MSLICE_SRC_DEV}/mslice/
           DESTINATION ${INBUNDLE}scripts/ExternalInterfaces/mslice )
-- 
GitLab