Re #13217 Support for User_files_description.xml

and unit tests for it. All seems work fine.

Code/Mantid/scripts/Inelastic/Direct/ISISDirecInelasticConfig.py

@@ -183,7 +183,7 @@ class MantidConfigDirectInelastic(object):
         # fall back files defined to use if USER_Files_description is for some reason not available or wrong
         self._sample_reduction_file = lambda InstrName : '{0}Reduction_Sample.py'.format(InstrName)
         # File name, used as target for copying to user folder for user to deploy as the base for his reduction script
-        self._target_reduction_file = lambda InstrName,cycleID : '{0}Reduction_{1}_{2}.py'.format(InstrName,cycleID[0],cycleID[1])
+        self._target_reduction_file = lambda InstrName,cycleID : '{0}Reduction_{1}.py'.format(InstrName,cycleID)
 
 
         # Static contents of the Mantid Config file
@@ -272,7 +272,7 @@ class MantidConfigDirectInelastic(object):
         else: # somebody have modified the target file. Leave it alone
             return False
 #
-    def _define_fullpath_to_copy(self,short_source_file=None,short_target_file=None):
+    def _fullpath_to_copy(self,short_source_file=None,short_target_file=None):
         """Append full path to source and target files """
 
         InstrName = self._user.instrument
@@ -289,39 +289,63 @@ class MantidConfigDirectInelastic(object):
 
         full_target = os.path.join(rb_folder,short_target_file)
         return full_source,full_target
-
 #
-    def copy_reduction_sample(self,InstrName,CycleID,rb_folder):
-        """Method copies sample reduction script from user script repository
+    def copy_reduction_sample(self, users_file_description=None):
+        """copy sample reduction scripts from user script repository
            to user folder.
         """
-        full_source,full_target = self._define_fullpath_to_copy(self._sample_reduction_file(InstrName),
-                                self._target_reduction_file(InstrName,CycleID))
+        if users_file_description is None:
+            users_file_description = self._user_files_descr
+        files_to_copy = self._parse_user_files_description(users_file_description)
+        for files in files_to_copy:
+            self._copy_user_file_job(files[0],files[1],files[2])
+
+
+    def _copy_and_parse_user_file(self,input_file,output_file,replacemets_list):
+        """Method processes file provided for user and replaces list of keywords, describing user
+           and experiment (See comments in User_files_description.xml) with their values
+        """
+        fh_targ    = open(output_file,'w')
+        if not fh_targ :
+            return
+        var_to_replace = replacemets_list.keys()
+        with open(input_file) as fh_source:
+            for line in fh_source:
+                rez = line
+                for var in var_to_replace:
+                    if var in rez:
+                        rez = rez.replace(var, replacemets_list[var])
+                fh_targ.write(rez)
+        fh_targ.close()
+
 
-        if not os.path.isfile(full_source):
+   
+
+    def _copy_user_file_job(self,input_file,output_file,replacement_list=None):
+        """Method processes file provided for user and replaces list of keywords, describing user
+           and experiment (See comments in User_files_description.xml) with their values if 
+           such replacement list is provided.
+        """
+        if not os.path.isfile(input_file):
             return
 
         # already have target file or modified by user
-        if not self.script_need_replacing(full_source,full_target):
+        if not self.script_need_replacing(input_file,output_file):
             return
-        if os.path.isfile(full_target):
-            os.remove(full_target)
-        shutil.copyfile(full_source,full_target)
-        os.chmod(full_target,0777)
+        if os.path.isfile(output_file):
+            os.remove(output_file)
+        if replacement_list is None:
+            shutil.copyfile(input_file,output_file)
+        else:
+            self._copy_and_parse_user_file(input_file,output_file,replacement_list)
+        os.chmod(output_file,0777)
 
-        if platform.system() != 'Windows':
-            os.system('chown '+self._fedid+':'+self._fedid+' '+full_target)
+        #if platform.system() != 'Windows':
+        #    os.system('chown '+self._fedid+':'+self._fedid+' '+output_file)
         # Set up the file creation and modification dates to the users start date
         start_date = self._user.start_date
         file_time = time.mktime(start_date.timetuple())
-        os.utime(full_target,(file_time,file_time))
-
-    def _copy_and_parse_user_file(self,input_file,output_file,replacemets_list):
-        """Method processes file provided for user and replaces list of keywords, describing user
-           and experiment (See comments in User_files_description.xml) with their values
-        """
-        shutil.copyfile(input_file,output_file)
-        pass
+        os.utime(output_file,(file_time,file_time))
 
     def _get_file_attributes(self,file_node):
         """processes xml file_node to retrieve file attributes to copy """
@@ -336,28 +360,54 @@ class MantidConfigDirectInelastic(object):
         else:
             if "$" in target_file:
                 target_file = self._user.replace_variables(target_file)
-        full_source,full_target  = self._define_fullpath_to_copy(source_file,target_file)
+        full_source,full_target  = self._fullpath_to_copy(source_file,target_file)
 
         return (full_source,full_target)      
+    def _parse_replacement_info(self,repl_info):
+        """process dom element 'replacement' and 
+           returns the  variables with its correspondent value
+           to replace variable by their value.
+
+           If value contains one or more of the supported variables as its part, this
+           variable is replaced by its value. 
+           Supported variables are defined by global list USER_PROPERTIES
+           and their values are taken from current self._user class
+        """
+        # what should be replaced in the file
+        source  = repl_info.getAttribute("var")
+        if len(source) == 0:
+            raise InvalidArgument('"replace" field of {0} file for instrument {1} has to contain attribute "var" and its value'\
+                                   .format(self._user_files_descr,self._user.instrument))
+        # what should be placed instead of the replacement
+        dest    = repl_info.getAttribute("by_var")
+        if len(dest) == 0:
+            raise InvalidArgument('"replace" field of {0} file for instrument {1} has to contain attribute "by_var" and its value'\
+                                   .format(self._user_files_descr,self._user.instrument))
+
+        # replace use-specific variables by their values
+        if '$' in dest:
+            dest = self._user.replace_variables(dest)
+        return (source,dest)
 
     def _parse_user_files_description(self,job_description_file):
         """ Method parses xml file used to describe files to provide to user"""
 
-        # does not work if user is not defined
-        copying_job =[]
+        # mainly for debugging purposes
+        filenames_to_copy=[]
 
+        # does not work if user is not defined
         if self._user is None:
             return None
-        # parse job description file
+        # parse job description file, fail down on default behavior if
+        # user files description is not there
         try:
             domObj=minidom.parse(job_description_file)
         except Exception as error:
-            input_file,output_file = self._define_fullpath_to_copy()
-            copying_job.append(lambda : shutil.copyfile(input_file,output_file))
-            return copying_job
+            input_file,output_file = self._fullpath_to_copy()
+            filenames_to_copy.append((input_file,output_file,None))
+            return filenames_to_copy
 
         files_to_copy = domObj.getElementsByTagName("file_to_copy")
-        copying_jobs = []
 
         # go through all files in the description and define file copying operations
         for file in files_to_copy :
@@ -367,20 +417,19 @@ class MantidConfigDirectInelastic(object):
                 continue
 
             # identify all replacements, defined for this file
-            replacements_info = job.getElementsByTagName('replace_in_file')
-            if replacements_info is None:
-                copying_job.append(lambda : shutil.copyfile(input_file,output_file))
+            replacements_info = file.getElementsByTagName('replace')
+            if len(replacements_info) == 0:
+                replacement_list = None
             else:
                 replacement_list = {}
                 for replacement in replacements_info:
                     source,dest = self._parse_replacement_info(replacement)
                     replacement_list[source]=dest
-                copying_job = lambda : self._copy_and_parse_user_file(input_file,output_file,replacement_list)
-                copying_jobs.append(copying_job)
-
-        return copying_jobs 
-
+            filenames_to_copy.append((input_file,output_file,replacement_list))
 
+        return filenames_to_copy
+#    
+       
     def get_data_folder_name(self,instr,cycle_ID):
         """Method to generate a data folder from instrument name and the cycle start date
            (cycle ID)
@@ -424,11 +473,10 @@ class MantidConfigDirectInelastic(object):
         self._dynamic_configuration = copy.deepcopy(self._dynamic_options_base)
         self._init_config()
     #
-
     def  _check_server_folders_present(self):
         """Routine checks all necessary server folder are present"""
         if not os.path.exists(self._mantid_path):
-            raise RuntimeError("SERVER ERROR: no correct mantid path defined at {0}".format(self._mantid_path))
+            raise RuntimeError("SERVER ERROR: no correct Mantid path defined at {0}".format(self._mantid_path))
         if not os.path.exists(self._home_path):
             raise RuntimeError("SERVER ERROR: no correct home path defined at {0}".format(self._home_path))
         if not os.path.exists(self._script_repo):
@@ -525,10 +573,7 @@ class MantidConfigDirectInelastic(object):
         if platform.system() != 'Windows':
             os.system('chown -R {0}:{0} {1}'.format(self._fedid,config_path))
 
-        InstrName = self._user.instrument
-        cycleID   = self._user.cycleID
-        rb_folder = self._user.rb_dir
-        self.copy_reduction_sample(InstrName,cycleID,rb_folder)
+        self.copy_reduction_sample(self._user_files_descr)
         #
         self.make_map_mask_links(user_path)
     #

Code/Mantid/scripts/test/ISISDirecInelasticConfigTest.py

@@ -22,7 +22,10 @@ class ISISDirectInelasticConfigTest(unittest.TestCase):
         self.rbnumber  = "RB" + nrbnumber
         self.start_date= '20150503'
         self.userID = 'tuf666699'
+        self._set_up()
         return super(ISISDirectInelasticConfigTest, self).__init__(methodName)
+    def __del__(self):
+        self._tear_down()
 
     def get_save_dir(self):
         targetDir = config['defaultsave.directory']
@@ -32,8 +35,22 @@ class ISISDirectInelasticConfigTest(unittest.TestCase):
             else:
                 targetDir = '/tmp'
         return targetDir 
+    def write_test_file(self,filename):
+        """Method writes test file with name provided and 
+           specified contents
+        """
+        dir_name = os.path.dirname(filename)
+        if not os.path.exists(dir_name):
+           os.makedirs(dir_name)
+
+        fh = open(filename,'w')
+        fh.write("****************************************\n")
+        fh.write("first test variable = AAAA # something\n")
+        fh.write("second test var  Test_reduction_file # other\n ")
+        fh.close()
+
 
-    def setUp(self):
+    def _set_up(self):
         # Create user's folder structure in default save directory.
         # the administrative script (not here) builds all this for real in /home
         targetDir = self.get_save_dir()
@@ -70,7 +87,7 @@ class ISISDirectInelasticConfigTest(unittest.TestCase):
         return full_file
 
 
-    def tearDown(self):
+    def _tear_down(self):
         # Clean-up user's folder structure
         if os.path.exists(self.rbdir):
             shutil.rmtree(self.rbdir,ignore_errors=True)
@@ -132,8 +149,10 @@ class ISISDirectInelasticConfigTest(unittest.TestCase):
         self.assertEqual(id,'2016-12-01')
         self.assertEqual(user._instrument[id],'MERLIN')
         self.assertEqual(user.instrument,'MERLIN')
+        
 
     def test_build_config(self):
+
         # script verifies the presence of a folder, not its contents.
         # for the script to work, let's run it on default save directory
         MantidDir = os.path.split(os.path.realpath(__file__))[0]
@@ -182,6 +201,7 @@ class ISISDirectInelasticConfigTest(unittest.TestCase):
         if os.path.exists(os.path.join(self.userRootDir,'.mantid')):
             shutil.rmtree(os.path.join(self.userRootDir,'.mantid'))
 
+
     def test_build_3Experiments_config(self):
         # script verifies the presence of a folder, not its contents.
         # for the script to work, let's run it on default save directory
@@ -276,6 +296,8 @@ class ISISDirectInelasticConfigTest(unittest.TestCase):
             shutil.rmtree(rbdir3,ignore_errors=True)
         if os.path.exists(user1RootDir):
             shutil.rmtree(user1RootDir,ignore_errors=True)
+        #
+
 
     def test_replace_user_variables(self):
         user = UserProperties("wkc26243")
@@ -284,6 +306,61 @@ class ISISDirectInelasticConfigTest(unittest.TestCase):
         targ_string = user.replace_variables('$instrument$ReductionScript$cycleID$.py')
         self.assertEqual(self.instrument+'ReductionScript2015_1.py',targ_string)
         
+    def test_parse_file_description(self):
+
+        this_file = os.path.realpath(__file__)
+        file_dir = os.path.dirname(this_file)
+        test_xml = os.path.join(file_dir,'User_files_description_test.xml')
+        self.assertTrue(os.path.exists(test_xml))
+
+        MantidDir = os.path.split(os.path.realpath(__file__))[0]
+        HomeRootDir = self.get_save_dir()
+        mcf = MantidConfigDirectInelastic(MantidDir,HomeRootDir,self.UserScriptRepoDir,self.MapMaskDir)
+
+        
+        user = UserProperties(self.userID)
+        user.set_user_properties(self.instrument,self.start_date,self.cycle,self.rbdir)
+        mcf.init_user(self.userID,user)
+
+        # test old defaults, deployed if no User_files_description are defined
+        files_to_copy = mcf._parse_user_files_description(None)
+        self.assertEqual(len(files_to_copy),1)
+        source = files_to_copy[0][0]
+        dest   = files_to_copy[0][1]
+        repl   = files_to_copy[0][2]
+        self.assertEqual(os.path.basename(source),'MERLINReduction_Sample.py')
+        self.assertEqual(os.path.basename(dest) ,'MERLINReduction_2015_1.py')
+        self.assertTrue(repl is None)
+
+        
+        # test files defined by test xml file
+        files_to_copy = mcf._parse_user_files_description(test_xml)
+        self.assertEqual(len(files_to_copy),2)
+
+        # define and generate test files to copy. 
+        # We know what files are and what their contents is from User_files_description_test.xml
+        for file_descr in files_to_copy:
+            source = file_descr[0]
+            dest   = file_descr[1]
+            if os.path.exists(dest):
+                os.remove(dest)
+            self.write_test_file(source)
+
+        # Check copy_reduction_sample 
+        mcf.copy_reduction_sample(test_xml)
+        for file_pair in files_to_copy:
+            dest   = file_pair[1]
+            self.assertTrue(os.path.exists(dest))
+
+        # Clean up
+        for file_pair in files_to_copy:
+            source = file_pair[0]
+            dest   = file_pair[1]
+            if os.path.exists(dest):
+                os.remove(dest)
+            if os.path.exists(source):
+                os.remove(source)
+
 
 
 

Code/Mantid/scripts/test/User_files_description_test.xml

@@ -16,10 +16,12 @@
  <!--Simple file copying -->  
   <file_to_copy file_name="Test_reduction_file1.py" copy_as="Test_reduction_file2.py"/>
 
- <!--Advanced file copying -->    
-  <file_to_copy file_name="Test_reduction_file.py" copy_as="Test_reduction_file$cycleID$.py">
-    <replace variable  ="Test_reduction_file" by_variable="Test_reduction_file$instrument$"/>
-    <replace variable  ="AAAA" by_variable="BBB"/>  
+ <!--Advanced file copying 
+    Variables have to be either strings or strings with variables, described above
+  -->    
+  <file_to_copy file_name="Test_reduction_file.py" copy_as="Test_reduction_file_$cycleID$.py">
+    <replace var  ="Test_reduction_file" by_var="$instrument$Test_reduction_file_$cycleID$"/>
+    <replace var  ="AAAA" by_var="BBB"/>  
   </file_to_copy>
 
 </user_files_description>