Loading src/topaz_reduction.py +140 −129 Original line number Diff line number Diff line Loading @@ -23,13 +23,13 @@ # ******************************************************************************************** # import os import subprocess import sys import threading import time import math import numpy as np from operator import itemgetter from itertools import groupby # sys.path.insert(0,"/opt/mantidnightly/bin") # sys.path.insert(0,"/opt/mantidnightly/lib") Loading @@ -37,8 +37,8 @@ import ReduceDictionary from mantid.simpleapi import * from mantid.kernel import * from mantid.geometry import PointGroupFactory, SpaceGroupFactory, UnitCell, Goniometer from mantid import config config['Q.convention'] = "Crystallography" DownloadInstrument(ForceUpdate=True) Loading @@ -60,7 +60,6 @@ from spectrumCalc import * from spectrum2 import * from absor_sphere import * print("API Version") print(apiVersion()) Loading Loading @@ -297,19 +296,24 @@ print('\nCompleted calculation of linear absorption coefficients\n') # how the the list of runs will be processed in parallel. print("\n*********************************************************************************") print("************** Start Peak Integration *******************************************") # # ProcessThread is a simple local class. Each instance of ProcessThread is # a thread that starts a command line process to reduce one run. # class ProcessThread(threading.Thread): command = "" def __init__(self): super().__init__() self.return_code = None def setCommand(self, command=""): self.command = command def run(self): print(('STARTING PROCESS: ' + self.command)) os.system( self.command ) print('STARTING PROCESS:', self.command) self.return_code = subprocess.run(self.command, shell=True) # ------------------------------------------------------------------------- # Loading @@ -321,7 +325,7 @@ list=[] index = 0 for r_num in run_nums: list.append(ProcessThread()) cmd = 'mantidpython -u ' + reduce_one_run_script + ' ' + config_file_name + ' ' + str(r_num) cmd = 'python -u ' + reduce_one_run_script + ' ' + config_file_name + ' ' + str(r_num) if slurm_queue_name is not None: console_file = output_directory + "/" + str(r_num) + "_output.txt" cmd = 'srun -p ' + slurm_queue_name + \ Loading @@ -335,6 +339,7 @@ for r_num in run_nums: # all_done = False active_list = [] return_codes = [] while not all_done: if (len(list) > 0 and len(active_list) < max_processes): thread = list[0] Loading @@ -345,6 +350,7 @@ while not all_done: for thread in active_list: if not thread.is_alive(): active_list.remove(thread) return_codes.append(thread.return_code) if len(list) == 0 and len(active_list) == 0: all_done = True Loading @@ -353,3 +359,8 @@ print("****************************** All DONE ********************************* print("**************************************************************************************\n") print('Config file: ' + config_file_name) for code in return_codes: if code != 0: print('Finished with errors') sys.exit(1) Loading
src/topaz_reduction.py +140 −129 Original line number Diff line number Diff line Loading @@ -23,13 +23,13 @@ # ******************************************************************************************** # import os import subprocess import sys import threading import time import math import numpy as np from operator import itemgetter from itertools import groupby # sys.path.insert(0,"/opt/mantidnightly/bin") # sys.path.insert(0,"/opt/mantidnightly/lib") Loading @@ -37,8 +37,8 @@ import ReduceDictionary from mantid.simpleapi import * from mantid.kernel import * from mantid.geometry import PointGroupFactory, SpaceGroupFactory, UnitCell, Goniometer from mantid import config config['Q.convention'] = "Crystallography" DownloadInstrument(ForceUpdate=True) Loading @@ -60,7 +60,6 @@ from spectrumCalc import * from spectrum2 import * from absor_sphere import * print("API Version") print(apiVersion()) Loading Loading @@ -297,19 +296,24 @@ print('\nCompleted calculation of linear absorption coefficients\n') # how the the list of runs will be processed in parallel. print("\n*********************************************************************************") print("************** Start Peak Integration *******************************************") # # ProcessThread is a simple local class. Each instance of ProcessThread is # a thread that starts a command line process to reduce one run. # class ProcessThread(threading.Thread): command = "" def __init__(self): super().__init__() self.return_code = None def setCommand(self, command=""): self.command = command def run(self): print(('STARTING PROCESS: ' + self.command)) os.system( self.command ) print('STARTING PROCESS:', self.command) self.return_code = subprocess.run(self.command, shell=True) # ------------------------------------------------------------------------- # Loading @@ -321,7 +325,7 @@ list=[] index = 0 for r_num in run_nums: list.append(ProcessThread()) cmd = 'mantidpython -u ' + reduce_one_run_script + ' ' + config_file_name + ' ' + str(r_num) cmd = 'python -u ' + reduce_one_run_script + ' ' + config_file_name + ' ' + str(r_num) if slurm_queue_name is not None: console_file = output_directory + "/" + str(r_num) + "_output.txt" cmd = 'srun -p ' + slurm_queue_name + \ Loading @@ -335,6 +339,7 @@ for r_num in run_nums: # all_done = False active_list = [] return_codes = [] while not all_done: if (len(list) > 0 and len(active_list) < max_processes): thread = list[0] Loading @@ -345,6 +350,7 @@ while not all_done: for thread in active_list: if not thread.is_alive(): active_list.remove(thread) return_codes.append(thread.return_code) if len(list) == 0 and len(active_list) == 0: all_done = True Loading @@ -353,3 +359,8 @@ print("****************************** All DONE ********************************* print("**************************************************************************************\n") print('Config file: ' + config_file_name) for code in return_codes: if code != 0: print('Finished with errors') sys.exit(1)