diff --git a/Framework/PythonInterface/plugins/algorithms/CreateCacheFilename.py b/Framework/PythonInterface/plugins/algorithms/CreateCacheFilename.py new file mode 100644 index 0000000000000000000000000000000000000000..3fef9760f2137e8ce14bc0762f724f877c13bbfe --- /dev/null +++ b/Framework/PythonInterface/plugins/algorithms/CreateCacheFilename.py @@ -0,0 +1,135 @@ +#pylint: disable=no-init,invalid-name,bare-except,too-many-arguments +from mantid.api import * +from mantid.kernel import * +import mantid, os + + +# See ticket #14716 + +class CreateCacheFilename(PythonAlgorithm): + """ Create cache filename + """ + def category(self): + """ + """ + return "Utils" + + def name(self): + """ + """ + return "CreateCacheFilename" + + def summary(self): + """ Return summary + """ + return """Create cache filename""" + + def require(self): + return + + def PyInit(self): + """ Declare properties + """ + # this is the requirement of using this plugin + # is there a place to register that? + self.require() + + self.declareProperty("PropertyManager", "", "name of a property manager from which properties are extracted from") + + self.declareProperty( + StringArrayProperty("Properties", Direction.Input), + "A list of property names to be included") + + self.declareProperty( + StringArrayProperty("OtherProperties", Direction.Input), + "A list of key=value strings for other properties not in the property manager") + + self.declareProperty( + "Prefix", "", "prefix to the output hash name") + + self.declareProperty( + "CacheDir", "", + "the directory in which the cache file will be created") + + self.declareProperty("OutputFilename", "", "output filename", Direction.Output) + + self.declareProperty("OutputSignature", "", "output signature", Direction.Output) + return + + def PyExec(self): + """ Main Execution Body + """ + # Inputs + prop_manager = self.getPropertyValue("PropertyManager") + other_props = self.getProperty("OtherProperties").value + if not prop_manager and not other_props: + raise ValueError("Either PropertyManager or OtherProperties should be supplied") + prop_manager = mantid.PropertyManagerDataService.retrieve(prop_manager)\ + if prop_manager else None + # default to all properties in the manager + props = self.getProperty("Properties").value + if not props and prop_manager: + props = prop_manager.keys() + # output settings + prefix = self.getPropertyValue("Prefix") + cache_dir = self.getPropertyValue("CacheDir") + if not cache_dir: + cache_dir = os.path.join( + ConfigService.getUserPropertiesDir(), + "cache" + ) + # calculate + fn = self._calculate( + prop_manager, props, other_props, prefix, cache_dir) + self.setProperty("OutputFilename", fn) + return + + def _get_signature(self, prop_manager, props, other_props): + # get matched properties + if prop_manager: + props = matched(prop_manager.keys(), props) + # create the list of key=value strings + kvpairs = [ + '%s=%s' % (prop, prop_manager.getPropertyValue(prop)) + for prop in props + ] + else: + kvpairs = [] + kvpairs += other_props + # sort + kvpairs.sort() + # one string out of the list + s = ','.join(kvpairs) + self.setProperty("OutputSignature", s) + return s + + def _calculate(self, prop_manager, props, other_props, prefix, cache_dir): + s = self._get_signature(prop_manager, props, other_props) + # hash + h = _hash(s) + # prefix + if prefix: + h = "%s_%s" % (prefix, h) + # filename + fn = "%s.nxs" % h + return os.path.join(cache_dir, fn) + + +def _hash(s): + import hashlib + return hashlib.sha1(s).hexdigest() + + +def matched(keys, patterns): + "return keys that match any of the given patterns" + import fnmatch + filtered = [] + for pat in patterns: + filtered += fnmatch.filter(keys, pat) + continue + return set(filtered) + + +# Register algorithm with Mantid +AlgorithmFactory.subscribe(CreateCacheFilename) + diff --git a/Framework/PythonInterface/test/python/plugins/algorithms/CreateCacheFilenameTest.py b/Framework/PythonInterface/test/python/plugins/algorithms/CreateCacheFilenameTest.py new file mode 100644 index 0000000000000000000000000000000000000000..42f19e710cdc4c3750709b05eeff892043aee79d --- /dev/null +++ b/Framework/PythonInterface/test/python/plugins/algorithms/CreateCacheFilenameTest.py @@ -0,0 +1,194 @@ +#pylint: disable=invalid-name,too-many-public-methods,too-many-arguments +import unittest +from mantid.kernel import * +from mantid.api import * +from testhelpers import run_algorithm + +import os, mantid, hashlib + +class CreateCacheFilename(unittest.TestCase): + + def test1(self): + """CreateCacheFilename: one prop + """ + pm = PropertyManager() + pm.declareProperty("a", 0) + pm.setProperty("a", 3) + mantid.PropertyManagerDataService.add("pm", pm) + # Execute + alg_test = run_algorithm( + "CreateCacheFilename", + PropertyManager = "pm", + Properties = [], + OtherProperties = [], + Prefix = "", + CacheDir = "", + ) + # executed? + self.assertTrue(alg_test.isExecuted()) + # Verify .... + expected = os.path.join( + ConfigService.getUserPropertiesDir(), "cache", + "%s.nxs" % hashlib.sha1("a=3").hexdigest() + ) + self.assertEqual( + alg_test.getPropertyValue("OutputFilename"), + expected) + + # Another test. don't specify the default values + alg_test = run_algorithm( + "CreateCacheFilename", + PropertyManager = "pm", + ) + # executed? + self.assertTrue(alg_test.isExecuted()) + # Verify .... + expected = os.path.join( + ConfigService.getUserPropertiesDir(), "cache", + "%s.nxs" % hashlib.sha1("a=3").hexdigest() + ) + self.assertEqual( + alg_test.getPropertyValue("OutputFilename"), + expected) + return + + def test_wronginput(self): + """CreateCacheFilename: wrong input + """ + # Execute + alg_test = run_algorithm( + "CreateCacheFilename", + ) + # executed? + self.assertFalse(alg_test.isExecuted()) + return + + def test_glob(self): + """CreateCacheFilename: globbing + """ + # glob pattern search anything with 'a' in it + # and leave other props out + pm = PropertyManager() + aprops = ["a", "alibaba", "taa", "sa", "a75"] + props = aprops + ['b', 'c', 'd'] + for p in props: + pm.declareProperty(p, 0) + pm.setProperty(p, 3) + continue + mantid.PropertyManagerDataService.add("test_glob", pm) + # Execute + alg_test = run_algorithm( + "CreateCacheFilename", + PropertyManager = "test_glob", + Properties = ['*a*'], + ) + # executed? + self.assertTrue(alg_test.isExecuted()) + # Verify .... + s = ','.join(sorted( ['%s=3' % p for p in aprops] )) + expected = os.path.join( + ConfigService.getUserPropertiesDir(), "cache", + "%s.nxs" % hashlib.sha1(s).hexdigest() + ) + self.assertEqual( + alg_test.getPropertyValue("OutputFilename"), + expected) + return + + def test_otherprops_only(self): + """CreateCacheFilename: other_props only + """ + # Execute + alg_test = run_algorithm( + "CreateCacheFilename", + OtherProperties = ["a=1", "b=2"], + ) + # executed? + self.assertTrue(alg_test.isExecuted()) + # Verify .... + expected = os.path.join( + ConfigService.getUserPropertiesDir(), "cache", + "%s.nxs" % hashlib.sha1("a=1,b=2").hexdigest() + ) + self.assertEqual( + alg_test.getPropertyValue("OutputFilename"), + expected) + return + + def test_bothprops(self): + """CreateCacheFilename: use both PropertyManager and OtherProperties + """ + pm = PropertyManager() + aprops = ["a", "alibaba", "taa", "sa", "a75"] + props = aprops + ['b', 'c', 'd'] + for p in props: + pm.declareProperty(p, '') + pm.setProperty(p, "fish") + continue + mantid.PropertyManagerDataService.add("test_bothprops", pm) + other_props = ["A=1", "B=2"] + # Execute + alg_test = run_algorithm( + "CreateCacheFilename", + PropertyManager = "test_bothprops", + Properties = ['*a*'], + OtherProperties = other_props, + ) + # executed? + self.assertTrue(alg_test.isExecuted()) + # Verify .... + s = ','.join(sorted( ['%s=fish' % p for p in aprops] + other_props )) + expected = os.path.join( + ConfigService.getUserPropertiesDir(), "cache", + "%s.nxs" % hashlib.sha1(s).hexdigest() + ) + self.assertEqual( + alg_test.getPropertyValue("OutputFilename"), + expected) + return + + def test_prefix(self): + """CreateCacheFilename: prefix + """ + # Execute + alg_test = run_algorithm( + "CreateCacheFilename", + OtherProperties = ["a=1", "b=2"], + Prefix = "vanadium", + ) + # executed? + self.assertTrue(alg_test.isExecuted()) + # Verify .... + expected = os.path.join( + ConfigService.getUserPropertiesDir(), "cache", + "vanadium_%s.nxs" % hashlib.sha1("a=1,b=2").hexdigest() + ) + self.assertEqual( + alg_test.getPropertyValue("OutputFilename"), + expected) + return + + def test_cache_dir(self): + """CreateCacheFilename: cache_dir + """ + # Execute + alg_test = run_algorithm( + "CreateCacheFilename", + OtherProperties = ["a=1", "b=2"], + CacheDir = "my_cache", + ) + # executed? + self.assertTrue(alg_test.isExecuted()) + # Verify .... + expected = os.path.join( + "my_cache", + "%s.nxs" % hashlib.sha1("a=1,b=2").hexdigest() + ) + self.assertEqual( + alg_test.getPropertyValue("OutputFilename"), + expected) + return + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/source/algorithms/CreateCacheFilename-v1.rst b/docs/source/algorithms/CreateCacheFilename-v1.rst new file mode 100644 index 0000000000000000000000000000000000000000..653d8b318114067aef364ba83f19b08806de9072 --- /dev/null +++ b/docs/source/algorithms/CreateCacheFilename-v1.rst @@ -0,0 +1,83 @@ +.. algorithm:: + +.. summary:: + +.. alias:: + +.. properties:: + +Description +----------- + +The purpose of this algorithm is to create a unique +filename for a cache so that a workflow can reuse +results from previous computations. + +The algorithm will accept a prefix, PropertyManager, list of properties +to use from the property manager (empty is use all), and +a string array (or List) of other properties to use, and +a directory for cache files to exist in (default described below). + +The list of property names will be used to select which of the properties +in the PropertyManager will be used to calculate the hash +and will be interpreted as globbing. + +The string array of other_properties will be key/value pairs of properties +that should be considered, but are not in the provided PropertyManager. + +If a directory is not specified, cache files will go into a cache +subdirectory of ConfigService::getUserPropertiesDir(). +On unix this will be ~/.mantid/cache. + +The algorithm will convert all properties to strings as +"%s=%s" % (property.name, property.valueAsStr), sort the list, +then convert it to a sha1. + +A filename with the form <location>/<prefix>_<sha1>.nxs +will be returned as the output property. +If no prefix is specified then file result will be <location>/<sha1>.nxs. + +* property_manager: an instance of PropertyManager from which property values + can be retrieved. None means we don't care about property manager + -- all properties will come from other_properties +* properties: a list of strings. each string is a property managed by the + given property_manager, or it can be glob pattern to match prop + names too. but empty list means taking all properties + from the property_manager +* other_properties: a list of strings. each string is in the form of + "key=value" for one property not managed by the property_manager. + no globbing here. +* prefix: prefix to the output hash name. when it is empty, just the hash. + when it is not empty, it will be <prefix>_<sha1> +* cache_dir: the directory in which the cach file will be created. + empty string means default as described above + + +Usage +----- + +**Example:** + +.. testcode:: ExCreateCacheFilename + + import mantid + from mantid.kernel import PropertyManager + pm = PropertyManager() + aprops = ["a", "alibaba", "taa", "sa", "a75"] + props = aprops + ['b', 'c', 'd'] + for p in props: + pm.declareProperty(p, '') + pm.setProperty(p, "fish") + continue + mantid.PropertyManagerDataService.add("excreatecachefilename", pm) + other_props = ["A=1", "B=2"] + # Execute + cache_path, signature = CreateCacheFilename( + PropertyManager = "excreatecachefilename", + Properties = ['*a*'], + OtherProperties = other_props, + ) + +.. categories:: + +.. sourcelink::