Added variable target filenames.

can also access everything defined the `reqs` dictionary of the `Job` class

(see `jobtypes.py`).

Substitutions follow a special order - so that variable references

will only work for variables declared earlier.  The order is:

  * members of the target (other than loop)

     - these are first, so not substituted

  * variables in the loop directive are substituted sequentially

  * members of the rule (other than script)

  * script directive (which also gets `{mpirun}` defined from the scheduler)

docs:

    resource: # 1

        time: 1.0

        nrs:  1

        cpu:  1

    out:

        txt: file_*n*.txt # 1

#   inp:

#       doc: test_{n}.rst # 2

    script: | # 3

        echo {n} >{out[txt]}

# These jobtypes are make-rules that know

# how to generate a group of output files from a group of input files.

# Classic `make` permits 1 output file per rule.

  replicas: 20

  out:

      xtc: run.xtc

  loop:

      n: [0, "{replicas}"] # range(0,20)

      out:

          txt: file_{n:02d}.txt

          #txt: cov_{n:02d}.txt

Cov2:

  dirname: Cov2

import re

_var_re = re.compile(r"\*([^*]*)\*")

# Note: a match-tree could be used to speed up the process

# of finding matching entries, but it's overkill here.

class FMatch:

    # Parse 'test_*u*.dat' into

    #   start: 'test_'

    #   end:   '.dat'

    #   var:   'u'

    # and 'test.dat' (no variable name) as:

    #   start: 'test.dat'

    #   end:   ''

    #   var:   None

    def __init__(self, fname):

        m = _var_re.search(fname)

        if m is None:

            self.start = fname

            self.end = ''

            self.var = None

        else:

            b, e = m.span()

            self.start = fname[:b]

            self.end = fname[e:]

            self.var = m[1]

        self.ls = len(self.start)

        self.le = len(self.end)

        self.n = self.ls + self.le

        if self.var is not None: # have to match >= 1 char

            self.n += 1

    # returns False if no match,

    # True for a match when self.var == None

    # and (varname, value) for a match with self.var != None

    # len(value) is guaranteed to be >= 1

    def match(self, fname):

        if self.var is None:

            return self.start == fname

        if len(fname) < self.n:

            return False

        if fname[:self.ls] == self.start and fname[-self.le:] == self.end:

            return (self.var, fname[self.ls:-self.le])

        return False

    # returns True if any file could match both names

    # False otherwise

    def same(a, b):

        if a.var is None and b.var is None: # neither are var

            return a.start == b.start

        if a.var is not None and b.var is not None: # both are var

            ls = min(a.ls, b.ls)

            le = min(a.le, b.le)

            return a.start[:ls] == b.start[:ls] and a.end[-le:] == b.end[-le:]

        if a.var is None:

            a, b = b, a

        # a is var, b is not

        if a.match(b.start):

            return True

        return False

    def __str__(self):

        if self.var is None:

            return self.start

        return '%s*%s*%s'%(self.start, self.var, self.end)

#   leaf-set of jobs ready to run

# G       : networkx.DiGraph of the current computation

# types   : dict of rulenames -> implementation classes

# types   : RuleTypes object

# rule    : Rule instance being currently implemented

# args    : dict of args to pass through to all descendant jobs (includes dirname)

def append_graph(G, types, rule, args, verb=False):

        p = rule.params['dirname'] / tname

        if p.exists():

            try:

                rulename, ftype = types.lookup[tname]

                nrule, ftype = types[tname]

            except KeyError:

                continue # no creating job found

            # Check for newer inputs.

            mtime = p.stat().st_mtime

            for t,f in types[rulename](args).params['inp'].items(): # need to re-run

            for t,f in nrule.params['inp'].items(): # need to re-run

                f = rule.params['dirname'] / f

                if f.exists() and f.stat().st_mtime > mtime:

                    print("File %s is newer than %s - re-running %s"%(f, p, rulename))

                    print("File %s is newer than %s - re-running %s"%(f, p, nrule.params['rulename']))

                    break

            continue

        try:

            rulename, ftype = types.lookup[tname]

            nrule, ftype = types[tname]

        except KeyError:

            raise TargetError("No rule to make target '%s' needed by '%s'."%(p, rule.id))

        rulename = nrule.params['rulename']

        if ftype != ttype:

            raise TargetError("Rule %s produces target file '%s' of type '%s' (but %s was requested)."%(rulename, tname, ftype, ttype))

        if verb:

            print("%s -> %s"%(rulename, rule))

        # instantiate the job

        job = types[rulename](args)

        # instantiate the Rule -- creating a job

        job = nrule(args)

        G.add_edge(job, rule, obj=p) # path is created, and rule is enabled by running 'job'

        addl.append(job) # follow-up with inputs of this job