Commit f7cd92c0 authored by David M. Rogers's avatar David M. Rogers
Browse files

Added -work option to separate working dir.

parent ea50b98f

helpers.py

+4 −3
Original line number Diff line number Diff line
@@ -93,13 +93,14 @@ def grep(fname, exp): 
    return False



class JobState:

    def __init__(self, dirname, jobname):

    def __init__(self, workdir, dirname, jobname):

        self.workdir = workdir

        self.dirname = dirname

        self.jobname = jobname

        self.err = False

        self.done = False

        self.job_in  = jobfile(dirname, jobname, 'sh')

        self.job_out = jobfile(dirname, jobname, 'log')

        self.job_in  = jobfile(workdir/dirname, jobname, 'sh')

        self.job_out = jobfile(workdir/dirname, jobname, 'log')



    def __str__(self):

        if self.err:

make.py

+10 −2
Original line number Diff line number Diff line
@@ -7,13 +7,21 @@ from machine import machine, Machine 
from manager import Manager

import logging



from pathlib import Path



def main(argv):

    workdir = Path()

    if len(argv) > 2 and argv[1] == '-log':

        logging.basicConfig(filename=argv[2], level=logging.DEBUG)

        del argv[1:3]

    if len(argv) > 2 and argv[1] == '-work':

        workdir = Path(argv[2])

        workdir.mkdir(exist_ok=True, parents=True)

        assert workdir.is_dir(), f"Unable to make directory {workdir}"

        del argv[1:3]



    assert len(argv) == 4, "Usage: %s [-log logfile.log] <rules.yaml> <targets.yaml> <minutes avail.>"%argv[0]

    rules = read_rules(argv[1])

    assert len(argv) == 4, "Usage: %s [-log logfile.log] [-work workdir] <rules.yaml> <targets.yaml> <minutes avail.>"%argv[0]

    rules = read_rules(argv[1], workdir)

    with open(argv[2]) as f:

        tgts = yaml.safe_load(f)

    T = float(argv[3])

make_test.py

+2 −1
Original line number Diff line number Diff line
@@ -5,10 +5,11 @@ from rules import read_rules 
from graph import TaskGraph

from machine import Machine

from manager import Manager

from pathlib import Path



def main(argv):

    assert len(argv) == 3, "Usage: %s <rules.yaml> <targets.yaml>"%argv[0]

    rules = read_rules(argv[1])

    rules = read_rules(argv[1], Path())

    with open(argv[2]) as f:

        tgts = yaml.safe_load(f)

requirements.txt

0 → 100644
+2 −0
Original line number Diff line number Diff line 
networkx

pyyaml

rules.py

+13 −4
Original line number Diff line number Diff line
@@ -63,6 +63,7 @@ class Rule: 
       to its `dirname`.

    """

    reqs = { 'dirname' : Path,

             'workdir' : Path,

             'rulename' : str,

             'resource': ResourceSet,

             'inp'     : dict,
@@ -158,7 +159,10 @@ class Rule: 
            if not isinstance(self.params[var], cls):

                raise SyntaxError("  Variable '%s' must have type %s (incorrect type)."%(var, str(cls)))



        assert self.params['dirname'].is_dir, "Job directory (%s) not present."%self.params['dirname']

        assert self.params['dirname'].is_dir, \

                "Job directory (%s) not present."%self.params['dirname']

        #assert self.params['workdir'].is_dir, \

        #        "Work directory (%s) not present."%self.params['workdir']



    def format(self, script):

        return script.format(**self.params)
@@ -168,6 +172,7 @@ class Rule: 
        if not self.next_job.all_inputs_present:

            log.error("Not all inputs for %s present.", self.id)

            return True

        self.next_job.job_in.parent.mkdir(parents=True, exist_ok=True)

        with open(self.next_job.job_in,'w') as f:

            f.write(self.format(jobscript % self.params['script']))

        return False
@@ -180,7 +185,9 @@ class Rule: 
            var = ""

        else:

            var = ".%s" % self.params[self.varname]

        self.next_job = JobState(self.params['dirname'], self.params['rulename']+var)

        self.next_job = JobState(self.params['workdir'],

                                 self.params['dirname'],

                                 self.params['rulename']+var)



        if testonly:

            self.f = None
@@ -252,7 +259,7 @@ class RuleTypes: 
        raise KeyError(fname)



# Read yaml-defined make-rules.

def read_rules(filename):

def read_rules(filename, workdir):

    types = RuleTypes()

    if filename is None:

        return types
@@ -261,7 +268,9 @@ def read_rules(filename): 
        x = yaml.safe_load(f)



    for name, info in x.items():

        params = {'rulename': name} # name of make-rule

        params = {'rulename': name,  # name of make-rule

                  'workdir': workdir # prefix for sh,log files

                 }

        dict_merge(params, info)

        types[name] = params

        #type(name, (Rule,), {'params': params})