Add instructions, inital configuration

README.md

-CESM-Workflow
+ACME-Workflow
 =============
 
-Pegasus workflow for CESM climate models
+Pegasus workflow for ACME climate models.
+
+On Hopper
+---------
+You need to create the case while logged into Hopper.
+
+In the following, assume that these variables are set to the values below:
+
+    ACMEPATH = /global/project/projectdirs/m2187/acme_code/ACME
+    CASENAME = F1850.g37.case2
+    SCRATCHDIR = /scratch/scratchdirs/juve/
+    CASEDIR = $SCRATCHDIR/$CASENAME
+
+Note that the "shared-scratch" directory in your site catalog must match the
+value for SCRATCHDIR above. This is because we use CASENAME as the --relative-dir
+option for the Pegasus planner so that CASEDIR will become Pegasus' scratch dir.
+
+1. Create the case directory. From $ACMEPATH/scripts run:
+
+    $ ./create_newcase -case $CASEDIR -mach hopper -compset F1850 -res T31_g37 -project m2187
+
+ In this case "m2187" is our Hopper project ID, and "mach" means "machine"
+ and should be set to "hopper". The "compset" defines what initial conditions
+ you want to use, and "res" specifies the resolution. Possible compsets: F1850, B1850.
+ Possible resolutions: ne30_g16, T31_g37.
+
+2. Make any manual changes to the case that are required for your simulation.
+
+3. Setup the case. From $CASEDIR run:
+
+    $ ./cesm_setup
+
+4. Compile the code. This should take about 20 minutes. From $CASEDIR run:
+
+    $ ./$CASENAME.build
+
+
+On Submit Host
+--------------
+
+1. Create/edit the configuration file (e.g. test.cfg)
+
+2. Generate the DAX
+
+    $ python daxgen.py test.cfg $CASENAME
+
+3. Edit the site catalog, sites.xml:
+
+ a. Update the "shared-scratch" directory entry to have your username
+
+ b. Update the "shared-storage" directory entry
+
+4. Plan the DAX
+
+    $ ./plan.sh $CASENAME
+
+5. Get NERSC grid proxy using:
+
+    $ myproxy-logon -s nerscca.nersc.gov:7512 -t 24 -T -l YOUR_NERSC_USERNAME
+
+6. Follow output of plan.sh to submit workflow
+
+    $ pegasus-run $CASENAME/submit/$CASENAME
+
+7. Monitor the workflow:
+
+    $ pegasus-status -l $CASENAME/submit/$CASENAME
+

bin/acme.sh

+#!/bin/bash
+
+function usage () {
+    echo "Usage: $0 -stop STOP_OPTION -n STOP_N -run RUN"
+}
+
+if [ "$#" == "0" ]; then
+    usage
+    exit 1
+fi
+
+while [ "$#" -ne 0 ]; do
+    case "$1" in
+        -case)
+            shift
+            CASE=$1
+            ;;
+        -stop)
+            shift
+            STOP_OPTION=$1
+            ;;
+        -n)
+            shift
+            STOP_N=$1
+            ;;
+        -run)
+            shift
+            RUN=$1
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+echo RUN $RUN
+echo STOP_N $STOP_N
+echo STOP_OPTION $STOP_OPTION
+
+exit 0
+
+cd $PBS_O_WORKDIR
+
+CASE=$(./xmlquery CASE -valonly -silent)
+
+./xmlchange -file env_run.xml -id RUNDIR -val $PWD/$RUN
+./xmlchange -file env_run.xml -id STOP_N -val $STOP_N
+./xmlchange -file env_run.xml -id STOP_OPTION -val $STOP_OPTION
+./xmlchange -file env_run.xml -id RESUBMIT -val 0
+./xmlchange -file env_run.xml -id DOUT_S_ROOT -val \$CASEROOT/archive
+
+exec $CASE.run
+

daxgen.py

+#!/usr/bin/env python
+import sys
+import string
+from ConfigParser import ConfigParser
+from Pegasus.DAX3 import *
+
+DAXGEN_DIR = os.path.dirname(os.path.realpath(__file__))
+TEMPLATE_DIR = os.path.join(DAXGEN_DIR, "templates")
+
+def format_template(name, outfile, **kwargs):
+    "This fills in the values for the template called 'name' and writes it to 'outfile'"
+    templatefile = os.path.join(TEMPLATE_DIR, name)
+    template = open(templatefile).read()
+    formatter = string.Formatter()
+    data = formatter.format(template, **kwargs)
+    f = open(outfile, "w")
+    try:
+        f.write(data)
+    finally:
+        f.close()
+
+class ACMEWorkflow(object):
+    def __init__(self, outdir, config):
+        "'outdir' is the directory where the workflow is written, and 'config' is a ConfigParser object"
+        self.outdir = outdir
+        self.config = config
+        self.daxfile = os.path.join(self.outdir, "dax.xml")
+        self.replicas = {}
+
+    def add_replica(self, name, path):
+        "Add a replica entry to the replica catalog for the workflow"
+        url = "file://%s" % path
+        self.replicas[name] = url
+
+    def generate_replica_catalog(self):
+        "Write the replica catalog for this workflow to a file"
+        path = os.path.join(self.outdir, "rc.txt")
+        f = open(path, "w")
+        try:
+            for name, url in self.replicas.items():
+                f.write('%-30s %-100s pool="local"\n' % (name, url))
+        finally:
+            f.close()
+
+    def generate_transformation_catalog(self):
+        "Write the transformation catalog for this workflow to a file"
+        path = os.path.join(self.outdir, "tc.txt")
+        f = open(path, "w")
+        try:
+            f.write("""
+tr acme {
+    site local {
+        pfn "file://%s/bin/acme.sh"
+        arch "x86_64"
+        os "linux"
+        type "STAGEABLE"
+    }
+}
+""" % DAXGEN_DIR)
+        finally:
+            f.close()
+
+    def generate_workflow(self):
+        "Generate a workflow (DAX, config files, and replica catalog)"
+        dax = ADAG("refinement")
+
+        stage = Job(name="acme")
+        stage.addArguments("-run stage1 -stop days -n 5")
+        dax.addJob(stage)
+
+        # Write the DAX file
+        dax.writeXMLFile(self.daxfile)
+
+        self.generate_replica_catalog()
+
+        self.generate_transformation_catalog()
+
+def main():
+    if len(sys.argv) != 3:
+        raise Exception("Usage: %s CONFIGFILE OUTDIR" % sys.argv[0])
+
+    configfile = sys.argv[1]
+    outdir = sys.argv[2]
+
+    if not os.path.isfile(configfile):
+        raise Exception("No such file: %s" % configfile)
+
+    if os.path.isdir(outdir):
+        raise Exception("Directory exists: %s" % outdir)
+
+    # Create the output directory
+    outdir = os.path.abspath(outdir)
+    os.makedirs(outdir)
+
+    # Read the config file
+    config = ConfigParser()
+    config.read(configfile)
+
+    # Generate the workflow in outdir based on the config file
+    workflow = ACMEWorkflow(outdir, config)
+    workflow.generate_workflow()
+
+
+if __name__ == '__main__':
+    main()
+

pegasus.properties

+pegasus.register = false
+pegasus.data.configuration = sharedfs
+pegasus.stagein.clusters = 1
+pegasus.stageout.clusters = 1

plan.sh

+#!/bin/bash
+
+SITE=hopper
+OUTPUT_SITE=hopper
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 WORKFLOW_DIR"
+    exit 1
+fi
+
+WORKFLOW_DIR=$1
+
+if [ -d "$WORKFLOW_DIR" ]; then
+    WORKFLOW_DIR=$(cd $WORKFLOW_DIR && pwd)
+else
+    echo "No such directory: $WORKFLOW_DIR"
+    exit 1
+fi
+
+DIR=$(cd $(dirname $0) && pwd)
+SUBMIT_DIR=$WORKFLOW_DIR/submit
+DAX=$WORKFLOW_DIR/dax.xml
+TC=$WORKFLOW_DIR/tc.txt
+RC=$WORKFLOW_DIR/rc.txt
+SC=$DIR/sites.xml
+PP=$DIR/pegasus.properties
+
+echo "Planning workflow..."
+pegasus-plan \
+    -Dpegasus.catalog.site.file=$SC \
+    -Dpegasus.catalog.replica=File \
+    -Dpegasus.catalog.replica.file=$RC \
+    -Dpegasus.catalog.transformation.file=$TC \
+    --conf $PP \
+    --dax $DAX \
+    --dir $SUBMIT_DIR \
+    --relative-dir MYCASE \
+    --sites $SITE \
+    --output-site $OUTPUT_SITE \
+    --cleanup none \
+    --force
+

sites.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<sitecatalog xmlns="http://pegasus.isi.edu/schema/sitecatalog"
+             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+             xsi:schemaLocation="http://pegasus.isi.edu/schema/sitecatalog http://pegasus.isi.edu/schema/sc-4.0.xsd"
+             version="4.0">
+
+    <site handle="local" arch="x86_64" os="LINUX">
+    </site>
+
+    <site handle="hopper" arch="x86_64" os="LINUX">
+        <grid type="gt5" contact="hoppergrid.nersc.gov/jobmanager" scheduler="Fork" jobtype="auxillary"/>
+        <grid type="gt5" contact="hoppergrid.nersc.gov/jobmanager-pbs" scheduler="PBS" jobtype="compute"/>
+        <!-- Your case dir must be in shared-scratch. Put your username here instead of "juve". -->
+        <directory type="shared-scratch" path="/scratch/scratchdirs/juve">
+            <file-server operation="all" url="gsiftp://hoppergrid.nersc.gov/scratch/scratchdirs/juve"/>
+        </directory>
+        <directory type="shared-storage" path="/project/projectdirs/m2187/pegasus">
+            <file-server operation="all" url="gsiftp://hoppergrid.nersc.gov/project/projectdirs/m2187/pegasus" />
+        </directory>
+        <profile namespace="env" key="PEGASUS_HOME">/project/projectdirs/m2187/pegasus/pegasus-4.4.0</profile>
+        <profile namespace="globus" key="project">m2187</profile>
+        <profile namespace="globus" key="queue">regular</profile>
+    </site>
+</sitecatalog>