Commit 88509201 authored by Juve, Gideon's avatar Juve, Gideon
Browse files

Add instructions, inital configuration

parent ec56a893
CESM-Workflow
ACME-Workflow
=============
Pegasus workflow for CESM climate models
Pegasus workflow for ACME climate models.
On Hopper
---------
You need to create the case while logged into Hopper.
In the following, assume that these variables are set to the values below:
ACMEPATH = /global/project/projectdirs/m2187/acme_code/ACME
CASENAME = F1850.g37.case2
SCRATCHDIR = /scratch/scratchdirs/juve/
CASEDIR = $SCRATCHDIR/$CASENAME
Note that the "shared-scratch" directory in your site catalog must match the
value for SCRATCHDIR above. This is because we use CASENAME as the --relative-dir
option for the Pegasus planner so that CASEDIR will become Pegasus' scratch dir.
1. Create the case directory. From $ACMEPATH/scripts run:
$ ./create_newcase -case $CASEDIR -mach hopper -compset F1850 -res T31_g37 -project m2187
In this case "m2187" is our Hopper project ID, and "mach" means "machine"
and should be set to "hopper". The "compset" defines what initial conditions
you want to use, and "res" specifies the resolution. Possible compsets: F1850, B1850.
Possible resolutions: ne30_g16, T31_g37.
2. Make any manual changes to the case that are required for your simulation.
3. Setup the case. From $CASEDIR run:
$ ./cesm_setup
4. Compile the code. This should take about 20 minutes. From $CASEDIR run:
$ ./$CASENAME.build
On Submit Host
--------------
1. Create/edit the configuration file (e.g. test.cfg)
2. Generate the DAX
$ python daxgen.py test.cfg $CASENAME
3. Edit the site catalog, sites.xml:
a. Update the "shared-scratch" directory entry to have your username
b. Update the "shared-storage" directory entry
4. Plan the DAX
$ ./plan.sh $CASENAME
5. Get NERSC grid proxy using:
$ myproxy-logon -s nerscca.nersc.gov:7512 -t 24 -T -l YOUR_NERSC_USERNAME
6. Follow output of plan.sh to submit workflow
$ pegasus-run $CASENAME/submit/$CASENAME
7. Monitor the workflow:
$ pegasus-status -l $CASENAME/submit/$CASENAME
#!/bin/bash
function usage () {
echo "Usage: $0 -stop STOP_OPTION -n STOP_N -run RUN"
}
if [ "$#" == "0" ]; then
usage
exit 1
fi
while [ "$#" -ne 0 ]; do
case "$1" in
-case)
shift
CASE=$1
;;
-stop)
shift
STOP_OPTION=$1
;;
-n)
shift
STOP_N=$1
;;
-run)
shift
RUN=$1
;;
*)
usage
exit 1
;;
esac
shift
done
echo RUN $RUN
echo STOP_N $STOP_N
echo STOP_OPTION $STOP_OPTION
exit 0
cd $PBS_O_WORKDIR
CASE=$(./xmlquery CASE -valonly -silent)
./xmlchange -file env_run.xml -id RUNDIR -val $PWD/$RUN
./xmlchange -file env_run.xml -id STOP_N -val $STOP_N
./xmlchange -file env_run.xml -id STOP_OPTION -val $STOP_OPTION
./xmlchange -file env_run.xml -id RESUBMIT -val 0
./xmlchange -file env_run.xml -id DOUT_S_ROOT -val \$CASEROOT/archive
exec $CASE.run
#!/usr/bin/env python
import sys
import string
from ConfigParser import ConfigParser
from Pegasus.DAX3 import *
DAXGEN_DIR = os.path.dirname(os.path.realpath(__file__))
TEMPLATE_DIR = os.path.join(DAXGEN_DIR, "templates")
def format_template(name, outfile, **kwargs):
"This fills in the values for the template called 'name' and writes it to 'outfile'"
templatefile = os.path.join(TEMPLATE_DIR, name)
template = open(templatefile).read()
formatter = string.Formatter()
data = formatter.format(template, **kwargs)
f = open(outfile, "w")
try:
f.write(data)
finally:
f.close()
class ACMEWorkflow(object):
def __init__(self, outdir, config):
"'outdir' is the directory where the workflow is written, and 'config' is a ConfigParser object"
self.outdir = outdir
self.config = config
self.daxfile = os.path.join(self.outdir, "dax.xml")
self.replicas = {}
def add_replica(self, name, path):
"Add a replica entry to the replica catalog for the workflow"
url = "file://%s" % path
self.replicas[name] = url
def generate_replica_catalog(self):
"Write the replica catalog for this workflow to a file"
path = os.path.join(self.outdir, "rc.txt")
f = open(path, "w")
try:
for name, url in self.replicas.items():
f.write('%-30s %-100s pool="local"\n' % (name, url))
finally:
f.close()
def generate_transformation_catalog(self):
"Write the transformation catalog for this workflow to a file"
path = os.path.join(self.outdir, "tc.txt")
f = open(path, "w")
try:
f.write("""
tr acme {
site local {
pfn "file://%s/bin/acme.sh"
arch "x86_64"
os "linux"
type "STAGEABLE"
}
}
""" % DAXGEN_DIR)
finally:
f.close()
def generate_workflow(self):
"Generate a workflow (DAX, config files, and replica catalog)"
dax = ADAG("refinement")
stage = Job(name="acme")
stage.addArguments("-run stage1 -stop days -n 5")
dax.addJob(stage)
# Write the DAX file
dax.writeXMLFile(self.daxfile)
self.generate_replica_catalog()
self.generate_transformation_catalog()
def main():
if len(sys.argv) != 3:
raise Exception("Usage: %s CONFIGFILE OUTDIR" % sys.argv[0])
configfile = sys.argv[1]
outdir = sys.argv[2]
if not os.path.isfile(configfile):
raise Exception("No such file: %s" % configfile)
if os.path.isdir(outdir):
raise Exception("Directory exists: %s" % outdir)
# Create the output directory
outdir = os.path.abspath(outdir)
os.makedirs(outdir)
# Read the config file
config = ConfigParser()
config.read(configfile)
# Generate the workflow in outdir based on the config file
workflow = ACMEWorkflow(outdir, config)
workflow.generate_workflow()
if __name__ == '__main__':
main()
pegasus.register = false
pegasus.data.configuration = sharedfs
pegasus.stagein.clusters = 1
pegasus.stageout.clusters = 1
#!/bin/bash
SITE=hopper
OUTPUT_SITE=hopper
if [ $# -ne 1 ]; then
echo "Usage: $0 WORKFLOW_DIR"
exit 1
fi
WORKFLOW_DIR=$1
if [ -d "$WORKFLOW_DIR" ]; then
WORKFLOW_DIR=$(cd $WORKFLOW_DIR && pwd)
else
echo "No such directory: $WORKFLOW_DIR"
exit 1
fi
DIR=$(cd $(dirname $0) && pwd)
SUBMIT_DIR=$WORKFLOW_DIR/submit
DAX=$WORKFLOW_DIR/dax.xml
TC=$WORKFLOW_DIR/tc.txt
RC=$WORKFLOW_DIR/rc.txt
SC=$DIR/sites.xml
PP=$DIR/pegasus.properties
echo "Planning workflow..."
pegasus-plan \
-Dpegasus.catalog.site.file=$SC \
-Dpegasus.catalog.replica=File \
-Dpegasus.catalog.replica.file=$RC \
-Dpegasus.catalog.transformation.file=$TC \
--conf $PP \
--dax $DAX \
--dir $SUBMIT_DIR \
--relative-dir MYCASE \
--sites $SITE \
--output-site $OUTPUT_SITE \
--cleanup none \
--force
<?xml version="1.0" encoding="UTF-8"?>
<sitecatalog xmlns="http://pegasus.isi.edu/schema/sitecatalog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://pegasus.isi.edu/schema/sitecatalog http://pegasus.isi.edu/schema/sc-4.0.xsd"
version="4.0">
<site handle="local" arch="x86_64" os="LINUX">
</site>
<site handle="hopper" arch="x86_64" os="LINUX">
<grid type="gt5" contact="hoppergrid.nersc.gov/jobmanager" scheduler="Fork" jobtype="auxillary"/>
<grid type="gt5" contact="hoppergrid.nersc.gov/jobmanager-pbs" scheduler="PBS" jobtype="compute"/>
<!-- Your case dir must be in shared-scratch. Put your username here instead of "juve". -->
<directory type="shared-scratch" path="/scratch/scratchdirs/juve">
<file-server operation="all" url="gsiftp://hoppergrid.nersc.gov/scratch/scratchdirs/juve"/>
</directory>
<directory type="shared-storage" path="/project/projectdirs/m2187/pegasus">
<file-server operation="all" url="gsiftp://hoppergrid.nersc.gov/project/projectdirs/m2187/pegasus" />
</directory>
<profile namespace="env" key="PEGASUS_HOME">/project/projectdirs/m2187/pegasus/pegasus-4.4.0</profile>
<profile namespace="globus" key="project">m2187</profile>
<profile namespace="globus" key="queue">regular</profile>
</site>
</sitecatalog>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment