Commit 27bbfbe0 authored by Juve, Gideon's avatar Juve, Gideon
Browse files

Add AMWG diagnostics stage

parent e59cdfd6
#!/bin/bash
set -e
if [ $# -ne 1 ]; then
echo "Usage: $0 diag140804.csh"
exit 1
fi
# Need to load the modules for the diagnostics code
module load ncl
module load nco
# Chmod the templated script for this stage
chmod 755 $1
# Run the script on a compute node
aprun -n 1 ./$1
#!/usr/bin/env python #!/usr/bin/env python
import sys import sys
import re
import string import string
from ConfigParser import ConfigParser from ConfigParser import ConfigParser
from Pegasus.DAX3 import * from Pegasus.DAX3 import *
...@@ -11,8 +12,13 @@ def format_template(name, outfile, **kwargs): ...@@ -11,8 +12,13 @@ def format_template(name, outfile, **kwargs):
"This fills in the values for the template called 'name' and writes it to 'outfile'" "This fills in the values for the template called 'name' and writes it to 'outfile'"
templatefile = os.path.join(TEMPLATE_DIR, name) templatefile = os.path.join(TEMPLATE_DIR, name)
template = open(templatefile).read() template = open(templatefile).read()
formatter = string.Formatter()
data = formatter.format(template, **kwargs) def repl(match):
key = match.group(1)
return str(kwargs[key])
data = re.sub("\{\{([a-z0-9A-Z-._]+)\}\}", repl, template)
f = open(outfile, "w") f = open(outfile, "w")
try: try:
f.write(data) f.write(data)
...@@ -86,16 +92,48 @@ tr acme-output { ...@@ -86,16 +92,48 @@ tr acme-output {
profile globus "jobtype" "single" profile globus "jobtype" "single"
} }
} }
""" % (DAXGEN_DIR, self.mppwidth, DAXGEN_DIR))
tr acme-amwg {
site local {
pfn "file://%s/bin/acme-amwg.sh"
arch "x86_64"
os "linux"
type "STAGEABLE"
profile globus "hostcount" "1"
profile globus "jobtype" "single"
}
}
""" % (DAXGEN_DIR, self.mppwidth, DAXGEN_DIR, DAXGEN_DIR))
finally: finally:
f.close() f.close()
def generate_amwg_script(self, stage, first_yr, nyrs):
"Generate the amwg script with the appropriate config"
name = "diag140804.stage%s.csh" % stage
path = os.path.join(self.outdir, name)
kw = {
"first_yr": first_yr,
"nyrs": nyrs,
"casename": self.casename,
"stage": stage
}
format_template("diag140804.csh", path, **kw)
self.add_replica(name, path)
return name
def generate_dax(self): def generate_dax(self):
"Generate a workflow (DAX, config files, and replica catalog)" "Generate a workflow (DAX, config files, and replica catalog)"
dax = ADAG(self.casename) dax = ADAG(self.casename)
last = None last = None
if self.stop_option in ["nyear", "nyears"]:
amwg = True
else:
print "WARNING: Diagnostics not added to workflow unles stop option is 'nyears'. Current setting is '%s'" % self.stop_option
amwg = False
tot_years = 0
i = 1 i = 1
for stop_n, walltime in zip(self.stop_n, self.walltime): for stop_n, walltime in zip(self.stop_n, self.walltime):
stage = Job(name="acme-run") stage = Job(name="acme-run")
...@@ -117,7 +155,34 @@ tr acme-output { ...@@ -117,7 +155,34 @@ tr acme-output {
dax.addJob(archive) dax.addJob(archive)
dax.depends(archive, stage) dax.depends(archive, stage)
# TODO Add data analysis job # Figure out how many years we have at this point
cur_years = int(stop_n)
tot_years = tot_years + cur_years
# Add diagnostics job for atmosphere
if amwg:
if tot_years <= 1:
print "WARNING: First stage does not have enough years for diagnostics"
else:
# The first year doesn't count, do no more than 5 years
nyrs = min(tot_years-1, 5)
# Years start at 1, not 0
first_yr = tot_years - nyrs + 1
# Create the amwg script
script_name = self.generate_amwg_script(i, first_yr, nyrs)
script = File(script_name)
diagnostics = File("amwg-stage%s" % i)
# Add the job
diag = Job(name="acme-amwg")
diag.addArguments(script)
diag.uses(script, link=Link.INPUT)
diag.uses(diagnostics, link=Link.OUTPUT, register=False, transfer=True)
dax.addJob(diag)
dax.depends(diag, stage)
last = archive last = archive
i+=1 i+=1
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
<file-server operation="all" url="gsiftp://hoppergrid.nersc.gov/project/projectdirs/m2187" /> <file-server operation="all" url="gsiftp://hoppergrid.nersc.gov/project/projectdirs/m2187" />
</directory> </directory>
<profile namespace="env" key="PEGASUS_HOME">/project/projectdirs/m2187/pegasus/pegasus-4.4.0</profile> <profile namespace="env" key="PEGASUS_HOME">/project/projectdirs/m2187/pegasus/pegasus-4.4.0</profile>
<profile namespace="env" key="DIAG_HOME">/project/projectdirs/m2187/amwg/amwg_diagnostics</profile>
<profile namespace="globus" key="project">m2187</profile> <profile namespace="globus" key="project">m2187</profile>
<profile namespace="globus" key="queue">regular</profile> <profile namespace="globus" key="queue">regular</profile>
</site> </site>
......
#!/bin/csh -f
# file: diag140804.csh
# Updated: 2014/08/04
unset echo verbose
setenv DIAG_VERSION 140804 # version number YYMMDD
#******************************************************************
# C-shell control script for AMWG Diagnostics Package. *
# Written by Dr. Mark J. Stevens, 2001-2003. *
# Updated by many people *
# e-mail: hannay@ucar.edu phone: 303-497-1327 *
# *
# - Please see the AMWG diagnostics package webpage at: *
# https://www2.cesm.ucar.edu/working-groups/amwg/amwg-diagnostics-package
# *
# - Subscribe to the ccsm-amwgdiag mailing list: *
# http://mailman.cgd.ucar.edu/mailman/listinfo/ccsm_amwgdiag *
# to receive updates from the AMWG diagnostic package. *
# *
# Implementation of parallel version with Swift sponsored by the *
# Office of Biological and Environmental Research of the *
# U.S. Department of Energy's Office of Science. *
# *
#******************************************************************
#
#
#******************************************************************
# PLEASE READ THIS *
#******************************************************************
# This script can be placed in any directory provided that
# access to the working directory and local directories are
# available (see below).
#
# Using this script your model output can be compared with
# observations (observed and reanalysis, data)
# or with another model run. With all the diagnostic sets
# turned on the package will produce over 600 plots and
# tables in your working directory. In addition, files are
# produced for the climatological monthly means and the DJF
# and JJA seasonal means, as well as the annual (ANN) means.
#
# Input file names are of the standard CCSM type and
# they must be in netcdf format. Filenames are of the
# form YYYY-MM.nc where YYYY are the model years and MM
# are the months, for example 00010-09.nc. The files
# can reside on the Mass Storage System (MSS), if they
# are on the MSS the script will get them using msrcp.
# If your files are not on the MSS they must be in a local
# directory.
#
# Normally 5 years of monthly means are used for the
# comparisons, however only 1 year of data will work.
# The December just prior to the starting year is also
# needed for the DJF seasonal mean, or Jan and Feb of
# the year following the last full year. For example,
# for 5-year means the following months are needed
#
# 0000-12.nc prior december
# 0001-01.nc first year (must be 0001 or greater)
# ...
# 0001-12.nc
# ...
# 0005-01.nc last year
# ...
# 0005-12.nc
#--> OR you can do this
#
# 0001-01.nc first year (must be 0001 or greater)
# ...
# 0001-12.nc
# ...
# 0005-01.nc last year full year
# ...
# 0005-12.nc
# 0006-01.nc following jan
# 0006-02.nc following feb
#
#******************************************************************
# USER MODIFY SECTION *
# Modify the following sections as needed. *
#******************************************************************
# In the following "test case" refers to the model run to be
# compared with the "control case", which may be model or obs data.
#******************************************************************
#******************************************************************
# *****************
# *** Test case ***
# *****************
# Set the identifying casename and paths for your test case run.
# The output netcdf files are in: $test_path_history
# The climatology files go in: $test_path_climo
# The diagnostic plots go in: $test_path_diag
# The HPSS path (if files doesn't exist locally): $test_path_HPSS
#
# Don t forget the trailing / when setting the paths
set test_casename = {{casename}}
set test_filetype = monthly_history
#set test_filetype = time_series
#set test_path_history = /glade/scratch/hannay/archive/$test_casename/atm/
#set test_path_climo = /glade/scratch/hannay/amwg/climo/$test_casename/
#set test_path_diag = /glade/scratch/hannay/amwg/diag/$test_casename/
set test_path_history = $PWD/run/
set test_path_climo = $PWD/amwg-stage{{stage}}/
set test_path_diag = $PWD/amwg-stage{{stage}}/
set test_path_HPSS = $test_path_history
#******************************************************************
# ********************
# *** Control case ***
# ********************
# Select the type of control case to be compared with your model
# test case (select one).
set CNTL = OBS # observed data (reanalysis etc)
#set CNTL = USER # user defined model control (see below)
#------------------------------------------------------------------
# FOR CNTL == USER ONLY (otherwise skip this section)
# Set the identifying casename and paths for your control case run.
# The output netcdf files are in: $cntl_path_history
# The climatology files are in: $cntl_path_climo
# The HPSS path (if files doesn't exist locally): $cntl_path_HPSS
#
# Don t forget the trailing / when setting the paths
set cntl_casename = f.e11.F2000C5.f09_f09.control.001
set cntl_filetype = monthly_history
#set cntl_filetype = time_series
set cntl_path_history = /glade/scratch/hannay/archive/$cntl_casename/atm/hist/
set cntl_path_climo = /glade/scratch/hannay/amwg/climo/$cntl_casename/
set cntl_path_HPSS = /CCSM/csm/$cntl_casename/atm/hist/
#******************************************************************
# *********************
# *** Climatologies ***
# *********************
# Use these settings if computing climatological means
# from the local test case data and/or local control case data
#-----------------------------------------------------------------
# Turn on/off the computation of climatologies
set test_compute_climo = 0 # (0=ON,1=OFF)
set cntl_compute_climo = 1 # (0=ON,1=OFF)
#-----------------------------------------------------------------
# If computing climatological means for test/cntl case, specify the first
# year of your data, and the number of years of data to be used.
# First year of data is: $test_first_yr (must be >= 1)
# Number of years is: $test_nyrs (must be >= 1)
set test_first_yr = {{first_yr}} # first year (must be >= 1)
set test_nyrs = {{nyrs}} # number of yrs (must be >= 1)
# FOR CNTL == USER ONLY (otherwise skip this section)
# First year of data is: $cntl_first_yr (must be >= 1)
# Number of years is: $cntl_nyrs (must be >= 1)
set cntl_first_yr = 2 # first year (must be >= 1)
set cntl_nyrs = 4 # number of yrs (must be >= 1)
#-----------------------------------------------------------------
# Strip off all the variables that are not required by the AMWG package
# in the computation of the climatology
# set to OFF for running with Chemistry
set strip_off_vars = 0 # (0=ON,1=OFF)
#-----------------------------------------------------------------
# Weight the months by their number of days when computing
# averages for ANN, DJF, JJA. This takes much longer to compute
# the climatologies. Many users might not care about the small
# differences and leave this turned off.
set weight_months = 0 # (0=ON,1=OFF)
#******************************************************************
# ******************************
# *** Select diagnostic sets ***
# ******************************
# Select the diagnostic sets to be done. You can do one at a
# time or as many as you want at one time, or all at once.
set all_sets = 1 # (0=ON,1=OFF) Do all the CAM sets (1-16)
set set_1 = 0 # (0=ON,1=OFF) tables of global,regional means
set set_2 = 0 # (0=ON,1=OFF) implied transport plots
set set_3 = 1 # (0=ON,1=OFF) zonal mean line plots
set set_4 = 1 # (0=ON,1=OFF) vertical zonal mean contour plots
set set_4a = 1 # (0=ON,1=OFF) vertical zonal mean contour plots
set set_5 = 1 # (0=ON,1=OFF) 2D-field contour plots
set set_6 = 1 # (0=ON,1=OFF) 2D-field vector plots
set set_7 = 1 # (0=ON,1=OFF) 2D-field polar plots
set set_8 = 1 # (0=ON,1=OFF) annual cycle (vs lat) contour plots
set set_9 = 1 # (0=ON,1=OFF) DJF-JJA difference plots
set set_10 = 1 # (0=ON,1=OFF) annual cycle line plots
set set_11 = 1 # (0=ON,1=OFF) miscellaneous plots
set set_12 = 1 # (0=selected stations: 1=NONE, 2=ALL stations
set set_13 = 1 # (0=ON,1=OFF) COSP cloud simulator plots
set set_14 = 1 # (0=ON,1=OFF) Taylor diagram plots
set set_15 = 1 # (0=ON,1=OFF) Annual Cycle Plots for Select stations
set set_16 = 1 # (0=ON,1=OFF) Budget Terms for Select stations
set all_waccm_sets = 1 # (0=ON,1=OFF) Do all the WACCM sets
set all_chem_sets = 1 # (0=ON,1=OFF) Do all the CHEM sets
set wset_1 = 0 # (0=ON,1=OFF) vertical zonal mean contour plots (log scale)
set cset_1 = 0 # (0=ON,1=OFF) tables of global budgets
set cset_2 = 0 # (0=ON,1=OFF) vertical zonal mean contour plots (log scale)
set cset_3 = 0 # (0=ON,1=OFF) Ozonesonde comparisions
set cset_4 = 0 # (0=ON,1=OFF) Column Ozone/CO Comparisons
set cset_5 = 0 # (0=ON,1=OFF) NOAA Aircraft comparisons
set cset_6 = 0 # (0=ON,1=OFF) Emmons Aircraft climatology
set cset_7 = 0 # (0=ON,1=OFF) surface comparisons (ozone, co, improve)
# Select the control case to compare against for Taylor Diagrams
# Cam run select cam3_5; coupled run select ccsm3_5
setenv TAYLOR_BASECASE ccsm3_5 # Base case to compare against
# Options are cam3_5, ccsm3_5
# They are both fv_1.9x2.5
#******************************************************************
# **************************************
# *** Customize plots (output/style) ***
# **************************************
# Select seasonal output to be plotted
# four_seasons = 0 # DJF, MAM, JJA, SON, ANN
# four_seasons = 1 # DJF, JJA, ANN
# four_seasons = 2 # Custom: Select the season you want to be plotted
# Note: four_seasons is not currently supported for model vs OBS diagnostics.
# if ($CNTL == OBS) then four_seasons is turned OFF.
set four_seasons = 1 # (0=ON; 1=OFF)
#------------------------------------------------------------------
# For four_seasons == 2 (otherwise skip this section)
# Select the seasons you want to be plotted
if ($four_seasons == 2) then
set plot_ANN_climo = 0 # (0=ON,1=OFF) used by sets 1-7,11
set plot_DJF_climo = 0 # (0=ON,1=OFF) used by sets 1,3-7,9,11
set plot_JJA_climo = 0 # (0=ON,1=OFF) used by sets 1,3-7,9,11
set plot_MAM_climo = 0 # (0=ON,1=OFF) used by sets 1,3-7,9,11
set plot_SON_climo = 0 # (0=ON,1=OFF) used by sets 1,3-7,9,11
set plot_MON_climo = 0 # (0=ON,1=OFF) used by sets 8,10,11,12
endif
#-----------------------------------------------------------------
# Select the output file type and style for plots.
set p_type = ps # postscript
#set p_type = pdf # portable document format (ncl ver 4.2.0.a028)
#set p_type = eps # encapsulated postscript
#set p_type = epsi # encapsulated postscript with bitmap
#set p_type = ncgm # ncar computer graphics metadata
#-------------------------------------------------------------------
# Select the output color type for plots.
set c_type = COLOR # color
#set c_type = MONO # black and white
# If needed select one of the following color schemes,
# you can see the colors by clicking on the links from
# http://www.cgd.ucar.edu/cms/diagnostics
set color_bar = default # the usual colors
set color_bar = blue_red # blue,red
#set color_bar = blue_yellow_red # blue,yellow,red (nice!)
#----------------------------------------------------------------
# Turn ON/OFF date/time stamp at bottom of plots.
# Leaving this OFF makes the plots larger.
set time_stamp = 1 # (0=ON,1=OFF)
#---------------------------------------------------------------
# Turn ON/OFF tick marks and labels for sets 5,6, and 7
# Turning these OFF make the areas plotted larger, which makes
# the images easier to look at.
set tick_marks = 1 # (0=ON,1=OFF)
#----------------------------------------------------------------
# Use custom case names for the PLOTS instead of the
# case names encoded in the netcdf files (default).
# Also useful for publications.
set custom_names = 1 # (0=ON,1=OFF)
# if needed set the names
set test_name = cam3_5_test # test case name
set cntl_name = cam3_5_cntl # control case name
#----------------------------------------------------------------
# Convert output postscript files to GIF, JPG or PNG image files
# and place them in subdirectories along with html files.
# Then make a tar file of the web pages and GIF,JPG or PNG files.
# On Dataproc and CGD Suns GIF images are smallest since I built
# ImageMagick from source and compiled in the LZW compression.
# On Linux systems JPG will be smallest if you have an rpm or
# binary distribution of ImageMagick (and hence convert) since
# NO LZW compression is the default. Only works if you have
# convert on your system and for postscript files (p_type = ps).
# NOTE: Unless you have rebuilt ImageMagick on your Linux system
# the GIF files can be as large as the postscript plots. I
# recommend that PNG always be used. The density option can be
# used with convert to make higher resolution images which will
# work better in powerpoint presentations, try density = 150.
set web_pages = 0 # (0=ON,1=OFF) make images and html files
set delete_ps = 0 # (0=ON,1=OFF) delete postscript files
set img_type = 0 # (0=PNG,1=GIF,2=JPG) select image type
set density = 85 # pixels/inch, use larger number for higher
# resolution images (default is 85)
#----------------------------------------------------------------
# Save the output netcdf files of the derived variables
# used to make the plots. These are normally deleted
# after the plots are made. If you want to save the
# netcdf files for your own uses then switch to ON.
set save_ncdfs = 1 # (0=ON,1=OFF)
#----------------------------------------------------------------
# Compute whether the means of the test case and control case
# are significantly different from each other at each grid point.
# Tests are performed only for model-to-model comparisons.
# REQUIRES at least 10 years of model data for each case.
# Number of years from above (test_nyrs and cntl_nyrs) is used.
# Also set the significance level for the t-test.
set significance = 1 # (0=ON,1=OFF)
# if needed set default level
set sig_lvl = 0.05 # level of significance
#******************************************************************
# ***************************
# *** Source code location ***
# ***************************
# Below is defined the amwg diagnostic package root location
# on CGD machines (tramhill,...), on old CSIL machines (mirage),
# on new CSIL machines (geyser), NERSC (euclid), and LBNL (lens).
#
# If you are installing the diagnostic package on your computer system.
# you need to set DIAG_HOME to the root location of the diagnostic code.
# The code is in $DIAG_HOME/code
# The obs data in $DIAG_HOME/obs_data
# The cam3.5 data in $DIAG_HOME/cam35_data
# CGD machines (tramhill, leehill...)
#setenv DIAG_HOME /project/amp/amwg/amwg_diagnostics
# CSIL machines (geyser, caldeira, ...)
#setenv DIAG_HOME /glade/p/cesm/amwg/amwg_diagnostics
# NERSC (euclid)
#setenv DIAG_HOME /global/homes/h/hannay/amwg/amwg_diagnostics
#setenv DIAG_HOME /project/projectdirs/m2187/amwg/amwg_diagnostics
# NCSS (lens)
#setenv DIAG_HOME /ccs/home/hannay/amwg/amwg_diagnostics
#*****************************************************************
# ****************************
# *** Additional settings ***
# ****************************
# Send yourself an e-mail message when everything is done.
set email = 1 # (0=ON,1=OFF)
set email_address = ${LOGNAME}@ucar.edu
#*****************************************************************
#*****************************************************************
# **************************
# *** Advanced settings ***
# **************************
#*****************************************************************
#-------------------------------------------------
# For CAM-SE grid, specify a lat/lon to interpolate to
#-------------------------------------------------
# By default, teh CAM-SE output is interpolated CAM_SE on a 1 degree grid
# You can select another grid below.
set test_res_out = 0.9x1.25
set cntl_res_out = 0.9x1.25
# Set the interpolation method for regridding: bilinear, patch, conserver
setenv INTERP_METHOD bilinear
#-------------------------------------------------
# Set to 0 to use swift
#-------------------------------------------------
setenv use_swift 1 # (0=ON,1=OFF)
setenv swift_scratch_dir /glade/scratch/$USER/swift_scratch/
set test_inst = -1
set cntl_inst = -1
#-------------------------------------------------
# For set 12:
#-------------------------------------------------
# Select vertical profiles to be computed. Select from list below,
# or do all stations, or none. You must have computed the monthly
# climatological means for this to work. Preset to the 17 selected
# stations.
# Specify selected stations for computing vertical profiles.
if ($set_12 == 0 || $all_sets == 0) then
# ARCTIC (60N-90N)
set western_alaska = 1 # (0=ON,1=OFF)
set whitehorse_canada = 1 # (0=ON,1=OFF)
set resolute_canada = 0 # (0=ON,1=OFF)
set thule_greenland = 0 # (0=ON,1=OFF)
# NORTHERN MIDLATITUDES (23N-60N)
set new_dehli_india = 1 # (0=ON,1=OFF)
set kagoshima_japan = 1 # (0=ON,1=OFF)
set tokyo_japan = 1 # (0=ON,1=OFF)
set midway_island = 0 # (0=ON,1=OFF)
set shipP_gulf_alaska = 0 # (0=ON,1=OFF)
set san_francisco_ca = 0 # (0=ON,1=OFF)
set denver_colorado = 1 # (0=ON,1=OFF)
set great_plains_usa = 0 # (0=ON,1=OFF)
set oklahoma_city_ok = 1 # (0=ON,1=OFF)
set miami_florida = 0 # (0=ON,1=OFF)
set new_york_usa = 1 # (0=ON,1=OFF)
set w_north_atlantic = 1 # (0=ON,1=OFF)
set shipC_n_atlantic = 1 # (0=ON,1=OFF)
set azores = 1 # (0=ON,1=OFF)
set gibraltor = 1 # (0=ON,1=OFF)
set london_england = 1 # (0=ON,1=OFF)
set western_europe = 0 # (0=ON,1=OFF)
set crete = 1 # (0=ON,1=OFF)
# TROPICS (23N-23S)
set central_india = 1 # (0=ON,1=OFF)
set madras_india = 1 # (0=ON,1=OFF)
set diego_garcia = 0 # (0=ON,1=OFF)
set cocos_islands = 1 # (0=ON,1=OFF)
set christmas_island = 1 # (0=ON,1=OFF)
set singapore = 1 # (0=ON,1=OFF)
set danang_vietnam = 1 # (0=ON,1=OFF)
set manila = 1 # (0=ON,1=OFF)
set darwin_australia = 1 # (0=ON,1=OFF)
set yap_island = 0 # (0=ON,1=OFF)
set port_moresby = 1 # (0=ON,1=OFF)
set truk_island = 0 # (0=ON,1=OFF)
set raoui_island = 1 # (0=ON,1=OFF)
set gilbert_islands = 1 # (0=ON,1=OFF)
set marshall_islands = 0 # (0=ON,1=OFF)
set samoa = 1 # (0=ON,1=OFF)
set hawaii = 0 # (0=ON,1=OFF)
set panama = 0 # (0=ON,1=OFF)
set mexico_city = 1 # (0=ON,1=OFF)
set lima_peru = 1 # (0=ON,1=OFF)