Commit 27bbfbe0 authored by Juve, Gideon's avatar Juve, Gideon
Browse files

Add AMWG diagnostics stage

parent e59cdfd6
#!/bin/bash
set -e
if [ $# -ne 1 ]; then
echo "Usage: $0 diag140804.csh"
exit 1
fi
# Need to load the modules for the diagnostics code
module load ncl
module load nco
# Chmod the templated script for this stage
chmod 755 $1
# Run the script on a compute node
aprun -n 1 ./$1
#!/usr/bin/env python
import sys
import re
import string
from ConfigParser import ConfigParser
from Pegasus.DAX3 import *
......@@ -11,8 +12,13 @@ def format_template(name, outfile, **kwargs):
"This fills in the values for the template called 'name' and writes it to 'outfile'"
templatefile = os.path.join(TEMPLATE_DIR, name)
template = open(templatefile).read()
formatter = string.Formatter()
data = formatter.format(template, **kwargs)
def repl(match):
key = match.group(1)
return str(kwargs[key])
data = re.sub("\{\{([a-z0-9A-Z-._]+)\}\}", repl, template)
f = open(outfile, "w")
try:
f.write(data)
......@@ -86,16 +92,48 @@ tr acme-output {
profile globus "jobtype" "single"
}
}
""" % (DAXGEN_DIR, self.mppwidth, DAXGEN_DIR))
tr acme-amwg {
site local {
pfn "file://%s/bin/acme-amwg.sh"
arch "x86_64"
os "linux"
type "STAGEABLE"
profile globus "hostcount" "1"
profile globus "jobtype" "single"
}
}
""" % (DAXGEN_DIR, self.mppwidth, DAXGEN_DIR, DAXGEN_DIR))
finally:
f.close()
def generate_amwg_script(self, stage, first_yr, nyrs):
"Generate the amwg script with the appropriate config"
name = "diag140804.stage%s.csh" % stage
path = os.path.join(self.outdir, name)
kw = {
"first_yr": first_yr,
"nyrs": nyrs,
"casename": self.casename,
"stage": stage
}
format_template("diag140804.csh", path, **kw)
self.add_replica(name, path)
return name
def generate_dax(self):
"Generate a workflow (DAX, config files, and replica catalog)"
dax = ADAG(self.casename)
last = None
if self.stop_option in ["nyear", "nyears"]:
amwg = True
else:
print "WARNING: Diagnostics not added to workflow unles stop option is 'nyears'. Current setting is '%s'" % self.stop_option
amwg = False
tot_years = 0
i = 1
for stop_n, walltime in zip(self.stop_n, self.walltime):
stage = Job(name="acme-run")
......@@ -117,7 +155,34 @@ tr acme-output {
dax.addJob(archive)
dax.depends(archive, stage)
# TODO Add data analysis job
# Figure out how many years we have at this point
cur_years = int(stop_n)
tot_years = tot_years + cur_years
# Add diagnostics job for atmosphere
if amwg:
if tot_years <= 1:
print "WARNING: First stage does not have enough years for diagnostics"
else:
# The first year doesn't count, do no more than 5 years
nyrs = min(tot_years-1, 5)
# Years start at 1, not 0
first_yr = tot_years - nyrs + 1
# Create the amwg script
script_name = self.generate_amwg_script(i, first_yr, nyrs)
script = File(script_name)
diagnostics = File("amwg-stage%s" % i)
# Add the job
diag = Job(name="acme-amwg")
diag.addArguments(script)
diag.uses(script, link=Link.INPUT)
diag.uses(diagnostics, link=Link.OUTPUT, register=False, transfer=True)
dax.addJob(diag)
dax.depends(diag, stage)
last = archive
i+=1
......
......@@ -18,6 +18,7 @@
<file-server operation="all" url="gsiftp://hoppergrid.nersc.gov/project/projectdirs/m2187" />
</directory>
<profile namespace="env" key="PEGASUS_HOME">/project/projectdirs/m2187/pegasus/pegasus-4.4.0</profile>
<profile namespace="env" key="DIAG_HOME">/project/projectdirs/m2187/amwg/amwg_diagnostics</profile>
<profile namespace="globus" key="project">m2187</profile>
<profile namespace="globus" key="queue">regular</profile>
</site>
......
#!/bin/csh -f
# file: diag140804.csh
# Updated: 2014/08/04
unset echo verbose
setenv DIAG_VERSION 140804 # version number YYMMDD
#******************************************************************
# C-shell control script for AMWG Diagnostics Package. *
# Written by Dr. Mark J. Stevens, 2001-2003. *
# Updated by many people *
# e-mail: hannay@ucar.edu phone: 303-497-1327 *
# *
# - Please see the AMWG diagnostics package webpage at: *
# https://www2.cesm.ucar.edu/working-groups/amwg/amwg-diagnostics-package
# *
# - Subscribe to the ccsm-amwgdiag mailing list: *
# http://mailman.cgd.ucar.edu/mailman/listinfo/ccsm_amwgdiag *
# to receive updates from the AMWG diagnostic package. *
# *
# Implementation of parallel version with Swift sponsored by the *
# Office of Biological and Environmental Research of the *
# U.S. Department of Energy's Office of Science. *
# *
#******************************************************************
#
#
#******************************************************************
# PLEASE READ THIS *
#******************************************************************
# This script can be placed in any directory provided that
# access to the working directory and local directories are
# available (see below).
#
# Using this script your model output can be compared with
# observations (observed and reanalysis, data)
# or with another model run. With all the diagnostic sets
# turned on the package will produce over 600 plots and
# tables in your working directory. In addition, files are
# produced for the climatological monthly means and the DJF
# and JJA seasonal means, as well as the annual (ANN) means.
#
# Input file names are of the standard CCSM type and
# they must be in netcdf format. Filenames are of the
# form YYYY-MM.nc where YYYY are the model years and MM
# are the months, for example 00010-09.nc. The files
# can reside on the Mass Storage System (MSS), if they
# are on the MSS the script will get them using msrcp.
# If your files are not on the MSS they must be in a local
# directory.
#
# Normally 5 years of monthly means are used for the
# comparisons, however only 1 year of data will work.
# The December just prior to the starting year is also
# needed for the DJF seasonal mean, or Jan and Feb of
# the year following the last full year. For example,
# for 5-year means the following months are needed
#
# 0000-12.nc prior december
# 0001-01.nc first year (must be 0001 or greater)
# ...
# 0001-12.nc
# ...
# 0005-01.nc last year
# ...
# 0005-12.nc
#--> OR you can do this
#
# 0001-01.nc first year (must be 0001 or greater)
# ...
# 0001-12.nc
# ...
# 0005-01.nc last year full year
# ...
# 0005-12.nc
# 0006-01.nc following jan
# 0006-02.nc following feb
#
#******************************************************************
# USER MODIFY SECTION *
# Modify the following sections as needed. *
#******************************************************************
# In the following "test case" refers to the model run to be
# compared with the "control case", which may be model or obs data.
#******************************************************************
#******************************************************************
# *****************
# *** Test case ***
# *****************
# Set the identifying casename and paths for your test case run.
# The output netcdf files are in: $test_path_history
# The climatology files go in: $test_path_climo
# The diagnostic plots go in: $test_path_diag
# The HPSS path (if files doesn't exist locally): $test_path_HPSS
#
# Don t forget the trailing / when setting the paths
set test_casename = {{casename}}
set test_filetype = monthly_history
#set test_filetype = time_series
#set test_path_history = /glade/scratch/hannay/archive/$test_casename/atm/
#set test_path_climo = /glade/scratch/hannay/amwg/climo/$test_casename/
#set test_path_diag = /glade/scratch/hannay/amwg/diag/$test_casename/
set test_path_history = $PWD/run/
set test_path_climo = $PWD/amwg-stage{{stage}}/
set test_path_diag = $PWD/amwg-stage{{stage}}/
set test_path_HPSS = $test_path_history
#******************************************************************
# ********************
# *** Control case ***
# ********************
# Select the type of control case to be compared with your model
# test case (select one).
set CNTL = OBS # observed data (reanalysis etc)
#set CNTL = USER # user defined model control (see below)
#------------------------------------------------------------------
# FOR CNTL == USER ONLY (otherwise skip this section)
# Set the identifying casename and paths for your control case run.
# The output netcdf files are in: $cntl_path_history
# The climatology files are in: $cntl_path_climo
# The HPSS path (if files doesn't exist locally): $cntl_path_HPSS
#
# Don t forget the trailing / when setting the paths
set cntl_casename = f.e11.F2000C5.f09_f09.control.001
set cntl_filetype = monthly_history
#set cntl_filetype = time_series
set cntl_path_history = /glade/scratch/hannay/archive/$cntl_casename/atm/hist/
set cntl_path_climo = /glade/scratch/hannay/amwg/climo/$cntl_casename/
set cntl_path_HPSS = /CCSM/csm/$cntl_casename/atm/hist/
#******************************************************************
# *********************
# *** Climatologies ***
# *********************
# Use these settings if computing climatological means
# from the local test case data and/or local control case data
#-----------------------------------------------------------------
# Turn on/off the computation of climatologies
set test_compute_climo = 0 # (0=ON,1=OFF)
set cntl_compute_climo = 1 # (0=ON,1=OFF)
#-----------------------------------------------------------------
# If computing climatological means for test/cntl case, specify the first
# year of your data, and the number of years of data to be used.
# First year of data is: $test_first_yr (must be >= 1)
# Number of years is: $test_nyrs (must be >= 1)
set test_first_yr = {{first_yr}} # first year (must be >= 1)
set test_nyrs = {{nyrs}} # number of yrs (must be >= 1)
# FOR CNTL == USER ONLY (otherwise skip this section)
# First year of data is: $cntl_first_yr (must be >= 1)
# Number of years is: $cntl_nyrs (must be >= 1)
set cntl_first_yr = 2 # first year (must be >= 1)
set cntl_nyrs = 4 # number of yrs (must be >= 1)
#-----------------------------------------------------------------
# Strip off all the variables that are not required by the AMWG package
# in the computation of the climatology
# set to OFF for running with Chemistry
set strip_off_vars = 0 # (0=ON,1=OFF)
#-----------------------------------------------------------------
# Weight the months by their number of days when computing
# averages for ANN, DJF, JJA. This takes much longer to compute
# the climatologies. Many users might not care about the small
# differences and leave this turned off.
set weight_months = 0 # (0=ON,1=OFF)
#******************************************************************
# ******************************
# *** Select diagnostic sets ***
# ******************************
# Select the diagnostic sets to be done. You can do one at a
# time or as many as you want at one time, or all at once.
set all_sets = 1 # (0=ON,1=OFF) Do all the CAM sets (1-16)
set set_1 = 0 # (0=ON,1=OFF) tables of global,regional means
set set_2 = 0 # (0=ON,1=OFF) implied transport plots
set set_3 = 1 # (0=ON,1=OFF) zonal mean line plots
set set_4 = 1 # (0=ON,1=OFF) vertical zonal mean contour plots
set set_4a = 1 # (0=ON,1=OFF) vertical zonal mean contour plots
set set_5 = 1 # (0=ON,1=OFF) 2D-field contour plots
set set_6 = 1 # (0=ON,1=OFF) 2D-field vector plots
set set_7 = 1 # (0=ON,1=OFF) 2D-field polar plots
set set_8 = 1 # (0=ON,1=OFF) annual cycle (vs lat) contour plots
set set_9 = 1 # (0=ON,1=OFF) DJF-JJA difference plots
set set_10 = 1 # (0=ON,1=OFF) annual cycle line plots
set set_11 = 1 # (0=ON,1=OFF) miscellaneous plots
set set_12 = 1 # (0=selected stations: 1=NONE, 2=ALL stations
set set_13 = 1 # (0=ON,1=OFF) COSP cloud simulator plots
set set_14 = 1 # (0=ON,1=OFF) Taylor diagram plots
set set_15 = 1 # (0=ON,1=OFF) Annual Cycle Plots for Select stations
set set_16 = 1 # (0=ON,1=OFF) Budget Terms for Select stations
set all_waccm_sets = 1 # (0=ON,1=OFF) Do all the WACCM sets
set all_chem_sets = 1 # (0=ON,1=OFF) Do all the CHEM sets
set wset_1 = 0 # (0=ON,1=OFF) vertical zonal mean contour plots (log scale)
set cset_1 = 0 # (0=ON,1=OFF) tables of global budgets
set cset_2 = 0 # (0=ON,1=OFF) vertical zonal mean contour plots (log scale)
set cset_3 = 0 # (0=ON,1=OFF) Ozonesonde comparisions
set cset_4 = 0 # (0=ON,1=OFF) Column Ozone/CO Comparisons
set cset_5 = 0 # (0=ON,1=OFF) NOAA Aircraft comparisons
set cset_6 = 0 # (0=ON,1=OFF) Emmons Aircraft climatology
set cset_7 = 0 # (0=ON,1=OFF) surface comparisons (ozone, co, improve)
# Select the control case to compare against for Taylor Diagrams
# Cam run select cam3_5; coupled run select ccsm3_5
setenv TAYLOR_BASECASE ccsm3_5 # Base case to compare against
# Options are cam3_5, ccsm3_5
# They are both fv_1.9x2.5
#******************************************************************
# **************************************
# *** Customize plots (output/style) ***
# **************************************
# Select seasonal output to be plotted
# four_seasons = 0 # DJF, MAM, JJA, SON, ANN
# four_seasons = 1 # DJF, JJA, ANN
# four_seasons = 2 # Custom: Select the season you want to be plotted
# Note: four_seasons is not currently supported for model vs OBS diagnostics.
# if ($CNTL == OBS) then four_seasons is turned OFF.
set four_seasons = 1 # (0=ON; 1=OFF)
#------------------------------------------------------------------
# For four_seasons == 2 (otherwise skip this section)
# Select the seasons you want to be plotted
if ($four_seasons == 2) then
set plot_ANN_climo = 0 # (0=ON,1=OFF) used by sets 1-7,11
set plot_DJF_climo = 0 # (0=ON,1=OFF) used by sets 1,3-7,9,11
set plot_JJA_climo = 0 # (0=ON,1=OFF) used by sets 1,3-7,9,11
set plot_MAM_climo = 0 # (0=ON,1=OFF) used by sets 1,3-7,9,11
set plot_SON_climo = 0 # (0=ON,1=OFF) used by sets 1,3-7,9,11
set plot_MON_climo = 0 # (0=ON,1=OFF) used by sets 8,10,11,12
endif
#-----------------------------------------------------------------
# Select the output file type and style for plots.
set p_type = ps # postscript
#set p_type = pdf # portable document format (ncl ver 4.2.0.a028)
#set p_type = eps # encapsulated postscript
#set p_type = epsi # encapsulated postscript with bitmap
#set p_type = ncgm # ncar computer graphics metadata
#-------------------------------------------------------------------
# Select the output color type for plots.
set c_type = COLOR # color
#set c_type = MONO # black and white
# If needed select one of the following color schemes,
# you can see the colors by clicking on the links from
# http://www.cgd.ucar.edu/cms/diagnostics
set color_bar = default # the usual colors
set color_bar = blue_red # blue,red
#set color_bar = blue_yellow_red # blue,yellow,red (nice!)
#----------------------------------------------------------------
# Turn ON/OFF date/time stamp at bottom of plots.
# Leaving this OFF makes the plots larger.
set time_stamp = 1 # (0=ON,1=OFF)
#---------------------------------------------------------------
# Turn ON/OFF tick marks and labels for sets 5,6, and 7
# Turning these OFF make the areas plotted larger, which makes
# the images easier to look at.
set tick_marks = 1 # (0=ON,1=OFF)
#----------------------------------------------------------------
# Use custom case names for the PLOTS instead of the
# case names encoded in the netcdf files (default).
# Also useful for publications.
set custom_names = 1 # (0=ON,1=OFF)
# if needed set the names
set test_name = cam3_5_test # test case name
set cntl_name = cam3_5_cntl # control case name
#----------------------------------------------------------------
# Convert output postscript files to GIF, JPG or PNG image files
# and place them in subdirectories along with html files.
# Then make a tar file of the web pages and GIF,JPG or PNG files.
# On Dataproc and CGD Suns GIF images are smallest since I built
# ImageMagick from source and compiled in the LZW compression.
# On Linux systems JPG will be smallest if you have an rpm or
# binary distribution of ImageMagick (and hence convert) since
# NO LZW compression is the default. Only works if you have
# convert on your system and for postscript files (p_type = ps).
# NOTE: Unless you have rebuilt ImageMagick on your Linux system
# the GIF files can be as large as the postscript plots. I
# recommend that PNG always be used. The density option can be
# used with convert to make higher resolution images which will
# work better in powerpoint presentations, try density = 150.
set web_pages = 0 # (0=ON,1=OFF) make images and html files
set delete_ps = 0 # (0=ON,1=OFF) delete postscript files
set img_type = 0 # (0=PNG,1=GIF,2=JPG) select image type
set density = 85 # pixels/inch, use larger number for higher
# resolution images (default is 85)
#----------------------------------------------------------------
# Save the output netcdf files of the derived variables
# used to make the plots. These are normally deleted
# after the plots are made. If you want to save the
# netcdf files for your own uses then switch to ON.
set save_ncdfs = 1 # (0=ON,1=OFF)
#----------------------------------------------------------------
# Compute whether the means of the test case and control case
# are significantly different from each other at each grid point.
# Tests are performed only for model-to-model comparisons.
# REQUIRES at least 10 years of model data for each case.
# Number of years from above (test_nyrs and cntl_nyrs) is used.
# Also set the significance level for the t-test.
set significance = 1 # (0=ON,1=OFF)
# if needed set default level
set sig_lvl = 0.05 # level of significance
#******************************************************************
# ***************************
# *** Source code location ***
# ***************************
# Below is defined the amwg diagnostic package root location
# on CGD machines (tramhill,...), on old CSIL machines (mirage),
# on new CSIL machines (geyser), NERSC (euclid), and LBNL (lens).
#
# If you are installing the diagnostic package on your computer system.
# you need to set DIAG_HOME to the root location of the diagnostic code.
# The code is in $DIAG_HOME/code
# The obs data in $DIAG_HOME/obs_data
# The cam3.5 data in $DIAG_HOME/cam35_data
# CGD machines (tramhill, leehill...)
#setenv DIAG_HOME /project/amp/amwg/amwg_diagnostics
# CSIL machines (geyser, caldeira, ...)
#setenv DIAG_HOME /glade/p/cesm/amwg/amwg_diagnostics
# NERSC (euclid)
#setenv DIAG_HOME /global/homes/h/hannay/amwg/amwg_diagnostics
#setenv DIAG_HOME /project/projectdirs/m2187/amwg/amwg_diagnostics
# NCSS (lens)
#setenv DIAG_HOME /ccs/home/hannay/amwg/amwg_diagnostics
#*****************************************************************
# ****************************
# *** Additional settings ***
# ****************************
# Send yourself an e-mail message when everything is done.
set email = 1 # (0=ON,1=OFF)
set email_address = ${LOGNAME}@ucar.edu
#*****************************************************************
#*****************************************************************
# **************************
# *** Advanced settings ***
# **************************
#*****************************************************************
#-------------------------------------------------
# For CAM-SE grid, specify a lat/lon to interpolate to
#-------------------------------------------------
# By default, teh CAM-SE output is interpolated CAM_SE on a 1 degree grid
# You can select another grid below.
set test_res_out = 0.9x1.25
set cntl_res_out = 0.9x1.25
# Set the interpolation method for regridding: bilinear, patch, conserver
setenv INTERP_METHOD bilinear
#-------------------------------------------------
# Set to 0 to use swift
#-------------------------------------------------
setenv use_swift 1 # (0=ON,1=OFF)
setenv swift_scratch_dir /glade/scratch/$USER/swift_scratch/
set test_inst = -1
set cntl_inst = -1
#-------------------------------------------------
# For set 12:
#-------------------------------------------------
# Select vertical profiles to be computed. Select from list below,
# or do all stations, or none. You must have computed the monthly
# climatological means for this to work. Preset to the 17 selected
# stations.
# Specify selected stations for computing vertical profiles.
if ($set_12 == 0 || $all_sets == 0) then
# ARCTIC (60N-90N)
set western_alaska = 1 # (0=ON,1=OFF)
set whitehorse_canada = 1 # (0=ON,1=OFF)
set resolute_canada = 0 # (0=ON,1=OFF)
set thule_greenland = 0 # (0=ON,1=OFF)
# NORTHERN MIDLATITUDES (23N-60N)
set new_dehli_india = 1 # (0=ON,1=OFF)
set kagoshima_japan = 1 # (0=ON,1=OFF)
set tokyo_japan = 1 # (0=ON,1=OFF)
set midway_island = 0 # (0=ON,1=OFF)
set shipP_gulf_alaska = 0 # (0=ON,1=OFF)
set san_francisco_ca = 0 # (0=ON,1=OFF)
set denver_colorado = 1 # (0=ON,1=OFF)
set great_plains_usa = 0 # (0=ON,1=OFF)
set oklahoma_city_ok = 1 # (0=ON,1=OFF)
set miami_florida = 0 # (0=ON,1=OFF)
set new_york_usa = 1 # (0=ON,1=OFF)
set w_north_atlantic = 1 # (0=ON,1=OFF)
set shipC_n_atlantic = 1 # (0=ON,1=OFF)
set azores = 1 # (0=ON,1=OFF)
set gibraltor = 1 # (0=ON,1=OFF)
set london_england = 1 # (0=ON,1=OFF)
set western_europe = 0 # (0=ON,1=OFF)
set crete = 1 # (0=ON,1=OFF)
# TROPICS (23N-23S)
set central_india = 1 # (0=ON,1=OFF)
set madras_india = 1 # (0=ON,1=OFF)
set diego_garcia = 0 # (0=ON,1=OFF)
set cocos_islands = 1 # (0=ON,1=OFF)
set christmas_island = 1 # (0=ON,1=OFF)
set singapore = 1 # (0=ON,1=OFF)
set danang_vietnam = 1 # (0=ON,1=OFF)
set manila = 1 # (0=ON,1=OFF)
set darwin_australia = 1 # (0=ON,1=OFF)
set yap_island = 0 # (0=ON,1=OFF)
set port_moresby = 1 # (0=ON,1=OFF)
set truk_island = 0 # (0=ON,1=OFF)
set raoui_island = 1 # (0=ON,1=OFF)
set gilbert_islands = 1 # (0=ON,1=OFF)
set marshall_islands = 0 # (0=ON,1=OFF)
set samoa = 1 # (0=ON,1=OFF)
set hawaii = 0 # (0=ON,1=OFF)
set panama = 0 # (0=ON,1=OFF)
set mexico_city = 1 # (0=ON,1=OFF)
set lima_peru = 1 # (0=ON,1=OFF)
set san_juan_pr = 1 # (0=ON,1=OFF)
set recife_brazil = 1 # (0=ON,1=OFF)
set ascension_island = 0 # (0=ON,1=OFF)
set ethiopia = 1 # (0=ON,1=OFF)
set nairobi_kenya = 1 # (0=ON,1=OFF)
# SOUTHERN MIDLATITUDES (23S-60S)
set heard_island = 1 # (0=ON,1=OFF)
set w_desert_australia = 1 # (0=ON,1=OFF)
set sydney_australia = 1 # (0=ON,1=OFF)
set christchurch_nz = 1 # (0=ON,1=OFF)
set easter_island = 0 # (0=ON,1=OFF)
set san_paulo_brazil = 1 # (0=ON,1=OFF)
set falkland_islands = 1 # (0=ON,1=OFF)
# ANTARCTIC (60S-90S)
set mcmurdo_antarctica = 0 # (0=ON,1=OFF)
endif
#-----------------------------------------------------------------
# PALEOCLIMATE coastlines
# Allows users to plot paleoclimate coastlines for sets 5,6,7,9.
# Two special files are created which contain the needed data