Commit 211475c2 authored by Juve, Gideon's avatar Juve, Gideon
Browse files

Use output directories instead of tar files

parent c69ff078
......@@ -41,16 +41,20 @@ RUNDIR=$(cd $RUNDIR && pwd)
# Get the workflow scratch directory
DIR=$(pwd)
# The tar works better from the run dir
cd $RUNDIR
# Make the stage output dir
STAGEDIR=$DIR/${CASE}-stage${STAGE}
mkdir $STAGEDIR
# Touch this file here so that it will always exist, even if it is empty
# for the first stage
touch outputs_to_ignore
# Tar up all the output files, but ignore outputs from previous stages
tar -czv -f $DIR/${CASE}.stage${STAGE}.tar.gz -X outputs_to_ignore ${CASE}.* rpointer.* *.log.*.gz
# For each of the new files
for f in $(ls ${RUNDIR}/${CASE}.* ${RUNDIR}/*.log.*.gz ${RUNDIR}/rpointer.* | grep -v -f outputs_to_ignore); do
# Symlink it to the output dir
ln -s $f ${STAGEDIR}/$(basename $f)
done
# Update the ignore file to ignore outputs from the current stage
ls ${CASE}.* *.log.* > outputs_to_ignore
ls ${RUNDIR}/${CASE}.* ${RUNDIR}/*.log.* > outputs_to_ignore
......@@ -149,7 +149,8 @@ tr acme-amwg {
if last is not None:
dax.depends(stage, last)
output = File("%s.stage%s.tar.gz" % (self.casename, i))
# This is actually a directory
output = File("%s-stage%s" % (self.casename, i))
archive = Job(name="acme-output")
archive.addArguments("-stage %s" % i)
......@@ -177,7 +178,8 @@ tr acme-amwg {
script_name = self.generate_amwg_script(i, first_yr, nyrs)
script = File(script_name)
diagnostics = File("amwg-stage%s" % i)
# This is actually a directory
diagnostics = File("%s-amwg-stage%s" % (self.casename, i))
# Add the job
diag = Job(name="acme-amwg")
......
......@@ -112,8 +112,8 @@ set test_filetype = monthly_history
#set test_path_climo = /glade/scratch/hannay/amwg/climo/$test_casename/
#set test_path_diag = /glade/scratch/hannay/amwg/diag/$test_casename/
set test_path_history = $PWD/run/
set test_path_climo = $PWD/amwg-stage{{stage}}/
set test_path_diag = $PWD/amwg-stage{{stage}}/
set test_path_climo = $PWD/{{casename}}-amwg-stage{{stage}}/
set test_path_diag = $PWD/{{casename}}-amwg-stage{{stage}}/
set test_path_HPSS = $test_path_history
#******************************************************************
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment