Use output directories instead of tar files

bin/acme-output.sh

@@ -41,16 +41,20 @@ RUNDIR=$(cd $RUNDIR && pwd)
 # Get the workflow scratch directory
 DIR=$(pwd)
 
-# The tar works better from the run dir
-cd $RUNDIR
+# Make the stage output dir
+STAGEDIR=$DIR/${CASE}-stage${STAGE}
+mkdir $STAGEDIR
 
 # Touch this file here so that it will always exist, even if it is empty
 # for the first stage
 touch outputs_to_ignore
 
-# Tar up all the output files, but ignore outputs from previous stages
-tar -czv -f $DIR/${CASE}.stage${STAGE}.tar.gz -X outputs_to_ignore ${CASE}.* rpointer.* *.log.*.gz
+# For each of the new files
+for f in $(ls ${RUNDIR}/${CASE}.* ${RUNDIR}/*.log.*.gz ${RUNDIR}/rpointer.* | grep -v -f outputs_to_ignore); do
+    # Symlink it to the output dir
+    ln -s $f ${STAGEDIR}/$(basename $f)
+done
 
 # Update the ignore file to ignore outputs from the current stage
-ls ${CASE}.* *.log.* > outputs_to_ignore
+ls ${RUNDIR}/${CASE}.* ${RUNDIR}/*.log.* > outputs_to_ignore
 

daxgen.py

@@ -149,7 +149,8 @@ tr acme-amwg {
             if last is not None:
                 dax.depends(stage, last)
 
-            output = File("%s.stage%s.tar.gz" % (self.casename, i))
+            # This is actually a directory
+            output = File("%s-stage%s" % (self.casename, i))
 
             archive = Job(name="acme-output")
             archive.addArguments("-stage %s" % i)
@@ -177,7 +178,8 @@ tr acme-amwg {
                     script_name = self.generate_amwg_script(i, first_yr, nyrs)
                     script = File(script_name)
 
-                    diagnostics = File("amwg-stage%s" % i)
+                    # This is actually a directory
+                    diagnostics = File("%s-amwg-stage%s" % (self.casename, i))
 
                     # Add the job
                     diag = Job(name="acme-amwg")

templates/diag140804.csh

@@ -112,8 +112,8 @@ set test_filetype = monthly_history
 #set test_path_climo   =  /glade/scratch/hannay/amwg/climo/$test_casename/ 
 #set test_path_diag    =  /glade/scratch/hannay/amwg/diag/$test_casename/
 set test_path_history =  $PWD/run/
-set test_path_climo   =  $PWD/amwg-stage{{stage}}/
-set test_path_diag    =  $PWD/amwg-stage{{stage}}/
+set test_path_climo   =  $PWD/{{casename}}-amwg-stage{{stage}}/
+set test_path_diag    =  $PWD/{{casename}}-amwg-stage{{stage}}/
 set test_path_HPSS    =  $test_path_history
 
 #******************************************************************