updates to run on cori and allow for more general data no owned by the user

postproc/e3sm1p0/atm.sh

@@ -5,21 +5,23 @@
 if [ "$postprocess" = "true" ]; then
 
 # regridding step to regrid to lat lon, currently regrids all years even if not requested
+# this block can be removed once mask files are included in sim output
 if [ "$version" == "e3sm" ]; then
 
-##ncwa -a -time $outpath/masks/mask_ne30np4.nc tmp.nc 
-##ncks -x -v time tmp.nc $outpath/masks/mask_ne30np4_notime.nc
+#in the event you get a mask file with a time dimension, remove it first
+#ncwa -a -time $outpath/masks/mask_ne30np4.nc tmp.nc 
+#ncks -x -v time tmp.nc $outpath/masks/mask_ne30np4_notime.nc
 
-#for fl in $path/atm/hist/*h0*; do
-#echo "first add mask in original grid to monthly atm output in $fl"
-##ncks -A -C -v gris_mask $outpath/masks/mask_ne30np4_notime.nc $fl   
-#ncks -A -C -v gris_mask $mask_file $fl   
-#done
+echo "first add mask in original grid to monthly atm output in $fl"
+	for fl in $path/atm/hist/*h0*; do
+	ncks --no_tmp_fl -A -C -v gris_mask $mask_file $fl 
+	done
+fi
 
 h0atm_path=$outpath/atm/hist_regrid
 
-#mkdir -p "$h0atm_path"
-#ls $path/atm/hist/*h0* | ncremap -m $map_file -O $h0atm_path
+mkdir -p "$h0atm_path"
+ls $path/atm/hist/*h0* | ncremap -m $map_file -O $h0atm_path
 
 else
 h0atm_path=$path/atm/hist

postproc/obs/atm_obs.sh → postproc/e3sm1p0/atm_obs.sh

@@ -8,9 +8,9 @@ for dataset in "${datasets[@]}"
 do
 
 # to get this working, need to apply to all climos and using cesm1/e3sm specific mask file that has already been created
-#echo "remapping ${dataset} climos onto CESM grid"
-#file_w_data_on_source_grid=$outpath/masks/${case}.clm2.gris_mask.nc
-#ncremap -i $file_w_data_on_source_grid -V CLDTOT -a bilinear -d /lustre/atlas1/cli900/world-shared/cesm/obs_data/${dataset}_01_climo.nc -v gris_mask -o $obspath/$case.gris_mask_${dataset}_grid.nc
+echo "remapping ${dataset} climos onto target grid"
+file_w_data_on_source_grid=$outpath/masks/${case}.clm2.gris_mask.nc
+ncremap -i $file_w_data_on_source_grid -V CLDTOT -a bilinear -d $obspath/${dataset}_01_climo.nc -v gris_mask -o $obspath/$case.gris_mask_${dataset}_grid.nc
 
 done
 

postproc/e3sm1p0/lnd.sh

@@ -4,18 +4,21 @@
 
 if [ "$postprocess" = "true" ]; then
 
+# regridding step to regrid to lat lon, currently regrids all years even if not requested
+# this is not needed unless a mask file is needed in the sim output AND analysis of \hist, else regrid after climos made
 if [ "$version" == "e3sm" ]; then
+
+#uncomment if you have a new mask file with a time dimension
 #ncwa -a -time $outpath/masks/mask_ne30np4_lnd.nc $outpath/masks/mask_ne30np4_lnd_notime.nc 
-#for fl in $path/lnd/hist/*h0*; do
-#echo "first add mask in original grid to monthly lnd output in $fl"
-#ncks -A -C -v gris_mask $outpath/masks/mask_ne30np4_lnd_notime.nc $fl
-#done
 
-# regridding step to regrid to lat lon, currently regrids all years even if not requested
-#h0lnd_path=$outpath/lnd/hist_regrid
-#mkdir -p "$h0lnd_path"
-#ls $path/lnd/hist/*h0* | ncremap -m $map_file -O $h0lnd_path
-h0lnd_path=$path/archive/lnd/hist
+for fl in $path/lnd/hist/*h0*; do
+echo "first add mask in original grid to monthly lnd output in $fl"
+ncks -A -C -v gris_mask $mask_file $fl
+done
+
+h0lnd_path=$outpath/lnd/hist_regrid
+mkdir -p "$h0lnd_path"
+ls $path/lnd/hist/*h0* | ncremap -m $map_file -O $h0lnd_path
 
 else 
 h0lnd_path=$path/lnd/hist
@@ -41,53 +44,22 @@ ncwa -O -a lat,lon -w area -B "gris_mask == 1" $outpath/lnd/climos/${case}_ANN_c
 ncwa -O -a lat,lon -w area -B "gris_mask == 1" $outpath/lnd/climos/${case}_JJA_climo.nc $outpath/lnd/climos/${case}_JJA_aavg_climo.nc
 ncwa -O -a lat,lon -w area -B "gris_mask == 1" $outpath/lnd/climos/${case}_DJF_climo.nc $outpath/lnd/climos/${case}_DJF_aavg_climo.nc
 
-mkdir -p "$outpath/lnd/davg"
-
-#echo "make annualized daily averages for land for all variables"
-#ls $path/lnd/hist/$case.$lnd.h1.*.nc |  /ccs/home/zender/bin_rhea/ncclimo --job_nbr=12 -c $case -C dly -s ${year_start} -e ${year_end} -o $outpath/lnd/davg -m $lnd 
-#/ccs/home/zender/bin_rhea/ncclimo --job_nbr=12 -c $case -C dly -s ${year_start} -e ${year_end} -o $outpath/lnd/davg -m $lnd -i $path/lnd/hist
-
-
-#echo "loop over each variable to create and process time series for h1 vars"
-#qsub rhea_tseries_lnd.sh 
-
-#for ivar in "${lnd_h1_vars[@]}"
-#do
-#echo $ivar
-
-#echo "concatenate all daily dates to be averaged into annualized land time series for $ivar"
-#ncrcat -O -F -v $ivar $outpath/lnd/davg/*$year_start*$year_end*_climo.nc $outpath/lnd/davg/$case.$lnd.h1.$ivar.davg.${year_start}-${year_end}.nc
-#
-#echo "add mask and area variables to annualized daily land averages for $ivar"
-#ncks -A -v gris_mask,area $outpath/masks/$case.$lnd.gris_mask.nc $outpath/lnd/davg/$case.$lnd.h1.$ivar.davg.${year_start}-${year_end}.nc
-
-#echo "average over ice sheet region for annualized avgd daily land values for $ivar"
-#ncwa -O -a lat,lon -w area -B "gris_mask == 1" $outpath/lnd/davg/$case.$lnd.h1.$ivar.davg.${year_start}-${year_end}.nc $outpath/lnd/davg/$case.$lnd.h1.$ivar.davg.aavg.${year_start}-${year_end}.nc
-#
-
-#if test -a $outpath/lnd/davg/$case.$lnd.h1.$ivar.davg.aavg.${year_start}-${year_end}.nc; then
-#        echo "Annualized daily masked $lnd area averages made"
-#else
-#        echo "Annualized daily masked $lnd area averages failed"
-#       exit
-#fi
+elif [ "$version" == "e3sm" ]; then
 
+# only needed if no mask in sim output
+#for fl in $outpath/lnd/climos/*.nc; do
+#echo "add mask in original grid to climos in $fl"
+#ncks -A -C -v gris_mask $mask_file $fl
 #done
 
-elif [ "$version" == "e3sm" ]; then
-
-for fl in $outpath/lnd/climos/*.nc; do
-echo "add mask in original grid to climos in $fl"
-ncks -A -C -v gris_mask $outpath/masks/mask_ne30np4_lnd_notime.nc $fl
-done
+#echo " remapping these land climo files"
+#climos_path=$outpath/lnd/climo_regrid
+#mkdir -p "$climos_path"
 
-# regrid climos to lat lon
-climos_path=$outpath/lnd/climo_regrid
-mkdir -p "$climos_path"
+#ls $outpath/lnd/climos/*_climo* 
+#ls $outpath/lnd/climos/*_climo* | ncremap -m $map_file -O $climos_path
 
-echo " remapping these land climo files"
-ls $outpath/lnd/climos/*_climo* 
-ls $outpath/lnd/climos/*_climo* | ncremap -m $map_file -O $climos_path
+climos_path=$outpath/lnd/climos
 
 echo "average land monthly climos over ice sheet region"
 ncwa -O -a lat,lon -w area -B "gris_mask == 1" $climos_path/${case}_ANN_climo.nc $climos_path/${case}_ANN_aavg_climo.nc

postproc/obs/lnd_obs.sh → postproc/e3sm1p0/lnd_obs.sh

 #!/bin/bash
 
-module load nco/4.6.9
+#if run stand alone
+#	module load nco/4.6.9
 
 #var=("t2m")
 var=("t2m" "swsn" "swsd" "lwsn" "lwsd" "smb" "latf" "senf" "gbot" "alb")
@@ -9,7 +10,8 @@ file_start_year=1958
 start_year=1980
 end_year=1999
 
-hist_path=$base/$casename
+hist_path=$obspath/$casename
+echo "$hist_path"
 lndpath=$hist_path/climos
 outseries=$hist_path/tseries
 mask_file=RACMO23_masks_ZGRN11_latlon.nc 
@@ -26,8 +28,6 @@ tseries_file=$casename.${ivar}.ann_tseries
 mkdir -p "$outseries"
 mkdir -p "$lndpath"
 
-#cd "$hist_path"
-
 let "go=($start_year-$file_start_year)*12"
 let "stop=($end_year-$file_start_year)*12"
 

postproc/e3sm1p0/main_e3sm.sh

-#!/bin/bash
 #clear
 
 # 8/18 to run this script on E3SM v1 on CORI at NERSC
@@ -11,30 +10,43 @@
 # update specialty files used for masking and regridding
 # total area of GrIS in model is 1.719e6
 
+postprocess=true
+make_plots=false
+
 if [[ $(hostname -s) = rhea* ]]; then
+#user settings
 	project=cli115  
 	case=20180215.DECKv1b_H1.ne30_oEC.edison
 	year_start=1979
 	year_end=2005
 	base=/lustre/atlas1/$project/proj-shared/salil/archive
 	path=$base/$case/archive
-	obspath=/lustre/atlas1/$project/world-shared/$USER/obs_data
 	outpath=/lustre/atlas1/$project/world-shared/$USER/$case/postproc
+# compute center settings
+	obspath=/lustre/atlas1/cli115/world-shared/4ue/obs_data
 	area_file=/lustre/atlas1/$project/world-shared/$USER/surfdata_0.9x1.25_simyr1850_c110725.nc
 	map_file=/ccs/home/zender/data/maps/map_ne30np4_to_fv129x256_aave.20150901.nc
-	mask_file=/lustre/atlas1/$project/world-shared/4ue/$case/postproc/mask_ne30np4_notime.nc
+	mask_file=/lustre/atlas1/$project/world-shared/4ue/$case/postproc/masks/mask_ne30np4_notime.nc
+	obs_mask_file=/lustre/atlas1/$project/world-shared/4ue/$case/postproc/masks/E3SMv1_ne30.clm2.gris_mask.nc
+	postprocess_obs=false
 elif [[ $(hostname -s) = cori* ]]; then
+#user settings
 	project=piscees  
 	case=20180612.B_case.T62_oEC60to30v3wLI.modified_runoff_mapping.edison
 	year_start=0070
 	year_end=0075
-	base=/global/cscratch1/sd/dcomeau/acme_scratch/edison/archive
-	path=$base/$case/run
+#need to specify prebase if using data created by someone else until masks are written with sim output
+	prebase=/global/cscratch1/sd/dcomeau/acme_scratch/edison/archive
+	base=/global/cscratch1/sd/$USER/acme_scratch/edison
+	path=$base/$case
+	outpath=/global/cscratch1/sd/$USER/acme_scratch/edison/$case/postproc
+# compute center settings
 	obspath=/project/projectdirs/piscees/postproc/obs_data
-	outpath=/global/cscratch1/sd/$USER/$case/postproc
 	area_file=/project/projectdirs/piscees/postproc/surfdata_0.9x1.25_simyr1850_c110725.nc
 	map_file=/project/projectdirs/piscees/postproc/map_ne30np4_to_fv129x256_aave.20150901.nc
-	#mask_file=/project/projectdirs/piscees/postproc/mask_ne30np4_notime.nc
+	mask_file=/project/projectdirs/piscees/postproc/mask_ne30np4_notime.nc
+	obs_mask_file=/project/projectdirs/piscees/postproc/E3SMv1_ne30.clm2.gris_mask.nc
+	postprocess_obs=false
 else
   echo "machine not recognized"
 fi
@@ -43,10 +55,6 @@ atm_vars=("PSL" "CLDLOW" "CLDHGH" "CLDMED" "CLDTOT" "TREFHT" "TS" "TSMX" "TSMN"
 lnd_h0_vars=("FSDS" "QRUNOFF" "QSOIL" "TSA" "RAIN" "SNOW" "FIRA" "FLDS" "FSH" "QSNOMELT")
 #glc_vars=("acab" "thk")
 
-postprocess=true
-postprocess_obs=false
-make_plots=true
-
 mkdir -p $outpath
 
 echo "$version"
@@ -56,6 +64,17 @@ echo "$version"
 	lnd=clm2 
 	atm=cam 
 
+# this copy to local dir can be removed once masks are written with sim output
+if test -d $prebase/$case; then
+	echo "Creating user owned target path for output data"
+	mkdir -p $base/$case/atm
+	mkdir -p $base/$case/lnd
+	cp -uvr $prebase/$case/atm $path
+	cp -uvr $prebase/$case/lnd $path
+else
+	exit 
+fi
+
 #run a check, is the requested data directory there?
 if test -d $path; then
 	echo "Directory $path exists"
@@ -93,7 +112,6 @@ elif [[ $(hostname -s) = cori* ]]; then
     module load gcc
     module load cray-netcdf/4.4.1.1.6 
     module load gsl
-#   module load hdf5-parallel/1.10.1 conflit with netcdf
     module load nco/4.7.4
 
     module load python/2.7-anaconda-4.4
@@ -121,9 +139,8 @@ else
 	exit
 fi
 
-# this is a hack- all files from E3SM runs we care about should have a gris_mask in the data
-#if [ "$version" == "cesm1" ]; then
-#
+if [ "$postprocess_obs" = "true" ]; then
+
 #if test -a $outpath/masks/${case}.$lnd.gris_mask.nc; then
 #        echo "already created gris_mask and area file, moving on"
 #else
@@ -135,14 +152,10 @@ fi
 #
 #fi # if mask file needs to be created
 
-#fi  # CESM
-
-if [ "$postprocess_obs" = "true" ]; then
-
 echo "process and remap cloud obs"
-../obs/atm_obs.sh 
+. atm_obs.sh 
 echo "process RACMO"
-../obs/lnd_obs.sh 
+. lnd_obs.sh 
 
 ./remap_RACMO_to_CESM2_inline.sh
 

postproc/e3sm1p0/tseries_atm_h0.sh

+#!/bin/bash
+
+if test -a $h0atm_path/${case}.${atm}.h0.${year_start}-01.nc; then
+        echo making $outpath/atm/tseries/$case.$atm.h0.$ivar.nc 
+echo "make time series of monthly averages for $ivar"
+if [ "$version" == "cesm1" ]; then
+        ncrcat -O -v date,time,lev,$ivar $h0atm_path/$case.$atm.h0.*.nc $outpath/atm/tseries/$case.$atm.h0.$ivar.nc 
+elif [ "$version" == "e3sm" ]; then
+        ncrcat -O -v date,time,gris_mask,lev,$ivar $h0atm_path/$case.$atm.h0.*.nc $outpath/atm/tseries/$case.$atm.h0.$ivar.nc 
+fi
+
+if test -a $outpath/atm/tseries/${case}.$atm.h0.$ivar.nc; then
+        echo "Monthly $atm averages of $ivar made"
+else
+        echo "Monthly $atm averages of $ivar failed"
+	exit
+fi
+else
+        echo "File $h0atm_path/$case.$atm.h0.$year_start-01.nc does not exist"
+        echo "No time series or monthly averages made"
+
+fi
+
+echo "make yearly average time series from monthly average time series of $ivar"
+ncra --mro -O -h -d time,,,12,12 $outpath/atm/tseries/$case.$atm.h0.$ivar.nc $outpath/atm/tseries/$case.$atm.h0.yrly.${ivar}.nc
+# since the monthly data time dimension is "days since 1851" and uses the last day of the month for its monthly average,
+# the time dimension needs to be altered to restate it properly when the data is averaged yearly.
+
+if [ "$version" == "cesm1" ]; then
+echo "add mask and area variables to time series of $ivar"
+ncks -A -v area,gris_mask $outpath/masks/$case.$lnd.gris_mask.nc -o $outpath/atm/tseries/$case.$atm.h0.yrly.$ivar.nc 
+fi
+
+echo "average over ice sheet for $ivar"
+ncwa -O -a lat,lon -w area -B "gris_mask == 1" $outpath/atm/tseries/$case.$atm.h0.yrly.$ivar.nc $outpath/atm/tseries/$case.$atm.aavg.h0.yrly.$ivar.nc
+
+if test -a $outpath/atm/tseries/${case}.$atm.aavg.h0.yrly.$ivar.nc; then
+        echo "Global avg yearly masked $atm averages of $ivar made"
+else
+        echo "Global avg yearly masked $atm averages of $ivar failed"
+       exit
+fi
+
+echo "$ivar yearly atm $time series complete"
+
+

postproc/e3sm1p0/tseries_lnd_h0.sh

@@ -47,7 +47,7 @@ fi
 elif [ "$version" == "e3sm" ]; then
 
 for fl in $outpath/lnd/tseries/*clm2.h0.yrly*; do
-ncks -A -C -v gris_mask $outpath/masks/mask_ne30np4_lnd_notime.nc $fl
+ncks -A -C -v gris_mask $mask_file $fl
 done
 
 echo "average over ice sheet region for yearly averaged lnd data for $ivar"