Commit 1b069a52 authored by Evans, Katherine's avatar Evans, Katherine
Browse files

updates to run on cori and allow for more general data no owned by the user

parent 55d5066f
......@@ -5,21 +5,23 @@
if [ "$postprocess" = "true" ]; then
# regridding step to regrid to lat lon, currently regrids all years even if not requested
# this block can be removed once mask files are included in sim output
if [ "$version" == "e3sm" ]; then
##ncwa -a -time $outpath/masks/mask_ne30np4.nc tmp.nc
##ncks -x -v time tmp.nc $outpath/masks/mask_ne30np4_notime.nc
#in the event you get a mask file with a time dimension, remove it first
#ncwa -a -time $outpath/masks/mask_ne30np4.nc tmp.nc
#ncks -x -v time tmp.nc $outpath/masks/mask_ne30np4_notime.nc
#for fl in $path/atm/hist/*h0*; do
#echo "first add mask in original grid to monthly atm output in $fl"
##ncks -A -C -v gris_mask $outpath/masks/mask_ne30np4_notime.nc $fl
#ncks -A -C -v gris_mask $mask_file $fl
#done
echo "first add mask in original grid to monthly atm output in $fl"
for fl in $path/atm/hist/*h0*; do
ncks --no_tmp_fl -A -C -v gris_mask $mask_file $fl
done
fi
h0atm_path=$outpath/atm/hist_regrid
#mkdir -p "$h0atm_path"
#ls $path/atm/hist/*h0* | ncremap -m $map_file -O $h0atm_path
mkdir -p "$h0atm_path"
ls $path/atm/hist/*h0* | ncremap -m $map_file -O $h0atm_path
else
h0atm_path=$path/atm/hist
......
......@@ -8,9 +8,9 @@ for dataset in "${datasets[@]}"
do
# to get this working, need to apply to all climos and using cesm1/e3sm specific mask file that has already been created
#echo "remapping ${dataset} climos onto CESM grid"
#file_w_data_on_source_grid=$outpath/masks/${case}.clm2.gris_mask.nc
#ncremap -i $file_w_data_on_source_grid -V CLDTOT -a bilinear -d /lustre/atlas1/cli900/world-shared/cesm/obs_data/${dataset}_01_climo.nc -v gris_mask -o $obspath/$case.gris_mask_${dataset}_grid.nc
echo "remapping ${dataset} climos onto target grid"
file_w_data_on_source_grid=$outpath/masks/${case}.clm2.gris_mask.nc
ncremap -i $file_w_data_on_source_grid -V CLDTOT -a bilinear -d $obspath/${dataset}_01_climo.nc -v gris_mask -o $obspath/$case.gris_mask_${dataset}_grid.nc
done
......
......@@ -4,18 +4,21 @@
if [ "$postprocess" = "true" ]; then
# regridding step to regrid to lat lon, currently regrids all years even if not requested
# this is not needed unless a mask file is needed in the sim output AND analysis of \hist, else regrid after climos made
if [ "$version" == "e3sm" ]; then
#uncomment if you have a new mask file with a time dimension
#ncwa -a -time $outpath/masks/mask_ne30np4_lnd.nc $outpath/masks/mask_ne30np4_lnd_notime.nc
#for fl in $path/lnd/hist/*h0*; do
#echo "first add mask in original grid to monthly lnd output in $fl"
#ncks -A -C -v gris_mask $outpath/masks/mask_ne30np4_lnd_notime.nc $fl
#done
# regridding step to regrid to lat lon, currently regrids all years even if not requested
#h0lnd_path=$outpath/lnd/hist_regrid
#mkdir -p "$h0lnd_path"
#ls $path/lnd/hist/*h0* | ncremap -m $map_file -O $h0lnd_path
h0lnd_path=$path/archive/lnd/hist
for fl in $path/lnd/hist/*h0*; do
echo "first add mask in original grid to monthly lnd output in $fl"
ncks -A -C -v gris_mask $mask_file $fl
done
h0lnd_path=$outpath/lnd/hist_regrid
mkdir -p "$h0lnd_path"
ls $path/lnd/hist/*h0* | ncremap -m $map_file -O $h0lnd_path
else
h0lnd_path=$path/lnd/hist
......@@ -41,53 +44,22 @@ ncwa -O -a lat,lon -w area -B "gris_mask == 1" $outpath/lnd/climos/${case}_ANN_c
ncwa -O -a lat,lon -w area -B "gris_mask == 1" $outpath/lnd/climos/${case}_JJA_climo.nc $outpath/lnd/climos/${case}_JJA_aavg_climo.nc
ncwa -O -a lat,lon -w area -B "gris_mask == 1" $outpath/lnd/climos/${case}_DJF_climo.nc $outpath/lnd/climos/${case}_DJF_aavg_climo.nc
mkdir -p "$outpath/lnd/davg"
#echo "make annualized daily averages for land for all variables"
#ls $path/lnd/hist/$case.$lnd.h1.*.nc | /ccs/home/zender/bin_rhea/ncclimo --job_nbr=12 -c $case -C dly -s ${year_start} -e ${year_end} -o $outpath/lnd/davg -m $lnd
#/ccs/home/zender/bin_rhea/ncclimo --job_nbr=12 -c $case -C dly -s ${year_start} -e ${year_end} -o $outpath/lnd/davg -m $lnd -i $path/lnd/hist
#echo "loop over each variable to create and process time series for h1 vars"
#qsub rhea_tseries_lnd.sh
#for ivar in "${lnd_h1_vars[@]}"
#do
#echo $ivar
#echo "concatenate all daily dates to be averaged into annualized land time series for $ivar"
#ncrcat -O -F -v $ivar $outpath/lnd/davg/*$year_start*$year_end*_climo.nc $outpath/lnd/davg/$case.$lnd.h1.$ivar.davg.${year_start}-${year_end}.nc
#
#echo "add mask and area variables to annualized daily land averages for $ivar"
#ncks -A -v gris_mask,area $outpath/masks/$case.$lnd.gris_mask.nc $outpath/lnd/davg/$case.$lnd.h1.$ivar.davg.${year_start}-${year_end}.nc
#echo "average over ice sheet region for annualized avgd daily land values for $ivar"
#ncwa -O -a lat,lon -w area -B "gris_mask == 1" $outpath/lnd/davg/$case.$lnd.h1.$ivar.davg.${year_start}-${year_end}.nc $outpath/lnd/davg/$case.$lnd.h1.$ivar.davg.aavg.${year_start}-${year_end}.nc
#
#if test -a $outpath/lnd/davg/$case.$lnd.h1.$ivar.davg.aavg.${year_start}-${year_end}.nc; then
# echo "Annualized daily masked $lnd area averages made"
#else
# echo "Annualized daily masked $lnd area averages failed"
# exit
#fi
elif [ "$version" == "e3sm" ]; then
# only needed if no mask in sim output
#for fl in $outpath/lnd/climos/*.nc; do
#echo "add mask in original grid to climos in $fl"
#ncks -A -C -v gris_mask $mask_file $fl
#done
elif [ "$version" == "e3sm" ]; then
for fl in $outpath/lnd/climos/*.nc; do
echo "add mask in original grid to climos in $fl"
ncks -A -C -v gris_mask $outpath/masks/mask_ne30np4_lnd_notime.nc $fl
done
#echo " remapping these land climo files"
#climos_path=$outpath/lnd/climo_regrid
#mkdir -p "$climos_path"
# regrid climos to lat lon
climos_path=$outpath/lnd/climo_regrid
mkdir -p "$climos_path"
#ls $outpath/lnd/climos/*_climo*
#ls $outpath/lnd/climos/*_climo* | ncremap -m $map_file -O $climos_path
echo " remapping these land climo files"
ls $outpath/lnd/climos/*_climo*
ls $outpath/lnd/climos/*_climo* | ncremap -m $map_file -O $climos_path
climos_path=$outpath/lnd/climos
echo "average land monthly climos over ice sheet region"
ncwa -O -a lat,lon -w area -B "gris_mask == 1" $climos_path/${case}_ANN_climo.nc $climos_path/${case}_ANN_aavg_climo.nc
......
#!/bin/bash
module load nco/4.6.9
#if run stand alone
# module load nco/4.6.9
#var=("t2m")
var=("t2m" "swsn" "swsd" "lwsn" "lwsd" "smb" "latf" "senf" "gbot" "alb")
......@@ -9,7 +10,8 @@ file_start_year=1958
start_year=1980
end_year=1999
hist_path=$base/$casename
hist_path=$obspath/$casename
echo "$hist_path"
lndpath=$hist_path/climos
outseries=$hist_path/tseries
mask_file=RACMO23_masks_ZGRN11_latlon.nc
......@@ -26,8 +28,6 @@ tseries_file=$casename.${ivar}.ann_tseries
mkdir -p "$outseries"
mkdir -p "$lndpath"
#cd "$hist_path"
let "go=($start_year-$file_start_year)*12"
let "stop=($end_year-$file_start_year)*12"
......
#!/bin/bash
#clear
# 8/18 to run this script on E3SM v1 on CORI at NERSC
......@@ -11,30 +10,43 @@
# update specialty files used for masking and regridding
# total area of GrIS in model is 1.719e6
postprocess=true
make_plots=false
if [[ $(hostname -s) = rhea* ]]; then
#user settings
project=cli115
case=20180215.DECKv1b_H1.ne30_oEC.edison
year_start=1979
year_end=2005
base=/lustre/atlas1/$project/proj-shared/salil/archive
path=$base/$case/archive
obspath=/lustre/atlas1/$project/world-shared/$USER/obs_data
outpath=/lustre/atlas1/$project/world-shared/$USER/$case/postproc
# compute center settings
obspath=/lustre/atlas1/cli115/world-shared/4ue/obs_data
area_file=/lustre/atlas1/$project/world-shared/$USER/surfdata_0.9x1.25_simyr1850_c110725.nc
map_file=/ccs/home/zender/data/maps/map_ne30np4_to_fv129x256_aave.20150901.nc
mask_file=/lustre/atlas1/$project/world-shared/4ue/$case/postproc/mask_ne30np4_notime.nc
mask_file=/lustre/atlas1/$project/world-shared/4ue/$case/postproc/masks/mask_ne30np4_notime.nc
obs_mask_file=/lustre/atlas1/$project/world-shared/4ue/$case/postproc/masks/E3SMv1_ne30.clm2.gris_mask.nc
postprocess_obs=false
elif [[ $(hostname -s) = cori* ]]; then
#user settings
project=piscees
case=20180612.B_case.T62_oEC60to30v3wLI.modified_runoff_mapping.edison
year_start=0070
year_end=0075
base=/global/cscratch1/sd/dcomeau/acme_scratch/edison/archive
path=$base/$case/run
#need to specify prebase if using data created by someone else until masks are written with sim output
prebase=/global/cscratch1/sd/dcomeau/acme_scratch/edison/archive
base=/global/cscratch1/sd/$USER/acme_scratch/edison
path=$base/$case
outpath=/global/cscratch1/sd/$USER/acme_scratch/edison/$case/postproc
# compute center settings
obspath=/project/projectdirs/piscees/postproc/obs_data
outpath=/global/cscratch1/sd/$USER/$case/postproc
area_file=/project/projectdirs/piscees/postproc/surfdata_0.9x1.25_simyr1850_c110725.nc
map_file=/project/projectdirs/piscees/postproc/map_ne30np4_to_fv129x256_aave.20150901.nc
#mask_file=/project/projectdirs/piscees/postproc/mask_ne30np4_notime.nc
mask_file=/project/projectdirs/piscees/postproc/mask_ne30np4_notime.nc
obs_mask_file=/project/projectdirs/piscees/postproc/E3SMv1_ne30.clm2.gris_mask.nc
postprocess_obs=false
else
echo "machine not recognized"
fi
......@@ -43,10 +55,6 @@ atm_vars=("PSL" "CLDLOW" "CLDHGH" "CLDMED" "CLDTOT" "TREFHT" "TS" "TSMX" "TSMN"
lnd_h0_vars=("FSDS" "QRUNOFF" "QSOIL" "TSA" "RAIN" "SNOW" "FIRA" "FLDS" "FSH" "QSNOMELT")
#glc_vars=("acab" "thk")
postprocess=true
postprocess_obs=false
make_plots=true
mkdir -p $outpath
echo "$version"
......@@ -56,6 +64,17 @@ echo "$version"
lnd=clm2
atm=cam
# this copy to local dir can be removed once masks are written with sim output
if test -d $prebase/$case; then
echo "Creating user owned target path for output data"
mkdir -p $base/$case/atm
mkdir -p $base/$case/lnd
cp -uvr $prebase/$case/atm $path
cp -uvr $prebase/$case/lnd $path
else
exit
fi
#run a check, is the requested data directory there?
if test -d $path; then
echo "Directory $path exists"
......@@ -93,7 +112,6 @@ elif [[ $(hostname -s) = cori* ]]; then
module load gcc
module load cray-netcdf/4.4.1.1.6
module load gsl
# module load hdf5-parallel/1.10.1 conflit with netcdf
module load nco/4.7.4
module load python/2.7-anaconda-4.4
......@@ -121,9 +139,8 @@ else
exit
fi
# this is a hack- all files from E3SM runs we care about should have a gris_mask in the data
#if [ "$version" == "cesm1" ]; then
#
if [ "$postprocess_obs" = "true" ]; then
#if test -a $outpath/masks/${case}.$lnd.gris_mask.nc; then
# echo "already created gris_mask and area file, moving on"
#else
......@@ -135,14 +152,10 @@ fi
#
#fi # if mask file needs to be created
#fi # CESM
if [ "$postprocess_obs" = "true" ]; then
echo "process and remap cloud obs"
../obs/atm_obs.sh
. atm_obs.sh
echo "process RACMO"
../obs/lnd_obs.sh
. lnd_obs.sh
./remap_RACMO_to_CESM2_inline.sh
......
#!/bin/bash
if test -a $h0atm_path/${case}.${atm}.h0.${year_start}-01.nc; then
echo making $outpath/atm/tseries/$case.$atm.h0.$ivar.nc
echo "make time series of monthly averages for $ivar"
if [ "$version" == "cesm1" ]; then
ncrcat -O -v date,time,lev,$ivar $h0atm_path/$case.$atm.h0.*.nc $outpath/atm/tseries/$case.$atm.h0.$ivar.nc
elif [ "$version" == "e3sm" ]; then
ncrcat -O -v date,time,gris_mask,lev,$ivar $h0atm_path/$case.$atm.h0.*.nc $outpath/atm/tseries/$case.$atm.h0.$ivar.nc
fi
if test -a $outpath/atm/tseries/${case}.$atm.h0.$ivar.nc; then
echo "Monthly $atm averages of $ivar made"
else
echo "Monthly $atm averages of $ivar failed"
exit
fi
else
echo "File $h0atm_path/$case.$atm.h0.$year_start-01.nc does not exist"
echo "No time series or monthly averages made"
fi
echo "make yearly average time series from monthly average time series of $ivar"
ncra --mro -O -h -d time,,,12,12 $outpath/atm/tseries/$case.$atm.h0.$ivar.nc $outpath/atm/tseries/$case.$atm.h0.yrly.${ivar}.nc
# since the monthly data time dimension is "days since 1851" and uses the last day of the month for its monthly average,
# the time dimension needs to be altered to restate it properly when the data is averaged yearly.
if [ "$version" == "cesm1" ]; then
echo "add mask and area variables to time series of $ivar"
ncks -A -v area,gris_mask $outpath/masks/$case.$lnd.gris_mask.nc -o $outpath/atm/tseries/$case.$atm.h0.yrly.$ivar.nc
fi
echo "average over ice sheet for $ivar"
ncwa -O -a lat,lon -w area -B "gris_mask == 1" $outpath/atm/tseries/$case.$atm.h0.yrly.$ivar.nc $outpath/atm/tseries/$case.$atm.aavg.h0.yrly.$ivar.nc
if test -a $outpath/atm/tseries/${case}.$atm.aavg.h0.yrly.$ivar.nc; then
echo "Global avg yearly masked $atm averages of $ivar made"
else
echo "Global avg yearly masked $atm averages of $ivar failed"
exit
fi
echo "$ivar yearly atm $time series complete"
......@@ -47,7 +47,7 @@ fi
elif [ "$version" == "e3sm" ]; then
for fl in $outpath/lnd/tseries/*clm2.h0.yrly*; do
ncks -A -C -v gris_mask $outpath/masks/mask_ne30np4_lnd_notime.nc $fl
ncks -A -C -v gris_mask $mask_file $fl
done
echo "average over ice sheet region for yearly averaged lnd data for $ivar"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment