saving work

atm/crm_main/Makefile.gnu

+include Makefile.inc
+
+# Makefile for KGEN-generated kernel
+
+# Originally used compiler(s)
+#FC_0 := /sw/summit/xl/16.1.1-5/xlf/16.1.1/bin/.orig/xlf2008_r
+FC_0 := /sw/summit/gcc/6.4.0/bin/gfortran
+#FC_0 := mpif90
+#FC_FLAGS_SET_0 := -F/sw/summit/xl/16.1.1-5/xlf/16.1.1/etc/xlf.cfg.rhel.7.6.gcc.4.8.5.cuda.10.1 -g -qfullpath -qmaxmem=-1 -qphsinfo -qzerosize -qfree=f90 -qxlf2003=polymorphic -qspillsize=2500 -qextname=flush -qsuffix=f=f90:cpp=F90 -g -O2 -qoffload -qthreaded -D_MPI
+#FC_FLAGS_SET_0 := -F/sw/summit/xl/16.1.1-5/xlf/16.1.1/etc/xlf.cfg.rhel.7.6.gcc.4.8.5.cuda.10.1 -g -qfullpath -qmaxmem=-1 -qphsinfo -qzerosize -qfree=f90 -qxlf2003=polymorphic -qspillsize=2500 -qextname=flush -qsuffix=f=f90:cpp=F90 -g -O2 -qoffload -qthreaded
+FC_FLAGS_SET_0 := -ffree-form -ffree-line-length-none -O3 -g -I $(EXTRAE_HOME)/include
+
+#CC_0 := /sw/summit/xl/16.1.1-5/xlC/16.1.1/bin/xlc_r
+CC_0 := /sw/summit/gcc/6.4.0/bin/gcc
+CC_FLAGS_SET_0 := -O3 -g -I $(EXTRAE_HOME)/include
+
+LFLAGS = -Wl,-rpath -Wl,$(EXTRAE_HOME)/lib -L$(EXTRAE_HOME)/lib -lseqtrace
+
+ALL_OBJS := crm_module.o vars.o grid.o domain.o crmdims.o sgs.o scalar_momentum.o params.o microphysics.o cloud.o precip_init.o task_util_NOMPI.o precip_proc.o micro_params.o damping.o crm_output_module.o kurant.o abcoefs.o buoyancy.o crm_surface.o zero.o forcing.o ice_fall.o shr_const_mod.o shr_kind_mod.o boundaries.o periodic.o bound_exchange.o advection.o tke_full.o shear_prod3D.o shear_prod2D.o sat.o advect_mom.o advect2_mom_z.o advect2_mom_xy.o diffuse_mom.o diffuse_mom2D.o diffuse_mom3D.o diagnose.o uvw.o coriolis.o pressure.o fft.o press_rhs.o bound_duvdt.o press_grad.o adams.o advect_all_scalars.o advect_scalar.o advect_scalar2D.o advect_scalar3D.o diffuse_scalar.o diffuse_scalar3D.o diffuse_scalar2D.o accelerate_crm.o kernel_driver.o kgen_utils.o tprof_mod.o gammafff.o
+
+build: ${ALL_OBJS}
+	${FC_0} ${FC_FLAGS_SET_0} ${LFLAGS} -o kernel.exe $^  
+
+
+PRERUN := source /sw/summit/extrae/3.7.1/rhel7.5_gnu6.4.0/etc/extrae.sh; export EXTRAE_CONFIG_FILE=extrae.xml;
+# export EXECUTABLE=./pi_instrumentedf # This is taken by extrae.xml
+#${EXECUTABLE}
+
+run: build
+	#mpirun -np 2 ./kernel.exe
+	${PRERUN}./kernel.exe
+
+crm_module.o: crm_module.F90 vars.o grid.o domain.o crmdims.o sgs.o scalar_momentum.o params.o microphysics.o cloud.o precip_init.o task_util_NOMPI.o precip_proc.o micro_params.o damping.o crm_output_module.o kurant.o abcoefs.o buoyancy.o crm_surface.o zero.o forcing.o ice_fall.o shr_const_mod.o shr_kind_mod.o boundaries.o periodic.o bound_exchange.o advection.o tke_full.o shear_prod3D.o shear_prod2D.o sat.o advect_mom.o advect2_mom_z.o advect2_mom_xy.o diffuse_mom.o diffuse_mom2D.o diffuse_mom3D.o diagnose.o uvw.o coriolis.o pressure.o fft.o press_rhs.o bound_duvdt.o press_grad.o adams.o advect_all_scalars.o advect_scalar.o advect_scalar2D.o advect_scalar3D.o diffuse_scalar.o diffuse_scalar3D.o diffuse_scalar2D.o accelerate_crm.o kgen_utils.o tprof_mod.o
+	$(FC_0) ${FC_FLAGS_SET_0} -c -o $@ $(EXTRAE_HOME)/include/extrae_module.f90
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+vars.o: vars.F90 kgen_utils.o tprof_mod.o grid.o params.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+grid.o: grid.F90 kgen_utils.o tprof_mod.o domain.o params.o advection.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+domain.o: domain.F90 kgen_utils.o tprof_mod.o crmdims.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+crmdims.o: crmdims.F90 kgen_utils.o tprof_mod.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+sgs.o: sgs.F90 kgen_utils.o tprof_mod.o params.o grid.o tke_full.o vars.o diffuse_mom.o diffuse_scalar.o microphysics.o scalar_momentum.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+scalar_momentum.o: scalar_momentum.F90 kgen_utils.o tprof_mod.o params.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+params.o: params.F90 kgen_utils.o tprof_mod.o shr_const_mod.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+microphysics.o: microphysics.F90 kgen_utils.o tprof_mod.o params.o vars.o micro_params.o grid.o precip_init.o cloud.o precip_proc.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+cloud.o: cloud.F90 kgen_utils.o tprof_mod.o micro_params.o vars.o params.o sat.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+precip_init.o: precip_init.F90 kgen_utils.o tprof_mod.o micro_params.o vars.o task_util_NOMPI.o sat.o params.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+task_util_NOMPI.o: task_util_NOMPI.F90 kgen_utils.o tprof_mod.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+precip_proc.o: precip_proc.F90 kgen_utils.o tprof_mod.o micro_params.o vars.o sat.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+micro_params.o: micro_params.F90 kgen_utils.o tprof_mod.o params.o grid.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+damping.o: damping.F90 kgen_utils.o tprof_mod.o params.o vars.o task_util_NOMPI.o microphysics.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+crm_output_module.o: crm_output_module.F90 kgen_utils.o tprof_mod.o params.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+kurant.o: kurant.F90 kgen_utils.o tprof_mod.o params.o vars.o sgs.o task_util_NOMPI.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+abcoefs.o: abcoefs.F90 kgen_utils.o tprof_mod.o params.o grid.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+buoyancy.o: buoyancy.F90 kgen_utils.o tprof_mod.o params.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+crm_surface.o: crm_surface.F90 kgen_utils.o tprof_mod.o params.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+zero.o: zero.F90 kgen_utils.o tprof_mod.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+forcing.o: forcing.F90 kgen_utils.o tprof_mod.o params.o vars.o microphysics.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+ice_fall.o: ice_fall.F90 kgen_utils.o tprof_mod.o params.o vars.o microphysics.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+shr_const_mod.o: shr_const_mod.F90 kgen_utils.o tprof_mod.o shr_kind_mod.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+shr_kind_mod.o: shr_kind_mod.F90 kgen_utils.o tprof_mod.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+boundaries.o: boundaries.F90 kgen_utils.o tprof_mod.o periodic.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+periodic.o: periodic.F90 kgen_utils.o tprof_mod.o bound_exchange.o vars.o sgs.o params.o microphysics.o scalar_momentum.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+bound_exchange.o: bound_exchange.F90 kgen_utils.o tprof_mod.o params.o grid.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+advection.o: advection.F90 kgen_utils.o tprof_mod.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+tke_full.o: tke_full.F90 kgen_utils.o tprof_mod.o params.o vars.o shear_prod3D.o shear_prod2D.o sat.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+shear_prod3D.o: shear_prod3D.F90 kgen_utils.o tprof_mod.o params.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+shear_prod2D.o: shear_prod2D.F90 kgen_utils.o tprof_mod.o params.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+sat.o: sat.F90 kgen_utils.o tprof_mod.o params.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+advect_mom.o: advect_mom.F90 kgen_utils.o tprof_mod.o advect2_mom_xy.o advect2_mom_z.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+advect2_mom_z.o: advect2_mom_z.F90 kgen_utils.o tprof_mod.o params.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+advect2_mom_xy.o: advect2_mom_xy.F90 kgen_utils.o tprof_mod.o params.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+diffuse_mom.o: diffuse_mom.F90 kgen_utils.o tprof_mod.o vars.o diffuse_mom3D.o diffuse_mom2D.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+diffuse_mom2D.o: diffuse_mom2D.F90 kgen_utils.o tprof_mod.o params.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+diffuse_mom3D.o: diffuse_mom3D.F90 kgen_utils.o tprof_mod.o params.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+diagnose.o: diagnose.F90 kgen_utils.o tprof_mod.o params.o vars.o sat.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+uvw.o: uvw.F90 kgen_utils.o tprof_mod.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+coriolis.o: coriolis.F90 kgen_utils.o tprof_mod.o params.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+pressure.o: pressure.F90 kgen_utils.o tprof_mod.o vars.o params.o press_rhs.o fft.o press_grad.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+fft.o: fft.F90 kgen_utils.o tprof_mod.o params.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+press_rhs.o: press_rhs.F90 kgen_utils.o tprof_mod.o params.o vars.o bound_duvdt.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+bound_duvdt.o: bound_duvdt.F90 kgen_utils.o tprof_mod.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+press_grad.o: press_grad.F90 kgen_utils.o tprof_mod.o vars.o params.o bound_duvdt.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+adams.o: adams.F90 kgen_utils.o tprof_mod.o params.o vars.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+advect_all_scalars.o: advect_all_scalars.F90 kgen_utils.o tprof_mod.o vars.o advect_scalar.o microphysics.o scalar_momentum.o sgs.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+advect_scalar.o: advect_scalar.F90 kgen_utils.o tprof_mod.o params.o grid.o advect_scalar3D.o vars.o advect_scalar2D.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+advect_scalar2D.o: advect_scalar2D.F90 kgen_utils.o tprof_mod.o params.o grid.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+advect_scalar3D.o: advect_scalar3D.F90 kgen_utils.o tprof_mod.o params.o grid.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+diffuse_scalar.o: diffuse_scalar.F90 kgen_utils.o tprof_mod.o params.o grid.o diffuse_scalar3D.o vars.o diffuse_scalar2D.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+diffuse_scalar3D.o: diffuse_scalar3D.F90 kgen_utils.o tprof_mod.o params.o grid.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+diffuse_scalar2D.o: diffuse_scalar2D.F90 kgen_utils.o tprof_mod.o params.o grid.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+accelerate_crm.o: accelerate_crm.F90 kgen_utils.o tprof_mod.o params.o shr_kind_mod.o grid.o vars.o microphysics.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+kernel_driver.o: kernel_driver.F90 crm_module.o vars.o grid.o domain.o crmdims.o sgs.o scalar_momentum.o params.o microphysics.o cloud.o precip_init.o task_util_NOMPI.o precip_proc.o micro_params.o damping.o crm_output_module.o kurant.o abcoefs.o buoyancy.o crm_surface.o zero.o forcing.o ice_fall.o shr_const_mod.o shr_kind_mod.o boundaries.o periodic.o bound_exchange.o advection.o tke_full.o shear_prod3D.o shear_prod2D.o sat.o advect_mom.o advect2_mom_z.o advect2_mom_xy.o diffuse_mom.o diffuse_mom2D.o diffuse_mom3D.o diagnose.o uvw.o coriolis.o pressure.o fft.o press_rhs.o bound_duvdt.o press_grad.o adams.o advect_all_scalars.o advect_scalar.o advect_scalar2D.o advect_scalar3D.o diffuse_scalar.o diffuse_scalar3D.o diffuse_scalar2D.o accelerate_crm.o kgen_utils.o tprof_mod.o
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+kgen_utils.o: kgen_utils.f90
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+tprof_mod.o: tprof_mod.f90
+	${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
+
+gammafff.o: gammafff.c
+	${CC_0} ${CC_FLAGS_SET_0} -c -o $@ $<
+
+clean:
+	rm -fr kernel.exe *.mod ${ALL_OBJS} TRACE*.mpit* TRACE.sym set-0
+

atm/crm_main/Makefile.inc

+# This file is automatically generated after the make install step.
+
+EXTRAE_HOME=/sw/summit/extrae/3.7.1/rhel7.5_gnu6.4.0
+MPI_HOME=/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi
+MPI_CFLAGS=-I/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/include
+MPI_LDFLAGS=-L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/lib
+MPI_LIBS=-lmpi_ibm
+PAPI_HOME=/autofs/nccs-svm1_sw/summit/.swci/0-core/opt/spack/20180914/linux-rhel7-ppc64le/gcc-4.8.5/papi-5.6.0-xhd3soyh4mzxlhqmtinhylnchxuqkshb/
+XML2_HOME= /usr
+XML2_LDFLAGS=-L/usr/lib64
+XML2_LIBS=-lxml2
+UNWIND_HOME=/sw/summit/scorep/libunwind-1.3.1/rhel7.5_gnu6.4.0/
+BFD_HOME=/usr
+LIBERTY_HOME=/usr
+BFD_LIBS=-L/usr/lib64 -lbfd -L/usr/lib64 -liberty
+UNWIND_LIBS=-L$(UNWIND_HOME)/lib -lunwind
+PAPI_LIBS=-L$(PAPI_HOME)/lib -lpapi
+CUPTI_LIBS=
+POSIX_CLOCK_LIBS=
+LIBEXECINFO_LIBS= # FreeBSD specific
+ZLIB_HOME=/usr
+ZLIB_LIBS=-lz
+DL_LIBS=-ldl
+OPENCL_HOME=
+OPENCL_LDFLAGS=
+
+CC=/sw/summit/gcc/6.4.0/bin/gcc
+FC=/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/bin/mpif90
+CXX=/sw/summit/gcc/6.4.0/bin/g++
+MPICC=/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/bin/mpicc
+MPIF77=/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/bin/mpif77
+MPIF90=/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/bin/mpif90
+NVCC=/sw/summit/cuda/10.1.105/bin/nvcc
+JAVAC=
+JAR=
+
+# For statically linked apps or systems
+
+EXTRAE_LDFLAGS_LIBSEQTRACE=-L$(EXTRAE_HOME)/lib -lseqtrace -L$(ZLIB_HOME)/lib -lz -L$(PAPI_HOME)/lib -lpapi -L$(XML2_HOME)/lib -lxml2 -L$(BFD_HOME)/lib -lbfd -liberty
+
+EXTRAE_LDFLAGS_LIBMPITRACE=-L$(EXTRAE_HOME)/lib -lmpitrace -L$(ZLIB_HOME)/lib -lz -L$(PAPI_HOME)/lib -lpapi -L$(XML2_HOME)/lib -lxml2 -L$(BFD_HOME)/lib -lbfd -liberty
+
+EXTRAE_LDFLAGS_LIBMPITRACEF=-L$(EXTRAE_HOME)/lib -lmpitracef -L$(ZLIB_HOME)/lib -lz -L$(PAPI_HOME)/lib -lpapi -L$(XML2_HOME)/lib -lxml2 -L$(BFD_HOME)/lib -lbfd -liberty
+
+

atm/crm_main/crm_module.F90

@@ -41,6 +41,9 @@ contains
 SUBROUTINE crm(kgen_unit, kgen_measure, kgen_isverified, kgen_filepath, ncrms, plev) 
     !-----------------------------------------------------------------------------------------------
     !-----------------------------------------------------------------------------------------------
+
+    USE EXTRAE_MODULE
+
     USE shr_kind_mod, ONLY: r8 => shr_kind_r8 
     USE vars 
     USE microphysics 
@@ -126,7 +129,7 @@ SUBROUTINE crm(kgen_unit, kgen_measure, kgen_isverified, kgen_filepath, ncrms, p
     INTEGER :: kgen_mpirank, kgen_openmptid, kgen_kernelinvoke 
     LOGICAL :: kgen_evalstage, kgen_warmupstage, kgen_mainstage 
     COMMON / state / kgen_mpirank, kgen_openmptid, kgen_kernelinvoke, kgen_evalstage, kgen_warmupstage, kgen_mainstage 
-    INTEGER, PARAMETER :: KGEN_MAXITER = 10
+    INTEGER, PARAMETER :: KGEN_MAXITER = 100
       
     TYPE(check_t) :: check_status 
     INTEGER*8 :: kgen_start_clock, kgen_stop_clock, kgen_rate_clock 
@@ -758,6 +761,8 @@ SUBROUTINE crm(kgen_unit, kgen_measure, kgen_isverified, kgen_filepath, ncrms, p
           
         CALL SYSTEM_CLOCK(kgen_start_clock, kgen_rate_clock) 
         DO kgen_intvar = 1, KGEN_MAXITER 
+
+            call extrae_user_function(1)
     nstep = nstep + 1
     !$omp target teams distribute parallel do 
     do icrm = 1 , ncrms
@@ -1075,6 +1080,10 @@ SUBROUTINE crm(kgen_unit, kgen_measure, kgen_isverified, kgen_filepath, ncrms, p
       enddo
     enddo
         END DO   
+
+            call extrae_user_function(0)
+			call extrae_next_hwc_set()
+
         CALL SYSTEM_CLOCK(kgen_stop_clock, kgen_rate_clock) 
         kgen_measure = 1.0D6*(kgen_stop_clock - kgen_start_clock)/DBLE(kgen_rate_clock*KGEN_MAXITER) 
 #ifdef _MPI 

atm/crm_main/crm_module.F90.extrae

+!KGEN-generated Fortran source file 
+  
+!Generated at : 2020-05-29 19:47:47 
+!KGEN version : 0.9.0 
+  
+
+
+module crm_module
+    USE abcoefs_mod, ONLY: abcoefs 
+    USE kurant_mod, ONLY: kurant 
+    USE boundaries_mod, ONLY: boundaries 
+    USE forcing_mod, ONLY: forcing 
+    USE advect_mom_mod, ONLY: advect_mom 
+    USE adams_mod, ONLY: adams 
+    USE advect_all_scalars_mod, ONLY: advect_all_scalars 
+    USE crm_surface_mod 
+    USE zero_mod 
+    USE buoyancy_mod 
+    USE pressure_mod 
+    USE uvw_mod 
+    USE diagnose_mod 
+    USE damping_mod 
+    USE ice_fall_mod 
+    USE coriolis_mod 
+    USE crm_output_module 
+
+!---------------------------------------------------------------
+!  Super-parameterization's main driver
+!  Marat Khairoutdinov, 2001-2009
+!---------------------------------------------------------------
+    USE kgen_utils_mod, ONLY: kgen_dp, kgen_array_sumcheck 
+    USE tprof_mod, ONLY: tstart, tstop, tnull, tprnt 
+    PUBLIC crm 
+#ifdef _MPI 
+    include "mpif.h" 
+#endif 
+      
+
+contains
+
+SUBROUTINE crm(kgen_unit, kgen_measure, kgen_isverified, kgen_filepath, ncrms, plev) 
+    !-----------------------------------------------------------------------------------------------
+    !-----------------------------------------------------------------------------------------------
+
+    USE EXTRAE_MODULE
+
+    USE shr_kind_mod, ONLY: r8 => shr_kind_r8 
+    USE vars 
+    USE microphysics 
+    USE sgs 
+    USE scalar_momentum_mod 
+    USE crmdims, ONLY: crm_nx_rad, crm_ny_rad 
+    USE accelerate_crm_mod, ONLY: use_crm_accel, accelerate_crm 
+    USE kgen_utils_mod, ONLY: kgen_dp, kgen_array_sumcheck 
+    USE kgen_utils_mod, ONLY: kgen_perturb_real 
+    USE vars, ONLY: kr_externs_out_vars 
+    USE grid, ONLY: kr_externs_out_grid 
+    USE sgs, ONLY: kr_externs_out_sgs 
+    USE params, ONLY: kr_externs_out_params 
+    USE microphysics, ONLY: kr_externs_out_microphysics 
+    USE micro_params, ONLY: kr_externs_out_micro_params 
+    USE crm_output_module, ONLY: kr_externs_out_crm_output_module 
+    USE accelerate_crm_mod, ONLY: kr_externs_out_accelerate_crm_mod 
+    USE kgen_utils_mod, ONLY: check_t, kgen_init_check, kgen_init_verify, kgen_tolerance, kgen_minvalue, kgen_verboselevel, &
+    &CHECK_IDENTICAL, CHECK_IN_TOL, CHECK_OUT_TOL 
+    USE vars, ONLY: kv_externs_vars 
+    USE grid, ONLY: kv_externs_grid 
+    USE params, ONLY: kv_externs_params 
+    USE microphysics, ONLY: kv_externs_microphysics 
+    USE micro_params, ONLY: kv_externs_micro_params 
+    USE crm_output_module, ONLY: kv_externs_crm_output_module 
+
+    IMPLICIT NONE 
+    !-----------------------------------------------------------------------------------------------
+    ! Interface variable declarations
+    !-----------------------------------------------------------------------------------------------
+
+
+    INTEGER, INTENT(INOUT) :: ncrms 
+    INTEGER, INTENT(INOUT) :: plev 
+    !-----------------------------------------------------------------------------------------------
+    ! Local variable declarations
+    !-----------------------------------------------------------------------------------------------
+
+
+    real(r8),       parameter :: wmin = 2.                      ! minimum up/downdraft velocity for stat
+    real(crm_rknd), parameter :: cwp_threshold = 0.001          ! threshold for cloud condensate for shaded fraction calculation
+    REAL(KIND=crm_rknd) :: tmp1, tmp 
+    INTEGER :: i, j, k, l, nn, icyc, icrm 
+    INTEGER :: kx 
+    REAL(KIND=crm_rknd) :: qsat 
+    REAL(KIND=crm_rknd), allocatable :: bflx(:) 
+    ! These should all be inputs
+
+    ! variables for radiation grouping method
+    INTEGER :: i_rad 
+    INTEGER :: j_rad 
+    LOGICAL :: crm_accel_ceaseflag 
+    ! Arrays
+
+    REAL(KIND=crm_rknd), allocatable :: cwp         (:,:,:) 
+    REAL(KIND=crm_rknd), allocatable :: cwph       (:,:,:) 
+    REAL(KIND=crm_rknd), allocatable :: cwpm       (:,:,:) 
+    REAL(KIND=crm_rknd), allocatable :: cwpl       (:,:,:) 
+    LOGICAL, allocatable :: flag_top(:,:,:) 
+    REAL(KIND=crm_rknd), allocatable :: cltemp   (:,:,:) 
+    REAL(KIND=crm_rknd), allocatable :: cmtemp   (:,:,:) 
+    REAL(KIND=crm_rknd), allocatable :: chtemp   (:,:,:) 
+    REAL(KIND=crm_rknd), allocatable :: cttemp   (:,:,:) 
+
+    REAL(KIND=r8), allocatable :: mui_crm(:,:) 
+    REAL(KIND=r8), allocatable :: mdi_crm(:,:) 
+
+    REAL(KIND=crm_rknd), pointer :: crm_rad_temperature   (:,:,:,:) 
+    REAL(KIND=crm_rknd), pointer :: crm_rad_qv                     (:,:,:,:) 
+    REAL(KIND=crm_rknd), pointer :: crm_rad_qc                     (:,:,:,:) 
+    REAL(KIND=crm_rknd), pointer :: crm_rad_qi                     (:,:,:,:) 
+    REAL(KIND=crm_rknd), pointer :: crm_rad_cld                   (:,:,:,:) 
+    REAL(KIND=crm_rknd), pointer :: crm_rad_qrad                 (:,:,:,:) 
+  !-----------------------------------------------------------------------------------------------
+  !-----------------------------------------------------------------------------------------------
+    INTEGER, INTENT(IN) :: kgen_unit 
+    REAL(KIND=kgen_dp), INTENT(OUT) :: kgen_measure 
+    LOGICAL, INTENT(OUT) :: kgen_isverified 
+    CHARACTER(LEN=*), INTENT(IN) :: kgen_filepath 
+    LOGICAL :: kgen_istrue 
+    REAL(KIND=8) :: kgen_array_sum 
+    INTEGER :: kgen_intvar, kgen_ierr 
+    INTEGER :: kgen_mpirank, kgen_openmptid, kgen_kernelinvoke 
+    LOGICAL :: kgen_evalstage, kgen_warmupstage, kgen_mainstage 
+    COMMON / state / kgen_mpirank, kgen_openmptid, kgen_kernelinvoke, kgen_evalstage, kgen_warmupstage, kgen_mainstage 
+    INTEGER, PARAMETER :: KGEN_MAXITER = 10
+      
+    TYPE(check_t) :: check_status 
+    INTEGER*8 :: kgen_start_clock, kgen_stop_clock, kgen_rate_clock 
+    REAL(KIND=kgen_dp) :: gkgen_measure 
+    REAL(KIND=crm_rknd) :: kgenref_tmp1 
+    REAL(KIND=crm_rknd) :: kgenref_tmp 
+    INTEGER :: kgenref_icrm 
+    INTEGER :: kgenref_icyc 
+    INTEGER :: kgenref_k 
+    INTEGER :: kgenref_j 
+    INTEGER :: kgenref_i 
+    INTEGER :: kgenref_nn 
+    INTEGER :: kgenref_l 
+    INTEGER :: kgenref_kx 
+    REAL(KIND=crm_rknd) :: kgenref_qsat 
+    INTEGER :: kgenref_i_rad 
+    INTEGER :: kgenref_j_rad 
+    LOGICAL :: kgenref_crm_accel_ceaseflag 
+    REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cwp 
+    REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cwph 
+    REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cwpm 
+    REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cwpl 
+    LOGICAL, allocatable, dimension(:,:,:) :: kgenref_flag_top 
+    REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cltemp 
+    REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cmtemp 
+    REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_chtemp 
+    REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cttemp 
+    REAL(KIND=r8), allocatable, dimension(:,:) :: kgenref_mui_crm 
+    REAL(KIND=r8), allocatable, dimension(:,:) :: kgenref_mdi_crm 
+    REAL(KIND=crm_rknd), pointer, dimension(:,:,:,:) :: kgenref_crm_rad_temperature 
+    REAL(KIND=crm_rknd), pointer, dimension(:,:,:,:) :: kgenref_crm_rad_qv 
+    REAL(KIND=crm_rknd), pointer, dimension(:,:,:,:) :: kgenref_crm_rad_qc 
+    REAL(KIND=crm_rknd), pointer, dimension(:,:,:,:) :: kgenref_crm_rad_qi 
+    REAL(KIND=crm_rknd), pointer, dimension(:,:,:,:) :: kgenref_crm_rad_cld 
+      
+    !parent block preprocessing 
+      
+#ifdef _MPI 
+    call mpi_comm_rank(mpi_comm_world, kgen_mpirank, kgen_ierr) 
+#else 
+    kgen_mpirank = 0 
+#endif 
+      
+      
+    !local input variables 
+    READ (UNIT = kgen_unit) tmp1 
+    READ (UNIT = kgen_unit) tmp 
+    READ (UNIT = kgen_unit) icrm 
+    READ (UNIT = kgen_unit) icyc 
+    READ (UNIT = kgen_unit) i 
+    READ (UNIT = kgen_unit) j 
+    READ (UNIT = kgen_unit) k 
+    READ (UNIT = kgen_unit) nn 
+    READ (UNIT = kgen_unit) l 
+    READ (UNIT = kgen_unit) kx 
+    READ (UNIT = kgen_unit) qsat 
+    CALL kr_crm_real__crm_rknd_dim1(bflx, kgen_unit, "bflx", .FALSE.) 
+    READ (UNIT = kgen_unit) i_rad 
+    READ (UNIT = kgen_unit) j_rad 
+    READ (UNIT = kgen_unit) crm_accel_ceaseflag 
+    CALL kr_crm_real__crm_rknd_dim3(cwp, kgen_unit, "cwp", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(cwph, kgen_unit, "cwph", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(cwpm, kgen_unit, "cwpm", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(cwpl, kgen_unit, "cwpl", .FALSE.) 
+    CALL kr_crm_logical___dim3(flag_top, kgen_unit, "flag_top", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(cltemp, kgen_unit, "cltemp", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(cmtemp, kgen_unit, "cmtemp", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(chtemp, kgen_unit, "chtemp", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(cttemp, kgen_unit, "cttemp", .FALSE.) 
+    CALL kr_crm_real__r8_dim2(mui_crm, kgen_unit, "mui_crm", .FALSE.) 
+    CALL kr_crm_real__r8_dim2(mdi_crm, kgen_unit, "mdi_crm", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_temperature, kgen_unit, "crm_rad_temperature", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_qv, kgen_unit, "crm_rad_qv", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_qc, kgen_unit, "crm_rad_qc", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_qi, kgen_unit, "crm_rad_qi", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_cld, kgen_unit, "crm_rad_cld", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_qrad, kgen_unit, "crm_rad_qrad", .FALSE.) 
+      
+    !extern output variables 
+    CALL kr_externs_out_vars(kgen_unit) 
+    CALL kr_externs_out_grid(kgen_unit) 
+    CALL kr_externs_out_sgs(kgen_unit) 
+    CALL kr_externs_out_params(kgen_unit) 
+    CALL kr_externs_out_microphysics(kgen_unit) 
+    CALL kr_externs_out_micro_params(kgen_unit) 
+    CALL kr_externs_out_crm_output_module(kgen_unit) 
+    CALL kr_externs_out_accelerate_crm_mod(kgen_unit) 
+      
+    !local output variables 
+    READ (UNIT = kgen_unit) kgenref_tmp1 
+    READ (UNIT = kgen_unit) kgenref_tmp 
+    READ (UNIT = kgen_unit) kgenref_icrm 
+    READ (UNIT = kgen_unit) kgenref_icyc 
+    READ (UNIT = kgen_unit) kgenref_k 
+    READ (UNIT = kgen_unit) kgenref_j 
+    READ (UNIT = kgen_unit) kgenref_i 
+    READ (UNIT = kgen_unit) kgenref_nn 
+    READ (UNIT = kgen_unit) kgenref_l 
+    READ (UNIT = kgen_unit) kgenref_kx 
+    READ (UNIT = kgen_unit) kgenref_qsat 
+    READ (UNIT = kgen_unit) kgenref_i_rad 
+    READ (UNIT = kgen_unit) kgenref_j_rad 
+    READ (UNIT = kgen_unit) kgenref_crm_accel_ceaseflag 
+    CALL kr_crm_real__crm_rknd_dim3(kgenref_cwp, kgen_unit, "kgenref_cwp", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(kgenref_cwph, kgen_unit, "kgenref_cwph", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(kgenref_cwpm, kgen_unit, "kgenref_cwpm", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(kgenref_cwpl, kgen_unit, "kgenref_cwpl", .FALSE.) 
+    CALL kr_crm_logical___dim3(kgenref_flag_top, kgen_unit, "kgenref_flag_top", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(kgenref_cltemp, kgen_unit, "kgenref_cltemp", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(kgenref_cmtemp, kgen_unit, "kgenref_cmtemp", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(kgenref_chtemp, kgen_unit, "kgenref_chtemp", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim3(kgenref_cttemp, kgen_unit, "kgenref_cttemp", .FALSE.) 
+    CALL kr_crm_real__r8_dim2(kgenref_mui_crm, kgen_unit, "kgenref_mui_crm", .FALSE.) 
+    CALL kr_crm_real__r8_dim2(kgenref_mdi_crm, kgen_unit, "kgenref_mdi_crm", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(kgenref_crm_rad_temperature, kgen_unit, "kgenref_crm_rad_temperature", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(kgenref_crm_rad_qv, kgen_unit, "kgenref_crm_rad_qv", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(kgenref_crm_rad_qc, kgen_unit, "kgenref_crm_rad_qc", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(kgenref_crm_rad_qi, kgen_unit, "kgenref_crm_rad_qi", .FALSE.) 
+    CALL kr_crm_real__crm_rknd_dim4_ptr(kgenref_crm_rad_cld, kgen_unit, "kgenref_crm_rad_cld", .FALSE.) 
+
+
+  !$omp target enter data map(alloc: t00      )
+  !$omp target enter data map(alloc: tln      )
+  !$omp target enter data map(alloc: qln      )
+  !$omp target enter data map(alloc: qccln    )
+  !$omp target enter data map(alloc: qiiln    )
+  !$omp target enter data map(alloc: uln      )
+  !$omp target enter data map(alloc: vln      )
+  !$omp target enter data map(alloc: cwp      )
+  !$omp target enter data map(alloc: cwph     )
+  !$omp target enter data map(alloc: cwpm     )
+  !$omp target enter data map(alloc: cwpl     )
+  !$omp target enter data map(alloc: flag_top )
+  !$omp target enter data map(alloc: cltemp   )
+  !$omp target enter data map(alloc: cmtemp   )
+  !$omp target enter data map(alloc: chtemp   )
+  !$omp target enter data map(alloc: cttemp   )
+  !$omp target enter data map(alloc: dd_crm   )
+  !$omp target enter data map(alloc: mui_crm  )
+  !$omp target enter data map(alloc: mdi_crm  )
+  !$omp target enter data map(alloc: ustar    )
+  !$omp target enter data map(alloc: bflx     )
+  !$omp target enter data map(alloc: wnd      )
+  !$omp target enter data map(alloc: qtot     )
+  !$omp target enter data map(alloc: colprec  )
+  !$omp target enter data map(alloc: colprecs )
+
+  !$omp target enter data map(alloc: crm_rad_temperature )
+  !$omp target enter data map(alloc: crm_rad_qv )
+  !$omp target enter data map(alloc: crm_rad_qc )
+  !$omp target enter data map(alloc: crm_rad_qi )
+  !$omp target enter data map(alloc: crm_rad_cld )
+  !$omp target enter data map(alloc: crm_rad_qrad )
+
+  !$omp target enter data map(alloc: crm_state_u_wind )
+  !$omp target enter data map(alloc: crm_state_v_wind )
+  !$omp target enter data map(alloc: crm_state_w_wind )
+  !$omp target enter data map(alloc: crm_state_temperature )
+  !$omp target enter data map(alloc: crm_state_qt )
+  !$omp target enter data map(alloc: crm_state_qp )
+  !$omp target enter data map(alloc: crm_state_qn )
+
+
+  !Loop over "vector columns"
+
+
+!-----------------------------------------------
+
+
+!-----------------------------------------
+
+
+  ! update data at device
+  !$omp taskwait
+  !  Initialize CRM fields:
+
+  !$omp target teams distribute parallel do collapse(4)
+
+  ! limit the velocity at the very first step:
+
+
+  ! Populate microphysics array from crm_state
+
+
+  !$omp target teams distribute parallel do collapse(4)
+
+
+  ! initialize sgs fields
+
+
+  !$omp target teams distribute parallel do 
+