Commit 669989b0 authored by Youngsung Kim's avatar Youngsung Kim
Browse files

saving work

parent 5bef05f3
include Makefile.inc
# Makefile for KGEN-generated kernel
# Originally used compiler(s)
#FC_0 := /sw/summit/xl/16.1.1-5/xlf/16.1.1/bin/.orig/xlf2008_r
FC_0 := /sw/summit/gcc/6.4.0/bin/gfortran
#FC_0 := mpif90
#FC_FLAGS_SET_0 := -F/sw/summit/xl/16.1.1-5/xlf/16.1.1/etc/xlf.cfg.rhel.7.6.gcc.4.8.5.cuda.10.1 -g -qfullpath -qmaxmem=-1 -qphsinfo -qzerosize -qfree=f90 -qxlf2003=polymorphic -qspillsize=2500 -qextname=flush -qsuffix=f=f90:cpp=F90 -g -O2 -qoffload -qthreaded -D_MPI
#FC_FLAGS_SET_0 := -F/sw/summit/xl/16.1.1-5/xlf/16.1.1/etc/xlf.cfg.rhel.7.6.gcc.4.8.5.cuda.10.1 -g -qfullpath -qmaxmem=-1 -qphsinfo -qzerosize -qfree=f90 -qxlf2003=polymorphic -qspillsize=2500 -qextname=flush -qsuffix=f=f90:cpp=F90 -g -O2 -qoffload -qthreaded
FC_FLAGS_SET_0 := -ffree-form -ffree-line-length-none -O3 -g -I $(EXTRAE_HOME)/include
#CC_0 := /sw/summit/xl/16.1.1-5/xlC/16.1.1/bin/xlc_r
CC_0 := /sw/summit/gcc/6.4.0/bin/gcc
CC_FLAGS_SET_0 := -O3 -g -I $(EXTRAE_HOME)/include
LFLAGS = -Wl,-rpath -Wl,$(EXTRAE_HOME)/lib -L$(EXTRAE_HOME)/lib -lseqtrace
ALL_OBJS := crm_module.o vars.o grid.o domain.o crmdims.o sgs.o scalar_momentum.o params.o microphysics.o cloud.o precip_init.o task_util_NOMPI.o precip_proc.o micro_params.o damping.o crm_output_module.o kurant.o abcoefs.o buoyancy.o crm_surface.o zero.o forcing.o ice_fall.o shr_const_mod.o shr_kind_mod.o boundaries.o periodic.o bound_exchange.o advection.o tke_full.o shear_prod3D.o shear_prod2D.o sat.o advect_mom.o advect2_mom_z.o advect2_mom_xy.o diffuse_mom.o diffuse_mom2D.o diffuse_mom3D.o diagnose.o uvw.o coriolis.o pressure.o fft.o press_rhs.o bound_duvdt.o press_grad.o adams.o advect_all_scalars.o advect_scalar.o advect_scalar2D.o advect_scalar3D.o diffuse_scalar.o diffuse_scalar3D.o diffuse_scalar2D.o accelerate_crm.o kernel_driver.o kgen_utils.o tprof_mod.o gammafff.o
build: ${ALL_OBJS}
${FC_0} ${FC_FLAGS_SET_0} ${LFLAGS} -o kernel.exe $^
PRERUN := source /sw/summit/extrae/3.7.1/rhel7.5_gnu6.4.0/etc/extrae.sh; export EXTRAE_CONFIG_FILE=extrae.xml;
# export EXECUTABLE=./pi_instrumentedf # This is taken by extrae.xml
#${EXECUTABLE}
run: build
#mpirun -np 2 ./kernel.exe
${PRERUN}./kernel.exe
crm_module.o: crm_module.F90 vars.o grid.o domain.o crmdims.o sgs.o scalar_momentum.o params.o microphysics.o cloud.o precip_init.o task_util_NOMPI.o precip_proc.o micro_params.o damping.o crm_output_module.o kurant.o abcoefs.o buoyancy.o crm_surface.o zero.o forcing.o ice_fall.o shr_const_mod.o shr_kind_mod.o boundaries.o periodic.o bound_exchange.o advection.o tke_full.o shear_prod3D.o shear_prod2D.o sat.o advect_mom.o advect2_mom_z.o advect2_mom_xy.o diffuse_mom.o diffuse_mom2D.o diffuse_mom3D.o diagnose.o uvw.o coriolis.o pressure.o fft.o press_rhs.o bound_duvdt.o press_grad.o adams.o advect_all_scalars.o advect_scalar.o advect_scalar2D.o advect_scalar3D.o diffuse_scalar.o diffuse_scalar3D.o diffuse_scalar2D.o accelerate_crm.o kgen_utils.o tprof_mod.o
$(FC_0) ${FC_FLAGS_SET_0} -c -o $@ $(EXTRAE_HOME)/include/extrae_module.f90
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
vars.o: vars.F90 kgen_utils.o tprof_mod.o grid.o params.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
grid.o: grid.F90 kgen_utils.o tprof_mod.o domain.o params.o advection.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
domain.o: domain.F90 kgen_utils.o tprof_mod.o crmdims.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
crmdims.o: crmdims.F90 kgen_utils.o tprof_mod.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
sgs.o: sgs.F90 kgen_utils.o tprof_mod.o params.o grid.o tke_full.o vars.o diffuse_mom.o diffuse_scalar.o microphysics.o scalar_momentum.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
scalar_momentum.o: scalar_momentum.F90 kgen_utils.o tprof_mod.o params.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
params.o: params.F90 kgen_utils.o tprof_mod.o shr_const_mod.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
microphysics.o: microphysics.F90 kgen_utils.o tprof_mod.o params.o vars.o micro_params.o grid.o precip_init.o cloud.o precip_proc.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
cloud.o: cloud.F90 kgen_utils.o tprof_mod.o micro_params.o vars.o params.o sat.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
precip_init.o: precip_init.F90 kgen_utils.o tprof_mod.o micro_params.o vars.o task_util_NOMPI.o sat.o params.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
task_util_NOMPI.o: task_util_NOMPI.F90 kgen_utils.o tprof_mod.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
precip_proc.o: precip_proc.F90 kgen_utils.o tprof_mod.o micro_params.o vars.o sat.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
micro_params.o: micro_params.F90 kgen_utils.o tprof_mod.o params.o grid.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
damping.o: damping.F90 kgen_utils.o tprof_mod.o params.o vars.o task_util_NOMPI.o microphysics.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
crm_output_module.o: crm_output_module.F90 kgen_utils.o tprof_mod.o params.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
kurant.o: kurant.F90 kgen_utils.o tprof_mod.o params.o vars.o sgs.o task_util_NOMPI.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
abcoefs.o: abcoefs.F90 kgen_utils.o tprof_mod.o params.o grid.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
buoyancy.o: buoyancy.F90 kgen_utils.o tprof_mod.o params.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
crm_surface.o: crm_surface.F90 kgen_utils.o tprof_mod.o params.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
zero.o: zero.F90 kgen_utils.o tprof_mod.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
forcing.o: forcing.F90 kgen_utils.o tprof_mod.o params.o vars.o microphysics.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
ice_fall.o: ice_fall.F90 kgen_utils.o tprof_mod.o params.o vars.o microphysics.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
shr_const_mod.o: shr_const_mod.F90 kgen_utils.o tprof_mod.o shr_kind_mod.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
shr_kind_mod.o: shr_kind_mod.F90 kgen_utils.o tprof_mod.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
boundaries.o: boundaries.F90 kgen_utils.o tprof_mod.o periodic.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
periodic.o: periodic.F90 kgen_utils.o tprof_mod.o bound_exchange.o vars.o sgs.o params.o microphysics.o scalar_momentum.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
bound_exchange.o: bound_exchange.F90 kgen_utils.o tprof_mod.o params.o grid.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
advection.o: advection.F90 kgen_utils.o tprof_mod.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
tke_full.o: tke_full.F90 kgen_utils.o tprof_mod.o params.o vars.o shear_prod3D.o shear_prod2D.o sat.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
shear_prod3D.o: shear_prod3D.F90 kgen_utils.o tprof_mod.o params.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
shear_prod2D.o: shear_prod2D.F90 kgen_utils.o tprof_mod.o params.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
sat.o: sat.F90 kgen_utils.o tprof_mod.o params.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
advect_mom.o: advect_mom.F90 kgen_utils.o tprof_mod.o advect2_mom_xy.o advect2_mom_z.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
advect2_mom_z.o: advect2_mom_z.F90 kgen_utils.o tprof_mod.o params.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
advect2_mom_xy.o: advect2_mom_xy.F90 kgen_utils.o tprof_mod.o params.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
diffuse_mom.o: diffuse_mom.F90 kgen_utils.o tprof_mod.o vars.o diffuse_mom3D.o diffuse_mom2D.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
diffuse_mom2D.o: diffuse_mom2D.F90 kgen_utils.o tprof_mod.o params.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
diffuse_mom3D.o: diffuse_mom3D.F90 kgen_utils.o tprof_mod.o params.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
diagnose.o: diagnose.F90 kgen_utils.o tprof_mod.o params.o vars.o sat.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
uvw.o: uvw.F90 kgen_utils.o tprof_mod.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
coriolis.o: coriolis.F90 kgen_utils.o tprof_mod.o params.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
pressure.o: pressure.F90 kgen_utils.o tprof_mod.o vars.o params.o press_rhs.o fft.o press_grad.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
fft.o: fft.F90 kgen_utils.o tprof_mod.o params.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
press_rhs.o: press_rhs.F90 kgen_utils.o tprof_mod.o params.o vars.o bound_duvdt.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
bound_duvdt.o: bound_duvdt.F90 kgen_utils.o tprof_mod.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
press_grad.o: press_grad.F90 kgen_utils.o tprof_mod.o vars.o params.o bound_duvdt.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
adams.o: adams.F90 kgen_utils.o tprof_mod.o params.o vars.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
advect_all_scalars.o: advect_all_scalars.F90 kgen_utils.o tprof_mod.o vars.o advect_scalar.o microphysics.o scalar_momentum.o sgs.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
advect_scalar.o: advect_scalar.F90 kgen_utils.o tprof_mod.o params.o grid.o advect_scalar3D.o vars.o advect_scalar2D.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
advect_scalar2D.o: advect_scalar2D.F90 kgen_utils.o tprof_mod.o params.o grid.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
advect_scalar3D.o: advect_scalar3D.F90 kgen_utils.o tprof_mod.o params.o grid.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
diffuse_scalar.o: diffuse_scalar.F90 kgen_utils.o tprof_mod.o params.o grid.o diffuse_scalar3D.o vars.o diffuse_scalar2D.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
diffuse_scalar3D.o: diffuse_scalar3D.F90 kgen_utils.o tprof_mod.o params.o grid.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
diffuse_scalar2D.o: diffuse_scalar2D.F90 kgen_utils.o tprof_mod.o params.o grid.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
accelerate_crm.o: accelerate_crm.F90 kgen_utils.o tprof_mod.o params.o shr_kind_mod.o grid.o vars.o microphysics.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
kernel_driver.o: kernel_driver.F90 crm_module.o vars.o grid.o domain.o crmdims.o sgs.o scalar_momentum.o params.o microphysics.o cloud.o precip_init.o task_util_NOMPI.o precip_proc.o micro_params.o damping.o crm_output_module.o kurant.o abcoefs.o buoyancy.o crm_surface.o zero.o forcing.o ice_fall.o shr_const_mod.o shr_kind_mod.o boundaries.o periodic.o bound_exchange.o advection.o tke_full.o shear_prod3D.o shear_prod2D.o sat.o advect_mom.o advect2_mom_z.o advect2_mom_xy.o diffuse_mom.o diffuse_mom2D.o diffuse_mom3D.o diagnose.o uvw.o coriolis.o pressure.o fft.o press_rhs.o bound_duvdt.o press_grad.o adams.o advect_all_scalars.o advect_scalar.o advect_scalar2D.o advect_scalar3D.o diffuse_scalar.o diffuse_scalar3D.o diffuse_scalar2D.o accelerate_crm.o kgen_utils.o tprof_mod.o
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
kgen_utils.o: kgen_utils.f90
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
tprof_mod.o: tprof_mod.f90
${FC_0} ${FC_FLAGS_SET_0} -c -o $@ $<
gammafff.o: gammafff.c
${CC_0} ${CC_FLAGS_SET_0} -c -o $@ $<
clean:
rm -fr kernel.exe *.mod ${ALL_OBJS} TRACE*.mpit* TRACE.sym set-0
# This file is automatically generated after the make install step.
EXTRAE_HOME=/sw/summit/extrae/3.7.1/rhel7.5_gnu6.4.0
MPI_HOME=/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi
MPI_CFLAGS=-I/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/include
MPI_LDFLAGS=-L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/lib
MPI_LIBS=-lmpi_ibm
PAPI_HOME=/autofs/nccs-svm1_sw/summit/.swci/0-core/opt/spack/20180914/linux-rhel7-ppc64le/gcc-4.8.5/papi-5.6.0-xhd3soyh4mzxlhqmtinhylnchxuqkshb/
XML2_HOME= /usr
XML2_LDFLAGS=-L/usr/lib64
XML2_LIBS=-lxml2
UNWIND_HOME=/sw/summit/scorep/libunwind-1.3.1/rhel7.5_gnu6.4.0/
BFD_HOME=/usr
LIBERTY_HOME=/usr
BFD_LIBS=-L/usr/lib64 -lbfd -L/usr/lib64 -liberty
UNWIND_LIBS=-L$(UNWIND_HOME)/lib -lunwind
PAPI_LIBS=-L$(PAPI_HOME)/lib -lpapi
CUPTI_LIBS=
POSIX_CLOCK_LIBS=
LIBEXECINFO_LIBS= # FreeBSD specific
ZLIB_HOME=/usr
ZLIB_LIBS=-lz
DL_LIBS=-ldl
OPENCL_HOME=
OPENCL_LDFLAGS=
CC=/sw/summit/gcc/6.4.0/bin/gcc
FC=/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/bin/mpif90
CXX=/sw/summit/gcc/6.4.0/bin/g++
MPICC=/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/bin/mpicc
MPIF77=/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/bin/mpif77
MPIF90=/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/gcc-8.1.1/spectrum-mpi-10.3.0.0-20190419-juwhiawazufuuoastohlsuewaklwmazi/bin/mpif90
NVCC=/sw/summit/cuda/10.1.105/bin/nvcc
JAVAC=
JAR=
# For statically linked apps or systems
EXTRAE_LDFLAGS_LIBSEQTRACE=-L$(EXTRAE_HOME)/lib -lseqtrace -L$(ZLIB_HOME)/lib -lz -L$(PAPI_HOME)/lib -lpapi -L$(XML2_HOME)/lib -lxml2 -L$(BFD_HOME)/lib -lbfd -liberty
EXTRAE_LDFLAGS_LIBMPITRACE=-L$(EXTRAE_HOME)/lib -lmpitrace -L$(ZLIB_HOME)/lib -lz -L$(PAPI_HOME)/lib -lpapi -L$(XML2_HOME)/lib -lxml2 -L$(BFD_HOME)/lib -lbfd -liberty
EXTRAE_LDFLAGS_LIBMPITRACEF=-L$(EXTRAE_HOME)/lib -lmpitracef -L$(ZLIB_HOME)/lib -lz -L$(PAPI_HOME)/lib -lpapi -L$(XML2_HOME)/lib -lxml2 -L$(BFD_HOME)/lib -lbfd -liberty
......@@ -41,6 +41,9 @@ contains
SUBROUTINE crm(kgen_unit, kgen_measure, kgen_isverified, kgen_filepath, ncrms, plev)
!-----------------------------------------------------------------------------------------------
!-----------------------------------------------------------------------------------------------
USE EXTRAE_MODULE
USE shr_kind_mod, ONLY: r8 => shr_kind_r8
USE vars
USE microphysics
......@@ -126,7 +129,7 @@ SUBROUTINE crm(kgen_unit, kgen_measure, kgen_isverified, kgen_filepath, ncrms, p
INTEGER :: kgen_mpirank, kgen_openmptid, kgen_kernelinvoke
LOGICAL :: kgen_evalstage, kgen_warmupstage, kgen_mainstage
COMMON / state / kgen_mpirank, kgen_openmptid, kgen_kernelinvoke, kgen_evalstage, kgen_warmupstage, kgen_mainstage
INTEGER, PARAMETER :: KGEN_MAXITER = 10
INTEGER, PARAMETER :: KGEN_MAXITER = 100
TYPE(check_t) :: check_status
INTEGER*8 :: kgen_start_clock, kgen_stop_clock, kgen_rate_clock
......@@ -758,6 +761,8 @@ SUBROUTINE crm(kgen_unit, kgen_measure, kgen_isverified, kgen_filepath, ncrms, p
CALL SYSTEM_CLOCK(kgen_start_clock, kgen_rate_clock)
DO kgen_intvar = 1, KGEN_MAXITER
call extrae_user_function(1)
nstep = nstep + 1
!$omp target teams distribute parallel do
do icrm = 1 , ncrms
......@@ -1075,6 +1080,10 @@ SUBROUTINE crm(kgen_unit, kgen_measure, kgen_isverified, kgen_filepath, ncrms, p
enddo
enddo
END DO
call extrae_user_function(0)
call extrae_next_hwc_set()
CALL SYSTEM_CLOCK(kgen_stop_clock, kgen_rate_clock)
kgen_measure = 1.0D6*(kgen_stop_clock - kgen_start_clock)/DBLE(kgen_rate_clock*KGEN_MAXITER)
#ifdef _MPI
......
!KGEN-generated Fortran source file
!Generated at : 2020-05-29 19:47:47
!KGEN version : 0.9.0
module crm_module
USE abcoefs_mod, ONLY: abcoefs
USE kurant_mod, ONLY: kurant
USE boundaries_mod, ONLY: boundaries
USE forcing_mod, ONLY: forcing
USE advect_mom_mod, ONLY: advect_mom
USE adams_mod, ONLY: adams
USE advect_all_scalars_mod, ONLY: advect_all_scalars
USE crm_surface_mod
USE zero_mod
USE buoyancy_mod
USE pressure_mod
USE uvw_mod
USE diagnose_mod
USE damping_mod
USE ice_fall_mod
USE coriolis_mod
USE crm_output_module
!---------------------------------------------------------------
! Super-parameterization's main driver
! Marat Khairoutdinov, 2001-2009
!---------------------------------------------------------------
USE kgen_utils_mod, ONLY: kgen_dp, kgen_array_sumcheck
USE tprof_mod, ONLY: tstart, tstop, tnull, tprnt
PUBLIC crm
#ifdef _MPI
include "mpif.h"
#endif
contains
SUBROUTINE crm(kgen_unit, kgen_measure, kgen_isverified, kgen_filepath, ncrms, plev)
!-----------------------------------------------------------------------------------------------
!-----------------------------------------------------------------------------------------------
USE EXTRAE_MODULE
USE shr_kind_mod, ONLY: r8 => shr_kind_r8
USE vars
USE microphysics
USE sgs
USE scalar_momentum_mod
USE crmdims, ONLY: crm_nx_rad, crm_ny_rad
USE accelerate_crm_mod, ONLY: use_crm_accel, accelerate_crm
USE kgen_utils_mod, ONLY: kgen_dp, kgen_array_sumcheck
USE kgen_utils_mod, ONLY: kgen_perturb_real
USE vars, ONLY: kr_externs_out_vars
USE grid, ONLY: kr_externs_out_grid
USE sgs, ONLY: kr_externs_out_sgs
USE params, ONLY: kr_externs_out_params
USE microphysics, ONLY: kr_externs_out_microphysics
USE micro_params, ONLY: kr_externs_out_micro_params
USE crm_output_module, ONLY: kr_externs_out_crm_output_module
USE accelerate_crm_mod, ONLY: kr_externs_out_accelerate_crm_mod
USE kgen_utils_mod, ONLY: check_t, kgen_init_check, kgen_init_verify, kgen_tolerance, kgen_minvalue, kgen_verboselevel, &
&CHECK_IDENTICAL, CHECK_IN_TOL, CHECK_OUT_TOL
USE vars, ONLY: kv_externs_vars
USE grid, ONLY: kv_externs_grid
USE params, ONLY: kv_externs_params
USE microphysics, ONLY: kv_externs_microphysics
USE micro_params, ONLY: kv_externs_micro_params
USE crm_output_module, ONLY: kv_externs_crm_output_module
IMPLICIT NONE
!-----------------------------------------------------------------------------------------------
! Interface variable declarations
!-----------------------------------------------------------------------------------------------
INTEGER, INTENT(INOUT) :: ncrms
INTEGER, INTENT(INOUT) :: plev
!-----------------------------------------------------------------------------------------------
! Local variable declarations
!-----------------------------------------------------------------------------------------------
real(r8), parameter :: wmin = 2. ! minimum up/downdraft velocity for stat
real(crm_rknd), parameter :: cwp_threshold = 0.001 ! threshold for cloud condensate for shaded fraction calculation
REAL(KIND=crm_rknd) :: tmp1, tmp
INTEGER :: i, j, k, l, nn, icyc, icrm
INTEGER :: kx
REAL(KIND=crm_rknd) :: qsat
REAL(KIND=crm_rknd), allocatable :: bflx(:)
! These should all be inputs
! variables for radiation grouping method
INTEGER :: i_rad
INTEGER :: j_rad
LOGICAL :: crm_accel_ceaseflag
! Arrays
REAL(KIND=crm_rknd), allocatable :: cwp (:,:,:)
REAL(KIND=crm_rknd), allocatable :: cwph (:,:,:)
REAL(KIND=crm_rknd), allocatable :: cwpm (:,:,:)
REAL(KIND=crm_rknd), allocatable :: cwpl (:,:,:)
LOGICAL, allocatable :: flag_top(:,:,:)
REAL(KIND=crm_rknd), allocatable :: cltemp (:,:,:)
REAL(KIND=crm_rknd), allocatable :: cmtemp (:,:,:)
REAL(KIND=crm_rknd), allocatable :: chtemp (:,:,:)
REAL(KIND=crm_rknd), allocatable :: cttemp (:,:,:)
REAL(KIND=r8), allocatable :: mui_crm(:,:)
REAL(KIND=r8), allocatable :: mdi_crm(:,:)
REAL(KIND=crm_rknd), pointer :: crm_rad_temperature (:,:,:,:)
REAL(KIND=crm_rknd), pointer :: crm_rad_qv (:,:,:,:)
REAL(KIND=crm_rknd), pointer :: crm_rad_qc (:,:,:,:)
REAL(KIND=crm_rknd), pointer :: crm_rad_qi (:,:,:,:)
REAL(KIND=crm_rknd), pointer :: crm_rad_cld (:,:,:,:)
REAL(KIND=crm_rknd), pointer :: crm_rad_qrad (:,:,:,:)
!-----------------------------------------------------------------------------------------------
!-----------------------------------------------------------------------------------------------
INTEGER, INTENT(IN) :: kgen_unit
REAL(KIND=kgen_dp), INTENT(OUT) :: kgen_measure
LOGICAL, INTENT(OUT) :: kgen_isverified
CHARACTER(LEN=*), INTENT(IN) :: kgen_filepath
LOGICAL :: kgen_istrue
REAL(KIND=8) :: kgen_array_sum
INTEGER :: kgen_intvar, kgen_ierr
INTEGER :: kgen_mpirank, kgen_openmptid, kgen_kernelinvoke
LOGICAL :: kgen_evalstage, kgen_warmupstage, kgen_mainstage
COMMON / state / kgen_mpirank, kgen_openmptid, kgen_kernelinvoke, kgen_evalstage, kgen_warmupstage, kgen_mainstage
INTEGER, PARAMETER :: KGEN_MAXITER = 10
TYPE(check_t) :: check_status
INTEGER*8 :: kgen_start_clock, kgen_stop_clock, kgen_rate_clock
REAL(KIND=kgen_dp) :: gkgen_measure
REAL(KIND=crm_rknd) :: kgenref_tmp1
REAL(KIND=crm_rknd) :: kgenref_tmp
INTEGER :: kgenref_icrm
INTEGER :: kgenref_icyc
INTEGER :: kgenref_k
INTEGER :: kgenref_j
INTEGER :: kgenref_i
INTEGER :: kgenref_nn
INTEGER :: kgenref_l
INTEGER :: kgenref_kx
REAL(KIND=crm_rknd) :: kgenref_qsat
INTEGER :: kgenref_i_rad
INTEGER :: kgenref_j_rad
LOGICAL :: kgenref_crm_accel_ceaseflag
REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cwp
REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cwph
REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cwpm
REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cwpl
LOGICAL, allocatable, dimension(:,:,:) :: kgenref_flag_top
REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cltemp
REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cmtemp
REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_chtemp
REAL(KIND=crm_rknd), allocatable, dimension(:,:,:) :: kgenref_cttemp
REAL(KIND=r8), allocatable, dimension(:,:) :: kgenref_mui_crm
REAL(KIND=r8), allocatable, dimension(:,:) :: kgenref_mdi_crm
REAL(KIND=crm_rknd), pointer, dimension(:,:,:,:) :: kgenref_crm_rad_temperature
REAL(KIND=crm_rknd), pointer, dimension(:,:,:,:) :: kgenref_crm_rad_qv
REAL(KIND=crm_rknd), pointer, dimension(:,:,:,:) :: kgenref_crm_rad_qc
REAL(KIND=crm_rknd), pointer, dimension(:,:,:,:) :: kgenref_crm_rad_qi
REAL(KIND=crm_rknd), pointer, dimension(:,:,:,:) :: kgenref_crm_rad_cld
!parent block preprocessing
#ifdef _MPI
call mpi_comm_rank(mpi_comm_world, kgen_mpirank, kgen_ierr)
#else
kgen_mpirank = 0
#endif
!local input variables
READ (UNIT = kgen_unit) tmp1
READ (UNIT = kgen_unit) tmp
READ (UNIT = kgen_unit) icrm
READ (UNIT = kgen_unit) icyc
READ (UNIT = kgen_unit) i
READ (UNIT = kgen_unit) j
READ (UNIT = kgen_unit) k
READ (UNIT = kgen_unit) nn
READ (UNIT = kgen_unit) l
READ (UNIT = kgen_unit) kx
READ (UNIT = kgen_unit) qsat
CALL kr_crm_real__crm_rknd_dim1(bflx, kgen_unit, "bflx", .FALSE.)
READ (UNIT = kgen_unit) i_rad
READ (UNIT = kgen_unit) j_rad
READ (UNIT = kgen_unit) crm_accel_ceaseflag
CALL kr_crm_real__crm_rknd_dim3(cwp, kgen_unit, "cwp", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(cwph, kgen_unit, "cwph", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(cwpm, kgen_unit, "cwpm", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(cwpl, kgen_unit, "cwpl", .FALSE.)
CALL kr_crm_logical___dim3(flag_top, kgen_unit, "flag_top", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(cltemp, kgen_unit, "cltemp", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(cmtemp, kgen_unit, "cmtemp", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(chtemp, kgen_unit, "chtemp", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(cttemp, kgen_unit, "cttemp", .FALSE.)
CALL kr_crm_real__r8_dim2(mui_crm, kgen_unit, "mui_crm", .FALSE.)
CALL kr_crm_real__r8_dim2(mdi_crm, kgen_unit, "mdi_crm", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_temperature, kgen_unit, "crm_rad_temperature", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_qv, kgen_unit, "crm_rad_qv", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_qc, kgen_unit, "crm_rad_qc", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_qi, kgen_unit, "crm_rad_qi", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_cld, kgen_unit, "crm_rad_cld", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(crm_rad_qrad, kgen_unit, "crm_rad_qrad", .FALSE.)
!extern output variables
CALL kr_externs_out_vars(kgen_unit)
CALL kr_externs_out_grid(kgen_unit)
CALL kr_externs_out_sgs(kgen_unit)
CALL kr_externs_out_params(kgen_unit)
CALL kr_externs_out_microphysics(kgen_unit)
CALL kr_externs_out_micro_params(kgen_unit)
CALL kr_externs_out_crm_output_module(kgen_unit)
CALL kr_externs_out_accelerate_crm_mod(kgen_unit)
!local output variables
READ (UNIT = kgen_unit) kgenref_tmp1
READ (UNIT = kgen_unit) kgenref_tmp
READ (UNIT = kgen_unit) kgenref_icrm
READ (UNIT = kgen_unit) kgenref_icyc
READ (UNIT = kgen_unit) kgenref_k
READ (UNIT = kgen_unit) kgenref_j
READ (UNIT = kgen_unit) kgenref_i
READ (UNIT = kgen_unit) kgenref_nn
READ (UNIT = kgen_unit) kgenref_l
READ (UNIT = kgen_unit) kgenref_kx
READ (UNIT = kgen_unit) kgenref_qsat
READ (UNIT = kgen_unit) kgenref_i_rad
READ (UNIT = kgen_unit) kgenref_j_rad
READ (UNIT = kgen_unit) kgenref_crm_accel_ceaseflag
CALL kr_crm_real__crm_rknd_dim3(kgenref_cwp, kgen_unit, "kgenref_cwp", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(kgenref_cwph, kgen_unit, "kgenref_cwph", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(kgenref_cwpm, kgen_unit, "kgenref_cwpm", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(kgenref_cwpl, kgen_unit, "kgenref_cwpl", .FALSE.)
CALL kr_crm_logical___dim3(kgenref_flag_top, kgen_unit, "kgenref_flag_top", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(kgenref_cltemp, kgen_unit, "kgenref_cltemp", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(kgenref_cmtemp, kgen_unit, "kgenref_cmtemp", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(kgenref_chtemp, kgen_unit, "kgenref_chtemp", .FALSE.)
CALL kr_crm_real__crm_rknd_dim3(kgenref_cttemp, kgen_unit, "kgenref_cttemp", .FALSE.)
CALL kr_crm_real__r8_dim2(kgenref_mui_crm, kgen_unit, "kgenref_mui_crm", .FALSE.)
CALL kr_crm_real__r8_dim2(kgenref_mdi_crm, kgen_unit, "kgenref_mdi_crm", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(kgenref_crm_rad_temperature, kgen_unit, "kgenref_crm_rad_temperature", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(kgenref_crm_rad_qv, kgen_unit, "kgenref_crm_rad_qv", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(kgenref_crm_rad_qc, kgen_unit, "kgenref_crm_rad_qc", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(kgenref_crm_rad_qi, kgen_unit, "kgenref_crm_rad_qi", .FALSE.)
CALL kr_crm_real__crm_rknd_dim4_ptr(kgenref_crm_rad_cld, kgen_unit, "kgenref_crm_rad_cld", .FALSE.)
!$omp target enter data map(alloc: t00 )
!$omp target enter data map(alloc: tln )
!$omp target enter data map(alloc: qln )
!$omp target enter data map(alloc: qccln )
!$omp target enter data map(alloc: qiiln )
!$omp target enter data map(alloc: uln )
!$omp target enter data map(alloc: vln )
!$omp target enter data map(alloc: cwp )
!$omp target enter data map(alloc: cwph )
!$omp target enter data map(alloc: cwpm )
!$omp target enter data map(alloc: cwpl )
!$omp target enter data map(alloc: flag_top )
!$omp target enter data map(alloc: cltemp )
!$omp target enter data map(alloc: cmtemp )
!$omp target enter data map(alloc: chtemp )
!$omp target enter data map(alloc: cttemp )
!$omp target enter data map(alloc: dd_crm )
!$omp target enter data map(alloc: mui_crm )
!$omp target enter data map(alloc: mdi_crm )
!$omp target enter data map(alloc: ustar )
!$omp target enter data map(alloc: bflx )
!$omp target enter data map(alloc: wnd )
!$omp target enter data map(alloc: qtot )
!$omp target enter data map(alloc: colprec )
!$omp target enter data map(alloc: colprecs )
!$omp target enter data map(alloc: crm_rad_temperature )
!$omp target enter data map(alloc: crm_rad_qv )
!$omp target enter data map(alloc: crm_rad_qc )
!$omp target enter data map(alloc: crm_rad_qi )
!$omp target enter data map(alloc: crm_rad_cld )
!$omp target enter data map(alloc: crm_rad_qrad )
!$omp target enter data map(alloc: crm_state_u_wind )
!$omp target enter data map(alloc: crm_state_v_wind )
!$omp target enter data map(alloc: crm_state_w_wind )
!$omp target enter data map(alloc: crm_state_temperature )
!$omp target enter data map(alloc: crm_state_qt )
!$omp target enter data map(alloc: crm_state_qp )
!$omp target enter data map(alloc: crm_state_qn )
!Loop over "vector columns"
!-----------------------------------------------
!-----------------------------------------
! update data at device
!$omp taskwait
! Initialize CRM fields:
!$omp target teams distribute parallel do collapse(4)
! limit the velocity at the very first step:
! Populate microphysics array from crm_state
!$omp target teams distribute parallel do collapse(4)
! initialize sgs fields
!$omp target teams distribute parallel do
!$omp target teams distribute parallel do collapse(2)