Commit 3fbb304d authored by Coon, Ethan's avatar Coon, Ethan
Browse files

adds running kokkos example (in Kokkos_SERIAL, will not yet run on device...

adds running kokkos example (in Kokkos_SERIAL, will not yet run on device without adding decorators to functors)
parent 6134a7de
cmake_minimum_required(VERSION 3.13.3)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
enable_testing()
......@@ -27,6 +27,13 @@ endif()
find_package(NetCDF REQUIRED)
message("Found NetCDF = ${NetCDF_C_LIBRARIES}")
# options for drivers
option(ENABLE_CC "Enable building with default C-style driver" ON)
option(ENABLE_Kokkos "Enable building with Kokkos driver" OFF)
if (ENABLE_Kokkos)
find_package(Kokkos REQUIRED)
endif()
add_subdirectory (src)
add_subdirectory (driver)
......@@ -18,6 +18,7 @@ sample configure/build/test looks like:
-DBUILD_SHARED_LIBS:BOOL=true \
-DCMAKE_INSTALL_PREFIX:FILEPATH=`pwd`/../install-dir
-DCMAKE_BUILD_TYPE:STRING=Debug \
-DENABLE_KOKKOS:BOOL=true \
`pwd`/../ELM_Kernels
make
......
add_subdirectory(utils)
add_subdirectory(cc)
if (ENABLE_CC)
add_subdirectory(cc)
endif()
if (ENABLE_Kokkos)
add_subdirectory(kokkos)
endif()
# include_directories(${NetCDF_INCLUDE_DIR})
# add_executable (ELM_driver driver.cc)
......
......@@ -3,89 +3,116 @@
#include "BareGroundFluxes.h"
#include "array.hh"
using ArrayD1 = ELM::Array<double,1>;
using ArrayI1 = ELM::Array<int,1>;
using ArrayD2 = ELM::Array<double,2>;
template<class Array_t>
Array_t create(const std::string& name, int D0) {
return Array_t(D0);
}
template<class Array_t>
Array_t create(const std::string& name, int D0, int D1) {
return Array_t(D0, D1);
}
template<class Array_t>
Array_t create(const std::string& name, int D0, int D1, int D2) {
return Array_t(D0, D1, D2);
}
template<class Array_t, typename Scalar_t>
void assign(Array_t& arr, Scalar_t val) {
arr = val;
}
int main(int argc, char** argv)
{
const int ncells = 3;
const int ntimes = 2;
// instantiate data
ELM::LandType land;
ELM::Array<int,1> frac_vec_nosno_in(ncells);
ELM::Array<double,1> forc_u(ncells);
ELM::Array<double,1> forc_v(ncells);
ELM::Array<double,1> forc_q_in(ncells);
ELM::Array<double,1> forc_th_in(ncells);
ELM::Array<double,1> forc_hgt_u_patch_in(ncells);
ELM::Array<double,1> thm_in(ncells);
ELM::Array<double,1> thv_in(ncells);
ELM::Array<double,1> t_grnd(ncells);
ELM::Array<double,1> qg(ncells);
ELM::Array<double,1> z0mg_in(ncells);
ELM::Array<double,1> dlrad(ncells);
ELM::Array<double,1> ulrad(ncells);
ELM::Array<double,1> forc_hgt_t_patch(ncells);
ELM::Array<double,1> forc_hgt_q_patch(ncells);
ELM::Array<double,1> z0mg(ncells);
ELM::Array<double,1> zii(ncells);
ELM::Array<double,1> beta(ncells);
ELM::Array<double,1> z0hg(ncells);
ELM::Array<double,1> z0qg(ncells);
ELM::Array<int,1> snl(ncells, 1); // fix me!
ELM::Array<double,1> forc_rho(ncells);
ELM::Array<double,1> soilbeta(ncells);
ELM::Array<double,1> dqgdT(ncells);
ELM::Array<double,1> htvp(ncells);
ELM::Array<double,1> t_h2osfc(ncells);
ELM::Array<double,1> qg_snow(ncells);
ELM::Array<double,1> qg_soil(ncells);
ELM::Array<double,1> qg_h2osfc(ncells);
ELM::Array<double,2> t_soisno(ncells, ELM::nlevsno+1); // is this correct? This is required by BareGroundFluxes_impl.hh:108
ELM::Array<double,1> forc_pbot(ncells);
ELM::Array<double,1> cgrnds(ncells);
ELM::Array<double,1> cgrndl(ncells);
ELM::Array<double,1> cgrnd(ncells);
ELM::Array<double,1> eflx_sh_grnd(ncells);
ELM::Array<double,1> eflx_sh_tot(ncells);
ELM::Array<double,1> eflx_sh_snow(ncells);
ELM::Array<double,1> eflx_sh_soil(ncells);
ELM::Array<double,1> eflx_sh_h2osfc(ncells);
ELM::Array<double,1> qflx_evap_soi(ncells);
ELM::Array<double,1> qflx_evap_tot(ncells);
ELM::Array<double,1> qflx_ev_snow(ncells);
ELM::Array<double,1> qflx_ev_soil(ncells);
ELM::Array<double,1> qflx_ev_h2osfc(ncells);
ELM::Array<double,1> t_ref2m(ncells);
ELM::Array<double,1> t_ref2m_r(ncells);
ELM::Array<double,1> q_ref2m(ncells);
ELM::Array<double,1> rh_ref2m(ncells);
ELM::Array<double,1> rh_ref2m_r(ncells);
auto frac_vec_nosno_in = create<ArrayI1>("frac_vec_nosno_in", ncells);
auto forc_u = create<ArrayD1>("forc_u", ncells);
auto forc_v = create<ArrayD1>("forc_v", ncells);
auto forc_q_in = create<ArrayD1>("forc_q_in", ncells);
auto forc_th_in = create<ArrayD1>("forc_th_in", ncells);
auto forc_hgt_u_patch_in = create<ArrayD1>("forc_hgt_u_patch_in", ncells);
auto thm_in = create<ArrayD1>("thm_in", ncells);
auto thv_in = create<ArrayD1>("thv_in", ncells);
auto t_grnd = create<ArrayD1>("t_grnd", ncells);
auto qg = create<ArrayD1>("qg", ncells);
auto z0mg_in = create<ArrayD1>("z0mg_in", ncells);
auto dlrad = create<ArrayD1>("dlrad", ncells);
auto ulrad = create<ArrayD1>("ulrad", ncells);
auto forc_hgt_t_patch = create<ArrayD1>("forc_hgt_t_patch", ncells);
auto forc_hgt_q_patch = create<ArrayD1>("forc_hgt_q_patch", ncells);
auto z0mg = create<ArrayD1>("z0mg", ncells);
auto zii = create<ArrayD1>("zii", ncells);
auto beta = create<ArrayD1>("beta", ncells);
auto z0hg = create<ArrayD1>("z0hg", ncells);
auto z0qg = create<ArrayD1>("z0qg", ncells);
auto snl = create<ArrayI1>("snl", ncells); assign(snl, 1); // fix me!
auto forc_rho = create<ArrayD1>("forc_rho", ncells);
auto soilbeta = create<ArrayD1>("soilbeta", ncells);
auto dqgdT = create<ArrayD1>("dqgdT", ncells);
auto htvp = create<ArrayD1>("htvp", ncells);
auto t_h2osfc = create<ArrayD1>("t_h2osfc", ncells);
auto qg_snow = create<ArrayD1>("qg_snow", ncells);
auto qg_soil = create<ArrayD1>("qg_soil", ncells);
auto qg_h2osfc = create<ArrayD1>("qg_h2osfc", ncells);
auto t_soisno = create<ArrayD2>("t_soisno", ncells, ELM::nlevsno+1); // is this correct? This is required by BareGroundFluxes_impl.hh:108
auto forc_pbot = create<ArrayD1>("forc_pbot", ncells);
auto cgrnds = create<ArrayD1>("cgrnds", ncells);
auto cgrndl = create<ArrayD1>("cgrndl", ncells);
auto cgrnd = create<ArrayD1>("cgrnd", ncells);
auto eflx_sh_grnd = create<ArrayD1>("eflx_sh_grnd", ncells);
auto eflx_sh_tot = create<ArrayD1>("eflx_sh_tot", ncells);
auto eflx_sh_snow = create<ArrayD1>("eflx_sh_snow", ncells);
auto eflx_sh_soil = create<ArrayD1>("eflx_sh_soil", ncells);
auto eflux_sh_h2osfc = create<ArrayD1>("eflx_sh_h2osfc", ncells);
auto qflx_evap_soi = create<ArrayD1>("qflx_evap_soi", ncells);
auto qflx_evap_tot = create<ArrayD1>("qflx_evap_tot", ncells);
auto qflx_ev_snow = create<ArrayD1>("qflx_ev_snow", ncells);
auto qflx_ev_soil = create<ArrayD1>("qflx_ev_soil", ncells);
auto qflx_ev_h2osfc = create<ArrayD1>("qflx_ev_h2osfc", ncells);
auto t_ref2m = create<ArrayD1>("t_ref2m", ncells);
auto t_ref2m_r = create<ArrayD1>("t_ref2m_r", ncells);
auto q_ref2m = create<ArrayD1>("q_ref2m", ncells);
auto rh_ref2m = create<ArrayD1>("rh_ref2m", ncells);
auto rh_ref2m_r = create<ArrayD1>("rh_ref2m_r", ncells);
// initialize
ELM::Array<ELM::BareGroundFluxes, 1> bg_fluxes(ncells);
for (int c=0; c!=ncells; ++c) {
bg_fluxes[c].InitializeFlux(land,
frac_vec_nosno_in[c],
forc_u[c],
forc_v[c],
forc_q_in[c],
forc_th_in[c],
forc_hgt_u_patch_in[c],
thm_in[c],
thv_in[c],
t_grnd[c],
qg[c],
z0mg_in[c],
dlrad[c],
ulrad[c]);
}
auto bg_fluxes = create<Kokkos::View<ELM::BareGroundFluxes*> >("bg_fluxes", ncells);
// iterate in time
for (int time=0; time!=ntimes; ++time) {
for (int c=0; c!=ncells; ++c) {
bg_fluxes[c].InitializeFlux(land,
frac_vec_nosno_in[c],
forc_u[c],
forc_v[c],
forc_q_in[c],
forc_th_in[c],
forc_hgt_u_patch_in[c],
thm_in[c],
thv_in[c],
t_grnd[c],
qg[c],
z0mg_in[c],
dlrad[c],
ulrad[c]);
bg_fluxes[c].StabilityIteration(land,
forc_hgt_t_patch[c],
forc_hgt_q_patch[c],
......
include_directories(${ELM_UTILS_SOURCE_DIR})
include_directories(${KOKKOS_INCLUDE_DIR})
add_executable (ELM_kokkos driver.cc)
target_link_libraries (ELM_kokkos LINK_PUBLIC elm_physics Kokkos::kokkos )
install(TARGETS ELM_kokkos)
add_test(NAME ELM_kokkos COMMAND ELM_kokkos)
#include "clm_constants.h"
#include "landtype.h"
#include "BareGroundFluxes.h"
#include "Kokkos_Core.hpp"
using ArrayD1 = Kokkos::View<double*>;
using ArrayI1 = Kokkos::View<int*>;
using ArrayD2 = Kokkos::View<double**>;
template<typename Array_t>
Array_t create(const std::string& name, int D0) {
return Array_t(name, D0);
}
template<typename Array_t>
Array_t create(const std::string& name, int D0, int D1) {
return Array_t(name, D0, D1);
}
template<typename Array_t>
Array_t create(const std::string& name, int D0, int D1, int D2) {
return Array_t(name, D0, D1, D2);
}
template<class Array_t, typename Scalar_t>
void assign(Array_t& arr, Scalar_t val) {
Kokkos::deep_copy(arr, val);
}
int main(int argc, char** argv)
{
Kokkos::initialize( argc, argv);
const int ncells = 3;
const int ntimes = 2;
{ // scope to make Kokkos happy
// instantiate data
ELM::LandType land;
auto frac_vec_nosno_in = create<ArrayI1>("frac_vec_nosno_in", ncells);
auto forc_u = create<ArrayD1>("forc_u", ncells);
auto forc_v = create<ArrayD1>("forc_v", ncells);
auto forc_q_in = create<ArrayD1>("forc_q_in", ncells);
auto forc_th_in = create<ArrayD1>("forc_th_in", ncells);
auto forc_hgt_u_patch_in = create<ArrayD1>("forc_hgt_u_patch_in", ncells);
auto thm_in = create<ArrayD1>("thm_in", ncells);
auto thv_in = create<ArrayD1>("thv_in", ncells);
auto t_grnd = create<ArrayD1>("t_grnd", ncells);
auto qg = create<ArrayD1>("qg", ncells);
auto z0mg_in = create<ArrayD1>("z0mg_in", ncells);
auto dlrad = create<ArrayD1>("dlrad", ncells);
auto ulrad = create<ArrayD1>("ulrad", ncells);
auto forc_hgt_t_patch = create<ArrayD1>("forc_hgt_t_patch", ncells);
auto forc_hgt_q_patch = create<ArrayD1>("forc_hgt_q_patch", ncells);
auto z0mg = create<ArrayD1>("z0mg", ncells);
auto zii = create<ArrayD1>("zii", ncells);
auto beta = create<ArrayD1>("beta", ncells);
auto z0hg = create<ArrayD1>("z0hg", ncells);
auto z0qg = create<ArrayD1>("z0qg", ncells);
auto snl = create<ArrayI1>("snl", ncells); assign(snl, 1); // fix me!
auto forc_rho = create<ArrayD1>("forc_rho", ncells);
auto soilbeta = create<ArrayD1>("soilbeta", ncells);
auto dqgdT = create<ArrayD1>("dqgdT", ncells);
auto htvp = create<ArrayD1>("htvp", ncells);
auto t_h2osfc = create<ArrayD1>("t_h2osfc", ncells);
auto qg_snow = create<ArrayD1>("qg_snow", ncells);
auto qg_soil = create<ArrayD1>("qg_soil", ncells);
auto qg_h2osfc = create<ArrayD1>("qg_h2osfc", ncells);
auto t_soisno = create<ArrayD2>("t_soisno", ncells, ELM::nlevsno+1); // is this correct? This is required by BareGroundFluxes_impl.hh:108
auto forc_pbot = create<ArrayD1>("forc_pbot", ncells);
auto cgrnds = create<ArrayD1>("cgrnds", ncells);
auto cgrndl = create<ArrayD1>("cgrndl", ncells);
auto cgrnd = create<ArrayD1>("cgrnd", ncells);
auto eflx_sh_grnd = create<ArrayD1>("eflx_sh_grnd", ncells);
auto eflx_sh_tot = create<ArrayD1>("eflx_sh_tot", ncells);
auto eflx_sh_snow = create<ArrayD1>("eflx_sh_snow", ncells);
auto eflx_sh_soil = create<ArrayD1>("eflx_sh_soil", ncells);
auto eflx_sh_h2osfc = create<ArrayD1>("eflx_sh_h2osfc", ncells);
auto qflx_evap_soi = create<ArrayD1>("qflx_evap_soi", ncells);
auto qflx_evap_tot = create<ArrayD1>("qflx_evap_tot", ncells);
auto qflx_ev_snow = create<ArrayD1>("qflx_ev_snow", ncells);
auto qflx_ev_soil = create<ArrayD1>("qflx_ev_soil", ncells);
auto qflx_ev_h2osfc = create<ArrayD1>("qflx_ev_h2osfc", ncells);
auto t_ref2m = create<ArrayD1>("t_ref2m", ncells);
auto t_ref2m_r = create<ArrayD1>("t_ref2m_r", ncells);
auto q_ref2m = create<ArrayD1>("q_ref2m", ncells);
auto rh_ref2m = create<ArrayD1>("rh_ref2m", ncells);
auto rh_ref2m_r = create<ArrayD1>("rh_ref2m_r", ncells);
// initialize
Kokkos::View<ELM::BareGroundFluxes*> bg_fluxes("bare ground fluxes", ncells);
// iterate in time
for (int time=0; time!=ntimes; ++time) {
Kokkos::parallel_for("BareGroundFluxes", ncells,
KOKKOS_LAMBDA(const int& c) {
bg_fluxes[c].InitializeFlux(land,
frac_vec_nosno_in[c],
forc_u[c],
forc_v[c],
forc_q_in[c],
forc_th_in[c],
forc_hgt_u_patch_in[c],
thm_in[c],
thv_in[c],
t_grnd[c],
qg[c],
z0mg_in[c],
dlrad[c],
ulrad[c]);
bg_fluxes[c].StabilityIteration(land,
forc_hgt_t_patch[c],
forc_hgt_q_patch[c],
z0mg[c],
zii[c],
beta[c],
z0hg[c],
z0qg[c]);
bg_fluxes[c].ComputeFlux(land,
snl[c],
forc_rho[c],
soilbeta[c],
dqgdT[c],
htvp[c],
t_h2osfc[c],
qg_snow[c],
qg_soil[c],
qg_h2osfc[c],
Kokkos::subview(t_soisno, c, Kokkos::ALL),
forc_pbot[c],
cgrnds[c],
cgrndl[c],
cgrnd[c],
eflx_sh_grnd[c],
eflx_sh_tot[c],
eflx_sh_snow[c],
eflx_sh_soil[c],
eflx_sh_h2osfc[c],
qflx_evap_soi[c],
qflx_evap_tot[c],
qflx_ev_snow[c],
qflx_ev_soil[c],
qflx_ev_h2osfc[c],
t_ref2m[c],
t_ref2m_r[c],
q_ref2m[c],
rh_ref2m[c],
rh_ref2m_r[c]);
});
} // loop in time
}
Kokkos::finalize();
return 0;
}
......@@ -29,11 +29,11 @@ min_max_sum1(const MPI_Comm& comm,
min_val = vc(i);
}
}, Kokkos::Min<double>(result));
double result_g;
MPI_Reduce(&result, &result_g, 1, MPI_DOUBLE, MPI_MIN, 0, comm);
results[0] = result_g;
}
}
{
double result;
Kokkos::parallel_reduce(
......@@ -43,11 +43,11 @@ min_max_sum1(const MPI_Comm& comm,
max_val = vc(i);
}
}, Kokkos::Max<double>(result));
double result_g;
MPI_Reduce(&result, &result_g, 1, MPI_DOUBLE, MPI_MAX, 0, comm);
results[1] = result_g;
}
}
{
double result;
Kokkos::parallel_reduce(
......@@ -55,12 +55,12 @@ min_max_sum1(const MPI_Comm& comm,
KOKKOS_LAMBDA(const int& i, double& sum_val) {
sum_val += vc(i);
}, result);
double result_g;
MPI_Reduce(&result, &result_g, 1, MPI_DOUBLE, MPI_SUM, 0, comm);
results[2] = result_g;
}
return results;
}
return results;
}
......@@ -80,11 +80,11 @@ min_max_sum2(const MPI_Comm& comm, const View_type& v)
KOKKOS_LAMBDA(const int& i, const int& j, double& min_val) {
if (vc(i,j) < min_val) min_val = vc(i,j);
}, Kokkos::Min<double>(result));
double result_g;
MPI_Reduce(&result, &result_g, 1, MPI_DOUBLE, MPI_MIN, 0, comm);
results[0] = result_g;
}
}
{
double result;
Kokkos::parallel_reduce(
......@@ -92,11 +92,11 @@ min_max_sum2(const MPI_Comm& comm, const View_type& v)
KOKKOS_LAMBDA(const int& i, const int& j, double& max_val) {
if (vc(i,j) > max_val) max_val = vc(i,j);
}, Kokkos::Max<double>(result));
double result_g;
MPI_Reduce(&result, &result_g, 1, MPI_DOUBLE, MPI_MAX, 0, comm);
results[1] = result_g;
}
}
{
double result;
Kokkos::parallel_reduce(
......@@ -104,12 +104,12 @@ min_max_sum2(const MPI_Comm& comm, const View_type& v)
KOKKOS_LAMBDA(const int& i, const int& j, double& sum_val) {
sum_val += vc(i,j);
}, result);
double result_g;
MPI_Reduce(&result, &result_g, 1, MPI_DOUBLE, MPI_SUM, 0, comm);
results[2] = result_g;
}
return results;
}
return results;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment