Skip to content
GitLab
Commit 3262843e authored by Pillai, Himanshu's avatar Pillai, Himanshu
Browse files

Legion and Kokkos implementation with C++ and Fortran Kernel

parent aa0f52d8
Hide whitespace changes
Inline Side-by-side
src/cpp/CanopyHydrology_SnowWater_impl.hh
View file @ 3262843e
......@@ -27,15 +27,15 @@ void CanopyHydrology_SnowWater(const double& dtime,
double& snow_depth,
double& h2osno,
double& int_snow,
Array_d& swe_old,
Array_d& h2osoi_liq,
Array_d& h2osoi_ice,
Array_d& t_soisno,
Array_d& frac_iceold,
int& snl,
Array_d& dz,
Array_d& z,
Array_d& zi,
Array_d swe_old,
Array_d h2osoi_liq,
Array_d h2osoi_ice,
Array_d t_soisno,
Array_d frac_iceold,
int& snow_level,
Array_d dz,
Array_d z,
Array_d zi,
int& newnode,
double& qflx_floodc,
double& qflx_snow_h2osfc,
......@@ -77,10 +77,10 @@ void CanopyHydrology_SnowWater(const double& dtime,
//set temporary variables prior to updating
temp_snow_depth=snow_depth;
//save initial snow content
for(j = -nlevsno+1; j < snl; j++) {
for(j = -nlevsno+1; j < snow_level; j++) {
swe_old[j] = 0.00;
}
for(j = snl+1; j < 0; j++) {
for(j = snow_level+1; j < 0; j++) {
swe_old[j]=h2osoi_liq[j]+h2osoi_ice[j];
}
......@@ -223,9 +223,9 @@ void CanopyHydrology_SnowWater(const double& dtime,
//Currently, the water temperature for the precipitation is simply set
//as the surface air temperature
newnode = 0 ; //flag for when snow node will be initialized
if (snl == 0 && qflx_snow_grnd_col > 0.00 && frac_sno*snow_depth >= 0.010) {
if (snow_level == 0 && qflx_snow_grnd_col > 0.00 && frac_sno*snow_depth >= 0.010) {
newnode = 1;
snl = -1;
snow_level = -1;
dz[0] = snow_depth ; //meter
z[0] = -0.50*dz[0];
zi[-1] = -dz[0];
......@@ -238,9 +238,9 @@ void CanopyHydrology_SnowWater(const double& dtime,
//The change of ice partial density of surface node due to precipitation.
//Only ice part of snowfall is added here, the liquid part will be added
//later.
if (snl < 0 && newnode == 0) {
h2osoi_ice[snl+1] = h2osoi_ice[snl+1]+newsnow;
dz[snl+1] = dz[snl+1]+dz_snowf*dtime;
if (snow_level < 0 && newnode == 0) {
h2osoi_ice[snow_level+1] = h2osoi_ice[snow_level+1]+newsnow;
dz[snow_level+1] = dz[snow_level+1]+dz_snowf*dtime;
}
}
}
......
src/cpp/CanopyHydrology_cpp.hh
View file @ 3262843e
......@@ -49,15 +49,15 @@ void CanopyHydrology_SnowWater(const double& dtime,
double& snow_depth,
double& h2osno,
double& int_snow,
Array_d& swe_old,
Array_d& h2osoi_liq,
Array_d& h2osoi_ice,
Array_d& t_soisno,
Array_d& frac_iceold,
int& snl,
Array_d& dz,
Array_d& z,
Array_d& zi,
Array_d swe_old,
Array_d h2osoi_liq,
Array_d h2osoi_ice,
Array_d t_soisno,
Array_d frac_iceold,
int& snow_level,
Array_d dz,
Array_d z,
Array_d zi,
int& newnode,
double& qflx_floodc,
double& qflx_snow_h2osfc,
......
tests/tests_kokkos_c/CanopyHydrology_kern1_multiple.cpp
View file @ 3262843e
......@@ -122,6 +122,7 @@ int main(int argc, char ** argv)
//h_h2o_can = 0.;
auto h2o_can1 = ELM::Utils::MatrixState();
// Array<int64_t, 2> a = h_h2o_can;
//const size_t n0 = h2o_can.extent_0 ();
// const int64_t begin0 = h_h2o_can.begin();
// const int64_t end0= h_h2o_can.end();
......@@ -142,9 +143,9 @@ int main(int argc, char ** argv)
std::cout << "Time\t Total Canopy Water\t Min Water\t Max Water" << std::endl;
auto min_max = std::minmax_element(h2o_can1.begin(), h2o_can1.end());
auto min_max = std::minmax_element(&h_h2o_can(0,0), &h_h2o_can(n_grid_cells, n_pfts));//h2o_can1.begin(), h2o_can1.end());
std::cout << std::setprecision(16)
<< 0 << "\t" << std::accumulate(h2o_can1.begin(), h2o_can1.end(), 0.)
<< 0 << "\t" << std::accumulate(&h_h2o_can(0,0), &h_h2o_can(n_grid_cells, n_pfts), 0.) //h2o_can1.begin(), h2o_can1.end(), 0.)
<< "\t" << *min_max.first
<< "\t" << *min_max.second << std::endl;
......@@ -168,7 +169,7 @@ int main(int argc, char ** argv)
// Kokkos::parallel_for("CanopyHydrology_Interception", Kokkos::MDRangePolicy<Kokkos::Rank<2,Kokkos::Iterate::Left>>({0,0},{n_grid_cells,n_pfts}),
// KOKKOS_LAMBDA (size_t g, size_t p) {
Kokkos::parallel_for("InitA", n_grid_cells, KOKKOS_LAMBDA (const size_t& g) {
Kokkos::parallel_for("n_grid_cells", n_grid_cells, KOKKOS_LAMBDA (const size_t& g) {
for (size_t p = 0; p != n_pfts; ++p) {
ELM::CanopyHydrology_Interception(dtime,
forc_rain(t,g), forc_snow(t,g), forc_irrig(t,g),
......@@ -187,9 +188,9 @@ int main(int argc, char ** argv)
}
});
auto min_max = std::minmax_element(h2o_can1.begin(), h2o_can1.end());
auto min_max = std::minmax_element(&h_h2o_can(0,0), &h_h2o_can(n_grid_cells, n_pfts));//h2o_can1.begin(), h2o_can1.end());
std::cout << std::setprecision(16)
<< t+1 << "\t" << std::accumulate(h2o_can1.begin(), h2o_can1.end(), 0.)
<< t+1 << "\t" << std::accumulate(&h_h2o_can(0,0), &h_h2o_can(n_grid_cells, n_pfts), 0.)//h2o_can1.begin(), h2o_can1.end(), 0.)
<< "\t" << *min_max.first
<< "\t" << *min_max.second << std::endl;
......
tests/tests_kokkos_c/CanopyHydrology_module.cpp
View file @ 3262843e
......@@ -42,6 +42,7 @@ int main(int argc, char ** argv)
using ELM::Utils::n_pfts;
using ELM::Utils::n_grid_cells;
using ELM::Utils::n_max_times;
using ELM::Utils::n_levels_snow;
// fixed magic parameters for now
const int ctype = 1;
......@@ -92,7 +93,7 @@ int main(int argc, char ** argv)
// mesh input (though can also change as snow layers evolve)
//
// NOTE: in a real case, these would be populated, but we don't actually
// need them to be for these kernels. --etc
// // need them to be for these kernels. --etc
// auto z = ELM::Utils::MatrixStateSoilColumn(0.);
// auto zi = ELM::Utils::MatrixStateSoilColumn(0.);
// auto dz = ELM::Utils::MatrixStateSoilColumn(0.);
......@@ -247,8 +248,8 @@ int main(int argc, char ** argv)
// PFT level operations
//for (size_t p = 0; p != n_pfts; ++p) {
Kokkos::parallel_for("CanopyHydrology_Functions", Kokkos::MDRangePolicy<Kokkos::Rank<2,Kokkos::Iterate::Left>>({0,0},{n_grid_cells,n_pfts}),
KOKKOS_LAMBDA (size_t g, size_t p) {
Kokkos::parallel_for("n_grid_cells", n_grid_cells, KOKKOS_LAMBDA (const size_t& g) {
for (size_t p = 0; p != n_pfts; ++p) {
//
// Calculate interception
//
......@@ -274,14 +275,14 @@ int main(int argc, char ** argv)
// --etc
double fwet = 0., fdry = 0.;
ELM::CanopyHydrology_FracWet(frac_veg_nosno, h2ocan(g,p), elai(g,p), esai(g,p), dewmx, fwet, fdry);
}); // end PFT loop
} // end PFT loop
// Column level operations
Kokkos::parallel_for("Initg", n_grid_cells, KOKKOS_LAMBDA (size_t& g) {
// NOTE: this is effectively an accumulation kernel/task! --etc
qflx_snow_grnd_col(g) = std::accumulate(qflx_snow_grnd_patch(g).begin(),
qflx_snow_grnd_patch(g).end(), 0.);
qflx_snow_grnd_col(g) = std::accumulate(&qflx_snow_grnd_patch(0,0),
&qflx_snow_grnd_patch(n_grid_cells, n_pfts), 0.);
// Calculate ?water balance? on the snow column, adding throughfall,
// removing melt, etc.
......@@ -292,9 +293,9 @@ int main(int argc, char ** argv)
ltype, ctype, urbpoi, do_capsnow, oldfflag,
forc_air_temp(t,g), t_grnd(g),
qflx_snow_grnd_col(g), qflx_snow_melt, n_melt, frac_h2osfc(g),
snow_depth(g), h2osno(g), int_snow(g), swe_old(g),
h2osoi_liq(g), h2osoi_ice(g), t_soisno(g), frac_iceold(g),
snow_level(g), dz(g), z(g), zi(g), newnode,
snow_depth(g), h2osno(g), int_snow(g), Kokkos::subview(swe_old, g , Kokkos::ALL),
Kokkos::subview(h2osoi_liq, g , Kokkos::ALL), Kokkos::subview(h2osoi_ice, g , Kokkos::ALL), Kokkos::subview(t_soisno, g , Kokkos::ALL), Kokkos::subview(frac_iceold, g , Kokkos::ALL),
snow_level(g), Kokkos::subview(dz, g , Kokkos::ALL), Kokkos::subview(z, g , Kokkos::ALL), Kokkos::subview(zi, g , Kokkos::ALL), newnode,
qflx_floodc(g), qflx_snow_h2osfc(g), frac_sno_eff(g), frac_sno(g));
// Calculate Fraction of Water to the Surface?
......
tests/tests_kokkos_c/KokkosCore_config.h
View file @ 3262843e
/* ---------------------------------------------
Makefile constructed configuration:
Tue Apr 9 11:49:16 EDT 2019
Thu Apr 11 08:28:33 EDT 2019
----------------------------------------------*/
#if !defined(KOKKOS_MACROS_HPP) || defined(KOKKOS_CORE_CONFIG_H)
#error "Do not include KokkosCore_config.h directly; include Kokkos_Macros.hpp instead."
......
tests/tests_kokkos_c/KokkosCore_config.tmp
View file @ 3262843e
/* ---------------------------------------------
Makefile constructed configuration:
Tue Apr 9 14:43:41 EDT 2019
Thu Apr 11 15:34:36 EDT 2019
----------------------------------------------*/
#if !defined(KOKKOS_MACROS_HPP) || defined(KOKKOS_CORE_CONFIG_H)
#error "Do not include KokkosCore_config.h directly; include Kokkos_Macros.hpp instead."
......
tests/tests_kokkos_c/Makefile
View file @ 3262843e
KOKKOS_PATH = ${HOME}/Downloads/kokkos
KOKKOS_DEVICES = "Cuda"
EXE_NAME = "CanopyHydrology_kern1_multiple"
EXE_NAME = "CanopyHydrology_module"
OBJECT = ../../src/
include $(OBJECT)config/Makefile.config
SRC = CanopyHydrology_kern1_multiple.cpp
SRC = CanopyHydrology_module.cpp
default: build
echo "Start Build"
......@@ -36,7 +36,7 @@ include $(KOKKOS_PATH)/Makefile.kokkos
build: $(EXE)
$(EXE): $(OBJ) $(KOKKOS_LINK_DEPENDS)
$(LINK) $(KOKKOS_LDFLAGS) ../../src/cpp/CanopyHydrology_Interception.cpp.o $(OBJ) $(KOKKOS_LIBS) $(LIB) -o $(EXE) $(LINKFLAGS) $(EXTRA_PATH)
$(LINK) $(KOKKOS_LDFLAGS) ../../src/cpp/CanopyHydrology_FracH2OSfc.cpp.o ../../src/cpp/CanopyHydrology_FracWet.cpp.o ../../src/cpp/CanopyHydrology_Interception.cpp.o $(OBJ) $(KOKKOS_LIBS) $(LIB) -o $(EXE) $(LINKFLAGS) $(EXTRA_PATH)
clean: kokkos-clean
rm -f *.o *.cuda *.host
......
tests/tests_legion_c/tasks.cc
View file @ 3262843e
......@@ -481,18 +481,18 @@ SumOverPFTs::launch(Context ctx, Runtime *runtime,
color_space, TaskArgument(&args, sizeof(args)), arg_map);
// -- permissions on output
// -- permissions on input
interception_launcher.add_region_requirement(
RegionRequirement(flux.logical_partition, flux.projection_id,
READ_WRITE, EXCLUSIVE, flux.logical_region));
std::vector<std::string> output{"qflx_snow_grnd_patch"};
for (auto fname : output)
READ_ONLY, EXCLUSIVE, flux.logical_region));
std::vector<std::string> input{"qflx_snow_grnd_patch"};
for (auto fname : input)
interception_launcher.add_field(2,flux.field_ids[fname]);
// -- permissions on output
interception_launcher.add_region_requirement(
RegionRequirement(surface.logical_partition, surface.projection_id,
READ_WRITE, EXCLUSIVE, surface.logical_region));
WRITE_DISCARD, EXCLUSIVE, surface.logical_region));
std::vector<std::string> output1{"qflx_snow_grnd_col"};
for (auto fname1 : output1)
interception_launcher.add_field(1,surface.field_ids[fname1]);
......@@ -515,13 +515,14 @@ SumOverPFTs::cpu_execute_task(const Task *task,
// get accessors
using AffineAccessorRO = FieldAccessor<READ_ONLY,double,2,coord_t,
Realm::AffineAccessor<double,2,coord_t> >;
using AffineAccessorWO = FieldAccessor<WRITE_DISCARD,double,2,coord_t,
Realm::AffineAccessor<double,2,coord_t> >;
using AffineAccessorWO = FieldAccessor<WRITE_DISCARD,double,1,coord_t,
Realm::AffineAccessor<double,1,coord_t> >;
// -- input
const AffineAccessorRO qflx_snow_grnd_patch (regions[0], task->regions[0].instance_fields[0]);
// -- output
const AffineAccessorWO qflx_snow_grnd_patch(regions[0], task->regions[0].instance_fields[0]);
const AffineAccessorWO qflx_snow_grnd_col(regions[1], task->regions[1].instance_fields[0]);
// -- output
const AffineAccessorRO qflx_snow_grnd_col(regions[1], task->regions[1].instance_fields[0]);
LogicalRegion lr = regions[0].get_logical_region();
IndexSpaceT<2> is(lr.get_index_space());
......@@ -532,9 +533,9 @@ SumOverPFTs::cpu_execute_task(const Task *task,
//int n = sizeof(qflx_snow_grnd_col) / sizeof(qflx_snow_grnd_patch[0]);
for (size_t g = bounds.lo[0]; g != bounds.hi[0]+1; ++g) {
for (int g = bounds.lo[0]; g != bounds.hi[0]+1; ++g) {
double sum = 0 ;
for (size_t p = bounds.lo[1]; p != bounds.hi[1]+1; ++p) {
for (int p = bounds.lo[1]; p != bounds.hi[1]+1; ++p) {
sum += qflx_snow_grnd_patch[g][p];
}
qflx_snow_grnd_col[g] = sum ;
......@@ -687,7 +688,7 @@ CanopyHydrology_SnowWater::cpu_execute_task(const Task *task,
int newnode = 0.;
for (size_t g = bounds.lo[0]; g != bounds.hi[0]+1; ++g) {
for (int g = bounds.lo[0]; g != bounds.hi[0]+1; ++g) {
ELM::CanopyHydrology_SnowWater(dtime, qflx_floodg,
ltype, ctype, urbpoi, do_capsnow, oldfflag,
......@@ -745,6 +746,15 @@ CanopyHydrology_FracH2OSfc::launch(Context ctx, Runtime *runtime,
IndexLauncher interception_launcher(taskid,
color_space, TaskArgument(&args, sizeof(args)), arg_map);
// -- permissions on input
interception_launcher.add_region_requirement(
RegionRequirement(surface.logical_partition, surface.projection_id,
READ_ONLY, EXCLUSIVE, surface.logical_region));
std::vector<std::string> input{"h2osno"};
for (auto fname1 : input)
interception_launcher.add_field(1,surface.field_ids[fname1]);
// -- permissions on output
interception_launcher.add_region_requirement(
RegionRequirement(soil.logical_partition, soil.projection_id,
......@@ -753,11 +763,10 @@ CanopyHydrology_FracH2OSfc::launch(Context ctx, Runtime *runtime,
for (auto fname : output)
interception_launcher.add_field(2,soil.field_ids[fname]);
// -- permissions on output
interception_launcher.add_region_requirement(
RegionRequirement(surface.logical_partition, surface.projection_id,
RegionRequirement(surface.logical_partition, surface.projection_id,
READ_WRITE, EXCLUSIVE, surface.logical_region));
std::vector<std::string> output1{"h2osno", "h2osfc", "frac_h2osfc", "qflx_h2osfc2topsoi", "frac_snow_eff", "frac_sno"}; //h2osnow rw thing
std::vector<std::string> output1{"h2osfc", "frac_h2osfc", "qflx_h2osfc2topsoi", "frac_snow_eff", "frac_sno"}; //h2osnow rw thing
for (auto fname1 : output1)
interception_launcher.add_field(1,surface.field_ids[fname1]);
......@@ -782,7 +791,7 @@ CanopyHydrology_FracH2OSfc::cpu_execute_task(const Task *task,
*((args_t*) task->args);
// get accessors
using AffineAccessorRO = FieldAccessor<READ_ONLY,double,2,coord_t,
using AffineAccessorRW1 = FieldAccessor<READ_WRITE,double,2,coord_t,
Realm::AffineAccessor<double,2,coord_t> >;
using AffineAccessorRO1 = FieldAccessor<READ_ONLY,double,1,coord_t,
Realm::AffineAccessor<double,1,coord_t> >;
......@@ -791,13 +800,13 @@ CanopyHydrology_FracH2OSfc::cpu_execute_task(const Task *task,
using AffineAccessorWO = FieldAccessor<WRITE_DISCARD,double,1,coord_t,
Realm::AffineAccessor<double,1,coord_t> >;
// -- output
const AffineAccessorRO h2osoi_liq(regions[0], task->regions[0].instance_fields[0]);
const AffineAccessorRO1 h2osno(regions[0], task->regions[0].instance_fields[0]);
// -- output
const AffineAccessorRW h2osno(regions[1], task->regions[1].instance_fields[0]);
const AffineAccessorRW1 h2osoi_liq(regions[1], task->regions[1].instance_fields[0]);
const AffineAccessorRW h2osfc(regions[1], task->regions[1].instance_fields[1]);
const AffineAccessorWO frac_h2osfc(regions[1], task->regions[1].instance_fields[2]);
const AffineAccessorWO qflx_h2osfc2topsoi(regions[1], task->regions[1].instance_fields[3]);
const AffineAccessorRW frac_h2osfc(regions[1], task->regions[1].instance_fields[2]);
const AffineAccessorRW qflx_h2osfc2topsoi(regions[1], task->regions[1].instance_fields[3]);
const AffineAccessorRW frac_snow_eff(regions[1], task->regions[1].instance_fields[4]);
const AffineAccessorRW frac_sno(regions[1], task->regions[1].instance_fields[5]);
......@@ -808,7 +817,7 @@ CanopyHydrology_FracH2OSfc::cpu_execute_task(const Task *task,
std::cout << "LOG: With bounds: " << bounds.lo << "," << bounds.hi << std::endl;
for (size_t g = bounds.lo[0]; g != bounds.hi[0]+1; ++g) {
for (int g = bounds.lo[0]; g != bounds.hi[0]+1; ++g) {
//for (size_t p = bounds.lo[1]; p != bounds.hi[1]+1; ++p) {
ELM::CanopyHydrology_FracH2OSfc(dtime, min_h2osfc, ltype, micro_sigma,
h2osno[g], h2osfc[g], h2osoi_liq[g][0], frac_sno[g], frac_snow_eff[g],qflx_h2osfc2topsoi[g], frac_h2osfc[g] );
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment