Commit e33ffbe9 authored by Pillai, Himanshu's avatar Pillai, Himanshu
Browse files

KOKKOS-HPX Repo : Initializes Kokkos on the main thread and uses the main...

 KOKKOS-HPX Repo : Initializes Kokkos on the main thread and uses the main thread executor to dispatch work from HPX to Kokkos
parent d40541a2
cmake_minimum_required(VERSION 3.10)
set(HPX_DIR "/home/7hp/Downloads/hpx/build")
set(HPX_INCLUDE_DIR "/home/7hp/Downloads/hpx/build/include")
set(HPX_LIB_DIR "/home/7hp/Downloads/hpx/build/lib/")
set(KOKKOS_ROOT "/home/7hp/Downloads/kokkos")
set(KOKKOS_SOURCE_DIR "/home/7hp/Downloads/kokkos")
set(ELM_ROOT "/home/7hp/Downloads/elm_kernels/src/cc/")
set(NETCDF "/usr/local")
set(HPX_IGNORE_COMPILER_COMPATIBILITY On)
set(HPX_CXX_COMPILER "/usr/bin/c++")
set(CMAKE_CXX_COMPILER "/usr/bin/c++")
set(CMAKE_CXX_FLAGS "-std=c++0x ${CMAKE_CXX_FLAGS} -g -ftest-coverage -fprofile-arcs")
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
set(HPX_CXX_COMPILER_ID "GNU")
set(CMAKE_CXX_COMPILER_ID "GNU")
set(CMAKE_MODULE_PATH $HPX_DIR)
if(NOT WIN32)
string(ASCII 27 Esc)
set(ColourReset "${Esc}[m")
set(Red "${Esc}[31m")
set(Green "${Esc}[32m")
set(Yellow "${Esc}[33m")
set(Blue "${Esc}[34m")
set(Magenta "${Esc}[35m")
set(Cyan "${Esc}[36m")
endif()
message(" ")
message(" ${Blue}EeeeeeeeeeeeeeE LLlL Mm Mm Kk kk LllL${ColourReset}")
message(" ${Blue}EeeeeeeeeeeeeeE LLlL Mm Mm Mm Mm Kk kk LllL${ColourReset}")
message(" ${Blue}Eee LLlL Mm Mm Mm Mm Kk kk LllL${ColourReset}")
message(" ${Blue}Eee LllL Mm Mm Mm Mm Kk kk LllL${ColourReset}")
message(" ${Blue}Eee LllL Mm Mm Mm Mm Kk kk LllL${ColourReset}")
message(" ${Blue}Eee LllL Mm Mm Mm Mm Kk kk eeeeeeeee RrRrRrRrRr Nnn Nn eeeeeeeee LllL${ColourReset}")
message(" ${Blue}EeeeeeeeE LllL Mm Mm Mm Kk kk kk eeeeeeeee Rr Rr Nn nn Nn eeeeeeeee LllL${ColourReset}")
message(" ${Blue}EeeeeeeeE LllL Mm Mm Kkk kk eeee Rr Rr Nn nn Nn eeee LllL${ColourReset}")
message(" ${Blue}Eee LllL Mm Mm Kk kk eeeeeeee RrRrRrRrRr Nn nn Nn eeeeeeee LllL${ColourReset}")
message(" ${Blue}Eee LllL Mm Mm Kk kk eeeeeeee RrR Nn nn Nn eeeeeeee LllL${ColourReset}")
message(" ${Blue}Eee LllL Mm Mm Kk kk eeee Rr Rr Nn nn Nn eeee LllL${ColourReset}")
message(" ${Blue}Eee LllL Mm Mm Kk kk eeee Rr Rr Nn nn Nn eeee LllL${ColourReset}")
message(" ${Blue}EeeeeeeeeeeeeeE LllllllllllLllL Mm Mm Kk kk eeeeeeeee Rr Rr Nn nn Nn eeeeeeeee LlllllllLllL${ColourReset}")
message(" ${Blue}EeeeeeeeeeeeeeE LllllllllllLllL Mm Mm Kk kk eeeeeeeee Rr Rr Nn nnNn eeeeeeeee LlllllllLllL${ColourReset}")
message(" ")
message(" ")
message(" ${Green} Copyright 2019, UT Battelle / Oak Ridge National Laboratory ${ColourReset}")
message(" ")
message(" ${Magenta} Collaboration ${ColourReset}")
message(" ${Cyan}Oak Ridge Leadership Computing Facility ${ColourReset}")
message(" ${Cyan}United States Department of Energy ${ColourReset}")
message(" ${Cyan}The Energy Exascale Earth System Model (E3SM) Project ${ColourReset}")
message(" ${Cyan}Coupling Approaches for Next-Generation Architectures (CANGA) - Scientific Discovery through Advanced Computing (SciDAC) ${ColourReset}")
message(" ")
message("${Yellow} Contact: ${ColourReset} ${Red}coonet@ornl.gov , pillaihk@ornl.gov ${ColourReset}")
#find_package(OpenMP)
#if (OPENMP_FOUND)
# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
# set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
# set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
#endif()
project(kokkos-hpx-c CXX)
list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
find_package(HPX REQUIRED)
if(KOKKOS_SOURCE_DIR)
add_subdirectory(${KOKKOS_SOURCE_DIR} ${kokkos-hpx-c_BINARY_DIR}/kokkos)
include_directories(${Kokkos_INCLUDE_DIRS_RET})
else()
message(FATAL_ERROR "KOKKOS_SOURCE_DIR is not set")
endif()
include_directories(
/home/7hp/Downloads/elm_kernels/src/cc/
/usr/local/include
)
link_directories(${HPX_LIBRARY_DIR})
#add_compile_options(-std=c++11)
add_hpx_executable(test_CanopyHydrology_kern1_multiple
ESSENTIAL
SOURCES CanopyHydrology_kern1_multiple.cpp
LINK_FLAGS "-I/usr/local/include -I/home/7hp/Downloads/elm_kernels/src/cc/"
COMPONENT_DEPENDENCIES iostreams init
DEPENDENCIES -g -L/usr/local/lib -lnetcdf)
target_link_libraries(test_CanopyHydrology_kern1_multiple_exe kokkos)
add_hpx_executable(test_CanopyHydrology_module
ESSENTIAL
SOURCES CanopyHydrology_module.cpp
LINK_FLAGS "-I/usr/local/include -I/home/7hp/Downloads/elm_kernels/src/cc/"
COMPONENT_DEPENDENCIES iostreams init
DEPENDENCIES -L/usr/local/lib -lnetcdf)
target_link_libraries(test_CanopyHydrology_module_exe kokkos)
if(KOKKOS_ENABLE_CUDA)
target_link_libraries(test_CanopyHydrology_kern1_multiple_exe cublas curand)
target_link_libraries(test_CanopyHydrology_module_exe cublas curand)
endif()
execute_process(
COMMAND ./test_CanopyHydrology_kern1_multiple
OUTPUT_FILE "test_CanopyHydrology_kern1_multiple.stdout"
)
execute_process(
COMMAND ./test_CanopyHydrology_module
OUTPUT_FILE "test_CanopyHydrology_module.stdout"
)
enable_testing()
add_test(NAME KOKKOS-HPXtest
COMMAND python ../compare_to_gold.py test_CanopyHydrology_kern1_multiple test_CanopyHydrology_module
)
add_custom_target(cleanall
COMMAND rm -r CMakeFiles/ Testing/
COMMAND rm test_* CMakeCache.txt cmake_* CTestTestfile.cmake Makefile *.soln *.stdout
)
\ No newline at end of file
#include <array>
#include <sstream>
#include <iterator>
#include <exception>
#include <string>
#include <stdlib.h>
#include <cstring>
#include <vector>
#include <iostream>
#include <iomanip>
#include <numeric>
#include <fstream>
#include <algorithm>
// #include <mpi.h>
#include <chrono>
#include <Kokkos_Core.hpp>
#include <hpx/hpx_start.hpp>
#include <hpx/include/async.hpp>
#include <hpx/include/apply.hpp>
#include <hpx/include/iostreams.hpp>
#include <hpx/include/parallel_for_loop.hpp>
#include <hpx/parallel/executors/service_executors.hpp>
#include "utils.hh"
#include "readers.hh"
#include "landunit_varcon.h"
#include "column_varcon.h"
#include "CanopyHydrology.hh"
#include "hpx_kokkos.hpp"
using namespace std::chrono;
namespace ELM {
namespace Utils {
static const int n_months = 12;
static const int n_pfts = 17;
static const int n_max_times = 31 * 24 * 2; // max days per month times hours per
// day * half hour timestep
static const int n_grid_cells = 24;
} // namespace
} // namespace
// Initializes Kokkos on the main thread and uses the main thread
// executor to dispatch work from HPX to Kokkos.
void work() {
using hpx::parallel::execution::par;
using namespace hpx::parallel;
using hpx::threads::executors::service_executor_type;
execution::service_executor exec(service_executor_type::main_thread);
using ELM::Utils::n_months;
using ELM::Utils::n_pfts;
using ELM::Utils::n_grid_cells;
using ELM::Utils::n_max_times;
using Kokkos::parallel_for;
using Kokkos::TeamPolicy;
using Kokkos::TeamThreadRange;
hpx::parallel::execution::async_execute(exec, []() {
// fixed magic parameters for now
const int ctype = 1;
const int ltype = 1;
const bool urbpoi = false;
const bool do_capsnow = false;
const int frac_veg_nosno = 1;
int n_irrig_steps_left = 0;
const double dewmx = 0.1;
double dtime = 1800.0;
// int myrank, numprocs;
// double mytime;
// MPI_Init(&argc,&argv);
// MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
// MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
// MPI_Barrier(MPI_COMM_WORLD);
typedef Kokkos::View<double**> ViewMatrixType;
//typedef Kokkos::Cuda ExecSpace;
//typedef Kokkos::Cuda MemSpace;
//typedef Kokkos::RangePolicy<ExecSpace> range_policy;
ViewMatrixType elai( "elai", n_grid_cells, n_pfts );
ViewMatrixType esai( "esai", n_grid_cells, n_pfts );
ViewMatrixType::HostMirror h_elai = Kokkos::create_mirror_view( elai );
ViewMatrixType::HostMirror h_esai = Kokkos::create_mirror_view( esai );
// phenology state
ELM::Utils::read_phenology("../links/surfacedataWBW.nc", n_months, n_pfts, 0, h_elai, h_esai);
ELM::Utils::read_phenology("../links/surfacedataBRW.nc", n_months, n_pfts, n_months, h_elai, h_esai);
// forcing state
ViewMatrixType forc_rain( "forc_rain", n_max_times,n_grid_cells );
ViewMatrixType forc_snow( "forc_snow", n_max_times,n_grid_cells );
ViewMatrixType forc_air_temp( "forc_air_temp", n_max_times,n_grid_cells );
ViewMatrixType::HostMirror h_forc_rain = Kokkos::create_mirror_view( forc_rain );
ViewMatrixType::HostMirror h_forc_snow = Kokkos::create_mirror_view( forc_snow );
ViewMatrixType::HostMirror h_forc_air_temp = Kokkos::create_mirror_view( forc_air_temp );
const int n_times = ELM::Utils::read_forcing("../links/forcing", n_max_times, 0, n_grid_cells, h_forc_rain, h_forc_snow, h_forc_air_temp);
ViewMatrixType forc_irrig( "forc_irrig", n_max_times,n_grid_cells );
ViewMatrixType::HostMirror h_forc_irrig = Kokkos::create_mirror_view( forc_irrig );
// output state by the grid cell
ViewMatrixType qflx_prec_intr( "qflx_prec_intr", n_grid_cells, n_pfts );
ViewMatrixType qflx_irrig( "qflx_irrig", n_grid_cells, n_pfts );
ViewMatrixType qflx_prec_grnd( "qflx_prec_grnd", n_grid_cells, n_pfts );
ViewMatrixType qflx_snwcp_liq( "qflx_snwcp_liq", n_grid_cells, n_pfts );
ViewMatrixType qflx_snwcp_ice ( "qflx_snwcp_ice ", n_grid_cells, n_pfts );
ViewMatrixType qflx_snow_grnd_patch( "qflx_snow_grnd_patch", n_grid_cells, n_pfts );
ViewMatrixType qflx_rain_grnd( "qflx_rain_grnd", n_grid_cells, n_pfts );
ViewMatrixType::HostMirror h_qflx_prec_intr = Kokkos::create_mirror_view( qflx_prec_intr );
ViewMatrixType::HostMirror h_qflx_irrig = Kokkos::create_mirror_view( qflx_irrig);
ViewMatrixType::HostMirror h_qflx_prec_grnd = Kokkos::create_mirror_view( qflx_prec_grnd);
ViewMatrixType::HostMirror h_qflx_snwcp_liq = Kokkos::create_mirror_view( qflx_snwcp_liq);
ViewMatrixType::HostMirror h_qflx_snwcp_ice = Kokkos::create_mirror_view( qflx_snwcp_ice );
ViewMatrixType::HostMirror h_qflx_snow_grnd_patch = Kokkos::create_mirror_view( qflx_snow_grnd_patch );
ViewMatrixType::HostMirror h_qflx_rain_grnd = Kokkos::create_mirror_view( qflx_rain_grnd );
// output state by the pft
ViewMatrixType h2o_can( "h2o_can", n_grid_cells, n_pfts );
ViewMatrixType::HostMirror h_h2o_can = Kokkos::create_mirror_view( h2o_can );
Kokkos::deep_copy( elai, h_elai);
Kokkos::deep_copy( esai, h_esai);
Kokkos::deep_copy( forc_rain, h_forc_rain);
Kokkos::deep_copy( forc_snow, h_forc_snow);
Kokkos::deep_copy( forc_air_temp, h_forc_air_temp);
double* end = &h_h2o_can(n_grid_cells-1, n_pfts-1) ;
std::ofstream soln_file;
soln_file.open("test_CanopyHydrology_kern1_multiple.soln");
soln_file << "Time\t Total Canopy Water\t Min Water\t Max Water" << std::endl;
std::cout << "Time\t Total Canopy Water\t Min Water\t Max Water" << std::endl;
auto min_max = std::minmax_element(&h_h2o_can(0,0), end+1);
soln_file << std::setprecision(16)
<< 0 << "\t" << std::accumulate(&h_h2o_can(0,0), end+1, 0.)
<< "\t" << *min_max.first
<< "\t" << *min_max.second << std::endl;
std::cout << std::setprecision(16)
<< 0 << "\t" << std::accumulate(&h_h2o_can(0,0), end+1, 0.)
<< "\t" << *min_max.first
<< "\t" << *min_max.second << std::endl;
Kokkos::Timer timer;
auto start = high_resolution_clock::now();
// mytime = MPI_Wtime();
// main loop
// -- the timestep loop cannot/should not be parallelized
for (size_t t = 0; t != n_times; ++t) {
typedef typename Kokkos::Experimental::MDRangePolicy< Kokkos::Experimental::Rank<2> > MDPolicyType_2D;
// Construct 2D MDRangePolicy: lower and upper bounds provided, tile dims defaulted
MDPolicyType_2D mdpolicy_2d( {{0,0}}, {{n_grid_cells,n_pfts}} );
Kokkos::parallel_for("md2d",mdpolicy_2d,KOKKOS_LAMBDA (const size_t& g, const size_t& p) {
ELM::CanopyHydrology_Interception(dtime,
forc_rain(t,g), forc_snow(t,g), forc_irrig(t,g),
ltype, ctype, urbpoi, do_capsnow,
elai(g,p), esai(g,p), dewmx, frac_veg_nosno,
h2o_can(g,p), n_irrig_steps_left,
qflx_prec_intr(g,p), qflx_irrig(g,p), qflx_prec_grnd(g,p),
qflx_snwcp_liq(g,p), qflx_snwcp_ice(g,p),
qflx_snow_grnd_patch(g,p), qflx_rain_grnd(g,p)); });
Kokkos::deep_copy( h_qflx_irrig, qflx_irrig);
Kokkos::deep_copy( h_qflx_prec_intr, qflx_prec_intr);
Kokkos::deep_copy( h_qflx_prec_grnd, qflx_prec_grnd);
Kokkos::deep_copy( h_qflx_snwcp_liq, qflx_snwcp_liq);
Kokkos::deep_copy( h_qflx_snwcp_ice, qflx_snwcp_ice);
Kokkos::deep_copy( h_qflx_snow_grnd_patch, qflx_snow_grnd_patch);
Kokkos::deep_copy( h_qflx_rain_grnd, qflx_rain_grnd);
Kokkos::deep_copy( h_h2o_can, h2o_can);
auto min_max = std::minmax_element(&h_h2o_can(0,0), end+1);
std::cout << std::setprecision(16)
<< t+1 << "\t" << std::accumulate(&h_h2o_can(0,0), end+1, 0.)
<< "\t" << *min_max.first
<< "\t" << *min_max.second << std::endl;
soln_file << std::setprecision(16)
<< t+1 << "\t" << std::accumulate(&h_h2o_can(0,0), end+1, 0.)
<< "\t" << *min_max.first
<< "\t" << *min_max.second << std::endl;
} soln_file.close();
double time = timer.seconds();
double Gbytes = 1.0e-9 * double( sizeof(double) * ( n_grid_cells + n_grid_cells * n_pfts + n_pfts ) );
printf( " n_pfts( %d ) n_grid_cells( %d ) n_times ( %d ) problem( %g MB ) time( %g s ) bandwidth( %g GB/s )\n",
n_pfts, n_grid_cells, n_times, Gbytes * 1000, time, Gbytes * n_times / time );
// mytime = MPI_Wtime() - mytime;
auto stop = high_resolution_clock::now();
// std::cout <<"Timing from node "<< myrank << " is "<< mytime << "seconds." << std::endl;
auto duration = duration_cast<microseconds>(stop - start);
std::cout << "Time taken by function: "<< duration.count() << " microseconds" << std::endl;
});
}
int hpx_main(int argc, char *argv[]) {
work();
return hpx::finalize();
}
int main(int argc, char *argv[]) {
Kokkos::initialize(argc, argv);
#if defined(KOKKOS_ENABLE_HPX)
hpx::apply(work);
#else
hpx::start(argc, argv);
hpx::stop();
#endif
Kokkos::finalize();
return 0;
}
#include <array>
#include <sstream>
#include <iterator>
#include <exception>
#include <string>
#include <stdlib.h>
#include <cstring>
#include <vector>
#include <iostream>
#include <iomanip>
#include <numeric>
#include <fstream>
// #include <mpi.h>
#include <chrono>
#include <Kokkos_Core.hpp>
#include <hpx/hpx_start.hpp>
#include <hpx/include/async.hpp>
#include <hpx/include/apply.hpp>
#include <hpx/include/iostreams.hpp>
#include <hpx/include/parallel_for_loop.hpp>
#include <hpx/parallel/executors/service_executors.hpp>
#include "utils.hh"
#include "readers.hh"
#include "CanopyHydrology.hh"
#include "CanopyHydrology_SnowWater_impl.hh"
#include "hpx_kokkos.hpp"
using namespace std::chrono;
namespace ELM {
namespace Utils {
static const int n_months = 12;
static const int n_pfts = 17;
static const int n_max_times = 31 * 24 * 2; // max days per month times hours per
// day * half hour timestep
static const int n_grid_cells = 24;
static const int n_levels_snow = 5;
} // namespace
} // namespace
void work() {
using hpx::parallel::execution::par;
using namespace hpx::parallel;
using hpx::threads::executors::service_executor_type;
execution::service_executor exec(service_executor_type::main_thread);
using ELM::Utils::n_months;
using ELM::Utils::n_pfts;
using ELM::Utils::n_grid_cells;
using ELM::Utils::n_max_times;
using ELM::Utils::n_levels_snow;
using Kokkos::parallel_for;
using Kokkos::TeamPolicy;
using Kokkos::TeamThreadRange;
hpx::parallel::execution::async_execute(exec, []() {
// fixed magic parameters for now
const int ctype = 1;
const int ltype = 1;
const bool urbpoi = false;
const bool do_capsnow = false;
const int frac_veg_nosno = 1;
int n_irrig_steps_left = 0;
const double dewmx = 0.1;
const double dtime = 1800.0;
// fixed magic parameters for SnowWater
const double qflx_snow_melt = 0.;
// fixed magic parameters for fracH2Osfc
const int oldfflag = 0;
const double micro_sigma = 0.1;
const double min_h2osfc = 1.0e-8;
const double n_melt = 0.7;
// int myrank, numprocs;
// double mytime;
// MPI_Init(&argc,&argv);
// MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
// MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
// MPI_Barrier(MPI_COMM_WORLD);
// phenology input
typedef Kokkos::View<double*> ViewVectorType;
typedef Kokkos::View<double**> ViewMatrixType;
typedef Kokkos::View<int**> ViewMatrixType1;
typedef Kokkos::View<int*> ViewVectorType1;
ViewMatrixType elai( "elai", n_grid_cells, n_pfts );
ViewMatrixType esai( "esai", n_grid_cells, n_pfts );
ViewMatrixType::HostMirror h_elai = Kokkos::create_mirror_view( elai );
ViewMatrixType::HostMirror h_esai = Kokkos::create_mirror_view( esai );
ELM::Utils::read_phenology("../links/surfacedataWBW.nc", n_months, n_pfts, 0, h_elai, h_esai);
ELM::Utils::read_phenology("../links/surfacedataBRW.nc", n_months, n_pfts, n_months, h_elai, h_esai);
// forcing input
ViewMatrixType forc_rain( "forc_rain", n_max_times,n_grid_cells );
ViewMatrixType forc_snow( "forc_snow", n_max_times,n_grid_cells );
ViewMatrixType forc_air_temp( "forc_air_temp", n_max_times,n_grid_cells );
ViewMatrixType::HostMirror h_forc_rain = Kokkos::create_mirror_view( forc_rain );
ViewMatrixType::HostMirror h_forc_snow = Kokkos::create_mirror_view( forc_snow );
ViewMatrixType::HostMirror h_forc_air_temp = Kokkos::create_mirror_view( forc_air_temp );
const int n_times = ELM::Utils::read_forcing("../links/forcing", n_max_times, 0, n_grid_cells, h_forc_rain, h_forc_snow, h_forc_air_temp);
ViewMatrixType forc_irrig( "forc_irrig", n_max_times,n_grid_cells );
ViewMatrixType::HostMirror h_forc_irrig = Kokkos::create_mirror_view( forc_irrig );
double qflx_floodg = 0.0;
// mesh input (though can also change as snow layers evolve)
//
// NOTE: in a real case, these would be populated, but we don't actually
// // need them to be for these kernels. --etc
ViewMatrixType z( "z", n_grid_cells, n_levels_snow );
ViewMatrixType zi( "zi", n_grid_cells, n_levels_snow );
ViewMatrixType dz( "dz", n_grid_cells, n_levels_snow );
ViewMatrixType::HostMirror h_z = Kokkos::create_mirror_view( z );
ViewMatrixType::HostMirror h_zi = Kokkos::create_mirror_view( zi );
ViewMatrixType::HostMirror h_dz = Kokkos::create_mirror_view( dz );
// state variables that require ICs and evolve (in/out)
ViewMatrixType h2ocan( "h2ocan", n_grid_cells, n_pfts );
ViewMatrixType swe_old( "swe_old", n_grid_cells, n_levels_snow );
ViewMatrixType h2osoi_liq( "h2osoi_liq", n_grid_cells, n_levels_snow );
ViewMatrixType h2osoi_ice( "h2osoi_ice", n_grid_cells, n_levels_snow );
ViewMatrixType t_soisno( "t_soisno", n_grid_cells, n_levels_snow );
ViewMatrixType frac_iceold( "frac_iceold", n_grid_cells, n_levels_snow );
ViewMatrixType::HostMirror h_h2ocan = Kokkos::create_mirror_view( h2ocan );
ViewMatrixType::HostMirror h_swe_old = Kokkos::create_mirror_view( swe_old );
ViewMatrixType::HostMirror h_h2osoi_liq = Kokkos::create_mirror_view( h2osoi_liq );
ViewMatrixType::HostMirror h_h2osoi_ice = Kokkos::create_mirror_view( h2osoi_ice );
ViewMatrixType::HostMirror h_t_soisno = Kokkos::create_mirror_view( t_soisno );
ViewMatrixType::HostMirror h_frac_iceold = Kokkos::create_mirror_view( frac_iceold );
ViewVectorType t_grnd( "t_grnd", n_grid_cells );
ViewVectorType h2osno( "h2osno", n_grid_cells );
ViewVectorType snow_depth( "snow_depth", n_grid_cells );
ViewVectorType1 snow_level( "snow_level", n_grid_cells );
ViewVectorType::HostMirror h_t_grnd = Kokkos::create_mirror_view( t_grnd);
ViewVectorType::HostMirror h_h2osno = Kokkos::create_mirror_view( h2osno);
ViewVectorType::HostMirror h_snow_depth = Kokkos::create_mirror_view( snow_depth);
ViewVectorType1::HostMirror h_snow_level = Kokkos::create_mirror_view( snow_level);
// auto h2osfc = ELM::Utils::VectorColumn(0.);
// auto frac_h2osfc = ELM::Utils::VectorColumn(0.);
ViewVectorType h2osfc( "h2osfc", n_grid_cells );
ViewVectorType frac_h2osfc( "frac_h2osfc", n_grid_cells );
ViewVectorType::HostMirror h_h2osfc = Kokkos::create_mirror_view( h2osfc);
ViewVectorType::HostMirror h_frac_h2osfc = Kokkos::create_mirror_view( frac_h2osfc);
// output fluxes by pft
ViewMatrixType qflx_prec_intr( "qflx_prec_intr", n_grid_cells, n_pfts );
ViewMatrixType qflx_irrig( "qflx_irrig", n_grid_cells, n_pfts );
ViewMatrixType qflx_prec_grnd( "qflx_prec_grnd", n_grid_cells, n_pfts );
ViewMatrixType qflx_snwcp_liq( "qflx_snwcp_liq", n_grid_cells, n_pfts );
ViewMatrixType qflx_snwcp_ice ( "qflx_snwcp_ice ", n_grid_cells, n_pfts );
ViewMatrixType qflx_snow_grnd_patch( "qflx_snow_grnd_patch", n_grid_cells, n_pfts );
ViewMatrixType qflx_rain_grnd( "qflx_rain_grnd", n_grid_cells, n_pfts );
ViewMatrixType::HostMirror h_qflx_prec_intr = Kokkos::create_mirror_view( qflx_prec_intr );
ViewMatrixType::HostMirror h_qflx_irrig = Kokkos::create_mirror_view( qflx_irrig);
ViewMatrixType::HostMirror h_qflx_prec_grnd = Kokkos::create_mirror_view( qflx_prec_grnd);
ViewMatrixType::HostMirror h_qflx_snwcp_liq = Kokkos::create_mirror_view( qflx_snwcp_liq);
ViewMatrixType::HostMirror h_qflx_snwcp_ice = Kokkos::create_mirror_view( qflx_snwcp_ice );
ViewMatrixType::HostMirror h_qflx_snow_grnd_patch = Kokkos::create_mirror_view( qflx_snow_grnd_patch );
ViewMatrixType::HostMirror h_qflx_rain_grnd = Kokkos::create_mirror_view( qflx_rain_grnd );
// FIXME: I have no clue what this is... it is inout on WaterSnow. For now I
// am guessing the data structure. Ask Scott. --etc
ViewVectorType integrated_snow( "integrated_snow", n_grid_cells );
ViewVectorType::HostMirror h_integrated_snow = Kokkos::create_mirror_view( integrated_snow);
// output fluxes, state by the column
ViewVectorType qflx_snow_grnd_col( "qflx_snow_grnd_col", n_grid_cells );
ViewVectorType qflx_snow_h2osfc( "qflx_snow_h2osfc", n_grid_cells );
ViewVectorType qflx_h2osfc2topsoi( "qflx_h2osfc2topsoi", n_grid_cells );
ViewVectorType qflx_floodc( "qflx_floodc", n_grid_cells );
ViewVectorType::HostMirror h_qflx_snow_grnd_col = Kokkos::create_mirror_view( qflx_snow_grnd_col);
ViewVectorType::HostMirror h_qflx_snow_h2osfc = Kokkos::create_mirror_view( qflx_snow_h2osfc);
ViewVectorType::HostMirror h_qflx_h2osfc2topsoi = Kokkos::create_mirror_view( qflx_h2osfc2topsoi);
ViewVectorType::HostMirror h_qflx_floodc = Kokkos::create_mirror_view( qflx_floodc);
ViewVectorType frac_sno_eff( "frac_sno_eff", n_grid_cells );
ViewVectorType frac_sno( "frac_sno", n_grid_cells );
ViewVectorType::HostMirror h_frac_sno_eff = Kokkos::create_mirror_view( frac_sno_eff);
ViewVectorType::HostMirror h_frac_sno = Kokkos::create_mirror_view( frac_sno);
Kokkos::deep_copy( elai, h_elai);
Kokkos::deep_copy( esai, h_esai);
Kokkos::deep_copy( forc_rain, h_forc_rain);
Kokkos::deep_copy( forc_snow, h_forc_snow);
Kokkos::deep_copy( forc_air_temp, h_forc_air_temp);
double* end1 = &h_h2ocan(n_grid_cells-1, n_pfts-1) ;
double* end2 = &h_h2osno(n_grid_cells-1) ;
double* end3 = &h_frac_h2osfc(n_grid_cells-1) ;
std::ofstream soln_file;
soln_file.open("test_CanopyHydrology_module.soln");
soln_file << "Time\t Total Canopy Water\t Min Water\t Max Water\t Total Snow\t Min Snow\t Max Snow\t Avg Frac Sfc\t Min Frac Sfc\t Max Frac Sfc" << std::endl;
std::cout << "Time\t Total Canopy Water\t Min Water\t Max Water\t Total Snow\t Min Snow\t Max Snow\t Avg Frac Sfc\t Min Frac Sfc\t Max Frac Sfc" << std::endl;
auto min_max_water = std::minmax_element(&h_h2ocan(0,0), end1+1);
auto sum_water = std::accumulate(&h_h2ocan(0,0), end1+1, 0.);
auto min_max_snow = std::minmax_element(&h_h2osno(0), end2+1);
auto sum_snow = std::accumulate(&h_h2osno(0), end2+1, 0.);
auto min_max_frac_sfc = std::minmax_element(&h_frac_h2osfc(0), end3+1);
auto avg_frac_sfc = std::accumulate(&h_frac_h2osfc(0), end3+1, 0.) / (end3+1 - &h_frac_h2osfc(0));
soln_file << std::setprecision(16)
<< 0 << "\t" << sum_water << "\t" << *min_max_water.first << "\t" << *min_max_water.second
<< "\t" << sum_snow << "\t" << *min_max_snow.first << "\t" << *min_max_snow.second