Make fixes and kern1_multiple works for both Kokkos and Legion

src/Makefile

@@ -5,7 +5,7 @@ OBJECT        = .
 
 default: all
 
-all: cc_fortran_wrappers fortran cc_implementation
+all: cc_fortran_wrappers fortran cc_serial cc_kokkos
 
 cc_fortran_wrappers: fortran
 	$(MAKE) -C cc_fortran_wrappers all

tests/compare_to_gold.py

@@ -25,7 +25,7 @@ def run_comparison(testname, full_message=False):
         gold = np.loadtxt(fname_gold, skiprows=1)
     except IOError:
         print 'ERROR: cannot find gold file "%s", bad testname?'%fname_gold
-        return 'G'
+        return 'G '
 
     # get my file
     fname_mine = '%s.soln'%testname
@@ -37,7 +37,7 @@ def run_comparison(testname, full_message=False):
             mine = np.loadtxt(fname_mine, skiprows=1)
         except IOError:
             print 'ERROR: cannot find my file "%s", failed test run?'%fname_mine
-            return 'D'
+            return 'D '
         
     # compare
     try:
@@ -46,12 +46,12 @@ def run_comparison(testname, full_message=False):
         if full_message:
             print ''
             print 'Test: "%s" FAILED with error: "%r"'%(testname, err)
-        return 'E'
+        return 'E '
 
     if close:
-        return 'PASS'
+        return 'PASS '
     else:
-        return 'FAIL'
+        return 'FAIL '
                       
 
 if __name__ == "__main__":
@@ -69,5 +69,5 @@ if __name__ == "__main__":
             results.append(run_comparison(test, args.verbose))
         except Exception as err:
             print err
-            results.append('N/A')
+            results.append('N/A ')
     print ''.join(results)

tests/tests_c_c/Makefile

 OBJECT        = ../../src/
-KERNEL_FOLDER = cc_implementation
+KERNEL_FOLDER = cc_serial
 SRCDIR	      =	$(OBJECT)$(KERNEL_FOLDER)
 
 include $(OBJECT)config/Makefile.config
@@ -26,13 +26,13 @@ test: $(EXEC_TESTS)
 
 
 CanopyHydrology_kern1_single: test_CanopyHydrology_kern1_single
-	./test_CanopyHydrology_kern1_single &> test_CanopyHydrology_kern1_single.stdout
+	./test_CanopyHydrology_kern1_single > test_CanopyHydrology_kern1_single.stdout
 
 CanopyHydrology_kern1_multiple: test_CanopyHydrology_kern1_multiple
-	./test_CanopyHydrology_kern1_multiple &> test_CanopyHydrology_kern1_multiple.stdout
+	./test_CanopyHydrology_kern1_multiple > test_CanopyHydrology_kern1_multiple.stdout
 
 CanopyHydrology_module: test_CanopyHydrology_module
-	./test_CanopyHydrology_module &> test_CanopyHydrology_module.stdout
+	./test_CanopyHydrology_module > test_CanopyHydrology_module.stdout
 
 
 test_%: %.cpp.o readers.hh utils.hh library

tests/tests_c_f/Makefile

@@ -28,13 +28,13 @@ test: $(EXEC_TESTS)
 	python ../compare_to_gold.py $(TESTS)
 
 CanopyHydrology_kern1_single: test_CanopyHydrology_kern1_single
-	./test_CanopyHydrology_kern1_single &> test_CanopyHydrology_kern1_single.stdout
+	./test_CanopyHydrology_kern1_single > test_CanopyHydrology_kern1_single.stdout
 
 CanopyHydrology_kern1_multiple: test_CanopyHydrology_kern1_multiple
-	./test_CanopyHydrology_kern1_multiple &> test_CanopyHydrology_kern1_multiple.stdout
+	./test_CanopyHydrology_kern1_multiple > test_CanopyHydrology_kern1_multiple.stdout
 
 CanopyHydrology_module: test_CanopyHydrology_module
-	./test_CanopyHydrology_module &> test_CanopyHydrology_module.stdout
+	./test_CanopyHydrology_module > test_CanopyHydrology_module.stdout
 
 
 test_%: %.cc.o readers.hh utils.hh library

tests/tests_f/CanopyHydrology_kern1_single.F90

@@ -29,7 +29,7 @@ program CanopyHydrology_kern1_single
   ! This will be the netCDF ID for the file and data variable.
   integer :: ncid, varid, dimid, status
 
-  status = nf90_open(SURFDAT_FILE_NAME, NF90_NOWRITE, ncid) 
+  status = nf90_open(SURFDAT_FILE_NAME, NF90_NOwrite, ncid) 
   if( status .ne. 0) call err_handle( status ) 
    
 
@@ -48,7 +48,7 @@ program CanopyHydrology_kern1_single
 
   status = nf90_close(ncid) 
 
-  status = nf90_open(FORCDAT_FILE_NAME, NF90_NOWRITE, ncid) 
+  status = nf90_open(FORCDAT_FILE_NAME, NF90_NOwrite, ncid) 
   if( status .ne. 0) call err_handle( status ) 
   status = nf90_inq_dimid(ncid, "time", dimid) 
   if( status .ne. 0) call err_handle( status ) 
@@ -65,6 +65,8 @@ program CanopyHydrology_kern1_single
 
   h2ocan = 0.0d0 
   print *, "Timestep, forc_rain, h2ocan, qflx_prec_grnd, qflx_prec_intr"
+  !open(1, file = 'test_CanopyHydrology_kern1_single.stdout', status = 'new')
+  !write(1,*)"Timestep, forc_rain, h2ocan, qflx_prec_grnd, qflx_prec_intr"
   do itime=1,ntimes 
 
 
@@ -87,8 +89,11 @@ program CanopyHydrology_kern1_single
      qflx_snwcp_liq, qflx_snwcp_ice, qflx_snow_grnd_patch, qflx_rain_grnd)
 
    print *, itime, forc_rain, h2ocan, qflx_prec_grnd, qflx_prec_intr 
+   
+   !open(1,file='test_CanopyHydrology_kern1_single.stdout',action='write',position='append')
+   !write(1,*) itime, forc_rain, h2ocan, qflx_prec_grnd, qflx_prec_intr
   end do 
-
+  !close(1)
 
   stop 
   contains 
@@ -97,6 +102,8 @@ program CanopyHydrology_kern1_single
       use netcdf
       integer :: status 
       print *, 'error ', status, nf90_strerror(status)
+      !open(1,file='test_CanopyHydrology_kern1_single.stdout',action='write',position='append')
+      !write(1,*) 'error ', status, nf90_strerror(status)
       stop 
     end subroutine err_handle
 

tests/tests_f/CanopyHydrology_module.F90

@@ -93,7 +93,12 @@ program test_CanopyHydrology_module
 
 
   h2ocan_pft = 0.0d0 
-  frac_h2osfc = 0.0d0 
+  frac_h2osfc = 0.0d0
+  print*, "Time", "Total Canopy Water", "Min Water", "Max Water", "Total Snow", &
+  "Min Snow", "Max Snow", "Avg Frac Sfc", "Min Frac Sfc", "Max Frac Sfc"
+  print*, 0, sum(h2ocan_pft), minval(h2ocan_pft), maxval(h2ocan_pft), sum(h2osno_grc), &
+  minval(h2osno_grc), maxval(h2osno_grc), sum(h2osoi_liq), minval(h2osoi_liq), maxval(h2osoi_liq)
+
   do itime=1,28*48  ! February is shortest month 
 
      do g=1,ngrcs ! grid cell loop 
@@ -159,6 +164,9 @@ program test_CanopyHydrology_module
         dz_grc(g,:) = dz 
 
      end do !mesh loop 
+     print*, itime, sum(h2ocan_pft), minval(h2ocan_pft), maxval(h2ocan_pft), &
+     sum(h2osno_grc), minval(h2osno_grc), maxval(h2osno_grc), &
+     sum(h2osoi_liq), minval(h2osoi_liq), maxval(h2osoi_liq)
 
   end do ! time loop  
 

tests/tests_f/Makefile

@@ -25,13 +25,13 @@ test: $(EXEC_TESTS)
 
 
 CanopyHydrology_kern1_single: test_CanopyHydrology_kern1_single
-	./test_CanopyHydrology_kern1_single &> test_CanopyHydrology_kern1_single.stdout
+	./test_CanopyHydrology_kern1_single > test_CanopyHydrology_kern1_single.stdout
 
 CanopyHydrology_kern1_multiple: test_CanopyHydrology_kern1_multiple
-	./test_CanopyHydrology_kern1_multiple &> test_CanopyHydrology_kern1_multiple.stdout
+	./test_CanopyHydrology_kern1_multiple > test_CanopyHydrology_kern1_multiple.stdout
 
 CanopyHydrology_module: test_CanopyHydrology_module
-	./test_CanopyHydrology_module &> test_CanopyHydrology_module.stdout
+	./test_CanopyHydrology_module > test_CanopyHydrology_module.stdout
 
 
 test_%: %.F90.o library

tests/tests_kokkos_c/CanopyHydrology_module.cpp

@@ -5,6 +5,7 @@
 #include <string>
 #include <stdlib.h>
 #include <cstring>
+#include <cmath>
 #include <vector>
 #include <iostream>
 #include <iomanip>
@@ -100,7 +101,7 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_Interception(double dtime,
 
 
                
-               h2ocan = std::max(0.0, h2ocan + dtime*qflx_prec_intr);
+               h2ocan = max(0.0, h2ocan + dtime*qflx_prec_intr);
 
                
                qflx_candrip = 0.0;
@@ -196,8 +197,8 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_FracWet(const int& frac_veg_nosno,
     if (h2ocan > 0.0) {
         vegt    = frac_veg_nosno*(elai + esai);
         dewmxi  = 1.00/dewmx;
-        fwet = std::pow(((dewmxi/vegt)*h2ocan), 2.0/3);
-        fwet = std::min(fwet,1.00);   
+        fwet = pow(((dewmxi/vegt)*h2ocan), 2.0/3);
+        fwet = min(fwet,1.00);   
         fdry = (1.0-fwet)*elai/(elai+esai);
       }
     else{
@@ -298,9 +299,9 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_SnowWater(const double& dtime,
   } else {
 
      if (forc_air_temp > tfrz + 2.) {
-        bifall=50. + 1.7*std::pow((17.0),1.5);
+        bifall=50. + 1.7*pow((17.0),1.5);
      } else if (forc_air_temp > tfrz - 15.) {
-        bifall=50. + 1.7*std::pow((forc_air_temp - tfrz + 15.),1.5);
+        bifall=50. + 1.7*pow((forc_air_temp - tfrz + 15.),1.5);
      } else {
         bifall=50.;
      }
@@ -309,7 +310,7 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_SnowWater(const double& dtime,
      newsnow = (1. - frac_h2osfc) * qflx_snow_grnd_col * dtime;
 
      // update integrated_snow
-     integrated_snow = std::max(integrated_snow,h2osno) ; //h2osno could be larger due to frost
+     integrated_snow = max(integrated_snow,h2osno) ; //h2osno could be larger due to frost
 
      // snowmelt from previous time step * dtime
      snowmelt = qflx_snow_melt * dtime;
@@ -324,9 +325,9 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_SnowWater(const double& dtime,
         // first compute change from melt during previous time step
         if(snowmelt > 0.) {
 
-           smr=std::min(1.,(h2osno)/(integrated_snow));
+           smr=min(1.,(h2osno)/(integrated_snow));
 
-           frac_sno = 1. - std::pow((acos(fmin(1.,(2.*smr - 1.)))/rpi),(n_melt)) ;
+           frac_sno = 1. - pow((acos(fmin(1.,(2.*smr - 1.)))/rpi),(n_melt)) ;
 
         }
 
@@ -336,8 +337,8 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_SnowWater(const double& dtime,
            frac_sno = fsno_new;
 
            // reset integrated_snow after accumulation events
-           temp_intsnow= (h2osno + newsnow) / (0.5*(cos(rpi*std::pow((1.0-std::max(frac_sno,1e-6)),(1.0/n_melt))+1.0))) ;
-           integrated_snow = std::min(1.e8,temp_intsnow) ;
+           temp_intsnow= (h2osno + newsnow) / (0.5*(cos(rpi*pow((1.0-max(frac_sno,1e-6)),(1.0/n_melt))+1.0))) ;
+           integrated_snow = min(1.e8,temp_intsnow) ;
         }
 
         //====================================================================
@@ -358,10 +359,10 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_SnowWater(const double& dtime,
         if (oldfflag == 1) { 
            // snow cover fraction in Niu et al. 2007
            if(snow_depth > 0.0)  {
-              frac_sno = tanh(snow_depth/(2.5*zlnd*std::pow((std::min(800.0,(h2osno+ newsnow)/snow_depth)/100.0),1.0)) ) ;
+              frac_sno = tanh(snow_depth/(2.5*zlnd*pow((min(800.0,(h2osno+ newsnow)/snow_depth)/100.0),1.0)) ) ;
            }
            if(h2osno < 1.0)  {
-              frac_sno=std::min(frac_sno,h2osno);
+              frac_sno=min(frac_sno,h2osno);
            }
         }
 
@@ -374,8 +375,8 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_SnowWater(const double& dtime,
 
            // make integrated_snow consistent w/ new fsno, h2osno
            integrated_snow = 0. ;//reset prior to adding newsnow below
-           temp_intsnow= (h2osno + newsnow) / (0.5*(cos(rpi*std::pow((1.0-std::max(frac_sno,1e-6)),(1.0/n_melt)))+1.0));
-           integrated_snow = std::min(1.e8,temp_intsnow);
+           temp_intsnow= (h2osno + newsnow) / (0.5*(cos(rpi*pow((1.0-max(frac_sno,1e-6)),(1.0/n_melt)))+1.0));
+           integrated_snow = min(1.e8,temp_intsnow);
 
            // update snow_depth and h2osno to be consistent with frac_sno, z_avg
            if (subgridflag ==1 && !urbpoi) {
@@ -387,7 +388,7 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_SnowWater(const double& dtime,
            if (oldfflag == 1) { 
               // snow cover fraction in Niu et al. 2007
               if(snow_depth > 0.0)  {
-                 frac_sno = tanh(snow_depth/(2.5*zlnd*std::pow((std::min(800.0,newsnow/snow_depth)/100.0),1.0)) );
+                 frac_sno = tanh(snow_depth/(2.5*zlnd*pow((min(800.0,newsnow/snow_depth)/100.0),1.0)) );
               }
            }
         } else {
@@ -488,7 +489,7 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_FracH2OSfc(const double& dtime,
 
           sigma=1.0e3 * micro_sigma ;
           for(int l = 0 ; l < 10; l++) {
-             fd = 0.5*d*(1.00+erf(d/(sigma*sqrt(2.0)))) + sigma/sqrt(2.0*shr_const_pi)*std::exp(-std::pow(d,2)/(2.0*std::pow(sigma,2))) -h2osfc;
+             fd = 0.5*d*(1.00+erf(d/(sigma*sqrt(2.0)))) + sigma/sqrt(2.0*shr_const_pi)*exp(-pow(d,2)/(2.0*pow(sigma,2))) -h2osfc;
              dfdd = 0.5*(1.00+erf(d/(sigma*sqrt(2.0))));
 
              d = d - fd/dfdd;
@@ -509,7 +510,7 @@ KOKKOS_INLINE_FUNCTION void CanopyHydrology_FracH2OSfc(const double& dtime,
           if (frac_sno > (1.0 - frac_h2osfc) && h2osno > 0) {
 
              if (frac_h2osfc > 0.010) {
-                frac_h2osfc = std::max(1.00 - frac_sno,0.010) ;
+                frac_h2osfc = max(1.00 - frac_sno,0.010) ;
                 frac_sno = 1.00 - frac_h2osfc; }
              else {
                 frac_sno = 1.00 - frac_h2osfc;
@@ -701,7 +702,7 @@ int main(int argc, char ** argv)
 
   // std::cout << "Time\t Total Canopy Water\t Min Water\t Max Water" << std::endl;
   // auto min_max = std::minmax_element(&h_h2ocan(0,0), end1+1);
-  // std::cout << std::setprecision(16)
+  // std::cout << std::minmax_element(16)
   //           << 0 << "\t" << std::accumulate(&h_h2ocan(0,0), end1+1, 0.)
   //           << "\t" << *min_max.first
   //           << "\t" << *min_max.second << std::endl;
@@ -772,13 +773,13 @@ int main(int argc, char ** argv)
 
       // PFT level operations
       //for (size_t p = 0; p != n_pfts; ++p) {
-        Kokkos::parallel_for("n_grid_cells", n_grid_cells, KOKKOS_LAMBDA (const size_t& g) {
+    Kokkos::parallel_for("n_grid_cells", n_grid_cells, KOKKOS_LAMBDA (const size_t& g) {
       for (size_t p = 0; p != n_pfts; ++p) {
         //
         // Calculate interception
         //
         // NOTE: this currently punts on what to do with the qflx variables!
-        // Surely they should be either accumulated or stored on PFTs as well.
+        // Surely they should be either std::accumulated or stored on PFTs as well.
         // --etc
         ELM::CanopyHydrology_Interception(dtime,
                 forc_rain(t,g), forc_snow(t,g), forc_irrig(t,g),
@@ -835,7 +836,7 @@ int main(int argc, char ** argv)
 
     
     // auto min_max = std::minmax_element(&h_h2ocan(0,0), end1+1);
-    // std::cout << std::setprecision(16)
+    // std::cout << std::minmax_element(16)
     //           << t+1 << "\t" << std::accumulate(&h_h2ocan(0,0), end1+1, 0.)
     //           << "\t" << *min_max.first
     //           << "\t" << *min_max.second << std::endl;

tests/tests_kokkos_c/Kokkos_OpenMP/CanopyHydrology_kern1_multiple.cpp

@@ -9,6 +9,7 @@
 #include <iostream>
 #include <iomanip>
 #include <numeric>
+#include <fstream>
 #include <algorithm>
 #include <Kokkos_Core.hpp>
 #include "utils.hh"
@@ -145,12 +146,14 @@ int main(int argc, char ** argv)
  
  double* end = &h_h2o_can(n_grid_cells-1, n_pfts-1) ;
 
-  std::cout << "Time\t Total Canopy Water\t Min Water\t Max Water" << std::endl;
+  std::ofstream soln_file;
+  soln_file.open("test_CanopyHydrology_kern1_multiple.soln");
+  soln_file << "Time\t Total Canopy Water\t Min Water\t Max Water" << std::endl;
   auto min_max = std::minmax_element(&h_h2o_can(0,0), end+1);//h2o_can1.begin(), h2o_can1.end());
-  std::cout << std::setprecision(16)
-            << 0 << "\t" << std::accumulate(&h_h2o_can(0,0), end+1, 0.) //h2o_can1.begin(), h2o_can1.end(), 0.)
-            << "\t" << *min_max.first
-            << "\t" << *min_max.second << std::endl;
+  soln_file << std::setprecision(16)
+          << 0 << "\t" << std::accumulate(&h_h2o_can(0,0), end+1, 0.) //h2o_can1.begin(), h2o_can1.end(), 0.)
+          << "\t" << *min_max.first
+          << "\t" << *min_max.second << std::endl;
 
   
 
@@ -192,12 +195,12 @@ int main(int argc, char ** argv)
     });
 
     auto min_max = std::minmax_element(&h_h2o_can(0,0), end+1);//h2o_can1.begin(), h2o_can1.end());
-    std::cout << std::setprecision(16)
+    soln_file << std::setprecision(16)
               << t+1 << "\t" << std::accumulate(&h_h2o_can(0,0), end+1, 0.)//h2o_can1.begin(), h2o_can1.end(), 0.)
               << "\t" << *min_max.first
               << "\t" << *min_max.second << std::endl;
 
-  }
+  } soln_file.close();
   }
   Kokkos::finalize();
   return 0;

tests/tests_kokkos_c/Kokkos_OpenMP/CanopyHydrology_module.cpp

@@ -9,6 +9,7 @@
 #include <iostream>
 #include <iomanip>
 #include <numeric>
+#include <fstream>
 #include <Kokkos_Core.hpp>
 #include "utils.hh"
 #include "readers.hh"
@@ -245,7 +246,9 @@ int main(int argc, char ** argv)
   double* end1 = &h_h2ocan(n_grid_cells-1, n_pfts-1) ;
   double* end2 = &h_h2osno(n_grid_cells-1) ;
   double* end3 = &h_frac_h2osfc(n_grid_cells-1) ;
-  std::cout << "Time\t Total Canopy Water\t Min Water\t Max Water\t Total Snow\t Min Snow\t Max Snow\t Avg Frac Sfc\t Min Frac Sfc\t Max Frac Sfc" << std::endl;
+  std::ofstream soln_file;
+  soln_file.open("test_CanopyHydrology_module.soln");
+  soln_file << "Time\t Total Canopy Water\t Min Water\t Max Water\t Total Snow\t Min Snow\t Max Snow\t Avg Frac Sfc\t Min Frac Sfc\t Max Frac Sfc" << std::endl;
   auto min_max_water = std::minmax_element(&h_h2ocan(0,0), end1+1);
   auto sum_water = std::accumulate(&h_h2ocan(0,0), end1+1, 0.);
 
@@ -255,7 +258,7 @@ int main(int argc, char ** argv)
   auto min_max_frac_sfc = std::minmax_element(&h_frac_h2osfc(0), end3+1);
   auto avg_frac_sfc = std::accumulate(&h_frac_h2osfc(0), end3+1, 0.) / (end3+1 - &h_frac_h2osfc(0));
 
-  std::cout << std::setprecision(16)
+  soln_file << std::setprecision(16)
             << 0 << "\t" << sum_water << "\t" << *min_max_water.first << "\t" << *min_max_water.second
             << "\t" << sum_snow << "\t" << *min_max_snow.first << "\t" << *min_max_snow.second
             << "\t" << avg_frac_sfc << "\t" << *min_max_frac_sfc.first << "\t" << *min_max_frac_sfc.second << std::endl;
@@ -346,12 +349,13 @@ int main(int argc, char ** argv)
     auto min_max_frac_sfc = std::minmax_element(&h_frac_h2osfc(0), end3+1);
     auto avg_frac_sfc = std::accumulate(&h_frac_h2osfc(0), end3+1, 0.) / (end3+1 - &h_frac_h2osfc(0));
                   
-    std::cout << std::setprecision(16)
+    soln_file << std::setprecision(16)
               << 0 << "\t" << sum_water << "\t" << *min_max_water.first << "\t" << *min_max_water.second
               << "\t" << sum_snow << "\t" << *min_max_snow.first << "\t" << *min_max_snow.second
               << "\t" << avg_frac_sfc << "\t" << *min_max_frac_sfc.first << "\t" << *min_max_frac_sfc.second << std::endl;
 
   } // end timestep loop
+  soln_file.close();
   }
   Kokkos::finalize();
   return 0;

tests/tests_kokkos_c/Kokkos_OpenMP/Makefile

-KOKKOS_PATH = ${HOME}/Downloads/kokkos
 KOKKOS_DEVICES = "Serial"
-EXE_NAME = "CanopyHydrology_module"
-OBJECT        = ../../../src/
+EXE_NAME1 = "test_CanopyHydrology_kern1_multiple"
+EXE_NAME1 = "test_CanopyHydrology_module"
+OBJECT  = ../../../src/
+KERNEL_LANG	  = cc_serial
+SRCDIR	      =	$(OBJECT)$(KERNEL_LANG)
 include $(OBJECT)config/Makefile.config
 
-SRC = CanopyHydrology_module.cpp
+SRC1 = CanopyHydrology_kern1_multiple.cpp
+SRC2 = CanopyHydrology_module.cpp
 
-default: build
+default: build1 build2
 	echo "Start Build"
 
 
 ifneq (,$(findstring Cuda,$(KOKKOS_DEVICES)))
 CXX = ${KOKKOS_PATH}/bin/nvcc_wrapper
-EXE = ${EXE_NAME}.cuda
-KOKKOS_ARCH = "HSW,Pascal60"
-KOKKOS_CUDA_OPTIONS = "enable_lambda,force_uvm"
+#EXE = ${EXE_NAME}.cuda
+KOKKOS_ARCH = "BSW,Pascal60"
+KOKKOS_CUDA_OPTIONS = "enable_lambda"
 else
 CXX = g++
-EXE = ${EXE_NAME}.host
-KOKKOS_ARCH = "HSW"
+#EXE = ${EXE_NAME}.host
+KOKKOS_ARCH = "BSW"
 endif
 
 CXXFLAGS = -g -O0
 LINK = ${CXX}
-LINKFLAGS = -lnetcdf  #-I../test_c_test
+LINKFLAGS = -lnetcdf
 EXTRA_PATH = -I/usr/local/include
 
 DEPFLAGS = -M
 
-OBJ =  $(SRC:.cpp=.o)
-LIB =
+OBJ1 =  $(SRC1:.cpp=.o)
+OBJ2 =  $(SRC2:.cpp=.o)
+LIB = -I$(NETCDF_ROOT)/include  -I$(SRCDIR)
 
 include $(KOKKOS_PATH)/Makefile.kokkos
 
-build: $(EXE)
+.PHONY: links library test
 
-$(EXE): $(OBJ) $(KOKKOS_LINK_DEPENDS)
-	$(LINK) $(KOKKOS_LDFLAGS) ../../../src/cpp/CanopyHydrology_Interception.cpp.o ../../../src/cpp/CanopyHydrology_FracH2OSfc.cpp.o  ../../../src/cpp/CanopyHydrology_FracWet.cpp.o $(OBJ) $(KOKKOS_LIBS) $(LIB) -o $(EXE) $(LINKFLAGS) $(EXTRA_PATH)
+default: all
+
+all: links library $(EXE_NAME1) $(EXE_NAME2)
+
+build1: $(SRC1)
+	python ../../compare_to_gold.py $(EXE_NAME1)
+
+CanopyHydrology_kern1_multiple: test_CanopyHydrology_kern1_multiple
+	./test_CanopyHydrology_kern1_multiple.host > test_CanopyHydrology_kern1_multiple.stdout
+
+build2: $(SRC2)
+	python ../../compare_to_gold.py $(EXE_NAME2)
+
+CanopyHydrology_module: test_CanopyHydrology_module
+	./test_CanopyHydrology_module.host > test_CanopyHydrology_module.stdout
+
+
+sandbox: test_sandbox_domain_template_magic
+	./test_sandbox_domain_template_magic
+
+$(EXE_NAME1): $(OBJ1) $(KOKKOS_LINK_DEPENDS)
+	$(LINK) $(KOKKOS_LDFLAGS) $(CC_LD_FLAGS) $(OBJ1) $(KOKKOS_LIBS) $(LIB) -o $(EXE_NAME1) $(LINKFLAGS) $(EXTRA_PATH)
+$(EXE_NAME2): $(OBJ2) $(KOKKOS_LINK_DEPENDS)
+	$(LINK) $(KOKKOS_LDFLAGS) $(CC_LD_FLAGS) $(OBJ2) $(KOKKOS_LIBS) $(LIB) -o $(EXE_NAME2) $(LINKFLAGS) $(EXTRA_PATH)
 
 clean: kokkos-clean
-	rm -f *.o *.cuda *.host
+	rm -f *.o *.cuda *.host test_*
+
+links:
+	@echo "making in links"
+	$(MAKE) -C ../../links links
+
+library:
+	$(MAKE) -C $(OBJECT) cc_serial
 
 # Compilation rules
 
 %.o:%.cpp $(KOKKOS_CPP_DEPENDS)
-	$(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -I../../../src/cpp $(EXTRA_PATH) $(EXTRA_INC) -c $<
+	$(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -I$(SRCDIR) $(EXTRA_PATH) $(EXTRA_INC) -c $<
 
-test: $(EXE)
-	./$(EXE)
\ No newline at end of file
+test: $(EXE_NAME1)
+	./$(EXE_NAME1)

tests/tests_legion_c/CanopyHydrology_module.cc

@@ -163,8 +163,8 @@ void top_level_task(const Task *task,
     // SumMinMaxReduction() both an actual reduction and dimension
     // independent?
     futures.push_back(SumMinMaxReduction().launch(ctx, runtime, flux, "h2ocan"));
-    futures.push_back(SumMinMaxReduction1().launch(ctx, runtime, surface, "h2osno"));
-    futures.push_back(SumMinMaxReduction1().launch(ctx, runtime, surface, "frac_h2osfc"));
+    futures.push_back(SumMinMaxReduction1D().launch(ctx, runtime, surface, "h2osno"));
+    futures.push_back(SumMinMaxReduction1D().launch(ctx, runtime, surface, "frac_h2osfc"));
   }
 
   int i = 0;
@@ -213,7 +213,7 @@ int main(int argc, char **argv)
   InitForcing::preregister();
   InitPhenology::preregister();
   SumMinMaxReduction::preregister();
-  SumMinMaxReduction1::preregister();
+  SumMinMaxReduction1D::preregister();
   
   
 

tests/tests_legion_c/Makefile

@@ -4,7 +4,7 @@ endif
 
 # from Makefile.config
 OBJECT        = ../../src/
-KERNEL_LANG	  = cc
+KERNEL_LANG	  = cc_kokkos
 SRCDIR	      =	$(OBJECT)$(KERNEL_LANG)
 # Flags for directing the runtime makefile what to include
 DEBUG           ?= 1		# Include debugging symbols
@@ -20,7 +20,7 @@ ALT_MAPPERS     ?= 0		# Include alternative mappers (not recommended)
 # You can modify these variables, some will be appended to by the runtime makefile
 GEN_SRC		?= tasks.cc
 
-INC_FLAGS	?= -I$(AMANZI_TPLS_DIR)/include -I../../src/cc -I../tests_c_legion
+INC_FLAGS	?= -I$(NETCDF_ROOT)/include -I$(SRCDIR) -I../tests_c_legion
 NVCC_FLAGS	?=
 GASNET_FLAGS	?=
 
@@ -59,7 +59,7 @@ test: $(EXEC_TESTS)
 
 
 CanopyHydrology_kern1_multiple: test_CanopyHydrology_kern1_multiple
-	./test_CanopyHydrology_kern1_multiple &> test_CanopyHydrology_kern1_multiple.stdout
+	./test_CanopyHydrology_kern1_multiple > test_CanopyHydrology_kern1_multiple.stdout
 
 CanopyHydrology_module: test_CanopyHydrology_module
 	./test_CanopyHydrology_module &> test_CanopyHydrology_module.stdout
@@ -110,6 +110,7 @@ $(GPU_RUNTIME_OBJS): %.cu.o : %.cu
 
 clean :
 	$(ELM_CLEAN)
+	rm -f test_*
 
 allclean : clean
 	$(MAKE) -C $(OBJECT) allclean

tests/tests_legion_c/tasks.cc

@@ -71,24 +71,24 @@ std::string SumMinMaxReduction::name = "sum_min_max_reduction";
 
 
 //
-// SumMinMaxReduction task1
+// SumMinMaxReduction1D task
 //
 // =============================================================================
 
 Future
-SumMinMaxReduction1::launch(Context ctx, Runtime *runtime,
+SumMinMaxReduction1D::launch(Context ctx, Runtime *runtime,
                            Data<1>& domain, const std::string& fname)
 {
-  TaskLauncher accumlate_launcher(taskid, TaskArgument());
-  accumlate_launcher.add_region_requirement(