Commit 0eb4c2a3 authored by Pillai, Himanshu's avatar Pillai, Himanshu
Browse files

RAJA working partially

parent a093730a
......@@ -3,16 +3,18 @@ Evaluation
| DRIVER | KERNEL | KERNEL |
|===================|=============|=============|
| PROGRAMMING MODEL | FORTRAN | C++ |
| --------------- | :---------: | :---------: |
|===================|=============|=============|
| FORTRAN | Works | X |
| C++ | Works | Works |
| LEGION FORTRAN | X | X |
| LEGION C++ | Partially | Works |
| KOKKOS Fortran | X | X |
| KOKKOS C++ | X | Works |
| CUDA | X | Debugging |
| CUDA | X | Works |
| OpenMP/OpenACC | N/A | N/A |
| FleCSI | N/A | Writing |
| Tpetra | N/A | N/A |
| HPX | N/A | N/A |
| HPX | N/A | Writing |
| Charm++ | X | Works |
| RAJA | X | Debugging |
......@@ -2,29 +2,26 @@
# Users should change these
# ------------------------------------------------------------------
#ifndef CONFIG_DONE
#define CONFIG_DONE
#
# COMPILERS
# =================================
#CC ?= gcc
CC ?= clang
CC ?= gcc
#CC ?= clang
#CXX ?= g++
CXX ?= clang++
CXX ?= g++
#CXX ?= clang++
CXXFLAGS ?= -g -Wall -Wshadow -std=c++11
STD_LIB_ROOT = /usr
STD_LIB_FLAG = -L$(STD_LIB_ROOT)/lib -lstdc++
#
# gfortran for fortran kernels
# ---------------------------------
FC ?= gfortran
FCFLAGS ?= -fimplicit-none -free -g3 -fbounds-check -fbacktrace
#FC_LIB_ROOT ?= /usr/lib/gcc/x86_64-linux-gnu/8
FC_LIB_ROOT ?= /usr/local/Cellar/gcc/8.2.0/lib/gcc/8
FC_FLAGS ?= -fimplicit-none -free -g3 -fbounds-check -fbacktrace
FC_LIB_ROOT ?= /usr/lib/gcc/x86_64-linux-gnu/8
GFORTRAN_FLAGS = -L$(FC_LIB_ROOT) -lgfortran
#FC_LIB_ROOT ?= /usr/local/Cellar/gcc/8.2.0/lib/gcc/8
#
......@@ -33,8 +30,8 @@ FC_LIB_ROOT ?= /usr/local/Cellar/gcc/8.2.0/lib/gcc/8
#
# requires MPI
# ---------------------------------
#MPI_LIB_ROOT ?= /usr/lib/x86_64-linux-gnu/openmpi
MPI_LIB_ROOT = /usr/local/Cellar/open-mpi/3.1.3
MPI_LIB_ROOT ?= /usr/lib/x86_64-linux-gnu/openmpi
#MPI_LIB_ROOT = /usr/local/Cellar/open-mpi/3.1.3
MPICC ?= mpicc
MPICXX ?= mpic++
MPI_INC_FLAGS = -I$(MPI_LIB_ROOT)/include
......@@ -42,9 +39,9 @@ MPI_INC_FLAGS = -I$(MPI_LIB_ROOT)/include
#
# requires NETCDF
# ---------------------------------
#NETCDF_ROOT ?= /usr/local
NETCDF_ROOT ?= $(AMANZI_TPLS_DIR)
NETCDF_FLAGS = -L$(NETCDF_ROOT)/lib -lnetcdf -lhdf5 -lhdf5_hl -lz
NETCDF_ROOT = /usr/local
#NETCDF_ROOT ?= $(AMANZI_TPLS_DIR)
NETCDF_FLAGS = -L$(NETCDF_ROOT)/lib -lnetcdf #-lhdf5 -lhdf5_hl -lz
NETCDFF_FLAGS = -L$(NETCDF_ROOT)/lib -lnetcdff
NETCDF_INC_FLAGS = -I$(NETCDF_ROOT)/include
......@@ -52,8 +49,9 @@ NETCDF_INC_FLAGS = -I$(NETCDF_ROOT)/include
# requires ELM datasets
# ---------------------------------
# Available from: git clone https://code.ornl.gov/isp/acme_inputdata
#ELM_DATA_LOCATION ?= /home/7hp/Downloads/acme_inputdata
ELM_DATA_LOCATION ?= /Users/uec/codes/elm/kernels/repos/acme_inputdata
ELM_DATA_LOCATION ?= /home/7hp/Downloads/acme_inputdata
#ELM_DATA_LOCATION ?= /Users/uec/codes/elm/kernels/repos/acme_inputdata
ELM_UTILS_DIR = cime_utils
# this library
ELM_FLAGS = -L$(ELM_ROOT) -lelm
......@@ -65,8 +63,8 @@ ELM_INC_FLAGS = -I$(ELM_ROOT)
#
# Kokkos tests
# ---------------------------------
#KOKKOS_PATH ?= ${HOME}/Downloads/kokkos
KOKKOS_PATH ?= ${AMANZI_TPLS_BUILD_DIR}/trilinos/trilinos-12-14-1_master-source/packages/kokkos
KOKKOS_PATH ?= ${HOME}/Downloads/kokkos
#KOKKOS_PATH ?= ${AMANZI_TPLS_BUILD_DIR}/trilinos/trilinos-12-14-1_master-source/packages/kokkos
# assumes you have a working CUDA for CUDA builds
NVCC=nvcc
......@@ -79,5 +77,4 @@ CUDA_INCDIRS=-I/usr/local/cuda-10.0/include
#
# Legion tests
# ---------------------------------
LG_RT_DIR ?= /Users/uec/codes/legion/repos/dev/runtime
#LG_RT_DIR ?= /Users/uec/codes/legion/repos/dev/runtime
\ No newline at end of file
......@@ -3,9 +3,12 @@ FCFLAGS += $(NETCDF_INC_FLAGS) $(ELM_INC_FLAGS)
CXXFLAGS += $(NETCDF_INC_FLAGS) $(ELM_INC_FLAGS)
# linking flags
FC_LD_FLAGS += $(NETCDF_FLAGS)
CXX_LD_FLAGS += $(NETCDF_FLAGS)
CUDA_LD_FLAGS = -L$(CUDA_LIBDIRS) -lstdc++ -lpthread -lcudart -lcublas
CC_LD_FLAGS += -L$(SRCDIR) -lelm -L$(NETCDF_ROOT) -lnetcdf -L$(STD_LIB_ROOT)/lib -lstdc++ -L$(FC_LIB_ROOT) -lgfortran
#CC_LD_FLAGS += -L$(SRCDIR) -lelm -L$(NETCDF_ROOT) -lnetcdf -L$(STD_LIB_ROOT)/lib -lstdc++ -L$(MPI_LIB_ROOT)/lib -L$(FC_LIB_ROOT) -lgfortran
FC_LD_FLAGS += -L$(ELM_ROOT) -lelm -L$(NETCDF_ROOT) -lnetcdff
CXX_LD_FLAGS += -L$(ELM_ROOT) -lelm -L$(NETCDF_ROOT) -lnetcdf
#CXX_LD_FLAGS += -L$(ELM_ROOT) -lelm -L$(NETCDF_ROOT) -lnetcdf -I$(MPI_LIB_ROOT)/include -L$(MPI_LIB_ROOT)/lib -lmpi
CUDA_LD_FLAGS = -L$(SRCDIR) -lelm -L$(NETCDF_ROOT) -lnetcdf -L$(CUDA_LIBDIRS) -lstdc++ -lpthread -lcudart -lcublas
#
# rules
......@@ -15,12 +18,16 @@ CUDA_LD_FLAGS = -L$(CUDA_LIBDIRS) -lstdc++ -lpthread -lcudart -lcublas
.SUFFIXES: .cc.o .F90.o .cpp.o .cu.o .cc .F90 .cpp .cu
%.F90.o: %.F90
$(FC) $(FCFLAGS) -c $< -o $@
$(FC) $(FC_FLAGS) -c $< -o $@
%.cc.o: %.cc
$(CC) $(CXXFLAGS) $(INC_FLAGS) -c $< -o $@
%.cpp.o: %.cpp
$(CXX) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) $(NETCDF_INC_FLAGS) $(ELM_INC_FLAGS) -c $< -o $@ #-I$(MPI_LIB_ROOT)/include
%.cu.o: %.cu
$(NVCC) $(CUDA_FLAGS) -I$(MPI_LIB_ROOT)/include $(INC_FLAGS) -c $< -o $@
SHELL = /bin/sh
LIB_SUFFIX=.a
......@@ -37,6 +44,3 @@ AR = ar
ELM_CLEAN=$(RM) -f *.o *.ii *.int.c *.log *.log.* stamp-* core core.* stats -r ti_files ii_files *.mod *.a
#endif
......@@ -4,6 +4,8 @@ OBJECT = ../
default: all
include $(OBJECT)config/Makefile.config
include $(OBJECT)config/Makefile.rules
FCFLAGS += -I$(ELM_UTILS_DIR)
# Order is important due to .mod file creation
......
SRCDIR = .
OBJECT = .
.PHONY: tests_c_c tests_legion_c tests_kokkos_c tests_c_f tests_f tests_legion_f tests_cuda
.PHONY: tests_c_c tests_legion_c tests_kokkos_c tests_c_f tests_f tests_legion_f tests_cuda tests_charm++_c tests_raja_c
default: test
default: all
test: tests_c_c tests_legion_c tests_kokkos_c tests_c_f tests_f tests_legion_f tests_cuda
all: tests_c_c tests_legion_c tests_kokkos_c tests_c_f tests_f tests_legion_f tests_cuda tests_charm++_c tests_raja_c
tests_c_c:
$(MAKE) -C tests_c_c test
$(MAKE) -C tests_c_c all
tests_cuda:
$(MAKE) -C tests_cuda test
$(MAKE) -C tests_cuda all
tests_legion_c:
$(MAKE) -C tests_legion_c test
$(MAKE) -C tests_legion_c all
tests_kokkos_c:
$(MAKE) -C tests_kokkos_c test
$(MAKE) -C tests_kokkos_c all
tests_c_f:
$(MAKE) -C tests_c_f test
$(MAKE) -C tests_c_f all
tests_f:
$(MAKE) -C tests_f test
$(MAKE) -C tests_f all
tests_legion_f:
$(MAKE) -C tests_legion_f test
$(MAKE) -C tests_legion_f all
tests_charm++_c:
$(MAKE) -C tests_charm++_c all
tests_raja_c:
$(MAKE) -C tests_raja_c all
clean:
$(MAKE) -C tests_c_c clean
......@@ -36,6 +42,8 @@ clean:
$(MAKE) -C tests_c_f clean
$(MAKE) -C tests_f clean
$(MAKE) -C tests_legion_f clean
$(MAKE) -C tests_charm++_c clean
$(MAKE) -C tests_raja_c clean
allclean:
$(MAKE) -C tests_c_c clean
......@@ -45,6 +53,8 @@ allclean:
$(MAKE) -C tests_c_f clean
$(MAKE) -C tests_f clean
$(MAKE) -C tests_legion_f clean
$(MAKE) -C tests_charm++_c clean
$(MAKE) -C tests_raja_c clean
......@@ -156,6 +156,6 @@ int main(int argc, char ** argv)
auto duration = duration_cast<microseconds>(stop - start);
std::cout << "Time taken by function: "<< duration.count() << " microseconds" << std::endl;
return 0;
// MPI_Finalize();
return 0;
}
......@@ -116,6 +116,7 @@ int main(int argc, char ** argv)
auto duration = duration_cast<microseconds>(stop - start);
std::cout << "Time taken by function: "<< duration.count() << " microseconds" << std::endl;
return 0;
// MPI_Finalize();
return 0;
}
......@@ -252,6 +252,6 @@ int main(int argc, char ** argv)
auto duration = duration_cast<microseconds>(stop - start);
std::cout << "Time taken by function: "<< duration.count() << " microseconds" << std::endl;
return 0;
// MPI_Finalize();
return 0;
}
......@@ -2,8 +2,9 @@ ELM_BASE = ../../src
KERNEL_FOLDER = cc
ELM_ROOT = $(ELM_BASE)/$(KERNEL_FOLDER)
CXXFLAGS = -std=c++11 -g
include $(ELM_BASE)/config/Makefile.config
CXX = $(MPICXX)
include $(ELM_BASE)/config/Makefile.rules
TESTS = test_CanopyHydrology_kern1_single \
......@@ -36,7 +37,7 @@ CanopyHydrology_module: test_CanopyHydrology_module
test_%: %.cpp.o readers.hh utils.hh
$(CXX) -o $@ $< $(CXXFLAGS)
$(CXX) -o $@ $< $(NETCDF_FLAGS)
clean:
......
......@@ -3,6 +3,7 @@ CSRCDIR = ../../src/cc_fortran_wrappers/
OBJECT = ../../src/
NATURE =
include $(OBJECT)config/Makefile.config
include $(OBJECT)config/Makefile.rules
INC_FLAGS = -I$(NETCDF_ROOT)/include -I$(CSRCDIR) -I../tests_c
......
......@@ -5,6 +5,7 @@ KERNEL_FOLDER = cc
ELM_ROOT = $(ELM_BASE)/$(KERNEL_FOLDER)
include $(ELM_BASE)/config/Makefile.config
include $(ELM_BASE)/config/Makefile.rules
TESTS = test_CanopyHydrology_kern1_single \
test_CanopyHydrology_kern1_multiple \
......
......@@ -4,6 +4,7 @@ SRCDIR = $(OBJECT)$(KERNEL_LANG)
CUDA_FLAGS += -DNATURE=__global__
NVCC = nvcc
include $(OBJECT)config/Makefile.config
include $(OBJECT)config/Makefile.rules
INC_FLAGS ?= -I$(NETCDF_ROOT)/include -I$(SRCDIR) -I../tests_cuda
......
......@@ -5,6 +5,7 @@ ELM_ROOT = $(ELM_BASE)/$(KERNEL_FOLDER)
FC = gfortran
include $(ELM_BASE)/config/Makefile.config
include $(ELM_BASE)/config/Makefile.rules
FC_FLAGS += -I$(ELM_ROOT) -I$(ELM_ROOT)/$(ELM_UTILS_DIR) -I$(NETCDF_ROOT)/include
......
......@@ -258,6 +258,7 @@ int main(int argc, char ** argv)
std::cout << "Time taken by function: "<< duration.count() << " microseconds" << std::endl;
}
Kokkos::finalize();
return 0;
// MPI_Finalize();
return 0;
}
......@@ -11,7 +11,7 @@
#include <iomanip>
#include <numeric>
#include <fstream>
#include <mpi.h>
//#include <mpi.h>
#include <chrono>
#include <algorithm>
#include <Kokkos_Core.hpp>
......@@ -54,13 +54,13 @@ int main(int argc, char ** argv)
const double dewmx = 0.1;
const double dtime = 1800.0;
int myrank, numprocs;
double mytime;
// int myrank, numprocs;
// double mytime;
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
MPI_Barrier(MPI_COMM_WORLD);
// MPI_Init(&argc,&argv);
// MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
// MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
// MPI_Barrier(MPI_COMM_WORLD);
Kokkos::initialize( argc, argv );
{
......@@ -113,9 +113,9 @@ int main(int argc, char ** argv)
soln_file << "Timestep, forc_rain, h2ocan, qflx_prec_grnd, qflx_prec_intr, total_precip_loop" << std::endl;
auto start = high_resolution_clock::now();
mytime = MPI_Wtime();
// mytime = MPI_Wtime();
for(size_t itime = 0; itime < n_times; itime += 1) { //Kokkos::parallel_for(n_times, KOKKOS_LAMBDA (const int itime) {
for(size_t itime = 0; itime < n_times; itime += 1) {
// note this call puts all precip as rain for testing
total_precip = forc_rain(itime,0) + forc_snow(itime,0);
......@@ -131,14 +131,14 @@ int main(int argc, char ** argv)
std::cout << std::setprecision(16) << itime+1 << "\t" << total_precip << "\t" << h2ocan<< "\t" << qflx_prec_grnd << "\t" << qflx_prec_intr << std::endl;
}soln_file.close();
mytime = MPI_Wtime() - mytime;
// mytime = MPI_Wtime() - mytime;
auto stop = high_resolution_clock::now();
std::cout <<"Timing from node "<< myrank << " is "<< mytime << "seconds." << std::endl;
//std::cout <<"Timing from node "<< myrank << " is "<< mytime << "seconds." << std::endl;
auto duration = duration_cast<microseconds>(stop - start);
std::cout << "Time taken by function: "<< duration.count() << " microseconds" << std::endl;
}
Kokkos::finalize();
// MPI_Finalize();
return 0;
MPI_Finalize();
}
......@@ -429,6 +429,6 @@ int main(int argc, char ** argv)
std::cout << "Time taken by function: "<< duration.count() << " microseconds" << std::endl;
}
Kokkos::finalize();
return 0;
// MPI_Finalize();
return 0;
}
/* ---------------------------------------------
Makefile constructed configuration:
Tue Jul 23 11:03:01 EDT 2019
Thu Aug 1 11:41:20 EDT 2019
----------------------------------------------*/
#if !defined(KOKKOS_MACROS_HPP) || defined(KOKKOS_CORE_CONFIG_H)
#error "Do not include KokkosCore_config.h directly; include Kokkos_Macros.hpp instead."
......
/* ---------------------------------------------
Makefile constructed configuration:
Tue Jul 23 11:29:55 EDT 2019
Thu Aug 1 12:19:53 EDT 2019
----------------------------------------------*/
#if !defined(KOKKOS_MACROS_HPP) || defined(KOKKOS_CORE_CONFIG_H)
#error "Do not include KokkosCore_config.h directly; include Kokkos_Macros.hpp instead."
......
......@@ -28,16 +28,14 @@ KOKKOS_ARCH = "BSW"
endif
include $(ELM_BASE)/config/Makefile.config
include $(ELM_BASE)/config/Makefile.rules
FC = mpifort
CXX = mpicxx
CC = mpicc
# FC = mpifort
# CXX = mpicxx
# CC = mpicc
CXXFLAGS += -DNATURE=KOKKOS_INLINE_FUNCTION
include $(ELM_BASE)/config/Makefile.rules
default: Serial
# include Kokkos
include $(KOKKOS_PATH)/Makefile.kokkos
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment