Commit 1cc01496 authored by Berrill, Mark's avatar Berrill, Mark
Browse files

Adding source code

parents

build/configure

0 → 100755
+16 −0
Original line number Diff line number Diff line 
# Sample configure file



cmake                          	        \

   -D CMAKE_BUILD_TYPE=Release          \

   -D CMAKE_C_COMPILER=gcc              \

   -D CMAKE_CXX_COMPILER=g++            \

   -D CXX_STD=11                        \

   -D USE_OPENACC=0                     \

   -D USE_KOKKOS=0                      \

      -D KOKKOS_DIRECTORY=/usr/local/kokkos \

      -D KOKKOS_WRAPPER=/usr/local/kokkos/config/nvcc_wrapper \

   -D USE_CUDA=1                        \

      -D CUDA_FLAGS="-arch sm_30"       \

   ../src

build/configure_titan_cray

0 → 100755
+22 −0
Original line number Diff line number Diff line 
module unload PrgEnv-pgi PrgEnv-cray PrgEnv-gnu PrgEnv-intel

module unload cray-libsci_acc cray-libsci

module load PrgEnv-cray

module swap cce cce/8.4.0.223

module load craype-accel-nvidia35

module load cudatoolkit

module load cmake



rm -rf CMake*



cmake                          \

   -D CMAKE_BUILD_TYPE=Release \

   -D CMAKE_C_COMPILER=cc      \

   -D CMAKE_CXX_COMPILER=CC    \

   -D CXX_STD=98               \

   -D CFLAGS="-h pragma=acc,msgs" \

   -D CXXFLAGS="-h pragma=acc,msgs" \

   -D USE_OPENACC=1            \

   -D USE_CUDA=0               \

   -D USE_KOKKOS=0             \

   ../src

build/configure_titan_gnu

0 → 100755
+22 −0
Original line number Diff line number Diff line 
module unload PrgEnv-pgi PrgEnv-cray PrgEnv-gnu PrgEnv-intel

module load PrgEnv-gnu 

module load cudatoolkit

module unload cmake

module load cmake3/3.1.0

module unload acml



rm -rf CMake*



cmake                          \

   -D CMAKE_BUILD_TYPE=Release \

   -D CMAKE_C_COMPILER=cc      \

   -D CMAKE_CXX_COMPILER=CC    \

   -D CXX_STD=11               \

   -D USE_OPENACC=0            \

   -D USE_CUDA=1               \

      -D CUDA_FLAGS="-arch sm_35"      \

   -D CUDA_HOST_COMPILER="/usr/bin/gcc" \

   -D USE_KOKKOS=0             \

      -D KOKKOS_DIRECTORY=~/stf006/install/titan/TPLs/opt/kokkos \

   ../src

src/CMakeLists.txt

0 → 100644
+161 −0
Original line number Diff line number Diff line 
CMAKE_MINIMUM_REQUIRED(VERSION 2.6)

IF ( ${CMAKE_MAJOR_VERSION} EQUAL 3 )

    IF ( NOT ( ${CMAKE_MINOR_VERSION} EQUAL 0 ) )

        CMAKE_POLICY(SET CMP0053 OLD)

    ENDIF()

ENDIF()



MESSAGE( "==============================" )

MESSAGE( "Configuring Ray Trace MiniApps" )

MESSAGE( "==============================" )





# Prevent users from building in place

IF ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}" )

    MESSAGE( FATAL_ERROR "Building code in place is a bad idea" )

ENDIF ()





# Set the project name

PROJECT( RAYTRACE-MINIAPPS )





# Set some common paths

SET( RAYTRACE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} )

SET( RAYTRACE_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR} )

IF( RAYTRACE_INSTALL_DIR )

    # Install path is set

ELSE()

    SET( RAYTRACE_INSTALL_DIR ${RAYTRACE_BUILD_DIR} )

ENDIF()





# Include macros

SET( PROJ RAYTRACE )

INCLUDE( "${RAYTRACE_SOURCE_DIR}/macros.cmake" )

IDENTIFY_COMPILER()

ADD_USER_FLAGS()

ADD_CXX_STD()





# Link external projects

INCLUDE( Find_TIMER.cmake )

CONFIGURE_TIMER( 0 "" )





# Enable OpenACC

IF ( USE_OPENACC )

    ADD_DEFINITIONS( -DUSE_OPENACC )

    IF ( USING_PGCC )

        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -acc")

        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -acc")

    ENDIF()

    MESSAGE( "Using OpenACC" )

ENDIF()





# Enable kokkos

IF ( USE_KOKKOS )

    ADD_DEFINITIONS( -DUSE_KOKKOS )

    VERIFY_PATH( ${KOKKOS_DIRECTORY} )

    VERIFY_PATH( ${KOKKOS_DIRECTORY}/include )

    VERIFY_PATH( ${KOKKOS_DIRECTORY}/lib )

    INCLUDE_DIRECTORIES( ${KOKKOS_DIRECTORY}/include )

    FIND_LIBRARY( KOKKOS_LIB  NAMES kokkos  PATHS "${KOKKOS_DIRECTORY}/lib"  NO_DEFAULT_PATH )

    IF ( NOT KOKKOS_LIB )

        MESSAGE(FATAL_ERROR "kokkos library not found in ${KOKKOS_DIRECTORY}/lib")

    ENDIF()

    IF ( EXISTS "${KOKKOS_DIRECTORY}/nvcc_wrapper" )

        SET( KOKKOS_WRAPPER "${KOKKOS_DIRECTORY}/nvcc_wrapper" )

    ENDIF()

    IF ( KOKKOS_WRAPPER )

        SET( CMAKE_CXX_COMPILER "${KOKKOS_WRAPPER}" )

    ELSE()

        MESSAGE("KOKKOS_WRAPPPER not set, set to the kokkos nvcc wrapper if compiling with CUDA")

    ENDIF()

    SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -fopenmp" )

    IF ( EXISTS KOKKOS_WRAPPER )

        SET( CMAKE_CXX_COMPILER "${KOKKOS_WRAPPER}" )

    ENDIF()

    MESSAGE( "Using kokkos" )

ELSE()

    NULL_USE(KOKKOS_DIRECTORY)

    NULL_USE(KOKKOS_WRAPPER)

ENDIF()





# Enable cuda

IF ( USE_CUDA )

    # Include FindCUDA

    INCLUDE( FindCUDA )

    IF ( NOT CUDA_FOUND )

        MESSAGE ( FATAL_ERROR "CUDA not found" )

    ENDIF()

    IF ( CUDA_FLAGS )

        SET( CUDA_NVCC_FLAGS "${CUDA_FLAGS} ${CXX_STD_FLAG}" )

    ENDIF()

    IF(NOT CUDA_NVCC_FLAGS)

        # Set minimum requirements

        SET( CUDA_NVCC_FLAGS "-arch=sm_20 ${CXX_STD_FLAG}" )

    ENDIF()

    IF ( ${CMAKE_BUILD_TYPE} STREQUAL "Debug" )

        SET( CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -lineinfo" )

    ENDIF()

    # SET( CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} ${CXX_STD_FLAG}" )

    IF( NOT CMAKE_BUILD_TYPE )

        MESSAGE(FATAL_ERROR "CMAKE_BUILD_TYPE is not set")

    ELSEIF( ${CMAKE_BUILD_TYPE} STREQUAL "Debug" )

        SET( CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -g -O0" )

    ELSEIF( ${CMAKE_BUILD_TYPE} STREQUAL "Release" )

        SET( CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3" )

    ENDIF()

    SET( CUDA_PROPAGATE_HOST_FLAGS OFF )

    SET( CUDA_FIND_QUIETLY )

    STRING( REPLACE " " ";" CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} )

    INCLUDE_DIRECTORIES ( ${CUDA_INCLUDE_DIRS} )

    ADD_DEFINITIONS( "-DUSE_CUDA" ) 

    MESSAGE( "Using CUDA ${CUDA_VERSION}" ) 

    MESSAGE( "  CUDA_LIBRARIES = ${CUDA_LIBRARIES}" )

    MESSAGE( "  CUDA_INCLUDE   = ${CUDA_INCLUDE_DIRS}" )

    MESSAGE( "  CUDA_NVCC_FLAGS = ${CUDA_NVCC_FLAGS}" )

    MESSAGE( "  CUDA_TOOLKIT_ROOT = ${CUDA_TOOLKIT_ROOT_DIR}" )

    SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${CUDA_LIBRARIES} )

ELSE()

    NULL_USE(CUDA_FLAGS)

ENDIF()





# Get the supported compiler features

INCLUDE( "${RAYTRACE_SOURCE_DIR}/WriteCompilerFeatures.cmake" )

WRITE_COMPILE_FEATURES( "${RAYTRACE_INSTALL_DIR}/include/CompilerFeatures.h" )

INCLUDE_DIRECTORIES( "${RAYTRACE_INSTALL_DIR}/include" )





# Create custom targets for distclean

ADD_DISTCLEAN( libRayTrace.* null_timer CreateImage* )





# Create the library

INCLUDE_DIRECTORIES( ${RAYTRACE_SOURCE_DIR} )

ADD_DEFINITIONS( -DDISABLE_WRITE_FAILED_RAYS )

SET( SOURCES RayTrace RayTrace.cpp RayTraceStructures.cpp utilities/RayUtilities.cpp interp.cpp RayTraceImageCPU.cpp )

IF ( USE_OPENACC )

    SET( SOURCES ${SOURCES} RayTraceImageOpenACC.cpp )

ENDIF()

IF ( USE_KOKKOS )

    #SET( SOURCES ${SOURCES} RayTraceImageKokkos.cpp )

ENDIF()

IF ( USE_CUDA )

    SET( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )

    SET( CMAKE_BUILD_WITH_INSTALL_RPATH TRUE )

    CUDA_COMPILE( CUBINS RayTraceImageCuda.cu )

ENDIF()

ADD_LIBRARY( ${SOURCES} ${CUBINS} )





# Add the applications

ADD_EXECUTABLE( CreateImage CreateImage.cpp )

TARGET_LINK_LIBRARIES( CreateImage RayTrace ${CUDA_LIBRARIES} ${KOKKOS_LIB} ${TIMER_LIBS} )

INSTALL( TARGETS CreateImage DESTINATION ${RAYTRACE_INSTALL_DIR}/bin )

src/CreateImage.cpp

0 → 100644
+184 −0
Original line number Diff line number Diff line 
// This is a mini-application to mimics part of the full Ray-Trace code

// This miniapp mimics the behavior of create_image



#include "RayTrace.h"

#include <iostream>

#include <stdint.h>

#include <math.h>



#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)

    #include <windows.h>

    #define get_time(x) QueryPerformanceCounter(x)

    #define get_frequency(f) QueryPerformanceFrequency(f)

    #define get_diff(start,end,f) \

        static_cast<double>(end.QuadPart-start.QuadPart)/static_cast<double>(f.QuadPart)

    #define TIME_TYPE LARGE_INTEGER

#else

    #include <sys/time.h>

    #define get_time(x) gettimeofday(x,NULL);

    #define get_frequency(f) (*f=timeval())

    #define get_diff(start,end,f) 1e-6*static_cast<double>( \

        0xF4240*(static_cast<int64_t>(end.tv_sec)-static_cast<int64_t>(start.tv_sec)) + \

                (static_cast<int64_t>(end.tv_usec)-static_cast<int64_t>(start.tv_usec)) )

    #define TIME_TYPE timeval

#endif



#ifdef USE_KOKKOS

    #include <Kokkos_Core.hpp>

#endif





inline void fread2( void *ptr, size_t size, size_t count, FILE *fid )

{

    size_t N = fread(ptr,size,count,fid);

    if ( N != count ) {

        std::cerr << "Failed to read desired count\n";

        exit(-1);

    }

}





inline int check_ans( const double *image0, const double *I_ang0,

    const RayTrace::create_image_struct& data )

{

    size_t N_image = data.euv_beam->nx*data.euv_beam->ny*data.euv_beam->nv;

    size_t N_ang   = data.euv_beam->na*data.euv_beam->nb;

    double error[2]={0,0};

    double norm0[2]={0,0};

    double norm1[2]={0,0};

    for (size_t i=0; i<N_image; i++) {

        error[0] += (image0[i]-data.image[i])*(image0[i]-data.image[i]);

        norm0[0] += image0[i]*image0[i];

        norm1[0] += data.image[i]*data.image[i];

    }

    for (size_t i=0; i<N_ang; i++) {

        error[1] += (I_ang0[i]-data.I_ang[i])*(I_ang0[i]-data.I_ang[i]);

        norm0[1] += I_ang0[i]*I_ang0[i];

        norm1[1] += data.I_ang[i]*data.I_ang[i];

    }

    norm0[0] = sqrt(norm0[0]);

    norm0[1] = sqrt(norm0[1]);

    norm1[0] = sqrt(norm1[0]);

    norm1[1] = sqrt(norm1[1]);

    error[0] = sqrt(error[0])/norm0[0];

    error[1] = sqrt(error[1])/norm0[1];

    const double tol = 5e-6;    // RayTrace uses single precision for some calculations (may need to adjust to 1e-5)

    //bool pass = error[0]<=tol && error[1]<=tol;

    bool pass = (norm0[0]-norm1[0])/norm0[0]<=tol && (norm0[1]-norm1[1])/norm0[1]<=tol;

    if ( !pass ) {

        std::cerr << "  Answers do not match:" << std::endl;

        std::cerr << "    image: " << error[0] << " " << norm0[0] << " " << norm1[0] << std::endl;

        std::cerr << "    I_ang: " << error[1] << " " << norm0[1] << " " << norm1[1] << std::endl;

    }

    return pass ? 0:1;

}







/******************************************************************

* The main program                                                *

******************************************************************/

int main(int argc, char *argv[]) 

{



    // Initialize kokkos

    #ifdef USE_KOKKOS

        /*int argc2 = 1;

        const char *argv2[10] = {NULL};

        argv2[0] = argv[0];

        #ifdef KOKKOS_HAVE_PTHREAD

            argv2[argc2] = "--kokkos-threads=4";

            argc2++;

        #endif

        Kokkos::initialize(argc2,(char**)argv2);*/

        Kokkos::initialize(argc,argv);

    #endif



    // Check the input arguments

    if ( argc != 2 ) {

        std::cerr << "CreateImage called with the wrong number of arguments\n";

        return -1;

    }



    // load the input file

    FILE *fid = fopen(argv[1],"rb");

    uint64_t N_bytes = 0;

    fread2(&N_bytes,sizeof(uint64_t),1,fid);

    char *data = new char[N_bytes];

    fread2(data,sizeof(char),N_bytes,fid);

    fclose(fid);

    

    // Create the image structure

    RayTrace::create_image_struct info;

    info.unpack(std::pair<char*,size_t>(data,N_bytes));

    delete [] data;  data = NULL;

    const double *image0 = info.image;

    const double *I_ang0 = info.I_ang;

    info.image = NULL;

    info.I_ang = NULL;



    // Get the list of methods to try

    std::vector<std::string> methods;

    methods.push_back("cpu");

    #if CXX_STD==11 || CXX_STD==14

        methods.push_back("threads");

    #endif

    #ifdef USE_CUDA

        methods.push_back("Cuda");

    #endif

    #ifdef USE_OPENACC

        methods.push_back("OpenAcc");

    #endif

    #ifdef USE_KOKKOS

        methods.push_back("Kokkos-Serial");

        #ifdef KOKKOS_HAVE_PTHREAD

            //methods.push_back("Kokkos-Thread");

        #endif

        #ifdef KOKKOS_HAVE_OPENMP

            methods.push_back("Kokkos-OpenMP");

        #endif

        #ifdef KOKKOS_HAVE_CUDA

            methods.push_back("Kokkos-Cuda");

        #endif

    #endif



    // Call create_image for each method

    int N_errors = 0;

    std::vector<double> time(methods.size());

    for (size_t i=0; i<methods.size(); i++) {

        printf("Running %s\n",methods[i].c_str());

        TIME_TYPE start, stop, f;

        get_frequency(&f);

        get_time(&start);

        RayTrace::create_image(&info,methods[i]);

        get_time(&stop);

        time[i] = get_diff(start,stop,f);

        // Check the results

        N_errors += check_ans( image0, I_ang0, info );

        free((void*)info.image);

        free((void*)info.I_ang);

        info.image = NULL;

        info.I_ang = NULL;

    }

    printf("\n      METHOD     TIME\n");

    for (size_t i=0; i<methods.size(); i++)

        printf("%14s   %0.3f\n",methods[i].c_str(),time[i]);



    // Free memory and return

    free((void*)image0);

    free((void*)I_ang0);

    delete info.euv_beam;

    delete info.seed_beam;

    delete [] info.gain;

    delete info.seed;

    if ( N_errors == 0 )

        std::cout << "All tests passed\n";

    else

        std::cout << "Some tests failed\n";

    #ifdef USE_KOKKOS

        Kokkos::finalize ();

    #endif

    return N_errors;

}