Commit 3a8c9cd8 authored by Patrick McCormick's avatar Patrick McCormick
Browse files

A few last-minute fixes before release: 

   1. -fkokkos mode forces -fno-exceptions to avoid code gen issues
      around parallelism code gen and internal exception mechanisms.
   2. following #1, added a patch to #ifdef out exceptions (try and
      catch blocks) in Kokkos memory spaces.
   3. fixed a bug in the realm ABI that only showed up for some
      examples -- would crash the compiler via an assertion on the
      types of a binary operator.
   4. tweaked some of the kokkos examples to actually behave like a
      good kokkos program should...
parent f39d3bf1

clang/lib/Frontend/CompilerInvocation.cpp

+2 −1
Original line number Diff line number Diff line
@@ -3146,7 +3146,8 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK, 


  // Set the flag to prevent the implementation from emitting device exception

  // handling code for those requiring so.

  if ((Opts.OpenMPIsDevice && T.isNVPTX()) || Opts.OpenCLCPlusPlus) {

  if ((Opts.OpenMPIsDevice && T.isNVPTX()) || Opts.OpenCLCPlusPlus ||

      Opts.Kokkos) {

    Opts.Exceptions = 0;

    Opts.CXXExceptions = 0;

  }

kitsune/CMakeLists.txt

+19 −9
Original line number Diff line number Diff line
@@ -17,12 +17,21 @@ set(KITSUNE_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include) 


if (KITSUNE_ENABLE_KOKKOS_SUPPORT)

  set(KITSUNE_KOKKOS_SRC_DIR  ${LLVM_BINARY_DIR}/kokkos)

  # TODO: Note that we have to patch Kokkos to disable some sections of code 

  # that are using exceptions.  In general, this is a bit messy with the

  # use of the 'fetch content' mechanisms in CMake -- in particular, the 

  # goofy system appears to continually want to run the patch even though

  # it has already been run previously.  This is a pain when developing. 

  # So...  We have hacked the patch command to always return success to 

  # avoid failed application of the patch from stopping the build.  Need to

  # find a safe way to accomplish this... 

  FetchContent_Declare(kokkos

    GIT_REPOSITORY      https://github.com/kokkos/kokkos.git

    GIT_TAG             3.3.01

    SOURCE_DIR        ${KITSUNE_KOKKOS_SRC_DIR})



  set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "")

    SOURCE_DIR          ${KITSUNE_KOKKOS_SRC_DIR}

    UPDATE_DISCONNECTED TRUE 

    CMAKE_ARGS -D Kokkos_ENABLE_SERIAL=ON -D BUILD_TESTING=OFF 

    PATCH_COMMAND patch -p0 --verbose --input=${KITSUNE_SOURCE_DIR}/patches/Kokkos_MemorySpace.patch)

  FetchContent_MakeAvailable(kokkos)

endif()
@@ -37,7 +46,8 @@ if (KITSUNE_ENABLE_RUNTIME_ABIS) 
    FetchContent_Declare(realm

      GIT_REPOSITORY   https://github.com/StanfordLegion/legion.git

      GIT_TAG          master

      SOURCE_DIR       ${KITSUNE_REALM_SOURCE_DIR})

      SOURCE_DIR       ${KITSUNE_REALM_SOURCE_DIR}

      UPDATE_DISCONNECTED TRUE)

    set(Legion_BUILD_REALM_ONLY ON)

    set(Legion_USE_LVVM OFF)

    set(Legion_LINK_LLVM_LIBS OFF)

kitsune/examples/kokkos/CMakeLists.txt

+44 −2
Original line number Diff line number Diff line
@@ -53,14 +53,18 @@ set(_example_srcs 
  complex.cpp

  matmult1.cpp

  normalize.cpp

#  raytrace.cpp

  vecadd.cpp)



set(_example_view_srcs

  raytrace.cpp

  vecadd-views.cpp

  )



# Build the pure-kokkos serial target version of all the examples first.

# NOTE: Do not build kokkos with parallel "back-ends" as part of the

# overall kitsune+tapir configure and build.  For performance comparisons

# and other related work you should use a separate kokkos build.

foreach(src_file IN ITEMS ${_example_srcs})

foreach(src_file IN ITEMS ${_example_srcs} ${_example_view_srcs})

  get_filename_component(exec_name kokkos.${src_file} NAME_WLE)

  message(STATUS "  ${src_file} --> ${exec_name}")

  add_executable(${exec_name} ${src_file})
@@ -120,6 +124,44 @@ foreach(rt IN ITEMS ${_tapir_rt_targets}) 
  endforeach()

endforeach()



# TODO: clean this up and write a function for the repeated case here 

# for views (that can't use -fkokkos-no-init due to the details of

# implementing views in Kokkos). 

foreach(rt IN ITEMS ${_tapir_rt_targets})

  foreach(src_file IN ITEMS ${_example_view_srcs})

    get_filename_component(exec_name ${src_file} NAME_WLE)

    set(exec_name "kokkos.${exec_name}.${rt}")

    message(STATUS "  ${src_file} --> ${exec_name}")

    add_executable(${exec_name} ${src_file})

    target_include_directories(${exec_name}

      BEFORE

      PRIVATE

      ${KITSUNE_INCLUDE_DIR}

      ${KITSUNE_KOKKOS_SRC_DIR}/core/src

      ${KITSUNE_KOKKOS_SRC_DIR}/containers/src)

    target_compile_options(${exec_name}

      PRIVATE

      -fkokkos

      -ftapir=${rt}

      -fno-exceptions

      -fno-cxx-exceptions)

    target_link_options(${exec_name}

      PRIVATE

      -fkokkos

      -ftapir=${rt})

    set(dep_list kokkos kokkoscore)

    set(target_libs kokkoscore)

    add_tapir_dependency(${exec_name} ${rt})

    message(STATUS "${exec_name} target libs + ${rt} libs: ${target_libs}")

    message(STATUS "${exec_name} dependencies: ${dep_list}")

    target_link_libraries(${exec_name} ${target_libs})

    set_target_properties(${exec_name}

      PROPERTIES

      RUNTIME_OUTPUT_DIRECTORY  ${LLVM_BINARY_DIR}/examples/kitsune)

    #

  endforeach()

endforeach()



# Do some clean up...

unset(_example_srcs)

unset(_tapir_rt_targets)

kitsune/examples/kokkos/raytrace.cpp

+12 −14
Original line number Diff line number Diff line
@@ -21,10 +21,10 @@ 
#include "Kokkos_DualView.hpp"

#include "kitsune/timer.h"



#define WIDTH  1920

#define HEIGHT 1080

#define BPP 3

typedef Kokkos::DualView<unsigned char**[3], Kokkos::LayoutRight, Kokkos::DefaultExecutionSpace> DualViewVector;

const size_t WIDTH = 640;

const size_t HEIGHT = 480;



typedef Kokkos::View<unsigned char**[3], Kokkos::LayoutRight, Kokkos::DefaultExecutionSpace> ViewVector;



struct Vec {

 float x,y,z;
@@ -75,7 +75,7 @@ KOKKOS_INLINE_FUNCTION float QueryDatabase(const Vec& position, int &hitType) { 
                9, 0, 9, 16,

                9, 0, 15, 0

        };

 for (int i = 0; i < sizeof(lines); i += 4) {

 for (size_t i = 0; i < sizeof(lines); i += 4) {

   Vec begin = Vec(lines[i], lines[i + 1]) * .5;

   Vec e = Vec(lines[i + 2], lines[i + 3]) * .5 + begin * -1;

   Vec o = f + (begin + e * fminf(-fminf((((begin + f * -1) % e )/(e % e)), 0),1)) * -1;
@@ -173,10 +173,9 @@ int main(int argc, char **argv) { 
  Kokkos::initialize(argc, argv);

  kitsune::timer t;

  {

    DualViewVector img = DualViewVector("img", WIDTH, HEIGHT);

    int samplesCount = 1 << 7;

    ViewVector img = ViewVector("img", WIDTH, HEIGHT);

    int samplesCount = 8;

    if (argc > 1 ) {samplesCount = atoi(argv[1]);}

    //img.modify_device();

    Kokkos::parallel_for(WIDTH*HEIGHT, KOKKOS_LAMBDA(const int i) {

        const int x = i % WIDTH;

        const int y = i / WIDTH;
@@ -199,21 +198,20 @@ int main(int argc, char **argv) { 
        color = color * (1.0f / samplesCount) + 14.0f / 241.0f;

        Vec o = color + 1.0f;

        color = Vec(color.x / o.x, color.y / o.y, color.z / o.z) * 255.0f;

        img.d_view(x,y,0) = color.x;

        img.d_view(x,y,1) = color.y;

        img.d_view(x,y,2) = color.z;

        img(x,y,0) = color.x;

        img(x,y,1) = color.y;

        img(x,y,2) = color.z;

      });

    double loop_secs = t.seconds();

    std::cout << "running time: " << loop_secs << std::endl;



    std::ofstream myfile;

    myfile.open ("example.ppm");

    //img.sync_host();



    myfile << "P6 " << WIDTH << " " << HEIGHT << " 255 ";

    for (int y = HEIGHT; y--;) {

      for (int x = WIDTH; x--;) {

        //int offset = y * w * BPP + x * BPP;

        myfile << (char)img.h_view(x,y,0) << (char)img.h_view(x,y,1) << (char)img.h_view(x,y,2);

        myfile << (char)img(x,y,0) << (char)img(x,y,1) << (char)img(x,y,2);

      }

    }

  }

kitsune/examples/kokkos/run-kokkos-examples.sh.in

deleted100755 → 0
+0 −16
Original line number Diff line number Diff line 
#/bin/bash



for exe in complex matmult1 normalize vecadd; do 

  for ncores in 1 2 4 6 8 10 12 14 16; do  

    echo -n "$exe.kokkos, kokkos, $ncores, " >> $exe.kokkos.log 

    ${CMAKE_INSTALL_PREFIX}/kitsune/examples/kokkos/$exe.kokkos --kokkos-threads=$ncores >> $exe.kokkos.log 

    # FIXME: really need to extra rt targets from build configuration...

    for rt in opencilk qthreads; do 

      echo -n "$exe.$rt,$rt, $ncores, " >> $exe.$rt.log 

      export CILK_NWORKERS=$ncores

      export QTHREAD_NUM_SHEPHERDS=$ncores

      ${KITSUNE_BINARY_DIR}/$exe.$rt >> $exe.$rt.log 

    done   

  done 

done