Commit 89da37ec authored by Berrill, Mark's avatar Berrill, Mark
Browse files

Syncing with main RayTrace code

parent 33270b99

build/configure

+7 −6
Original line number Diff line number Diff line 
# Sample configure file



export KOKKOS_DIR=/packages/TPLs/install/opt/kokkos

export KOKKOS_DIR=/packages/TPLs/install/debug/kokkos



rm -rf CMake*
@@ -9,13 +9,14 @@ cmake \ 
   -D CMAKE_CXX_COMPILER=mpic++         \

   -D CMAKE_CXX_STANDARD=11             \

   -D USE_OPENACC=0                     \

   -D USE_OPENMP=0                      \

   -D USE_OPENMP=1                      \

   -D USE_KOKKOS=0                      \

      -D KOKKOS_DIRECTORY=${KOKKOS_DIR} \

      -D KOKKOS_WRAPPER=${KOKKOS_DIR}/nvcc_wrapper \

   -D USE_CUDA=0                        \

      -D CUDA_FLAGS="-arch sm_30"       \

   -D USE_HIP=1                         \

   -D USE_CUDA=1                        \

      -D CMAKE_CUDA_FLAGS="--expt-extended-lambda -I/usr/lib/x86_64-linux-gnu/openmpi/include" \

      -D CMAKE_CUDA_ARCHITECTURES=30    \

   -D USE_HIP=0                         \

      -D HIP_NVCC_OPTIONS="-arch sm_30" \

   ../src

src/CMakeLists.txt

+15 −2
Original line number Diff line number Diff line
@@ -63,6 +63,11 @@ IF ( USE_MPI ) 
ENDIF()





# Enable threads

FIND_PACKAGE( Threads )

SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${CMAKE_THREAD_LIBS_INIT} )





# Enable OpenMP

IF ( USE_OPENMP )

    ADD_DEFINITIONS( -DUSE_OPENMP )
@@ -187,6 +192,7 @@ ENDFOREACH() 
# Create the library

INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )

ADD_DEFINITIONS( -DDISABLE_WRITE_FAILED_RAYS )

SET( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${RAYTRACE_INSTALL_DIR}/lib )

SET( SOURCES RayTrace RayTraceImage.cpp RayTraceStructures.cpp utilities/RayUtilities.cpp AtomicModel/interp.cpp RayTraceImageCPU.cpp CreateImageHelpers.cpp MPI_helpers.cpp )

IF ( USE_OPENACC )

    SET( SOURCES ${SOURCES} RayTraceImageOpenACC.cpp )
@@ -213,7 +219,14 @@ ENDIF() 


# Add the applications

ADD_EXECUTABLE( CreateImage CreateImage.cpp )

TARGET_LINK_LIBRARIES( CreateImage RayTrace RayTraceHIP  ${KOKKOS_LIB} ${TIMER_LIBS} ${LDFLAGS} ${LDLIBS} )

INSTALL( TARGETS CreateImage DESTINATION ${RAYTRACE_INSTALL_DIR}/bin )

#TARGET_LINK_LIBRARIES( CreateImage RayTrace RayTraceHIP  ${KOKKOS_LIB} ${TIMER_LIBS} ${LDFLAGS} ${LDLIBS} )

TARGET_LINK_LIBRARIES( CreateImage RayTrace ${KOKKOS_LIB} ${TIMER_LIBS} ${LDFLAGS} ${LDLIBS} )





# Install

INSTALL( TARGETS CreateImage DESTINATION ${RAYTRACE_INSTALL_DIR}/bin )

INSTALL( TARGETS RayTrace DESTINATION ${RAYTRACE_INSTALL_DIR}/lib )

IF ( USE_HIP )

    INSTALL( TARGETS RayTraceHIP DESTINATION ${RAYTRACE_INSTALL_DIR}/lib )

ENDIF()

src/CreateImageHelpers.cpp

+10 −38
Original line number Diff line number Diff line
@@ -162,42 +162,14 @@ bool check_ans( 
template<class TYPE>

void scale_beam( TYPE &beam, double scale )

{

    const double x[2] = { beam.x[0] - 0.5 * beam.dx, beam.x[beam.nx - 1] + 0.5 * beam.dx };

    const double y[2] = { beam.y[0] - 0.5 * beam.dy, beam.y[beam.ny - 1] + 0.5 * beam.dy };

    const double a[2] = { beam.a[0] - 0.5 * beam.da, beam.a[beam.na - 1] + 0.5 * beam.da };

    const double b[2] = { beam.b[0] - 0.5 * beam.db, beam.b[beam.nb - 1] + 0.5 * beam.db };

    int nx            = static_cast<int>( beam.nx * scale );

    int ny            = static_cast<int>( beam.ny * scale );

    int na            = static_cast<int>( beam.na * scale );

    int nb            = static_cast<int>( beam.nb * scale );

    delete[] beam.x;

    beam.x = new double[nx];

    delete[] beam.y;

    beam.y = new double[ny];

    delete[] beam.a;

    beam.a = new double[na];

    delete[] beam.b;

    beam.b  = new double[nb];

    beam.nx = nx;

    beam.dx = ( x[1] - x[0] ) / nx;

    beam.ny = ny;

    beam.dy = ( y[1] - y[0] ) / ny;

    beam.na = na;

    beam.da = ( a[1] - a[0] ) / na;

    beam.nb = nb;

    beam.db = ( b[1] - b[0] ) / nb;

    for ( int i = 0; i < nx; i++ ) {

        beam.x[i] = x[0] + ( 0.5 + i ) * beam.dx;

    }

    for ( int i = 0; i < ny; i++ ) {

        beam.y[i] = y[0] + ( 0.5 + i ) * beam.dy;

    }

    for ( int i = 0; i < na; i++ ) {

        beam.a[i] = a[0] + ( 0.5 + i ) * beam.da;

    }

    for ( int i = 0; i < nb; i++ ) {

        beam.b[i] = b[0] + ( 0.5 + i ) * beam.db;

    }

    beam.nx = static_cast<int>( beam.nx * scale );

    beam.ny = static_cast<int>( beam.ny * scale );

    beam.na = static_cast<int>( beam.na * scale );

    beam.nb = static_cast<int>( beam.nb * scale );

    beam.dx = ( beam.x_range[1] - beam.x_range[0] ) / beam.nx;

    beam.dy = ( beam.y_range[1] - beam.y_range[0] ) / beam.ny;

    beam.da = ( beam.a_range[1] - beam.a_range[0] ) / beam.na;

    beam.db = ( beam.b_range[1] - beam.b_range[0] ) / beam.nb;

}

void scale_problem( RayTrace::create_image_struct &info, double scale )

{
@@ -372,8 +344,8 @@ Options::Options( int argc, char *argv[] ) : iterations( 1 ), scale( 1.0 ) 
        "  CreateImage <args> file.dat\n"

        "Optional arguments:\n"

        "  -methods=METHODS  Comma seperated list of methods to test\n"

        "                    cpu, threads, OpenMP, Cuda, Cuda-MultiGPU, HIP, OpenAcc, Kokkos-Serial, "

        "Kokkos-Thread, Kokkos-OpenMP, Kokkos-Cuda\n"

        "                    cpu, threads, OpenMP, Cuda, Cuda-MultiGPU, HIP, OpenAcc,"

        "                    Kokkos-Serial, Kokkos-Thread, Kokkos-OpenMP, Kokkos-Cuda\n"

        "                    all - run all availible tests (default)\n"

        "                    parallel - run all availible parallel tests\n"

        "  -iterations=N     Number of iterations to run.\n"

src/RayTraceImage.cpp

+27 −38
Original line number Diff line number Diff line
@@ -78,6 +78,17 @@ extern void RayTraceImageHIPLoop( int N, const RayTrace::EUV_beam_struct &euv_be 
#endif





// Function to create a uniform grid

static inline std::vector<double> createGrid( size_t N, double dx, const double range[2] )

{

    RAY_ASSERT( range[1] - range[0] - N * dx < 0.1 * dx );

    std::vector<double> x( N );

    for ( size_t i = 0; i < N; i++ )

        x[i] = range[0] + ( i + 0.5 ) * dx;

    return x;

}





/**********************************************************************

 * Call RayTraceImage function from a thread loop                      *

 **********************************************************************/
@@ -208,30 +219,10 @@ void RayTrace::create_image( create_image_struct *info, std::string compute_meth 
    const int nb = info->euv_beam->nb;

    const int nv = info->euv_beam->nv;



    // Check the euv_beam grid

    bool grid_error = false;

    grid_error =

        grid_error || check_grid( info->euv_beam->nx, info->euv_beam->dx, info->euv_beam->x );

    grid_error =

        grid_error || check_grid( info->euv_beam->ny, info->euv_beam->dy, info->euv_beam->y );

    grid_error =

        grid_error || check_grid( info->euv_beam->na, info->euv_beam->da, info->euv_beam->a );

    grid_error =

        grid_error || check_grid( info->euv_beam->nb, info->euv_beam->db, info->euv_beam->b );

    if ( grid_error )

        RAY_ERROR( "Only uniform grid spacings are currently supported (euv_beam)" );



    // Check the seed_beam grid

    if ( info->seed_beam != nullptr ) {

        const seed_beam_struct *seed_beam = info->seed_beam;

        grid_error                        = false;

        grid_error = grid_error || check_grid( seed_beam->nx, seed_beam->dx, seed_beam->x );

        grid_error = grid_error || check_grid( seed_beam->ny, seed_beam->dy, seed_beam->y );

        grid_error = grid_error || check_grid( seed_beam->na, seed_beam->da, seed_beam->a );

        grid_error = grid_error || check_grid( seed_beam->nb, seed_beam->db, seed_beam->b );

        if ( grid_error )

            RAY_ERROR( "Only uniform grid spacings are currently supported (seed_beam)" );

        if ( ( info->euv_beam->y[0] >= 0.0 ) != ( seed_beam->y[0] >= 0.0 ) )

        if ( ( info->euv_beam->y_range[0] >= 0.0 ) != ( seed_beam->y_range[0] >= 0.0 ) )

            RAY_ERROR( "Negitive y positions in seed_beam or euv_beam, but not both" );

    }
@@ -250,14 +241,10 @@ void RayTrace::create_image( create_image_struct *info, std::string compute_meth 
    std::vector<ray_struct> rays;

    double scale = 0;

    int method   = 0;

    int N2[4]    = { nx, ny, na, nb };

    std::string timer_name;

    std::vector<double> x, y, a, b;

    if ( info->seed != nullptr ) {

        method         = 2;

        N2[0]          = info->seed_beam->nx;

        N2[1]          = info->seed_beam->ny;

        N2[2]          = info->seed_beam->na;

        N2[3]          = info->seed_beam->nb;

        double seed_dx = info->seed_beam->dx;

        double seed_dy = info->seed_beam->dy;

        double seed_da = info->seed_beam->da;
@@ -265,14 +252,23 @@ void RayTrace::create_image( create_image_struct *info, std::string compute_meth 
        scale =

            ( seed_dx * seed_dy * seed_da * seed_db ) / ( info->euv_beam->dx * info->euv_beam->dy );

        timer_name = "propagate_seed";

        x = createGrid( info->seed_beam->nx, info->seed_beam->dx, info->seed_beam->x_range );

        y = createGrid( info->seed_beam->ny, info->seed_beam->dy, info->seed_beam->y_range );

        a = createGrid( info->seed_beam->na, info->seed_beam->da, info->seed_beam->a_range );

        b = createGrid( info->seed_beam->nb, info->seed_beam->db, info->seed_beam->b_range );

    } else {

        method     = 1;

        scale      = 1.0;

        timer_name = "propagate_ASE";

        x          = createGrid( info->euv_beam->nx, info->euv_beam->dx, info->euv_beam->x_range );

        y          = createGrid( info->euv_beam->ny, info->euv_beam->dy, info->euv_beam->y_range );

        a          = createGrid( info->euv_beam->na, info->euv_beam->da, info->euv_beam->a_range );

        b          = createGrid( info->euv_beam->nb, info->euv_beam->db, info->euv_beam->b_range );

    }

    size_t N2[4]     = { x.size(), y.size(), a.size(), b.size() };

    int Nt           = N2[0] * N2[1] * N2[2] * N2[3];

    const int skip   = info->N_parallel;

    const int offset = info->N_start;

    int Nt           = N2[0] * N2[1] * N2[2] * N2[3];

    rays.reserve( ( Nt / skip ) + 1 );

    for ( int it = 0; it < ( Nt / skip ) + 1; ++it ) {

        int ijkm = offset + it * skip;
@@ -286,17 +282,10 @@ void RayTrace::create_image( create_image_struct *info, std::string compute_meth 
        int i = ijkm / ( N2[1] * N2[2] * N2[3] );     // x

        // Create ray information

        ray_struct ray;

        if ( info->seed_beam == nullptr ) {

            ray.x = (float) info->euv_beam->x[i];

            ray.y = (float) info->euv_beam->y[j];

            ray.a = (float) info->euv_beam->a[k];

            ray.b = (float) info->euv_beam->b[m];

        } else {

            ray.x = (float) info->seed_beam->x[i];

            ray.y = (float) info->seed_beam->y[j];

            ray.a = (float) info->seed_beam->a[k];

            ray.b = (float) info->seed_beam->b[m];

        }

        ray.x = (float) x[i];

        ray.y = (float) y[j];

        ray.a = (float) a[k];

        ray.b = (float) b[m];

        rays.push_back( ray );

    }

src/RayTraceImageCPU.cpp

+14 −13
Original line number Diff line number Diff line
@@ -10,11 +10,12 @@ 




// Get the index

inline int getIndex( int n, const double *x, double dx, double y )

inline int getIndex( int n, const double x_range[2], double dx, double y )

{

    if ( y < x[0] - 0.5 * dx || y > x[n - 1] + 0.5 * dx )

    if ( y < x_range[0] || y > x_range[1] )

        return -1;

    return findfirstsingle( x, n, y - 0.5 * dx );

    double i = ( y - x_range[0] ) / dx;

    return std::min<int>( i, n - 1 );

}
@@ -44,16 +45,16 @@ void RayTraceImageCPULoop( int N, const RayTrace::EUV_beam_struct &beam, 
            // Note: The sign of the angle is reversed with respect to the euv_beam

            ray2.a = -ray2.a;

            ray2.b = -ray2.b;

            if ( ray2.y < 0.0 && beam.y[0] >= 0.0 ) {

            if ( ray2.y < 0.0 && beam.y_range[0] >= 0.0 ) {

                // We need to change the sign of y

                ray2.y = -ray2.y;

            }

        }

        // Get the indicies to the cells in image and I_ang

        int i1 = getIndex( beam.nx, beam.x, beam.dx, ray2.x );

        int i2 = getIndex( beam.ny, beam.y, beam.dy, ray2.y );

        int i3 = getIndex( beam.na, beam.a, beam.da, ray2.a );

        int i4 = getIndex( beam.nb, beam.b, beam.db, ray2.b );

        int i1 = getIndex( beam.nx, beam.x_range, beam.dx, ray2.x );

        int i2 = getIndex( beam.ny, beam.y_range, beam.dy, ray2.y );

        int i3 = getIndex( beam.na, beam.a_range, beam.da, ray2.a );

        int i4 = getIndex( beam.nb, beam.b_range, beam.db, ray2.b );

        // Copy I_out into image

        if ( i1 >= 0 && i2 >= 0 ) {

            double *Iv2 = &image[beam.nv * ( i1 + i2 * beam.nx )];
@@ -101,16 +102,16 @@ void RayTraceImageOpenMPLoop( int N, const RayTrace::EUV_beam_struct &beam, 
            // Note: The sign of the angle is reversed with respect to the euv_beam

            ray2.a = -ray2.a;

            ray2.b = -ray2.b;

            if ( ray2.y < 0.0 && beam.y[0] >= 0.0 ) {

            if ( ray2.y < 0.0 && beam.y_range[0] >= 0.0 ) {

                // We need to change the sign of y

                ray2.y = -ray2.y;

            }

        }

        // Get the indicies to the cells in image and I_ang

        int i1 = getIndex( beam.nx, beam.x, beam.dx, ray2.x );

        int i2 = getIndex( beam.ny, beam.y, beam.dy, ray2.y );

        int i3 = getIndex( beam.na, beam.a, beam.da, ray2.a );

        int i4 = getIndex( beam.nb, beam.b, beam.db, ray2.b );

        int i1 = getIndex( beam.nx, beam.x_range, beam.dx, ray2.x );

        int i2 = getIndex( beam.ny, beam.y_range, beam.dy, ray2.y );

        int i3 = getIndex( beam.na, beam.a_range, beam.da, ray2.a );

        int i4 = getIndex( beam.nb, beam.b_range, beam.db, ray2.b );

        // Copy I_out into image

        if ( i1 >= 0 && i2 >= 0 ) {

            double *Iv2 = &image[beam.nv * ( i1 + i2 * beam.nx )];