Commit d95cf8d8 authored by Kim, Youngsung's avatar Kim, Youngsung
Browse files

add ensemble run support

parent 037d223a

.gitmodules

+3 −0
Original line number Diff line number Diff line 
[submodule "external/kokkos"]

	path = external/kokkos

	url = https://github.com/kokkos/kokkos.git

[submodule "external/yaml-cpp"]

	path = external/yaml-cpp

	url = https://github.com/jbeder/yaml-cpp.git

CMakeLists.txt

+8 −3
Original line number Diff line number Diff line
@@ -22,10 +22,15 @@ endif() 
add_subdirectory(${TRITON_SOURCE_DIR}/external)



# build triton

process_target(${TRITON_EXECUTABLE} ./src/main.cpp)

target_include_directories(${TRITON_EXECUTABLE} PUBLIC ${Kokkos_INCLUDE_DIRS_RET})

add_executable(${TRITON_EXECUTABLE} ./src/main.cpp)

target_link_libraries(${TRITON_EXECUTABLE} PUBLIC kokkos)



if (TRITON_ENABLE_GDAL)

if (ENSEMBLE_BUILD)

  target_link_libraries(${TRITON_EXECUTABLE} PUBLIC yaml-cpp)

  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENSEMBLE_BUILD")

endif()



if (ENABLE_GDAL)

  target_include_directories(${TRITON_EXECUTABLE} PUBLIC ${GDAL_INCLUDE_DIR})

  target_link_libraries(${TRITON_EXECUTABLE} PUBLIC ${GDAL_LIBRARY})

  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DTRITON_GDAL")

cmake/util.cmake

+38 −16
Original line number Diff line number Diff line 


macro(process_target tname src)

  add_executable(${tname} ${src})

  target_compile_options(${tname} PUBLIC $<$<COMPILE_LANGUAGE:CXX>:${TRITON_CXX_FLAGS}>)

  target_link_libraries(${tname} PUBLIC ${TRITON_LINK_FLAGS})

  target_link_libraries(${tname} PUBLIC kokkos)

endmacro()



macro(run_bash_command command outvar)

  execute_process(

    COMMAND bash -c "${command}"
@@ -34,6 +26,25 @@ macro(add_build_and_run_scripts) 
  file(APPEND ${_BuildScript} "cmake --build . -j ${N_PHYSICAL_CORES}\n\n")

  execute_process(COMMAND chmod +x ${_BuildScript})



  if (ENSEMBLE_BUILD)

    # create an ensemble run script           

    set(_RunScript ${CMAKE_BINARY_DIR}/triton_ensrun.sh)

    file(WRITE ${_RunScript}  "#!/usr/bin/env bash\n\n")

    file(APPEND ${_RunScript} "source ./${ENVFILE}\n\n")

    file(APPEND ${_RunScript} "ENSCFG_FILE=\${1:-./ensembles_ensify.yaml}\n")

    file(APPEND ${_RunScript} "MPI_CMD=\${2:-${RUN_COMMAND}}\n")

    file(APPEND ${_RunScript} "\${MPI_CMD} ./${TRITON_EXECUTABLE} \${ENSCFG_FILE}\n\n")

    execute_process(COMMAND chmod +x ${_RunScript})



    configure_file(${CMAKE_SOURCE_DIR}/test/ensembles_ensify.yaml

      ${CMAKE_BINARY_DIR}/ensembles_ensify.yaml COPYONLY)

    configure_file(${CMAKE_SOURCE_DIR}/test/paraboloid_ens1.cfg

      ${CMAKE_BINARY_DIR}/paraboloid_ens1.cfg COPYONLY)

    configure_file(${CMAKE_SOURCE_DIR}/test/paraboloid_ens2.cfg

      ${CMAKE_BINARY_DIR}/paraboloid_ens2.cfg COPYONLY)



  else()



    # create a run script           

    set(_RunScript ${CMAKE_BINARY_DIR}/triton_run.sh)

    file(WRITE ${_RunScript}  "#!/usr/bin/env bash\n\n")
@@ -42,6 +53,7 @@ macro(add_build_and_run_scripts) 
    file(APPEND ${_RunScript} "MPI_CMD=\${2:-${RUN_COMMAND}}\n")

    file(APPEND ${_RunScript} "\${MPI_CMD} ./${TRITON_EXECUTABLE} \${CFG_FILE}\n\n")

    execute_process(COMMAND chmod +x ${_RunScript})

  endif()



  execute_process(

    COMMAND ${CMAKE_COMMAND} -E create_symlink
@@ -85,6 +97,16 @@ macro(add_clean_script) 
  file(APPEND ${_CleanScript} "    output_allatoona \\\n")

  file(APPEND ${_CleanScript} "    output_circular_dambreak \\\n")

  file(APPEND ${_CleanScript} "    output_paraboloid \\\n")

  file(APPEND ${_CleanScript} "    triton_ensrun.sh \\\n")

  file(APPEND ${_CleanScript} "    ensembles_ensify.yaml \\\n")

  file(APPEND ${_CleanScript} "    paraboloid_ens1.cfg \\\n")

  file(APPEND ${_CleanScript} "    paraboloid_ens2.cfg \\\n")

  file(APPEND ${_CleanScript} "    output_ens1 \\\n")

  file(APPEND ${_CleanScript} "    output_ens2 \\\n")

  file(APPEND ${_CleanScript} "    ens1.err \\\n")

  file(APPEND ${_CleanScript} "    ens1.out \\\n")

  file(APPEND ${_CleanScript} "    ens2.err \\\n")

  file(APPEND ${_CleanScript} "    ens2.out \\\n")

  file(APPEND ${_CleanScript} "    external \\\n")

  file(APPEND ${_CleanScript} "    input \\\n")

  file(APPEND ${_CleanScript} "    output")

doc/ensemble_run.rst

0 → 100644
+46 −0
Original line number Diff line number Diff line 
.. _ensemble_run:



Running TRITON in Ensemble Mode

==================================



The ensemble run feature allows users to execute multiple TRITON runs within a single job submission, saving considerable time and effort. This feature is based on **Ensify** (`https://github.com/mrnorman/Ensify <https://github.com/mrnorman/Ensify>`_). 



STEP 1: Build TRITON for Ensemble Run

---------------------------------------



To build TRITON for an ensemble run, add **-DENSEMBLE_BUILD=ON** to the CMake command line along with the other arguments shown below.



.. code-block:: bash



    cmake -DENSEMBLE_BUILD=ON <other cmake arguments>

    ./triton_build.sh 



STEP 2: Configure Ensemble Run

---------------------------------------



The TRITON ensemble configuration file is a YAML file used to define an ensemble run, using the syntax explained below.



.. code-block:: yaml



    # rank_beg , rank_end , arguments_list , stdout_filename , stderr_filename

    ensembles:  [

                    [ 0, 7, ["./ensemble1.cfg", ...] , "ens1.out" , "ens1.err" ],

                    [ 8, 16, ["./ensemble2.cfg", ...] , "ens2.out" , "ens2.err" ],

                    ...

                ]



Ensembles are defined as a list of ensemble entries. Each ensemble entry includes the following fields in order: rank_beg, rank_end, arguments_list, stdout_filename, and stderr_filename. In the example above, the first two entries specify the first entry for MPI ranks 0–7 and the second entry for MPI ranks 8–16, respectively.



See `Ensify ensemble input format <https://github.com/mrnorman/Ensify#enssify-ensemble-input-format>`_ for details.



STEP 3: Run TRITON in Ensemble Mode

---------------------------------------



Running TRITON in ensemble mode is the same as a regular run, except that the ensemble configuration file is the only argument to the triton.exe executable. The ensemble configuration file is the one defined in the previous step.



.. code-block:: bash



    source ./triton_env.sh

    <MPI command and arguments> ./triton.exe <ensemble configuration file>



As an example of an ensemble run, **triton_ensrun.sh** is created with a sample configuration for the Paraboloid case.

doc/index.rst

+1 −0
Original line number Diff line number Diff line
@@ -162,6 +162,7 @@ Project Website 
   configuration_variable_index

   triton_run

   docker_run

   ensemble_run

   casestudy



.. toctree::