Merge branch '8yk-ensify' into 'master'

[submodule "external/kokkos"]

	path = external/kokkos

	url = https://github.com/kokkos/kokkos.git

[submodule "external/yaml-cpp"]

	path = external/yaml-cpp

	url = https://github.com/jbeder/yaml-cpp.git

add_subdirectory(${TRITON_SOURCE_DIR}/external)

# build triton

process_target(${TRITON_EXECUTABLE} ./src/main.cpp)

target_include_directories(${TRITON_EXECUTABLE} PUBLIC ${Kokkos_INCLUDE_DIRS_RET})

add_executable(${TRITON_EXECUTABLE} ./src/main.cpp)

target_link_libraries(${TRITON_EXECUTABLE} PUBLIC kokkos)

if (TRITON_ENABLE_GDAL)

if (ENSEMBLE_BUILD)

  target_link_libraries(${TRITON_EXECUTABLE} PUBLIC yaml-cpp)

  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENSEMBLE_BUILD")

endif()

if (ENABLE_GDAL)

  target_include_directories(${TRITON_EXECUTABLE} PUBLIC ${GDAL_INCLUDE_DIR})

  target_link_libraries(${TRITON_EXECUTABLE} PUBLIC ${GDAL_LIBRARY})

  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DTRITON_GDAL")

macro(process_target tname src)

  add_executable(${tname} ${src})

  target_compile_options(${tname} PUBLIC $<$<COMPILE_LANGUAGE:CXX>:${TRITON_CXX_FLAGS}>)

  target_link_libraries(${tname} PUBLIC ${TRITON_LINK_FLAGS})

  target_link_libraries(${tname} PUBLIC kokkos)

endmacro()

macro(run_bash_command command outvar)

  execute_process(

    COMMAND bash -c "${command}"

  file(APPEND ${_BuildScript} "cmake --build . -j ${N_PHYSICAL_CORES}\n\n")

  execute_process(COMMAND chmod +x ${_BuildScript})

  if (ENSEMBLE_BUILD)

    # create an ensemble run script           

    set(_RunScript ${CMAKE_BINARY_DIR}/triton_ensrun.sh)

    file(WRITE ${_RunScript}  "#!/usr/bin/env bash\n\n")

    file(APPEND ${_RunScript} "source ./${ENVFILE}\n\n")

    file(APPEND ${_RunScript} "ENSCFG_FILE=\${1:-./ensembles_ensify.yaml}\n")

    file(APPEND ${_RunScript} "MPI_CMD=\${2:-${RUN_COMMAND}}\n")

    file(APPEND ${_RunScript} "\${MPI_CMD} ./${TRITON_EXECUTABLE} \${ENSCFG_FILE}\n\n")

    execute_process(COMMAND chmod +x ${_RunScript})

    configure_file(${CMAKE_SOURCE_DIR}/test/ensembles_ensify.yaml

      ${CMAKE_BINARY_DIR}/ensembles_ensify.yaml COPYONLY)

    configure_file(${CMAKE_SOURCE_DIR}/test/paraboloid_ens1.cfg

      ${CMAKE_BINARY_DIR}/paraboloid_ens1.cfg COPYONLY)

    configure_file(${CMAKE_SOURCE_DIR}/test/paraboloid_ens2.cfg

      ${CMAKE_BINARY_DIR}/paraboloid_ens2.cfg COPYONLY)

  else()

    # create a run script           

    set(_RunScript ${CMAKE_BINARY_DIR}/triton_run.sh)

    file(WRITE ${_RunScript}  "#!/usr/bin/env bash\n\n")

    file(APPEND ${_RunScript} "MPI_CMD=\${2:-${RUN_COMMAND}}\n")

    file(APPEND ${_RunScript} "\${MPI_CMD} ./${TRITON_EXECUTABLE} \${CFG_FILE}\n\n")

    execute_process(COMMAND chmod +x ${_RunScript})

  endif()

  execute_process(

    COMMAND ${CMAKE_COMMAND} -E create_symlink

  file(APPEND ${_CleanScript} "    output_allatoona \\\n")

  file(APPEND ${_CleanScript} "    output_circular_dambreak \\\n")

  file(APPEND ${_CleanScript} "    output_paraboloid \\\n")

  file(APPEND ${_CleanScript} "    triton_ensrun.sh \\\n")

  file(APPEND ${_CleanScript} "    ensembles_ensify.yaml \\\n")

  file(APPEND ${_CleanScript} "    paraboloid_ens1.cfg \\\n")

  file(APPEND ${_CleanScript} "    paraboloid_ens2.cfg \\\n")

  file(APPEND ${_CleanScript} "    output_ens1 \\\n")

  file(APPEND ${_CleanScript} "    output_ens2 \\\n")

  file(APPEND ${_CleanScript} "    ens1.err \\\n")

  file(APPEND ${_CleanScript} "    ens1.out \\\n")

  file(APPEND ${_CleanScript} "    ens2.err \\\n")

  file(APPEND ${_CleanScript} "    ens2.out \\\n")

  file(APPEND ${_CleanScript} "    external \\\n")

  file(APPEND ${_CleanScript} "    input \\\n")

  file(APPEND ${_CleanScript} "    output")

.. _ensemble_run:

Running TRITON in Ensemble Mode

==================================

The ensemble run feature allows users to execute multiple TRITON runs within a single job submission, saving considerable time and effort. This feature is based on **Ensify** (`https://github.com/mrnorman/Ensify <https://github.com/mrnorman/Ensify>`_). 

STEP 1: Build TRITON for Ensemble Run

---------------------------------------

To build TRITON for an ensemble run, add **-DENSEMBLE_BUILD=ON** to the CMake command line along with the other arguments shown below.

.. code-block:: bash

    cmake -DENSEMBLE_BUILD=ON <other cmake arguments>

    ./triton_build.sh 

STEP 2: Configure Ensemble Run

---------------------------------------

The TRITON ensemble configuration file is a YAML file used to define an ensemble run, using the syntax explained below.

.. code-block:: yaml

    # rank_beg , rank_end , arguments_list , stdout_filename , stderr_filename

    ensembles:  [

                    [ 0, 7, ["./ensemble1.cfg", ...] , "ens1.out" , "ens1.err" ],

                    [ 8, 16, ["./ensemble2.cfg", ...] , "ens2.out" , "ens2.err" ],

                    ...

                ]

Ensembles are defined as a list of ensemble entries. Each ensemble entry includes the following fields in order: rank_beg, rank_end, arguments_list, stdout_filename, and stderr_filename. In the example above, the first two entries specify the first entry for MPI ranks 0–7 and the second entry for MPI ranks 8–16, respectively.

See `Ensify ensemble input format <https://github.com/mrnorman/Ensify#enssify-ensemble-input-format>`_ for details.

STEP 3: Run TRITON in Ensemble Mode

---------------------------------------

Running TRITON in ensemble mode is the same as a regular run, except that the ensemble configuration file is the only argument to the triton.exe executable. The ensemble configuration file is the one defined in the previous step.

.. code-block:: bash

    source ./triton_env.sh

    <MPI command and arguments> ./triton.exe <ensemble configuration file>

As an example of an ensemble run, **triton_ensrun.sh** is created with a sample configuration for the Paraboloid case.

   configuration_variable_index

   triton_run

   docker_run

   ensemble_run

   casestudy

.. toctree::