Commit 29d8e4d9 authored by Simon Spannagel's avatar Simon Spannagel
Browse files

Merge branch 'named_args' into 'master' 

Testing: Renovate CMake, Introduce Templating

See merge request allpix-squared/allpix-squared!661
parents 94d2ce9e 09fb0b4a

CMakeLists.txt

+1 −1
Original line number Diff line number Diff line
@@ -109,7 +109,7 @@ IF(BUILD_DOCS_ONLY) 
        GET_FILENAME_COMPONENT(title ${test} NAME_WE)

        GET_FILENAME_COMPONENT(dir "${test}/../.." ABSOLUTE)

        GET_FILENAME_COMPONENT(mod "${dir}" NAME)

        ADD_ALLPIX_TEST(${CMAKE_CURRENT_SOURCE_DIR}/src/modules/${test} "modules/${mod}/${title}")

        ADD_ALLPIX_TEST(NAME "modules/${mod}/${title}" FILE ${CMAKE_CURRENT_SOURCE_DIR}/src/modules/${test})

    ENDFOREACH()



    SET(MODULES_TEST_DESCRIPTIONS "${TEST_DESCRIPTIONS}")

cmake/AllpixMacros.cmake

+82 −33
Original line number Diff line number Diff line
@@ -166,35 +166,53 @@ FUNCTION(add_default_fail_conditions name) 
ENDFUNCTION()



# Add a test to the unit test suite and parse its configuration file for options

FUNCTION(add_allpix_test test name)

FUNCTION(add_allpix_test)

    SET(options IGNORE_PASS_CONDITION)

    SET(oneValueArgs NAME FILE EXECUTABLE WORKING_DIRECTORY)

    SET(multiValueArgs DEPENDS)

    CMAKE_PARSE_ARGUMENTS(TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})



    # Set default working directory

    IF(NOT DEFINED TEST_WORKING_DIRECTORY)

        # cmake-lint: disable=C0103

        SET(TEST_WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testing")

    ENDIF()

    FILE(MAKE_DIRECTORY ${TEST_WORKING_DIRECTORY})



    # Default executable is "allpix"

    IF(NOT DEFINED TEST_EXECUTABLE)

        # cmake-lint: disable=C0103

        SET(TEST_EXECUTABLE "allpix")

    ENDIF()



    # Allow the test to specify additional module CLI parameters:

    FILE(STRINGS ${test} OPTS REGEX "#OPTION ")

    FILE(STRINGS ${TEST_FILE} OPTS REGEX "#OPTION ")

    FOREACH(opt ${OPTS})

        STRING(REPLACE "#OPTION " "" opt "${opt}")

        SET(clioptions "${clioptions} -o ${opt}")

    ENDFOREACH()

    # Allow the test to specify additional geometry CLI parameters:

    FILE(STRINGS ${test} OPTS REGEX "#DETOPTION ")

    FILE(STRINGS ${TEST_FILE} OPTS REGEX "#DETOPTION ")

    FOREACH(opt ${OPTS})

        STRING(REPLACE "#DETOPTION " "" opt "${opt}")

        SET(clioptions "${clioptions} -g ${opt}")

    ENDFOREACH()

    # Allow the test to specify additional CLI parameters:

    FILE(STRINGS ${test} OPTS REGEX "#CLIOPTION ")

    FILE(STRINGS ${TEST_FILE} OPTS REGEX "#CLIOPTION ")

    FOREACH(opt ${OPTS})

        STRING(REPLACE "#CLIOPTION " "" opt "${opt}")

        SET(clioptions "${clioptions} ${opt}")

    ENDFOREACH()



    # Register the test for inclusion in the documentation:

    FILE(STRINGS ${test} DESC REGEX "#DESC ")

    FILE(STRINGS ${TEST_FILE} DESC REGEX "#DESC ")

    LIST(LENGTH DESC listcount_desc)

    IF(listcount_desc EQUAL 0)

        MESSAGE(WARNING "Test ${name} does not provide a description")

        MESSAGE(WARNING "Test ${TEST_NAME} does not provide a description")

    ELSEIF(listcount_desc GREATER 1)

        MESSAGE(FATAL_ERROR "More than one DESC expressions defined in test ${name}")

        MESSAGE(FATAL_ERROR "More than one DESC expressions defined in test ${TEST_NAME}")

    ELSE()

        STRING(REPLACE "#DESC " "\\item[\\file{${name}}] " DESC "${DESC}")

        STRING(REPLACE "#DESC " "\\item[\\file{${TEST_NAME}}] " DESC "${DESC}")

        LIST(APPEND TEST_DESCRIPTIONS ${DESC})

    ENDIF()

    SET(TEST_DESCRIPTIONS
@@ -202,68 +220,82 @@ FUNCTION(add_allpix_test test name) 
        PARENT_SCOPE)



    # Parse possible commands to be run before

    FILE(STRINGS ${test} OPTS REGEX "#BEFORE_SCRIPT ")

    FILE(STRINGS ${TEST_FILE} OPTS REGEX "#BEFORE_SCRIPT ")

    FOREACH(opt ${OPTS})

        STRING(REPLACE "#BEFORE_SCRIPT " "" opt "${opt}")

        LIST(APPEND before_script ${opt})

    ENDFOREACH()



    ADD_TEST(

        NAME "${name}"

        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/etc/unittests

        COMMAND ${PROJECT_SOURCE_DIR}/etc/unittests/run_directory.sh "output/${name}"

                "${CMAKE_INSTALL_PREFIX}/bin/allpix -c ${CMAKE_CURRENT_SOURCE_DIR}/${test} ${clioptions}"

        NAME "${TEST_NAME}"

        WORKING_DIRECTORY ${TEST_WORKING_DIRECTORY}

        COMMAND ${PROJECT_SOURCE_DIR}/etc/unittests/run_directory.sh "${TEST_NAME}"

                "${CMAKE_INSTALL_PREFIX}/bin/${TEST_EXECUTABLE} -c ${TEST_FILE} ${clioptions}"

                ${before_script})



    # Parse configuration file for pass/fail conditions:

    FILE(STRINGS ${test} PASS_LST_ REGEX "#PASS ")

    FILE(STRINGS ${test} FAIL_LST_ REGEX "#FAIL ")

    FILE(STRINGS ${TEST_FILE} PASS_LST_ REGEX "#PASS ")

    FILE(STRINGS ${TEST_FILE} FAIL_LST_ REGEX "#FAIL ")



    # Check for number of pass or fail conditions - we should have at least one of them

    LIST(LENGTH PASS_LST_ listcount_pass)

    LIST(LENGTH FAIL_LST_ listcount_fail)

    IF(listcount_pass EQUAL 0 AND listcount_fail EQUAL 0)

        MESSAGE(FATAL_ERROR "Neither PASS nor FAIL defined for test \"${name}\"")

        MESSAGE(FATAL_ERROR "Neither PASS nor FAIL defined for test \"${TEST_NAME}\"")

    ENDIF()



    IF(NOT TEST_IGNORE_PASS_CONDITION)

        # Escape possible regex patterns in the expected output:

        FOREACH(pass ${PASS_LST_})

            ESCAPE_REGEX("${pass}" pass)

            SET_PROPERTY(

            TEST ${name}

                TEST ${TEST_NAME}

                APPEND

                PROPERTY PASS_REGULAR_EXPRESSION "${pass}")

        ENDFOREACH()

    ENDIF()

    FOREACH(fail ${FAIL_LST_})

        ESCAPE_REGEX("${fail}" fail)

        SET_PROPERTY(

            TEST ${name}

            TEST ${TEST_NAME}

            APPEND

            PROPERTY FAIL_REGULAR_EXPRESSION "${fail}")

    ENDFOREACH()



    # Add default fail conditions

    ADD_DEFAULT_FAIL_CONDITIONS(${name})

    ADD_DEFAULT_FAIL_CONDITIONS(${TEST_NAME})



    # Some tests might depend on others:

    FILE(STRINGS ${test} DEPENDENCY REGEX "#DEPENDS ")

    FILE(STRINGS ${TEST_FILE} DEPENDENCY REGEX "#DEPENDS ")

    IF(DEPENDENCY)

        STRING(REPLACE "#DEPENDS " "" DEPENDENCY "${DEPENDENCY}")

        SET_PROPERTY(TEST ${name} PROPERTY DEPENDS "${DEPENDENCY}")

        SET_PROPERTY(TEST ${TEST_NAME} PROPERTY DEPENDS "${DEPENDENCY}")

    ENDIF()

    FOREACH(depend ${TEST_DEPENDS})

        MESSAGE(STATUS "DEP ${depend}")

        SET_PROPERTY(TEST ${TEST_NAME} PROPERTY DEPENDS "${depend}")

    ENDFOREACH()



    # Add individual timeout criteria:

    FILE(STRINGS ${test} TESTTIMEOUT REGEX "#TIMEOUT ")

    FILE(STRINGS ${TEST_FILE} TESTTIMEOUT REGEX "#TIMEOUT ")

    IF(TESTTIMEOUT)

        STRING(REPLACE "#TIMEOUT " "" TESTTIMEOUT "${TESTTIMEOUT}")

        SET_PROPERTY(TEST ${name} PROPERTY TIMEOUT_AFTER_MATCH "${TESTTIMEOUT}" "Starting event loop")

        SET_PROPERTY(TEST ${TEST_NAME} PROPERTY TIMEOUT_AFTER_MATCH "${TESTTIMEOUT}" "Starting event loop")

    ENDIF()



    # Allow to add test labels

    FILE(STRINGS ${test} TESTLABEL REGEX "#LABEL ")

    FILE(STRINGS ${TEST_FILE} TESTLABEL REGEX "#LABEL ")

    IF(TESTLABEL)

        STRING(REPLACE "#LABEL " "" TESTLABEL "${TESTLABEL}")

        SET_PROPERTY(TEST ${name} PROPERTY LABELS "${TESTLABEL}")

        SET_PROPERTY(TEST ${TEST_NAME} PROPERTY LABELS "${TESTLABEL}")

    ENDIF()



    # Add required files if specified

    FILE(STRINGS ${TEST_FILE} TESTDATA REGEX "#DATA ")

    IF(TESTDATA)

        STRING(REPLACE "#DATA " "" TESTDATA "${TESTDATA}")

        STRING(REPLACE " " ";" TESTDATA ${TESTDATA})

        SET_PROPERTY(TEST ${TEST_NAME} PROPERTY REQUIRED_FILES "${TESTDATA}")

    ENDIF()

ENDFUNCTION()
@@ -272,15 +304,32 @@ MACRO(ALLPIX_MODULE_TESTS name directory) 
    # Get the name of the module

    GET_FILENAME_COMPONENT(_allpix_module_dir ${CMAKE_CURRENT_SOURCE_DIR} NAME)



    SET(TEST_BASE_DIR "${PROJECT_BINARY_DIR}/testing")

    IF(TEST_MODULES)

        FILE(

            GLOB AUX_FILES_MODULES

            RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}

            ${directory}/[!00-99]*)



        # Copy and configure auxiliary files

        FOREACH(file ${AUX_FILES_MODULES})

            CONFIGURE_FILE(${file} "${CMAKE_CURRENT_BINARY_DIR}/${file}" @ONLY)

        ENDFOREACH()



        FILE(

            GLOB TEST_LIST_MODULES

            RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}

            ${directory}/[00-99]*)



        # Copy and configure tests

        FOREACH(test ${TEST_LIST_MODULES})

            GET_FILENAME_COMPONENT(title ${test} NAME_WE)

            ADD_ALLPIX_TEST(${test} "modules/${_allpix_module_dir}/${title}")

            SET(TEST_DIR "${TEST_BASE_DIR}/modules/${_allpix_module_dir}/${title}")



            CONFIGURE_FILE(${test} "${CMAKE_CURRENT_BINARY_DIR}/${test}" @ONLY)

            ADD_ALLPIX_TEST(NAME "modules/${_allpix_module_dir}/${title}"

                            FILE ${CMAKE_CURRENT_BINARY_DIR}/${test}

                            WORKING_DIRECTORY ${TEST_BASE_DIR})



            GET_PROPERTY(old_count GLOBAL PROPERTY COUNT_TESTS_MODULES)

            MATH(EXPR new_count "${old_count}+1")

doc/usermanual/chapters/testing.tex

+29 −1
Original line number Diff line number Diff line
@@ -246,7 +246,7 @@ $ ctest -R test_performance 
The configuration of the tests can be found in the \dir{etc/unittests/test_*} directories of the repository and are automatically discovered by CMake.

CMake automatically searches for \apsq configuration files in the different directories and passes them to the \apsq executable~(cf.\ Section~\ref{sec:allpix_executable}).



Adding a new test is as simple as adding a new configuration file to one of the different subdirectories and specifying the pass or fail conditions based on the tags described in the following paragraph.

Adding a new test is as simple as adding a new configuration file to one of the different subdirectories and specifying the pass or fail conditions based on the tags described in the following paragraphs.



\paragraph{Test Tags, Pass and Fail Conditions}
@@ -296,3 +296,31 @@ If a test is expected to create multiple error or warning messages which cannot 
#PASS (ERROR) Multithreading disabled since the current module configuration does not support it

#FAIL FATAL

\end{minted}



\paragraph{Directory Variables in Tests}



Sometimes it is necessary to pass directories or file names as test input.

To facilitate this, the test files can contain variables which are replaced with the respective paths before being executed.

All variable names have to be enclosed in \parameter{@} symbols to be detected and parsed correctly.

Variables can be used both in test files and the auxiliary configuration files such as detector geometry definitions.



The following variables are available:



\begin{description}

  \item[\parameter{@TEST_DIR@}:] Directory in which the current test is executed, i.e. where all output files will be placed.

  \item[\parameter{@TEST_BASE_DIR@}:] Base directory under which all tests are being executed. This can be used to reference the output files from another test. it should be noted that the respective test has to be referenced using the \parameter{#DEPENDS} keyword to ensure that it successfully ran before.

  \item[\parameter{@PROJECT_SOURCE_DIR@}:] The root directory of the project. This can for example be used to call a script provided in the \dir{etc/scripts} directory of the repository.

\end{description}



The following example demonstrates the use of these variables.

A script is called before executing the test and an input file is expected:



\begin{minted}[frame=single,framesep=3pt,breaklines=true,tabsize=2,linenos]{ini}

[Allpix]

detectors_file = "detector.conf"



[DepositionReader]

file_name = "@TEST_DIRECTORY@/deposition.root"



#BEFORE_SCRIPT python @PROJECT_SOURCE_DIR@/etc/scripts/create_deposition_file.py --type a --detector mydetector

\end{minted}

etc/unittests/CMakeLists.txt

+2 −2
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ IF(TEST_PERFORMANCE) 
    SET_PROPERTY(GLOBAL PROPERTY COUNT_TESTS_PERFORMANCE "${NUM_TEST_PERFORMANCE}")

    FOREACH(test ${TEST_LIST_PERFORMANCE})

        GET_FILENAME_COMPONENT(title ${test} NAME_WE)

        ADD_ALLPIX_TEST(${test} "performance/${title}")

        ADD_ALLPIX_TEST(NAME "performance/${title}" FILE ${CMAKE_CURRENT_SOURCE_DIR}/${test})

    ENDFOREACH()

    SET_PROPERTY(GLOBAL PROPERTY PERF_TEST_DESCRIPTIONS "${TEST_DESCRIPTIONS}")

    SET(TEST_DESCRIPTIONS "")
@@ -30,7 +30,7 @@ IF(TEST_CORE) 
    SET_PROPERTY(GLOBAL PROPERTY COUNT_TESTS_CORE "${NUM_TEST_CORE}")

    FOREACH(test ${TEST_LIST_CORE})

        GET_FILENAME_COMPONENT(title ${test} NAME_WE)

        ADD_ALLPIX_TEST(${test} "core/${title}")

        ADD_ALLPIX_TEST(NAME "core/${title}" FILE ${CMAKE_CURRENT_SOURCE_DIR}/${test})

    ENDFOREACH()

    SET_PROPERTY(GLOBAL PROPERTY CORE_TEST_DESCRIPTIONS "${TEST_DESCRIPTIONS}")

    SET(TEST_DESCRIPTIONS "")

src/modules/CorryvreckanWriter/tests/01-corry.conf

+1 −1
Original line number Diff line number Diff line
@@ -30,4 +30,4 @@ reference = mydetector 
output_mctruth = false

log_level = TRACE



#PASS /etc/unittests/output/modules/CorryvreckanWriter/01-corry/output/CorryvreckanWriter/
 
#PASS /modules/CorryvreckanWriter/01-corry/output/CorryvreckanWriter/