Commit 1e8f734a authored by Joel E. Denny's avatar Joel E. Denny
Browse files

[Clacc][lit] Make ALLOW_RETRIES retry a timed out test 

A timeout is a kind of test failure.  If a test is found to
occasionally hang, it makes sense to be able to kill it and retry it
to see if it will succeed on the next attempt.
parent edfe96f0

llvm/utils/lit/lit/TestRunner.py

+1 −1
Original line number Diff line number Diff line
@@ -2054,7 +2054,7 @@ def _runShTest(test, litConfig, useExternalSh, script, tmpBase): 
            return res



        out,err,exitCode,timeoutInfo,status = res

        if status != Test.FAIL:

        if status != Test.FAIL and status != Test.TIMEOUT:

            break



    # If we had to run the test more than once, count it as a flaky pass. These

llvm/utils/lit/tests/Inputs/shtest-timeout/allow-retries.py

0 → 100644
+25 −0
Original line number Diff line number Diff line 
# ALLOW_RETRIES: 1



# RUN: "%{python}" "%s" "%{counter}"



import sys

import os



counter_file = sys.argv[1]



# The first time the test is run, initialize the counter to 1.

if not os.path.exists(counter_file):

    with open(counter_file, 'w') as counter:

        counter.write("1")



# Succeed if this is the second time we're being run.

with open(counter_file, 'r') as counter:

    num = int(counter.read())

    if num == 2:

        sys.exit(0)



# Otherwise, increment the counter and force a timeout

with open(counter_file, 'w') as counter:

    counter.write(str(num + 1))

while True:

    pass

llvm/utils/lit/tests/Inputs/shtest-timeout/lit.cfg

+2 −0
Original line number Diff line number Diff line
@@ -38,3 +38,5 @@ if 'PYTHONPATH' in config.environment: 
config.environment['PYTHONPATH'] = os.pathsep.join(pythonpath_list)



config.substitutions.append(('%{python}', '"%s"' % (sys.executable)))

config.substitutions.append(('%{counter}', lit_config.params.get('counter',

                                                                 '')))

llvm/utils/lit/tests/shtest-timeout.py

+13 −0
Original line number Diff line number Diff line
@@ -78,3 +78,16 @@ 
# RUN: FileCheck --check-prefix=CHECK-CMDLINE-OVERRIDE-ERR < %t.pass.cmdover.err %s



# CHECK-CMDLINE-OVERRIDE-ERR: Forcing timeout to be 3600 seconds





###############################################################################

# Check that ALLOW_RETRIES will retry after a timeout

###############################################################################



# RUN: rm -f %t.counter

# RUN: %{lit} %{inputs}/shtest-timeout/allow-retries.py -v --timeout 5 \

# RUN:     -Dcounter=%t.counter -Dexternal=0 2>&1 | \

# RUN:   FileCheck --check-prefix=CHECK-RETRY %s



# CHECK-RETRY: FLAKYPASS: per_test_timeout :: allow-retries.py

# CHECK-RETRY: Passed With Retry: 1