-
Martyn Gigg authoredMartyn Gigg authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
doctest.py 18.13 KiB
"""
Defines a handler for the Sphinx 'build-finished' event.
If the builder is doctest then it post-processes the
output file to produce an XUnit-style XML file that can be
more easily parse by CI servers such as Jenkins.
Output file structure
~~~~~~~~~~~~~~~~~~~~~
The following outcomes are possible for a given
document:
- all tests pass;
- all tests fail;
- and some test pass and some fail.
Below are examples of the output for each of the above
outcomes, given a document named 'FooDoc' in a
directory 'bar' relative to the documentation root.
- All Passed:
============
Document: algorithms/AllPassed
------------------------------
2 items passed all tests:
1 tests in Ex 2
2 tests in default
3 tests in 2 items.
3 passed and 0 failed.
Test passed.
Doctest summary
===============
3 tests
0 failures in tests
0 failures in setup code
0 failures in cleanup code
- All Failed:
============
Document: bar/FooDoc
--------------------
**********************************************************************
File "bar/FooDoc.rst", line 127, in Ex2[31]
Failed example:
print "Multi-line failed"
print "test"
Expected:
No match
Got:
Multi-line failed
test
**********************************************************************
File "bar/FooDoc.rst", line 111, in Ex1
Failed example:
print "Single line failed test"
Expected:
No match
Got:
Single line failed test
**********************************************************************
2 items had failures:
1 of 1 in Ex1
1 of 1 in Ex2[31]
2 tests in 2 items.
0 passed and 2 failed.
***Test Failed*** 2 failures.
2 items passed all tests:
1 tests in Ex1 (cleanup code)
1 tests in Ex2[31] (cleanup code)
2 tests in 2 items.
2 passed and 0 failed.
Test passed.
Doctest summary
===============
2 tests
2 failures in tests
0 failures in setup code
0 failures in cleanup code
- Some pass some fail:
====================
Document: bar/FooDoc
--------------------
**********************************************************************
File "bar/FooDoc.rst", line 127, in default
Failed example:
print "A failed test"
Expected:
Not a success
Got:
A failed test
**********************************************************************
File "bar/FooDoc.rst", line 143, in Ex1
Failed example:
print "Second failed test"
Expected:
Not a success again
Got:
Second failed test
1 items passed all tests:
1 tests in Ex3
**********************************************************************
2 items had failures:
1 of 1 in Ex1
1 of 2 in default
4 tests in 3 items.
2 passed and 2 failed.
***Test Failed*** 2 failures.
1 items passed all tests:
1 tests in Ex (cleanup code)
1 tests in 1 items.
1 passed and 0 failed.
Test passed.
Doctest summary
===============
4 tests
2 failures in tests
0 failures in setup code
0 failures in cleanup code
"""
import re
try:
import lxml.etree as ElementTree
except ImportError:
import xml.etree.ElementTree as ElementTree
# Name of file produced by doctest target. It is assumed that it is created
# in app.outdir
DOCTEST_OUTPUT = "output.txt"
# Name of output file that the resultant XUnit output is saved
# @todo make this a configuration variable
XUNIT_OUTPUT = "TEST-doctest.xml"
# Error type string
TEST_FAILURE_TYPE = "UsageFailure"
# Package name
PACKAGE_NAME = "docs"
#-------------------------------------------------------------------------------
# Define parts of lines that denote a document
DOCTEST_DOCUMENT_BEGIN = "Document:"
DOCTEST_SUMMARY_TITLE = "Doctest summary"
FAILURE_MARKER = "*"*70
# Regexes
ALLPASS_TEST_NAMES_RE = re.compile(r"^\s+(\d+) tests in (.+)$")
NUMBER_PASSED_RE = re.compile(r"^(\d+) items passed all tests:$")
TEST_PASSED_END_RE = re.compile(r"Test passed.")
TEST_FAILED_END_RE = re.compile(r"\*\*\*Test Failed\*\*\* (\d+) failures.")
FAILURE_LOC_RE = re.compile(r"^File \"(.+)\",\s+line\s+(\d+),\s+in\s+(\S+)(\s\((setup|cleanup) code\))?$")
MIX_FAIL_RE = re.compile(r'^\s+(\d+)\s+of\s+(\d+)\s+in\s+(\w+)$')
#-------------------------------------------------------------------------------
class TestSuiteReport(object):
def __init__(self, name, cases, package=None):
if len(cases) == 0:
raise ValueError("No test cases provided")
self.name = name
self.testcases = cases
self.package = package
@property
def ntests(self):
return len(self.testcases)
@property
def nfailed(self):
def sum_failure(fails, case):
if case.failed:
return fails + 1
else:
return fails
return reduce(sum_failure, self.testcases, 0)
@property
def npassed(self):
return self.ntests - self.nfailed
#-------------------------------------------------------------------------------
class TestCaseReport(object):
def __init__(self, classname, name, failure_descr):
self.classname = classname
self.name = name
if failure_descr is not None:
self.failure_descr = failure_descr
else:
self.failure_descr = ""
def __repr__(self):
return "%s: %d" % (self.name, self.passed)
@property
def passed(self):
return self.failure_descr == ""
@property
def failed(self):
return not self.passed
def issetup(self):
return "setup code" in self.failure_descr
def iscleanup(self):
return "cleanup code" in self.failure_descr
#-------------------------------------------------------------------------------
class DocTestOutputParser(object):
"""
Process a doctest output file and convert it
to a different format
"""
def __init__(self, doctest_output, isfile = True):
"""
Parses the given doctest output
Args:
doctest_output (str): String giving either doctest output as plain
text or a filename
isfile (bool): If True then the doctest_output argument is treated
as a filename
"""
if isfile:
with open(doctest_output,'r') as results:
self.testsuite = self.__parse(results)
else:
self.testsuite = self.__parse(doctest_output.splitlines())
def as_xunit(self, filename):
"""
Write out the test results in Xunit-style format
"""
suite_node = ElementTree.Element("testsuite")
suite_node.attrib["name"] = self.testsuite.name
suite_node.attrib["tests"] = str(self.testsuite.ntests)
suite_node.attrib["failures"] = str(self.testsuite.nfailed)
if self.testsuite.package:
suite_node.attrib["package"] = self.testsuite.package
cases = self.testsuite.testcases
for testcase in cases:
case_node = ElementTree.SubElement(suite_node, "testcase")
case_node.attrib["classname"] = testcase.classname
case_node.attrib["name"] = testcase.name
if testcase.failed:
failure_node = ElementTree.SubElement(case_node, "failure")
failure_node.attrib["type"] = TEST_FAILURE_TYPE
failure_node.text = testcase.failure_descr
# Serialize to file
tree = ElementTree.ElementTree(suite_node)
tree.write(filename, encoding="utf-8", xml_declaration=True)
def __parse(self, results):
"""
Parse a doctest output file and a TestSuiteReport
object that describe the results of the
all tests on a single document
Arguments:
results (iterable): Iterable where each element contains
a line of the results
Returns:
TestSuite: TestSuite object
"""
in_doc = False
document_txt = None
cases = []
for line in results:
line = line.rstrip()
if line.startswith(DOCTEST_DOCUMENT_BEGIN):
# parse previous results
if document_txt:
cases.extend(self.__parse_document(document_txt))
document_txt = [line]
in_doc = True
continue
if line.startswith(DOCTEST_SUMMARY_TITLE): # end of tests
in_doc = False
cases.extend(self.__parse_document(document_txt))
document_txt = None
if in_doc and line != "":
document_txt.append(line)
# endfor
return TestSuiteReport(name="doctests", cases=cases,
package=PACKAGE_NAME)
def __parse_document(self, results):
"""
Create a list of TestCaseReport object for this document
Args:
results (list): List of lines of doctest output
for a single document
Returns:
list: List of test cases in the document
"""
fullname = self.__extract_fullname(results[0])
if not results[1].startswith("-"):
raise ValueError("Invalid second line of output: '%s'. "\
"Expected a title underline."
% text[1])
results = results[2:] # trim off top two lines of header information
maintests, cleanup = self.__split_on_cleanup(results)
overall_success = not maintests[0] == FAILURE_MARKER
if overall_success:
testcases = self.__parse_success(fullname, maintests)
else:
testcases = self.__parse_failures(fullname, maintests)
return testcases
def __extract_fullname(self, line):
"""
Extract the document name from the line of text.
Args:
line (str): Line to test for title
"""
if not line.startswith(DOCTEST_DOCUMENT_BEGIN):
raise ValueError("First line of output text should be a line "
"beginning '%s'" % DOCTEST_DOCUMENT_BEGIN)
return line.replace(DOCTEST_DOCUMENT_BEGIN, "").strip()
def __split_on_cleanup(self, results):
"""
Split the list of text into 2 parts, giving results for:
- testcode
- testcleanup
An empty list is returned if that section has no text
Args:
results (list): A list of string lines giving all results
Returns:
2-tuple: Each element is a list of result lines for that section
"""
# Look for first line containing either '***Test Failed***' or
# 'Test passed' as this is the final line of the main test results
summaryline_idx = None
for idx, line in enumerate(results):
if TEST_PASSED_END_RE.match(line) or \
TEST_FAILED_END_RE.match(line):
summaryline_idx = idx
break
if summaryline_idx:
# +1 includes the summary line itself
return (results[:summaryline_idx+1], results[summaryline_idx+1:])
else:
return results
def __parse_success(self, fullname, results):
"""
Parse results for a success case
Args:
fullname (str): String containing full name of document
results (line): List containing lines of doctest output for
document
Returns:
A list of test cases that were parsed from the results
"""
match = NUMBER_PASSED_RE.match(results[0])
if not match:
raise ValueError("All passed line incorrect: '%s'"
% results[0])
classname = self.__create_classname(fullname)
nitems = int(match.group(1))
cases = []
for line in results[1:1+nitems]:
match = ALLPASS_TEST_NAMES_RE.match(line)
if not match:
raise ValueError("Unexpected information line in "
"all pass case: %s" % line)
ntests, name = int(match.group(1)), match.group(2)
for idx in range(ntests):
cases.append(TestCaseReport(classname, name, failure_descr=None))
#endfor
return cases
def __parse_failures(self, fullname, results):
"""
Parse text for failures cases for a single document
Args:
fullname (str): String containing full name of document
results (line): List containing lines of doctest output for
document
Returns:
A list of test cases that were parsed from the results
"""
classname = self.__create_classname(fullname)
# Find index marker lines that delineate failures or
# a line containing 'items passed all tests:'. It looks as if
# this is a bug in sphinx.doctest output that doesn't delineate
# the failures and successes properly.
fail_markers = []
success_markers = []
for idx, line in enumerate(results):
if line == FAILURE_MARKER:
if len(success_markers) > 0:
success_markers.append(idx)
else:
fail_markers.append(idx)
if line.endswith("items passed all tests:"):
success_markers.append(idx)
fail_markers.append(idx)
# Parse failure text first as the last section can contain
# information about other tests that have passed.
nmarkers = len(fail_markers)
failcases = []
for i in range(0, nmarkers - 1):
start, end = fail_markers[i] + 1, fail_markers[i+1]
failcases.append(self.__create_failure_report(classname,
results[start:end]))
if len(success_markers) == 0:
return failcases
elif len(success_markers) == 1:
# Just take the rest
success_markers.append(len(results))
# Parse successful tests that have unique names
start, end = success_markers[0], success_markers[1]
passcases = self.__parse_success(fullname, results[start:end])
# The final puzzle piece is that some tests that have failed
# may have the same names as those that have passed.
for line in results[end+1:]:
match = MIX_FAIL_RE.match(line)
if not match:
continue
nfails, ntotal = int(match.group(1)), int(match.group(2))
npasses = ntotal - nfails
name = match.group(3)
for i in range(npasses):
passcases.append(TestCaseReport(classname, name,
failure_descr=None))
return self.__merge_passfail(passcases, failcases)
def __create_classname(self, fullname):
"""
Given a fullname, that can include path separators,
produce a classname for the document
Args:
fullname (str): Fullname of document (including paths)
"""
return PACKAGE_NAME + "." + fullname
def __create_failure_report(self, classname, failure_desc):
"""
Create a TestCaseReport from a description of the failure. The
name is retrieved from the end of first line of description.
Args:
classname (str): A string name for the 'class'
failure_desrc (list): A list of lines giving describing the failure
Returns:
A new TestCaseReport object
"""
match = FAILURE_LOC_RE.match(failure_desc[0])
if not match:
raise ValueError("Unexpected failure description format.\n"
"Expected the first line to contain details "
"of the location of the error.\n"
"Found '%s'" % failure_desc[0])
name = match.group(3)
return TestCaseReport(classname, name, "\n".join(failure_desc))
def __merge_passfail(self, passes, fails):
"""
Merge the two set of test reports together. Tests with the same name
but marked either as setup/cleanup will be treated as the same
test and only marked as pass if all parts passed.
Args:
passes (list): A list of test cases that have passed
fails (list): A list of test cases that have failed
Returns:
A merged list of cases
"""
merged = passes
for case in fails:
if case.issetup() or case.iscleanup():
for maincase in passes:
if case.name == maincase.name:
maincase.failure_descr = case.failure_descr
break # only do the first match
else:
merged.append(case)
return merged
#-------------------------------------------------------------------------------
def doctest_to_xunit(app, exception):
"""
If the runner was 'doctest'then parse the "output.txt"
file and produce an XUnit-style XML file, otherwise it does
nothing.
Arguments:
app (Sphinx.application): Sphinx application object
exception: (Exception): If an exception was raised then it is given here.
It is simply re-raised if an error occurred
"""
if exception:
raise exception
if app.builder.name != "doctest":
app.debug("Skipping xunit parsing for builder '%s'" % app.builder.name)
return
import os
doctest_file = os.path.join(app.builder.outdir, DOCTEST_OUTPUT)
app.debug("Parsing doctest output file '%s'" % doctest_file)
doctests = DocTestOutputParser(doctest_file)
app.debug("Saving doctest as xunit to file '%s'" % doctest_file)
xunit_file = os.path.join(app.builder.outdir, XUNIT_OUTPUT)
doctests.as_xunit(xunit_file)
#-------------------------------------------------------------------------------
def setup(app):
"""
Connect the 'build-finished' event to the handler function.
"""
app.connect('build-finished', doctest_to_xunit)