Commit 9e8d6bbe authored by Stefan Hertrampf's avatar Stefan Hertrampf
Browse files

nixos/test-driver: add junit-xml logger 

We add a new logger that allows generating a junit-xml compatible report
listing the subtests used in the nixos integration test. Junit-xml is a
widely used standard for test reports. The report can be used for quick
evaluation of which subtest failed.
parent 9d90df51

nixos/lib/test-driver/test_driver/__init__.py

+15 −2
Original line number Diff line number Diff line
@@ -6,7 +6,7 @@ from pathlib import Path 
import ptpython.repl



from test_driver.driver import Driver

from test_driver.logger import rootlog

from test_driver.logger import JunitXMLLogger, XMLLogger, rootlog





class EnvDefault(argparse.Action):
@@ -92,6 +92,11 @@ def main() -> None: 
        default=Path.cwd(),

        type=writeable_dir,

    )

    arg_parser.add_argument(

        "--junit-xml",

        help="Enable JunitXML report generation to the given path",

        type=Path,

    )

    arg_parser.add_argument(

        "testscript",

        action=EnvDefault,
@@ -102,6 +107,14 @@ def main() -> None: 


    args = arg_parser.parse_args()



    output_directory = args.output_directory.resolve()



    if "LOGFILE" in os.environ.keys():

        rootlog.add_logger(XMLLogger(os.environ["LOGFILE"]))



    if args.junit_xml:

        rootlog.add_logger(JunitXMLLogger(output_directory / args.junit_xml))



    if not args.keep_vm_state:

        rootlog.info("Machine state will be reset. To keep it, pass --keep-vm-state")
@@ -109,7 +122,7 @@ def main() -> None: 
        args.start_scripts,

        args.vlans,

        args.testscript.read_text(),

        args.output_directory.resolve(),

        output_directory,

        args.keep_vm_state,

        args.global_timeout,

    ) as driver:

nixos/lib/test-driver/test_driver/logger.py

+75 −1
Original line number Diff line number Diff line 
import atexit

import codecs

import os

import sys
@@ -5,12 +6,14 @@ import time 
import unicodedata

from abc import ABC, abstractmethod

from contextlib import ExitStack, contextmanager

from pathlib import Path

from queue import Empty, Queue

from typing import Any, Dict, Iterator, List

from xml.sax.saxutils import XMLGenerator

from xml.sax.xmlreader import AttributesImpl



from colorama import Fore, Style

from junit_xml import TestCase, TestSuite





class AbstractLogger(ABC):
@@ -49,6 +52,77 @@ class AbstractLogger(ABC): 
        pass





class JunitXMLLogger(AbstractLogger):



    class TestCaseState:

        def __init__(self) -> None:

            self.stdout = ""

            self.stderr = ""

            self.failure = False



    def __init__(self, outfile: Path) -> None:

        self.tests: dict[str, JunitXMLLogger.TestCaseState] = {

            "main": self.TestCaseState()

        }

        self.currentSubtest = "main"

        self.outfile: Path = outfile

        self._print_serial_logs = True

        atexit.register(self.close)



    def log(self, message: str, attributes: Dict[str, str] = {}) -> None:

        self.tests[self.currentSubtest].stdout += message + os.linesep



    @contextmanager

    def subtest(self, name: str, attributes: Dict[str, str] = {}) -> Iterator[None]:

        old_test = self.currentSubtest

        self.tests.setdefault(name, self.TestCaseState())

        self.currentSubtest = name



        yield



        self.currentSubtest = old_test



    @contextmanager

    def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]:

        self.log(message)

        yield



    def info(self, *args, **kwargs) -> None:  # type: ignore

        self.tests[self.currentSubtest].stdout += args[0] + os.linesep



    def warning(self, *args, **kwargs) -> None:  # type: ignore

        self.tests[self.currentSubtest].stdout += args[0] + os.linesep



    def error(self, *args, **kwargs) -> None:  # type: ignore

        self.tests[self.currentSubtest].stderr += args[0] + os.linesep

        self.tests[self.currentSubtest].failure = True



    def log_serial(self, message: str, machine: str) -> None:

        if not self._print_serial_logs:

            return



        self.log(f"{machine} # {message}")



    def print_serial_logs(self, enable: bool) -> None:

        self._print_serial_logs = enable



    def close(self) -> None:

        with open(self.outfile, "w") as f:

            test_cases = []

            for name, test_case_state in self.tests.items():

                tc = TestCase(

                    name,

                    stdout=test_case_state.stdout,

                    stderr=test_case_state.stderr,

                )

                if test_case_state.failure:

                    tc.add_failure_info("test case failed")



                test_cases.append(tc)

            ts = TestSuite("NixOS integration test", test_cases)

            f.write(TestSuite.to_xml_string([ts]))





class CompositeLogger(AbstractLogger):

    def __init__(self, logger_list: List[AbstractLogger]) -> None:

        self.logger_list = logger_list
@@ -238,4 +312,4 @@ class XMLLogger(AbstractLogger): 


terminal_logger = TerminalLogger()

xml_logger = XMLLogger()

rootlog: AbstractLogger = CompositeLogger([terminal_logger, xml_logger])
 
rootlog: CompositeLogger = CompositeLogger([terminal_logger, xml_logger])