nixos/test-driver: improve error reporting and assertions (#390996) (d0c304d4) · Commits · nix / nixpkgs

nixos/doc/manual/development/writing-nixos-tests.section.md

+3 −2

Original line number	Diff line number	Diff line
		@@ -121,8 +121,7 @@ and checks that the output is more-or-less correct:
		```py
		machine.start()
		machine.wait_for_unit("default.target")
		if not "Linux" in machine.succeed("uname"):
		raise Exception("Wrong OS")
		t.assertIn("Linux", machine.succeed("uname"), "Wrong OS")
		```

		The first line is technically unnecessary; machines are implicitly started
		@@ -134,6 +133,8 @@ starting them in parallel:
		start_all()
		```

		Under the variable `t`, all assertions from [`unittest.TestCase`](https://docs.python.org/3/library/unittest.html) are available.

		If the hostname of a node contains characters that can't be used in a
		Python variable name, those characters will be replaced with
		underscores in the variable name, so `nodes.machine-a` will be exposed

nixos/lib/test-driver/default.nix

+1 −0

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@ python3Packages.buildPythonApplication {
		colorama
		junit-xml
		ptpython
		ipython
		]
		++ extraPythonPackages python3Packages;

nixos/lib/test-driver/src/pyproject.toml

+1 −1

Original line number	Diff line number	Diff line
		@@ -21,7 +21,7 @@ target-version = "py312"
		line-length = 88

		lint.select = ["E", "F", "I", "U", "N"]
		lint.ignore = ["E501"]
		lint.ignore = ["E501", "N818"]

		# xxx: we can import https://pypi.org/project/types-colorama/ here
		[[tool.mypy.overrides]]

nixos/lib/test-driver/src/test_driver/init.py

+4 −5

Original line number	Diff line number	Diff line
		@@ -3,7 +3,7 @@ import os
		import time
		from pathlib import Path

		import ptpython.repl
		import ptpython.ipython

		from test_driver.driver import Driver
		from test_driver.logger import (
		@@ -136,11 +136,10 @@ def main() -> None:
		if args.interactive:
		history_dir = os.getcwd()
		history_path = os.path.join(history_dir, ".nixos-test-history")
		ptpython.repl.embed(
		driver.test_symbols(),
		{},
		ptpython.ipython.embed(
		user_ns=driver.test_symbols(),
		history_filename=history_path,
		)
		) # type:ignore
		else:
		tic = time.time()
		driver.run_tests()

nixos/lib/test-driver/src/test_driver/driver.py

+48 −2

Original line number	Diff line number	Diff line
		import os
		import re
		import signal
		import sys
		import tempfile
		import threading
		import traceback
		from collections.abc import Callable, Iterator
		from contextlib import AbstractContextManager, contextmanager
		from pathlib import Path
		from typing import Any
		from unittest import TestCase

		from test_driver.errors import MachineError, RequestedAssertionFailed
		from test_driver.logger import AbstractLogger
		from test_driver.machine import Machine, NixStartScript, retry
		from test_driver.polling_condition import PollingCondition
		@@ -16,6 +20,18 @@ from test_driver.vlan import VLan
		SENTINEL = object()


		class AssertionTester(TestCase):
		"""
		Subclass of `unittest.TestCase` which is used in the
		`testScript` to perform assertions.

		It throws a custom exception whose parent class
		gets special treatment in the logs.
		"""

		failureException = RequestedAssertionFailed


		def get_tmp_dir() -> Path:
		"""Returns a temporary directory that is defined by TMPDIR, TEMP, TMP or CWD
		Raises an exception in case the retrieved temporary directory is not writeable
		@@ -115,7 +131,7 @@ class Driver:
		try:
		yield
		except Exception as e:
		self.logger.error(f'Test "{name}" failed with error: "{e}"')
		self.logger.log_test_error(f'Test "{name}" failed with error: "{e}"')
		raise e

		def test_symbols(self) -> dict[str, Any]:
		@@ -140,6 +156,7 @@ class Driver:
		serial_stdout_on=self.serial_stdout_on,
		polling_condition=self.polling_condition,
		Machine=Machine, # for typing
		t=AssertionTester(),
		)
		machine_symbols = {pythonize_name(m.name): m for m in self.machines}
		# If there's exactly one machine, make it available under the name
		@@ -163,7 +180,36 @@ class Driver:
		"""Run the test script"""
		with self.logger.nested("run the VM test script"):
		symbols = self.test_symbols() # call eagerly
		try:
		exec(self.tests, symbols, None)
		except MachineError:
		for line in traceback.format_exc().splitlines():
		self.logger.log_test_error(line)
		sys.exit(1)
		except RequestedAssertionFailed:
		exc_type, exc, tb = sys.exc_info()
		# We manually print the stack frames, keeping only the ones from the test script
		# (note: because the script is not a real file, the frame filename is `<string>`)
		filtered = [
		frame
		for frame in traceback.extract_tb(tb)
		if frame.filename == "<string>"
		]

		self.logger.log_test_error("Traceback (most recent call last):")

		code = self.tests.splitlines()
		for frame, line in zip(filtered, traceback.format_list(filtered)):
		self.logger.log_test_error(line.rstrip())
		if lineno := frame.lineno:
		self.logger.log_test_error(f" {code[lineno - 1].strip()}")

		self.logger.log_test_error("") # blank line for readability
		exc_prefix = exc_type.__name__ if exc_type is not None else "Error"
		for line in f"{exc_prefix}: {exc}".splitlines():
		self.logger.log_test_error(line)

		sys.exit(1)

		def run_tests(self) -> None:
		"""Run the test script (for non-interactive test runs)"""