Merge pull request #196251 from hercules-ci/testers-build-failure-and-equal-contents (29352751) · Commits · nix / nixpkgs

doc/builders/testers.chapter.md

+64 −0

Original line number	Diff line number	Diff line
		@@ -35,6 +35,70 @@ passthru.tests.version = testers.testVersion {
		};
		```

		## `testBuildFailure` {#tester-testBuildFailure}

		Make sure that a build does not succeed. This is useful for testing testers.

		This returns a derivation with an override on the builder, with the following effects:

		- Fail the build when the original builder succeeds
		- Move `$out` to `$out/result`, if it exists (assuming `out` is the default output)
		- Save the build log to `$out/testBuildFailure.log` (same)

		Example:

		```nix
		runCommand "example" {
		failed = testers.testBuildFailure (runCommand "fail" {} ''
		echo ok-ish >$out
		echo failing though
		exit 3
		'');
		} ''
		grep -F 'ok-ish' $failed/result
		grep -F 'failing though' $failed/testBuildFailure.log
		[[ 3 = $(cat $failed/testBuildFailure.exit) ]]
		touch $out
		'';
		```

		While `testBuildFailure` is designed to keep changes to the original builder's
		environment to a minimum, some small changes are inevitable.

		- The file `$TMPDIR/testBuildFailure.log` is present. It should not be deleted.
		- `stdout` and `stderr` are a pipe instead of a tty. This could be improved.
		- One or two extra processes are present in the sandbox during the original
		builder's execution.
		- The derivation and output hashes are different, but not unusual.
		- The derivation includes a dependency on `buildPackages.bash` and
		`expect-failure.sh`, which is built to include a transitive dependency on
		`buildPackages.coreutils` and possibly more. These are not added to `PATH`
		or any other environment variable, so they should be hard to observe.

		## `testEqualContents` {#tester-equalContents}

		Check that two paths have the same contents.

		Example:

		```nix
		testers.testEqualContents {
		assertion = "sed -e performs replacement";
		expected = writeText "expected" ''
		foo baz baz
		'';
		actual = runCommand "actual" {
		# not really necessary for a package that's in stdenv
		nativeBuildInputs = [ gnused ];
		base = writeText "base" ''
		foo bar baz
		'';
		} ''
		sed -e 's/bar/baz/g' $base >$out
		'';
		}
		```

		## `testEqualDerivation` {#tester-testEqualDerivation}

		Checks that two packages produce the exact same build instructions.

pkgs/build-support/testers/default.nix

+53 −1

Original line number	Diff line number	Diff line
		{ pkgs, lib, callPackage, runCommand, stdenv }:
		{ pkgs, buildPackages, lib, callPackage, runCommand, stdenv, substituteAll, }:
		# Documentation is in doc/builders/testers.chapter.md
		{
		# See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailure
		# or doc/builders/testers.chapter.md
		testBuildFailure = drv: drv.overrideAttrs (orig: {
		builder = buildPackages.bash;
		args = [
		(substituteAll { coreutils = buildPackages.coreutils; src = ./expect-failure.sh; })
		orig.realBuilder or stdenv.shell
		] ++ orig.args or ["-e" (orig.builder or ../../stdenv/generic/default-builder.sh)];
		});

		# See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualDerivation
		# or doc/builders/testers.chapter.md
		testEqualDerivation = callPackage ./test-equal-derivation.nix { };

		# See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualContents
		# or doc/builders/testers.chapter.md
		testEqualContents = {
		assertion,
		actual,
		expected,
		}: runCommand "equal-contents-${lib.strings.toLower assertion}" {
		inherit assertion actual expected;
		} ''
		echo "Checking:"
		echo "$assertion"
		if ! diff -U5 -r "$actual" "$expected" --color=always
		then
		echo
		echo 'Contents must be equal, but were not!'
		echo
		echo "+: expected, at $expected"
		echo "-: unexpected, at $actual"
		exit 1
		else
		find "$expected" -type f -executable > expected-executables \| sort
		find "$actual" -type f -executable > actual-executables \| sort
		if ! diff -U0 actual-executables expected-executables --color=always
		then
		echo
		echo "Contents must be equal, but some files' executable bits don't match"
		echo
		echo "+: make this file executable in the actual contents"
		echo "-: make this file non-executable in the actual contents"
		exit 1
		else
		echo "expected $expected and actual $actual match."
		echo 'OK'
		touch $out
		fi
		fi
		'';

		# See https://nixos.org/manual/nixpkgs/unstable/#tester-testVersion
		# or doc/builders/testers.chapter.md
		testVersion =
		{ package,
		command ? "${package.meta.mainProgram or package.pname or package.name} --version",

pkgs/build-support/testers/expect-failure.sh

0 → 100644

+62 −0

Original line number	Diff line number	Diff line
		# Run a builder, flip exit code, save log and fix outputs
		#
		# Sub-goals:
		# - Delegate to another original builder passed via args
		# - Save the build log to output for further checks
		# - Make the derivation succeed if the original builder fails
		# - Make the derivation fail if the original builder returns exit code 0
		#
		# Requirements:
		# This runs before, without and after stdenv. Do not modify the environment;
		# especially not before invoking the original builder. For example, use
		# "@" substitutions instead of PATH.
		# Do not export any variables.

		# Stricter bash
		set -eu

		# ------------------------
		# Run the original builder

		echo "testBuildFailure: Expecting non-zero exit from builder and args: ${*@Q}"

		("$@" 2>&1) \| @coreutils@/bin/tee $TMPDIR/testBuildFailure.log \
		\| while read ln; do
		echo "original builder: $ln"
		done

		r=${PIPESTATUS[0]}
		if [[ $r = 0 ]]; then
		echo "testBuildFailure: The builder did not fail, but a failure was expected!"
		exit 1
		fi
		echo "testBuildFailure: Original builder produced exit code: $r"

		# -----------------------------------------
		# Write the build log to the default output

		outs=( $outputs )
		defOut=${outs[0]}
		defOutPath=${!defOut}

		if [[ ! -d $defOutPath ]]; then
		if [[ -e $defOutPath ]]; then
		@coreutils@/bin/mv $defOutPath $TMPDIR/out-node
		@coreutils@/bin/mkdir $defOutPath
		@coreutils@/bin/mv $TMPDIR/out-node $defOutPath/result
		fi
		fi

		@coreutils@/bin/mkdir -p $defOutPath
		@coreutils@/bin/mv $TMPDIR/testBuildFailure.log $defOutPath/testBuildFailure.log
		echo $r >$defOutPath/testBuildFailure.exit

		# ------------------------------------------------------
		# Put empty directories in place for any missing outputs

		for outputName in ${outputs:-out}; do
		outputPath="${!outputName}"
		if [[ ! -e "${outputPath}" ]]; then
		@coreutils@/bin/mkdir "${outputPath}";
		fi
		done

pkgs/build-support/testers/test/default.nix

+164 −1

Original line number	Diff line number	Diff line
		{ testers, lib, pkgs, ... }:
		{ testers, lib, pkgs, hello, runCommand, ... }:
		let
		pkgs-with-overlay = pkgs.extend(final: prev: {
		proof-of-overlay-hello = prev.hello;
		@@ -24,4 +24,167 @@ lib.recurseIntoAttrs {
		machine.succeed("hello \| figlet >/dev/console")
		'';
		});

		testBuildFailure = lib.recurseIntoAttrs {
		happy = runCommand "testBuildFailure-happy" {
		failed = testers.testBuildFailure (runCommand "fail" {} ''
		echo ok-ish >$out
		echo failing though
		echo also stderr 1>&2
		exit 3
		'');
		} ''
		grep -F 'failing though' $failed/testBuildFailure.log
		grep -F 'also stderr' $failed/testBuildFailure.log
		grep -F 'ok-ish' $failed/result
		[[ 3 = $(cat $failed/testBuildFailure.exit) ]]
		touch $out
		'';

		helloDoesNotFail = runCommand "testBuildFailure-helloDoesNotFail" {
		failed = testers.testBuildFailure (testers.testBuildFailure hello);

		# Add hello itself as a prerequisite, so we don't try to run this test if
		# there's an actual failure in hello.
		inherit hello;
		} ''
		echo "Checking $failed/testBuildFailure.log"
		grep -F 'testBuildFailure: The builder did not fail, but a failure was expected' $failed/testBuildFailure.log
		[[ 1 = $(cat $failed/testBuildFailure.exit) ]]
		touch $out
		'';

		multiOutput = runCommand "testBuildFailure-multiOutput" {
		failed = testers.testBuildFailure (runCommand "fail" {
		# dev will be the default output
		outputs = ["dev" "doc" "out"];
		} ''
		echo i am failing
		exit 1
		'');
		} ''
		grep -F 'i am failing' $failed/testBuildFailure.log >/dev/null
		[[ 1 = $(cat $failed/testBuildFailure.exit) ]]

		# Checking our note that dev is the default output
		echo $failed/_ \| grep -- '-dev/_' >/dev/null
		echo 'All good.'
		touch $out
		'';
		};

		testEqualContents = lib.recurseIntoAttrs {
		happy = testers.testEqualContents {
		assertion = "The same directory contents at different paths are recognized as equal";
		expected = runCommand "expected" {} ''
		mkdir -p $out/c
		echo a >$out/a
		echo b >$out/b
		echo d >$out/c/d
		'';
		actual = runCommand "actual" {} ''
		mkdir -p $out/c
		echo a >$out/a
		echo b >$out/b
		echo d >$out/c/d
		'';
		};

		unequalExe =
		runCommand "testEqualContents-unequalExe" {
		log = testers.testBuildFailure (testers.testEqualContents {
		assertion = "The same directory contents at different paths are recognized as equal";
		expected = runCommand "expected" {} ''
		mkdir -p $out/c
		echo a >$out/a
		chmod a+x $out/a
		echo b >$out/b
		echo d >$out/c/d
		'';
		actual = runCommand "actual" {} ''
		mkdir -p $out/c
		echo a >$out/a
		echo b >$out/b
		chmod a+x $out/b
		echo d >$out/c/d
		'';
		});
		} ''
		(
		set -x
		grep -F -- "executable bits don't match" $log/testBuildFailure.log
		grep -E -- '+.*-actual/a' $log/testBuildFailure.log
		grep -E -- '-.*-actual/b' $log/testBuildFailure.log
		grep -F -- "--- actual-executables" $log/testBuildFailure.log
		grep -F -- "+++ expected-executables" $log/testBuildFailure.log
		) \|\| {
		echo "Test failed: could not find pattern in build log $log"
		exit 1
		}
		echo 'All good.'
		touch $out
		'';

		fileDiff =
		runCommand "testEqualContents-fileDiff" {
		log = testers.testBuildFailure (testers.testEqualContents {
		assertion = "The same directory contents at different paths are recognized as equal";
		expected = runCommand "expected" {} ''
		mkdir -p $out/c
		echo a >$out/a
		echo b >$out/b
		echo d >$out/c/d
		'';
		actual = runCommand "actual" {} ''
		mkdir -p $out/c
		echo a >$out/a
		echo B >$out/b
		echo d >$out/c/d
		'';
		});
		} ''
		(
		set -x
		grep -F -- "Contents must be equal but were not" $log/testBuildFailure.log
		grep -E -- '+++ .*-actual/b' $log/testBuildFailure.log
		grep -E -- '--- .*-actual/b' $log/testBuildFailure.log
		grep -F -- "-B" $log/testBuildFailure.log
		grep -F -- "+b" $log/testBuildFailure.log
		) \|\| {
		echo "Test failed: could not find pattern in build log $log"
		exit 1
		}
		echo 'All good.'
		touch $out
		'';

		fileMissing =
		runCommand "testEqualContents-fileMissing" {
		log = testers.testBuildFailure (testers.testEqualContents {
		assertion = "The same directory contents at different paths are recognized as equal";
		expected = runCommand "expected" {} ''
		mkdir -p $out/c
		echo a >$out/a
		echo b >$out/b
		echo d >$out/c/d
		'';
		actual = runCommand "actual" {} ''
		mkdir -p $out/c
		echo a >$out/a
		echo d >$out/c/d
		'';
		});
		} ''
		(
		set -x
		grep -F -- "Contents must be equal but were not" $log/testBuildFailure.log
		grep -E -- 'Only in .*-expected: b' $log/testBuildFailure.log
		) \|\| {
		echo "Test failed: could not find pattern in build log $log"
		exit 1
		}
		echo 'All good.'
		touch $out
		'';
		};
		}