testers.testBuildFailure: init

};

```

## `testBuildFailure` {#tester-testBuildFailure}

Make sure that a build does not succeed. This is useful for testing testers.

This returns a derivation with an override on the builder, with the following effects:

 - Fail the build when the original builder succeeds

 - Move `$out` to `$out/result`, if it exists (assuming `out` is the default output)

 - Save the build log to `$out/testBuildFailure.log` (same)

Example:

```nix

runCommand "example" {

  failed = testers.testBuildFailure (runCommand "fail" {} ''

    echo ok-ish >$out

    echo failing though

    exit 3

  '');

} ''

  grep -F 'ok-ish' $failed/result

  grep -F 'failing though' $failed/testBuildFailure.log

  [[ 3 = $(cat $failed/testBuildFailure.exit) ]]

  touch $out

'';

```

While `testBuildFailure` is designed to keep changes to the original builder's 

environment to a minimum, some small changes are inevitable.

 - The file `$TMPDIR/testBuildFailure.log` is present. It should not be deleted.

 - `stdout` and `stderr` are a pipe instead of a tty. This could be improved.

 - One or two extra processes are present in the sandbox during the original

   builder's execution.

 - The derivation and output hashes are different, but not unusual.

 - The derivation includes a dependency on `buildPackages.bash` and

   `expect-failure.sh`, which is built to include a transitive dependency on

   `buildPackages.coreutils` and possibly more. These are not added to `PATH`

   or any other environment variable, so they should be hard to observe.

## `testEqualDerivation` {#tester-testEqualDerivation}

Checks that two packages produce the exact same build instructions.

{ pkgs, lib, callPackage, runCommand, stdenv }:

{ pkgs, buildPackages, lib, callPackage, runCommand, stdenv, substituteAll, }:

# Documentation is in doc/builders/testers.chapter.md

{

  # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailure

  # or doc/builders/testers.chapter.md

  testBuildFailure = drv: drv.overrideAttrs (orig: {

    builder = buildPackages.bash;

    args = [

      (substituteAll { coreutils = buildPackages.coreutils; src = ./expect-failure.sh; })

      orig.realBuilder or stdenv.shell

    ] ++ orig.args or ["-e" (orig.builder or ../../stdenv/generic/default-builder.sh)];

  });

  testEqualDerivation = callPackage ./test-equal-derivation.nix { };

  testVersion =

# Run a builder, flip exit code, save log and fix outputs

#

# Sub-goals:

# - Delegate to another original builder passed via args

# - Save the build log to output for further checks

# - Make the derivation succeed if the original builder fails

# - Make the derivation fail if the original builder returns exit code 0

#

# Requirements:

# This runs before, without and after stdenv. Do not modify the environment;

# especially not before invoking the original builder. For example, use

# "@" substitutions instead of PATH.

# Do not export any variables.

# Stricter bash

set -eu

# ------------------------

# Run the original builder

echo "testBuildFailure: Expecting non-zero exit from builder and args: ${*@Q}"

("$@" 2>&1) | @coreutils@/bin/tee $TMPDIR/testBuildFailure.log \

  | while read ln; do

    echo "original builder: $ln"

  done

r=${PIPESTATUS[0]}

if [[ $r = 0 ]]; then

  echo "testBuildFailure: The builder did not fail, but a failure was expected!"

  exit 1

fi

echo "testBuildFailure: Original builder produced exit code: $r"

# -----------------------------------------

# Write the build log to the default output

outs=( $outputs )

defOut=${outs[0]}

defOutPath=${!defOut}

if [[ ! -d $defOutPath ]]; then

  if [[ -e $defOutPath ]]; then

    @coreutils@/bin/mv $defOutPath $TMPDIR/out-node

    @coreutils@/bin/mkdir $defOutPath

    @coreutils@/bin/mv $TMPDIR/out-node $defOutPath/result

  fi

fi

@coreutils@/bin/mkdir -p $defOutPath

@coreutils@/bin/mv $TMPDIR/testBuildFailure.log $defOutPath/testBuildFailure.log

echo $r >$defOutPath/testBuildFailure.exit

# ------------------------------------------------------

# Put empty directories in place for any missing outputs

for outputName in ${outputs:-out}; do

  outputPath="${!outputName}"

  if [[ ! -e "${outputPath}" ]]; then

    @coreutils@/bin/mkdir "${outputPath}";

  fi

done

{ testers, lib, pkgs, ... }:

{ testers, lib, pkgs, hello, runCommand, ... }:

let

  pkgs-with-overlay = pkgs.extend(final: prev: {

    proof-of-overlay-hello = prev.hello;

      machine.succeed("hello | figlet >/dev/console")

    '';

  });

  testBuildFailure = lib.recurseIntoAttrs {

    happy = runCommand "testBuildFailure-happy" {

      failed = testers.testBuildFailure (runCommand "fail" {} ''

        echo ok-ish >$out

        echo failing though

        echo also stderr 1>&2

        exit 3

      '');

    } ''

      grep -F 'failing though' $failed/testBuildFailure.log

      grep -F 'also stderr' $failed/testBuildFailure.log

      grep -F 'ok-ish' $failed/result

      [[ 3 = $(cat $failed/testBuildFailure.exit) ]]

      touch $out

    '';

    helloDoesNotFail = runCommand "testBuildFailure-helloDoesNotFail" {

      failed = testers.testBuildFailure (testers.testBuildFailure hello);

      # Add hello itself as a prerequisite, so we don't try to run this test if

      # there's an actual failure in hello.

      inherit hello;

    } ''

      echo "Checking $failed/testBuildFailure.log"

      grep -F 'testBuildFailure: The builder did not fail, but a failure was expected' $failed/testBuildFailure.log

      [[ 1 = $(cat $failed/testBuildFailure.exit) ]]

      touch $out

    '';

    multiOutput = runCommand "testBuildFailure-multiOutput" {

      failed = testers.testBuildFailure (runCommand "fail" {

        # dev will be the default output

        outputs = ["dev" "doc" "out"];

      } ''

        echo i am failing

        exit 1

      '');

    } ''

      grep -F 'i am failing' $failed/testBuildFailure.log >/dev/null

      [[ 1 = $(cat $failed/testBuildFailure.exit) ]]

      # Checking our note that dev is the default output

      echo $failed/_ | grep -- '-dev/_' >/dev/null

      touch $out

    '';

  };

}