Merge pull request #280592 from 9999years/write-shell-application-options

## `writeShellApplication` {#trivial-builder-writeShellApplication}

This can be used to easily produce a shell script that has some dependencies (`runtimeInputs`). It automatically sets the `PATH` of the script to contain all of the listed inputs, sets some sanity shellopts (`errexit`, `nounset`, `pipefail`), and checks the resulting script with [`shellcheck`](https://github.com/koalaman/shellcheck).

`writeShellApplication` is similar to `writeShellScriptBin` and `writeScriptBin` but supports runtime dependencies with `runtimeInputs`.

Writes an executable shell script to `/nix/store/<store path>/bin/<name>` and checks its syntax with [`shellcheck`](https://github.com/koalaman/shellcheck) and the `bash`'s `-n` option.

Some basic Bash options are set by default (`errexit`, `nounset`, and `pipefail`), but can be overridden with `bashOptions`.

For example, look at the following code:

Extra arguments may be passed to `stdenv.mkDerivation` by setting `derivationArgs`; note that variables set in this manner will be set when the shell script is _built,_ not when it's run.

Runtime environment variables can be set with the `runtimeEnv` argument.

For example, the following shell application can refer to `curl` directly, rather than needing to write `${curl}/bin/curl`:

```nix

writeShellApplication {

}

```

Unlike with normal `writeShellScriptBin`, there is no need to manually write out `${curl}/bin/curl`, setting the PATH

was handled by `writeShellApplication`. Moreover, the script is being checked with `shellcheck` for more strict

validation.

## `symlinkJoin` {#trivial-builder-symlinkJoin}

This can be used to put many derivations into the same directory structure. It works by creating a new derivation and adding symlinks to each of the paths listed. It expects two arguments, `name`, and `paths`. `name` is the name used in the Nix store path for the created derivation. `paths` is a list of paths that will be symlinked. These paths can be to Nix store derivations or any other subdirectory contained within.

    , meta ? { }

    , allowSubstitutes ? false

    , preferLocalBuild ? true

    , derivationArgs ? { } # Extra arguments to pass to `stdenv.mkDerivation`

    }:

    let

      matches = builtins.match "/bin/([^/]+)" destination;

    in

    runCommand name

      {

      ({

        inherit text executable checkPhase allowSubstitutes preferLocalBuild;

        passAsFile = [ "text" ];

        passAsFile = [ "text" ]

          ++ derivationArgs.passAsFile or [ ];

        meta = lib.optionalAttrs (executable && matches != null)

          {

            mainProgram = lib.head matches;

          } // meta;

      }

          } // meta // derivationArgs.meta or {};

      } // removeAttrs derivationArgs [ "passAsFile" "meta" ])

      ''

        target=$out${lib.escapeShellArg destination}

        mkdir -p "$(dirname "$target")"

      meta.mainProgram = name;

    };

  # See doc/build-helpers/trivial-build-helpers.chapter.md

  # or https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-text-writing

  writeShellApplication =

    {

      /*

    Similar to writeShellScriptBin and writeScriptBin.

    Writes an executable Shell script to /nix/store/<store path>/bin/<name> and

    checks its syntax with shellcheck and the shell's -n option.

    Individual checks can be foregone by putting them in the excludeShellChecks

    list, e.g. [ "SC2016" ].

    Automatically includes sane set of shellopts (errexit, nounset, pipefail)

    and handles creation of PATH based on runtimeInputs

    Note that the checkPhase uses stdenv.shell for the test run of the script,

    while the generated shebang uses runtimeShell. If, for whatever reason,

    those were to mismatch you might lose fidelity in the default checks.

         The name of the script to write.

    Example:

         Type: String

       */

      name,

      /*

         The shell script's text, not including a shebang.

    Writes my-file to /nix/store/<store path>/bin/my-file and makes executable.

         Type: String

       */

      text,

      /*

         Inputs to add to the shell script's `$PATH` at runtime.

         Type: [String|Derivation]

       */

      runtimeInputs ? [ ],

      /*

         Extra environment variables to set at runtime.

    writeShellApplication {

      name = "my-file";

      runtimeInputs = [ curl w3m ];

      text = ''

        curl -s 'https://nixos.org' | w3m -dump -T text/html

       '';

    }

         Type: AttrSet

       */

      runtimeEnv ? null,

      /*

         `stdenv.mkDerivation`'s `meta` argument.

         Type: AttrSet

       */

  writeShellApplication =

    { name

    , text

    , runtimeInputs ? [ ]

    , meta ? { }

    , checkPhase ? null

    , excludeShellChecks ? [ ]

      meta ? { },

      /*

         The `checkPhase` to run. Defaults to `shellcheck` on supported

         platforms and `bash -n`.

         The script path will be given as `$target` in the `checkPhase`.

         Type: String

       */

      checkPhase ? null,

      /*

         Checks to exclude when running `shellcheck`, e.g. `[ "SC2016" ]`.

         See <https://www.shellcheck.net/wiki/> for a list of checks.

         Type: [String]

       */

      excludeShellChecks ? [ ],

      /*

         Bash options to activate with `set -o` at the start of the script.

         Defaults to `[ "errexit" "nounset" "pipefail" ]`.

         Type: [String]

       */

      bashOptions ? [ "errexit" "nounset" "pipefail" ],

      /* Extra arguments to pass to `stdenv.mkDerivation`.

         :::{.caution}

         Certain derivation attributes are used internally,

         overriding those could cause problems.

         :::

         Type: AttrSet

       */

      derivationArgs ? { },

    }:

    writeTextFile {

      inherit name meta;

      inherit name meta derivationArgs;

      executable = true;

      destination = "/bin/${name}";

      allowSubstitutes = true;

      preferLocalBuild = false;

      text = ''

        #!${runtimeShell}

        set -o errexit

        set -o nounset

        set -o pipefail

      '' + lib.optionalString (runtimeInputs != [ ]) ''

        ${lib.concatMapStringsSep "\n" (option: "set -o ${option}") bashOptions}

      '' + lib.optionalString (runtimeEnv != null)

        (lib.concatStrings

          (lib.mapAttrsToList

            (name: value: ''

              ${lib.toShellVar name value}

              export ${name}

            '')

            runtimeEnv))

      + lib.optionalString (runtimeInputs != [ ]) ''

        export PATH="${lib.makeBinPath runtimeInputs}:$PATH"

      '' + ''

/*

  Run with:

# Run with:

# nix-build -A tests.trivial-builders.writeShellApplication

{ writeShellApplication

, writeTextFile

, runCommand

, lib

, linkFarm

, diffutils

, hello

}:

let

  checkShellApplication = args@{name, expected, ...}:

    let

      writeShellApplicationArgs = builtins.removeAttrs args ["expected"];

      script = writeShellApplication writeShellApplicationArgs;

      executable = lib.getExe script;

      expected' = writeTextFile {

        name = "${name}-expected";

        text = expected;

      };

      actual = "${name}-actual";

    in

    runCommand name { } ''

      echo "Running test executable ${name}"

      ${executable} > ${actual}

      echo "Got output from test executable:"

      cat ${actual}

      echo "Checking test output against expected output:"

      ${diffutils}/bin/diff --color --unified ${expected'} ${actual}

      touch $out

    '';

in

linkFarm "writeShellApplication-tests" {

  test-meta =

    let

      script = writeShellApplication {

        name = "test-meta";

        text = "";

        meta.description = "A test for the `writeShellApplication` `meta` argument.";

      };

    in

    assert script.meta.mainProgram == "test-meta";

    assert script.meta.description == "A test for the `writeShellApplication` `meta` argument.";

    script;

      cd nixpkgs

      nix-build -A tests.trivial-builders.writeShellApplication

*/

  test-runtime-inputs =

    checkShellApplication {

      name = "test-runtime-inputs";

      text = ''

        hello

      '';

      runtimeInputs = [ hello ];

      expected = "Hello, world!\n";

    };

{ lib, writeShellApplication, runCommand }:

let

  pkg = writeShellApplication {

    name = "test-script";

    excludeShellChecks = [ "SC2016" ];

  test-runtime-env =

    checkShellApplication {

      name = "test-runtime-env";

      runtimeEnv = {

        MY_COOL_ENV_VAR = "my-cool-env-value";

        MY_OTHER_COOL_ENV_VAR = "my-other-cool-env-value";

        # Check that we can serialize a bunch of different types:

        BOOL = true;

        INT = 1;

        LIST = [1 2 3];

        MAP = {

          a = "a";

          b = "b";

        };

      };

      text = ''

      echo -e '#!/usr/bin/env bash\n' \

       'echo "$SHELL"' > /tmp/something.sh  # this line would normally

                                            # ...cause shellcheck error

        echo "$MY_COOL_ENV_VAR"

        echo "$MY_OTHER_COOL_ENV_VAR"

      '';

      expected = ''

        my-cool-env-value

        my-other-cool-env-value

      '';

    };

in

  assert pkg.meta.mainProgram == "test-script";

  runCommand "test-writeShellApplication" { } ''

    echo Testing if writeShellApplication builds without shellcheck error...

  test-check-phase =

    checkShellApplication {

      name = "test-check-phase";

      text = "";

      checkPhase = ''

        echo "echo -n hello" > $target

      '';

      expected = "hello";

    };

    target=${lib.getExe pkg}

  test-argument-forwarding =

    checkShellApplication {

      name = "test-argument-forwarding";

      text = "";

      derivationArgs.MY_BUILD_TIME_VARIABLE = "puppy";

      derivationArgs.postCheck = ''

        if [[ "$MY_BUILD_TIME_VARIABLE" != puppy ]]; then

          echo "\$MY_BUILD_TIME_VARIABLE is not set to 'puppy'!"

          exit 1

        fi

      '';

      meta.description = "A test checking that `writeShellApplication` forwards extra arguments to `stdenv.mkDerivation`.";

      expected = "";

    };

    touch $out

  ''

  test-exclude-shell-checks = writeShellApplication {

    name = "test-exclude-shell-checks";

    excludeShellChecks = [ "SC2016" ];

    text = ''

      # Triggers SC2016: Expressions don't expand in single quotes, use double

      # quotes for that.

      echo '$SHELL'

    '';

  };

  test-bash-options-pipefail = checkShellApplication {

    name = "test-bash-options-pipefail";

    text = ''

      touch my-test-file

      echo puppy | grep doggy | sed 's/doggy/puppy/g'

      #            ^^^^^^^^^^ This will fail.

      true

    '';

    # Don't use `pipefail`:

    bashOptions = ["errexit" "nounset"];

    expected = "";

  };

  test-bash-options-nounset = checkShellApplication {

    name = "test-bash-options-nounset";

    text = ''

      echo -n "$someUndefinedVariable"

    '';

    # Don't use `nounset`:

    bashOptions = [];

    # Don't warn about the undefined variable at build time:

    excludeShellChecks = [ "SC2154" ];

    expected = "";

  };

}