Merge pull request #264891 from tweag/fileset.gitTracked

    _printFileset

    _intersection

    _difference

    _mirrorStorePath

    _fetchGitSubmodulesMinver

    ;

  inherit (builtins)

    isBool

    isList

    isPath

    pathExists

    seq

    typeOf

    nixVersion

    ;

  inherit (lib.lists)

  inherit (lib.strings)

    isStringLike

    versionOlder

    ;

  inherit (lib.filesystem)

  inherit (lib.trivial)

    isFunction

    pipe

    inPureEvalMode

    ;

in {

      # We could also return the original fileset argument here,

      # but that would then duplicate work for consumers of the fileset, because then they have to coerce it again

      actualFileset;

  /*

    Create a file set containing all [Git-tracked files](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository) in a repository.

    This function behaves like [`gitTrackedWith { }`](#function-library-lib.fileset.gitTrackedWith) - using the defaults.

    Type:

      gitTracked :: Path -> FileSet

    Example:

      # Include all files tracked by the Git repository in the current directory

      gitTracked ./.

      # Include only files tracked by the Git repository in the parent directory

      # that are also in the current directory

      intersection ./. (gitTracked ../.)

  */

  gitTracked =

    /*

      The [path](https://nixos.org/manual/nix/stable/language/values#type-path) to the working directory of a local Git repository.

      This directory must contain a `.git` file or subdirectory.

    */

    path:

    # See the gitTrackedWith implementation for more explanatory comments

    let

      fetchResult = builtins.fetchGit path;

    in

    if inPureEvalMode then

      throw "lib.fileset.gitTracked: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292."

    else if ! isPath path then

      throw "lib.fileset.gitTracked: Expected the argument to be a path, but it's a ${typeOf path} instead."

    else if ! pathExists (path + "/.git") then

      throw "lib.fileset.gitTracked: Expected the argument (${toString path}) to point to a local working tree of a Git repository, but it's not."

    else

      _mirrorStorePath path fetchResult.outPath;

  /*

    Create a file set containing all [Git-tracked files](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository) in a repository.

    The first argument allows configuration with an attribute set,

    while the second argument is the path to the Git working tree.

    If you don't need the configuration,

    you can use [`gitTracked`](#function-library-lib.fileset.gitTracked) instead.

    This is equivalent to the result of [`unions`](#function-library-lib.fileset.unions) on all files returned by [`git ls-files`](https://git-scm.com/docs/git-ls-files)

    (which uses [`--cached`](https://git-scm.com/docs/git-ls-files#Documentation/git-ls-files.txt--c) by default).

    :::{.warning}

    Currently this function is based on [`builtins.fetchGit`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-fetchGit)

    As such, this function causes all Git-tracked files to be unnecessarily added to the Nix store,

    without being re-usable by [`toSource`](#function-library-lib.fileset.toSource).

    This may change in the future.

    :::

    Type:

      gitTrackedWith :: { recurseSubmodules :: Bool ? false } -> Path -> FileSet

    Example:

      # Include all files tracked by the Git repository in the current directory

      # and any submodules under it

      gitTracked { recurseSubmodules = true; } ./.

  */

  gitTrackedWith =

    {

      /*

        (optional, default: `false`) Whether to recurse into [Git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) to also include their tracked files.

        If `true`, this is equivalent to passing the [--recurse-submodules](https://git-scm.com/docs/git-ls-files#Documentation/git-ls-files.txt---recurse-submodules) flag to `git ls-files`.

      */

      recurseSubmodules ? false,

    }:

    /*

      The [path](https://nixos.org/manual/nix/stable/language/values#type-path) to the working directory of a local Git repository.

      This directory must contain a `.git` file or subdirectory.

    */

    path:

    let

      # This imports the files unnecessarily, which currently can't be avoided

      # because `builtins.fetchGit` is the only function exposing which files are tracked by Git.

      # With the [lazy trees PR](https://github.com/NixOS/nix/pull/6530),

      # the unnecessarily import could be avoided.

      # However a simpler alternative still would be [a builtins.gitLsFiles](https://github.com/NixOS/nix/issues/2944).

      fetchResult = builtins.fetchGit {

        url = path;

        # This is the only `fetchGit` parameter that makes sense in this context.

        # We can't just pass `submodules = recurseSubmodules` here because

        # this would fail for Nix versions that don't support `submodules`.

        ${if recurseSubmodules then "submodules" else null} = true;

      };

    in

    if inPureEvalMode then

      throw "lib.fileset.gitTrackedWith: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292."

    else if ! isBool recurseSubmodules then

      throw "lib.fileset.gitTrackedWith: Expected the attribute `recurseSubmodules` of the first argument to be a boolean, but it's a ${typeOf recurseSubmodules} instead."

    else if recurseSubmodules && versionOlder nixVersion _fetchGitSubmodulesMinver then

      throw "lib.fileset.gitTrackedWith: Setting the attribute `recurseSubmodules` to `true` is only supported for Nix version ${_fetchGitSubmodulesMinver} and after, but Nix version ${nixVersion} is used."

    else if ! isPath path then

      throw "lib.fileset.gitTrackedWith: Expected the second argument to be a path, but it's a ${typeOf path} instead."

    # We can identify local working directories by checking for .git,

    # see https://git-scm.com/docs/gitrepository-layout#_description.

    # Note that `builtins.fetchGit` _does_ work for bare repositories (where there's no `.git`),

    # even though `git ls-files` wouldn't return any files in that case.

    else if ! pathExists (path + "/.git") then

      throw "lib.fileset.gitTrackedWith: Expected the second argument (${toString path}) to point to a local working tree of a Git repository, but it's not."

    else

      _mirrorStorePath path fetchResult.outPath;

}

        ${baseNameOf root} =

          fromFile (baseNameOf root) rootType;

      };

  # Support for `builtins.fetchGit` with `submodules = true` was introduced in 2.4

  # https://github.com/NixOS/nix/commit/55cefd41d63368d4286568e2956afd535cb44018

  _fetchGitSubmodulesMinver = "2.4";

  # Mirrors the contents of a Nix store path relative to a local path as a file set.

  # Some notes:

  # - The store path is read at evaluation time.

  # - The store path must not include files that don't exist in the respective local path.

  #

  # Type: Path -> String -> FileSet

  _mirrorStorePath = localPath: storePath:

    let

      recurse = focusedStorePath:

        mapAttrs (name: type:

          if type == "directory" then

            recurse (focusedStorePath + "/${name}")

          else

            type

        ) (builtins.readDir focusedStorePath);

    in

    _create localPath

      (recurse storePath);

}

    cut -d \" -f2

}

prefixExpression='let

  lib = import <nixpkgs/lib>;

prefixExpression() {

    echo 'let

      lib =

        (import <nixpkgs/lib>)

    '

    if [[ "${1:-}" == "--simulate-pure-eval" ]]; then

        echo '

        .extend (final: prev: {

          trivial = prev.trivial // {

            inPureEvalMode = true;

          };

        })'

    fi

    echo '

      ;

      internal = import <nixpkgs/lib/fileset/internal.nix> {

        inherit lib;

      };

    with lib;

    with internal;

    with lib.fileset;'

}

# Check that two nix expression successfully evaluate to the same value.

# The expressions have `lib.fileset` in scope.

    local actualExpr=$1

    local expectedExpr=$2

    if actualResult=$(nix-instantiate --eval --strict --show-trace 2>"$tmp"/actualStderr \

        --expr "$prefixExpression ($actualExpr)"); then

        --expr "$(prefixExpression) ($actualExpr)"); then

        actualExitCode=$?

    else

        actualExitCode=$?

    actualStderr=$(< "$tmp"/actualStderr)

    if expectedResult=$(nix-instantiate --eval --strict --show-trace 2>"$tmp"/expectedStderr \

        --expr "$prefixExpression ($expectedExpr)"); then

        --expr "$(prefixExpression) ($expectedExpr)"); then

        expectedExitCode=$?

    else

        expectedExitCode=$?

# Usage: expectStorePath NIX

expectStorePath() {

    local expr=$1

    if ! result=$(nix-instantiate --eval --strict --json --read-write-mode --show-trace \

        --expr "$prefixExpression ($expr)"); then

    if ! result=$(nix-instantiate --eval --strict --json --read-write-mode --show-trace 2>"$tmp"/stderr \

        --expr "$(prefixExpression) ($expr)"); then

        cat "$tmp/stderr" >&2

        die "$expr failed to evaluate, but it was expected to succeed"

    fi

    # This is safe because we assume to get back a store path in a string

# The expression has `lib.fileset` in scope.

# Usage: expectFailure NIX REGEX

expectFailure() {

    if [[ "$1" == "--simulate-pure-eval" ]]; then

        maybePure="--simulate-pure-eval"

        shift

    else

        maybePure=""

    fi

    local expr=$1

    local expectedErrorRegex=$2

    if result=$(nix-instantiate --eval --strict --read-write-mode --show-trace 2>"$tmp/stderr" \

        --expr "$prefixExpression $expr"); then

        --expr "$(prefixExpression $maybePure) $expr"); then

        die "$expr evaluated successfully to $result, but it was expected to fail"

    fi

    stderr=$(<"$tmp/stderr")

    local expectedTrace=$2

    nix-instantiate --eval --show-trace >/dev/null 2>"$tmp"/stderrTrace \

        --expr "$prefixExpression trace ($expr)" || true

        --expr "$(prefixExpression) trace ($expr)" || true

    actualTrace=$(sed -n 's/^trace: //p' "$tmp/stderrTrace")

    nix-instantiate --eval --show-trace >/dev/null 2>"$tmp"/stderrTraceVal \

        --expr "$prefixExpression traceVal ($expr)" || true

        --expr "$(prefixExpression) traceVal ($expr)" || true

    actualTraceVal=$(sed -n 's/^trace: //p' "$tmp/stderrTraceVal")

}))) null' 'trace ./a/b null'

rm -rf -- *

## lib.fileset.gitTracked/gitTrackedWith

# The first/second argument has to be a path

expectFailure 'gitTracked null' 'lib.fileset.gitTracked: Expected the argument to be a path, but it'\''s a null instead.'

expectFailure 'gitTrackedWith {} null' 'lib.fileset.gitTrackedWith: Expected the second argument to be a path, but it'\''s a null instead.'

# The path has to contain a .git directory

expectFailure 'gitTracked ./.' 'lib.fileset.gitTracked: Expected the argument \('"$work"'\) to point to a local working tree of a Git repository, but it'\''s not.'

expectFailure 'gitTrackedWith {} ./.' 'lib.fileset.gitTrackedWith: Expected the second argument \('"$work"'\) to point to a local working tree of a Git repository, but it'\''s not.'

# recurseSubmodules has to be a boolean

expectFailure 'gitTrackedWith { recurseSubmodules = null; } ./.' 'lib.fileset.gitTrackedWith: Expected the attribute `recurseSubmodules` of the first argument to be a boolean, but it'\''s a null instead.'

# recurseSubmodules = true is not supported on all Nix versions

if [[ "$(nix-instantiate --eval --expr "$(prefixExpression) (versionAtLeast builtins.nixVersion _fetchGitSubmodulesMinver)")" == true ]]; then

    fetchGitSupportsSubmodules=1

else

    fetchGitSupportsSubmodules=

    expectFailure 'gitTrackedWith { recurseSubmodules = true; } ./.' 'lib.fileset.gitTrackedWith: Setting the attribute `recurseSubmodules` to `true` is only supported for Nix version 2.4 and after, but Nix version [0-9.]+ is used.'

fi

# Checks that `gitTrackedWith` contains the same files as `git ls-files`

# for the current working directory.

# If --recurse-submodules is passed, the flag is passed through to `git ls-files`

# and as `recurseSubmodules` to `gitTrackedWith`

checkGitTrackedWith() {

    if [[ "${1:-}" == "--recurse-submodules" ]]; then

        gitLsFlags="--recurse-submodules"

        gitTrackedArg="{ recurseSubmodules = true; }"

    else

        gitLsFlags=""

        gitTrackedArg="{ }"

    fi

    # All files listed by `git ls-files`

    expectedFiles=()

    while IFS= read -r -d $'\0' file; do

        # If there are submodules but --recurse-submodules isn't passed,

        # `git ls-files` lists them as empty directories,

        # we need to filter that out since we only want to check/count files

        if [[ -f "$file" ]]; then

            expectedFiles+=("$file")

        fi

    done < <(git ls-files -z $gitLsFlags)

    storePath=$(expectStorePath 'toSource { root = ./.; fileset = gitTrackedWith '"$gitTrackedArg"' ./.; }')

    # Check that each expected file is also in the store path with the same content

    for expectedFile in "${expectedFiles[@]}"; do

        if [[ ! -e "$storePath"/"$expectedFile" ]]; then

            die "Expected file $expectedFile to exist in $storePath, but it doesn't.\nGit status:\n$(git status)\nStore path contents:\n$(find "$storePath")"

        fi

        if ! diff "$expectedFile" "$storePath"/"$expectedFile"; then

            die "Expected file $expectedFile to have the same contents as in $storePath, but it doesn't.\nGit status:\n$(git status)\nStore path contents:\n$(find "$storePath")"

        fi

    done

    # This is a cheap way to verify the inverse: That all files in the store path are also expected

    # We just count the number of files in both and verify they're the same

    actualFileCount=$(find "$storePath" -type f -printf . | wc -c)

    if [[ "${#expectedFiles[@]}" != "$actualFileCount" ]]; then

        die "Expected ${#expectedFiles[@]} files in $storePath, but got $actualFileCount.\nGit status:\n$(git status)\nStore path contents:\n$(find "$storePath")"

    fi

}

# Runs checkGitTrackedWith with and without --recurse-submodules

# Allows testing both variants together

checkGitTracked() {

    checkGitTrackedWith

    if [[ -n "$fetchGitSupportsSubmodules" ]]; then

        checkGitTrackedWith --recurse-submodules

    fi

}

createGitRepo() {

    git init -q "$1"

    # Only repo-local config

    git -C "$1" config user.name "Nixpkgs"

    git -C "$1" config user.email "nixpkgs@nixos.org"

    # Get at least a HEAD commit, needed for older Nix versions

    git -C "$1" commit -q --allow-empty -m "Empty commit"

}

# Check the error message for pure eval mode

createGitRepo .

expectFailure --simulate-pure-eval 'toSource { root = ./.; fileset = gitTracked ./.; }' 'lib.fileset.gitTracked: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292.'

expectFailure --simulate-pure-eval 'toSource { root = ./.; fileset = gitTrackedWith {} ./.; }' 'lib.fileset.gitTrackedWith: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292.'

rm -rf -- *

# Go through all stages of Git files

# See https://www.git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository

# Empty repository

createGitRepo .

checkGitTracked

# Untracked file

echo a > a

checkGitTracked

# Staged file

git add a

checkGitTracked

# Committed file

git commit -q -m "Added a"

checkGitTracked

# Edited file

echo b > a

checkGitTracked

# Removed file

git rm -f -q a

checkGitTracked

rm -rf -- *

# gitignored file

createGitRepo .

echo a > .gitignore

touch a

git add -A

checkGitTracked

# Add it regardless (needs -f)

git add -f a

checkGitTracked

rm -rf -- *

# Directory

createGitRepo .

mkdir -p d1/d2/d3

touch d1/d2/d3/a

git add d1

checkGitTracked

rm -rf -- *

# Submodules

createGitRepo .

createGitRepo sub

# Untracked submodule

git -C sub commit -q --allow-empty -m "Empty commit"

checkGitTracked

# Tracked submodule

git submodule add ./sub sub >/dev/null

checkGitTracked

# Untracked file

echo a > sub/a

checkGitTracked

# Staged file

git -C sub add a

checkGitTracked

# Committed file

git -C sub commit -q -m "Add a"

checkGitTracked

# Changed file

echo b > sub/b

checkGitTracked

# Removed file

git -C sub rm -f -q a

checkGitTracked

rm -rf -- *

# TODO: Once we have combinators and a property testing library, derive property tests from https://en.wikipedia.org/wiki/Algebra_of_sets

echo >&2 tests ok

      ];

      nativeBuildInputs = [

        nix

        pkgs.gitMinimal

      ] ++ lib.optional pkgs.stdenv.isLinux pkgs.inotify-tools;

      strictDeps = true;

    } ''

      datadir="${nix}/share"

      export TEST_ROOT=$(pwd)/test-tmp

      export HOME=$(mktemp -d)

      export NIX_BUILD_HOOK=

      export NIX_CONF_DIR=$TEST_ROOT/etc

      export NIX_LOCALSTATE_DIR=$TEST_ROOT/var