Commit 7f3ca3e2 authored by John Ericson's avatar John Ericson
Browse files

stdenv: Fix handling of dependencies and hooks 

4 far-reaching changes: Smaller PATH, New vars, different propagation
logic, and different hook logic

Smaller PATH
------------

`buildInputs` no longer go on the PATH at build time, as they cannot be
run when cross compiling and we don't want to special case. Simply make
a `nativeBuildInput` too if one needs them on the PATH. Fixes #21191.

Many new depedendency variables
-------------------------------

See the stdenv chapter of the nixpkgs manual. I pulled out the existing
documentation of dependency specification into a new section, and added
language for these two (and their propagated equivalents) along side
the others'.

More complex propagation logic
------------------------------

Before a propagated*XXX*Input always acted as if it was specified
directly as a *XXX*Input downstream. That's simple enough, but violates
the intended roles of each sort of dep, which has functional and not
just stylistic consequences.

The new algorithm is detailed in the manual, and ensures everything
ends up in the right place. I tried to give both an informal and formal
description, but I suspect in practice it will not make much sense
until one tries cross compiling, after which it will immediately make
sense as the only sane option.

Simplified hook logic
---------------------

Rather than `envHook` and `crossEnvHook`, whose behavior differs
depending on whether we are cross compiling or not, there is now one
hook per sort (or rather non-propagated and propagated pair of sorts)
of dependency. These new hooks have the same meaning regardless of
cross compilation. See the setup hook section of stdenv chapter of the
Nixpkgs manual for more details.
parent bb18a3b5

pkgs/stdenv/generic/make-derivation.nix

+60 −16
Original line number Diff line number Diff line
@@ -14,11 +14,27 @@ rec { 
  mkDerivation =

    { name ? ""



    , nativeBuildInputs ? []

    , buildInputs ? []

    # These types of dependencies are all exhaustively documented in

    # the "Specifying Dependencies" section of the "Standard

    # Environment" chapter of the Nixpkgs manual.



    , propagatedNativeBuildInputs ? []

    , propagatedBuildInputs ? []

    # TODO(@Ericson2314): Stop using legacy dep attribute names



    #                           host offset -> target offset

    , depsBuildBuild              ? [] # -1 -> -1

    , depsBuildBuildPropagated    ? [] # -1 -> -1

    , nativeBuildInputs           ? [] # -1 ->  0  N.B. Legacy name

    , propagatedNativeBuildInputs ? [] # -1 ->  0  N.B. Legacy name

    , depsBuildTarget             ? [] # -1 ->  1

    , depsBuildTargetPropagated   ? [] # -1 ->  1



    , depsHostHost                ? [] #  0 ->  0

    , depsHostHostPropagated      ? [] #  0 ->  0

    , buildInputs                 ? [] #  0 ->  1  N.B. Legacy name

    , propagatedBuildInputs       ? [] #  0 ->  1  N.B. Legacy name



    , depsTargetTarget            ? [] #  1 ->  1

    , depsTargetTargetPropagated  ? [] #  1 ->  1



    , configureFlags ? []

    , # Target is not included by default because most programs don't care.
@@ -56,15 +72,35 @@ rec { 
      inherit erroneousHardeningFlags hardeningDisable hardeningEnable supportedHardeningFlags;

    })

    else let

      dependencies = map lib.chooseDevOutputs [

      dependencies = map (map lib.chooseDevOutputs) [

        [

          (map (drv: drv.__spliced.buildBuild or drv) depsBuildBuild)

          (map (drv: drv.nativeDrv or drv) nativeBuildInputs

             ++ lib.optional separateDebugInfo ../../build-support/setup-hooks/separate-debug-info.sh

             ++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh)

          (map (drv: drv.__spliced.buildTarget or drv) depsBuildTarget)

        ]

        [

          (map (drv: drv.__spliced.hostHost or drv) depsHostHost)

          (map (drv: drv.crossDrv or drv) buildInputs)

        ]

        [

          (map (drv: drv.__spliced.targetTarget or drv) depsTargetTarget)

        ]

      ];

      propagatedDependencies = map lib.chooseDevOutputs [

      propagatedDependencies = map (map lib.chooseDevOutputs) [

        [

          (map (drv: drv.__spliced.buildBuild or drv) depsBuildBuildPropagated)

          (map (drv: drv.nativeDrv or drv) propagatedNativeBuildInputs)

          (map (drv: drv.__spliced.buildTarget or drv) depsBuildTargetPropagated)

        ]

        [

          (map (drv: drv.__spliced.hostHost or drv) depsHostHostPropagated)

          (map (drv: drv.crossDrv or drv) propagatedBuildInputs)

        ]

        [

          (map (drv: drv.__spliced.targetTarget or drv) depsTargetTargetPropagated)

        ]

      ];



      outputs' =
@@ -105,11 +141,19 @@ rec { 
          userHook = config.stdenv.userHook or null;

          __ignoreNulls = true;



          nativeBuildInputs = lib.elemAt dependencies 0;

          buildInputs = lib.elemAt dependencies 1;

          depsBuildBuild              = lib.elemAt (lib.elemAt dependencies 0) 0;

          nativeBuildInputs           = lib.elemAt (lib.elemAt dependencies 0) 1;

          depsBuildTarget             = lib.elemAt (lib.elemAt dependencies 0) 2;

          depsHostBuild               = lib.elemAt (lib.elemAt dependencies 1) 0;

          buildInputs                 = lib.elemAt (lib.elemAt dependencies 1) 1;

          depsTargetTarget            = lib.elemAt (lib.elemAt dependencies 2) 0;



          propagatedNativeBuildInputs = lib.elemAt propagatedDependencies 0;

          propagatedBuildInputs = lib.elemAt propagatedDependencies 1;

          depsBuildBuildPropagated    = lib.elemAt (lib.elemAt propagatedDependencies 0) 0;

          propagatedNativeBuildInputs = lib.elemAt (lib.elemAt propagatedDependencies 0) 1;

          depsBuildTargetPropagated   = lib.elemAt (lib.elemAt propagatedDependencies 0) 2;

          depsHostBuildPropagated     = lib.elemAt (lib.elemAt propagatedDependencies 1) 0;

          propagatedBuildInputs       = lib.elemAt (lib.elemAt propagatedDependencies 1) 1;

          depsTargetTargetPropagated  = lib.elemAt (lib.elemAt propagatedDependencies 2) 0;



          # This parameter is sometimes a string, sometimes null, and sometimes a list, yuck

          configureFlags = let inherit (lib) optional elem; in

pkgs/stdenv/generic/setup.sh

+232 −76
Original line number Diff line number Diff line
@@ -300,11 +300,72 @@ runHook preHook 
runHook addInputsHook





# Recursively find all build inputs.

# Package accumulators



# shellcheck disable=SC2034

declare -a pkgsBuildBuild pkgsBuildHost pkgsBuildTarget

declare -a pkgsHostHost pkgsHostTarget

declare -a pkgsTargetTarget



declare -ra pkgBuildAccumVars=(pkgsBuildBuild pkgsBuildHost pkgsBuildTarget)

declare -ra pkgHostAccumVars=(pkgsHostHost pkgsHostTarget)

declare -ra pkgTargetAccumVars=(pkgsTargetTarget)



declare -ra pkgAccumVarVars=(pkgBuildAccumVars pkgHostAccumVars pkgTargetAccumVars)





# Hooks



declare -a envBuildBuildHooks envBuildHostHooks envBuildTargetHooks

declare -a envHostHostHooks envHostTargetHooks

declare -a envTargetTargetHooks



declare -ra pkgBuildHookVars=(envBuildBuildHook envBuildHostHook envBuildTargetHook)

declare -ra pkgHostHookVars=(envHostHostHook envHostTargetHook)

declare -ra pkgTargetHookVars=(envTargetTargetHook)



declare -ra pkgHookVarVars=(pkgBuildHookVars pkgHostHookVars pkgTargetHookVars)





# Propagated dep files



declare -ra propagatedBuildDepFiles=(

    propagated-build-build-deps

    propagated-native-build-inputs # Legacy name for back-compat

    propagated-build-target-deps

)

declare -ra propagatedHostDepFiles=(

    propagated-host-host-deps

    propagated-build-inputs # Legacy name for back-compat

)

declare -ra propagatedTargetDepFiles=(

    propagated-target-target-deps

)

declare -ra propagatedDepFilesVars=(

    propagatedBuildDepFiles

    propagatedHostDepFiles

    propagatedTargetDepFiles

)



# Platform offsets: build = -1, host = 0, target = 1

declare -ra allPlatOffsets=(-1 0 1)





# Mutually-recursively find all build inputs. See the dependency section of the

# stdenv chapter of the Nixpkgs manual for the specification this algorithm

# implements.

findInputs() {

    local pkg="$1"; shift

    local var="$1"; shift

    local propagatedBuildInputsFiles=("$@")

    local -r pkg="$1"

    local -ri hostOffset="$2"

    local -ri targetOffset="$3"



    # Sanity check

    (( "$hostOffset" <= "$targetOffset" )) || exit -1



    local varVar="${pkgAccumVarVars[$hostOffset + 1]}"

    local varRef="$varVar[\$targetOffset - \$hostOffset]"

    local var="${!varRef}"

    unset -v varVar varRef



    # TODO(@Ericson2314): Restore using associative array once Darwin

    # nix-shell doesn't use impure bash. This should replace the O(n)
@@ -324,21 +385,106 @@ findInputs() { 
        exit 1

    fi



    local file

    for file in "${propagatedBuildInputsFiles[@]}"; do

        file="$pkg/nix-support/$file"

        [[ -f "$file" ]] || continue

    # The current package's host and target offset together

    # provide a <=-preserving homomorphism from the relative

    # offsets to current offset

    function mapOffset() {

        local -ri inputOffset="$1"

        if (( "$inputOffset" <= 0 )); then

            local -ri outputOffset="$inputOffset + $hostOffset"

        else

            local -ri outputOffset="$inputOffset - 1 + $targetOffset"

        fi

        echo "$outputOffset"

    }



    # Host offset relative to that of the package whose immediate

    # dependencies we are currently exploring.

    local -i relHostOffset

    for relHostOffset in "${allPlatOffsets[@]}"; do

        # `+ 1` so we start at 0 for valid index

        local files="${propagatedDepFilesVars[$relHostOffset + 1]}"



        # Host offset relative to the package currently being

        # built---as absolute an offset as will be used.

        local -i hostOffsetNext

        hostOffsetNext="$(mapOffset relHostOffset)"



        # Ensure we're in bounds relative to the package currently

        # being built.

        [[ "${allPlatOffsets[*]}" = *"$hostOffsetNext"*  ]] || continue



        # Target offset relative to the *host* offset of the package

        # whose immediate dependencies we are currently exploring.

        local -i relTargetOffset

        for relTargetOffset in "${allPlatOffsets[@]}"; do

            (( "$relHostOffset" <= "$relTargetOffset" )) || continue



            local fileRef="${files}[$relTargetOffset - $relHostOffset]"

            local file="${!fileRef}"

            unset -v fileRef



            # Target offset relative to the package currently being

            # built.

            local -i targetOffsetNext

            targetOffsetNext="$(mapOffset relTargetOffset)"



            # Once again, ensure we're in bounds relative to the

            # package currently being built.

            [[ "${allPlatOffsets[*]}" = *"$targetOffsetNext"* ]] || continue



            [[ -f "$pkg/nix-support/$file" ]] || continue



            local pkgNext

        for pkgNext in $(< "$file"); do

            findInputs "$pkgNext" "$var" "${propagatedBuildInputsFiles[@]}"

            for pkgNext in $(< "$pkg/nix-support/$file"); do

                findInputs "$pkgNext" "$hostOffsetNext" "$targetOffsetNext"

            done

        done

    done

}



# Make sure all are at least defined as empty

: ${depsBuildBuild=} ${depsBuildBuildPropagated=}

: ${nativeBuildInputs=} ${propagatedNativeBuildInputs=} ${defaultNativeBuildInputs=}

: ${depsBuildTarget=} ${depsBuildTargetPropagated=}

: ${depsHostHost=} ${depsHostHostPropagated=}

: ${buildInputs=} ${propagatedBuildInputs=} ${defaultBuildInputs=}

: ${depsTargetTarget=} ${depsTargetTargetPropagated=}



for pkg in $depsBuildBuild $depsBuildBuildPropagated; do

    findInputs "$pkg" -1 -1

done

for pkg in $nativeBuildInputs $propagatedNativeBuildInputs; do

    findInputs "$pkg" -1  0

done

for pkg in $depsBuildTarget $depsBuildTargetPropagated; do

    findInputs "$pkg" -1  1

done

for pkg in $depsHostHost $depsHostHostPropagated; do

    findInputs "$pkg"  0  0

done

for pkg in $buildInputs $propagatedBuildInputs ; do

    findInputs "$pkg"  0  1

done

for pkg in $depsTargetTarget $depsTargetTargetPropagated; do

    findInputs "$pkg"  1  1

done

# Default inputs must be processed last

for pkg in $defaultNativeBuildInputs; do

    findInputs "$pkg" -1  0

done

for pkg in $defaultBuildInputs; do

    findInputs "$pkg"  0  1

done



# Add package to the future PATH and run setup hooks

activatePackage() {

    local pkg="$1"

    local -ri hostOffset="$2"

    local -ri targetOffset="$3"



    # Sanity check

    (( "$hostOffset" <= "$targetOffset" )) || exit -1



    if [ -f "$pkg" ]; then

        local oldOpts="$(shopt -po nounset)"
@@ -347,11 +493,16 @@ activatePackage() { 
        eval "$oldOpts"

    fi



    if [ -d "$pkg/bin" ]; then

    # Only dependencies whose host platform is guaranteed to match the

    # build platform are included here. That would be `depsBuild*`,

    # and legacy `nativeBuildInputs`. Other aren't because of cross

    # compiling, and we want to have consistent rules whether or not

    # we are cross compiling.

    if [[ "$hostOffset" -le -1 && -d "$pkg/bin" ]]; then

        addToSearchPath _PATH "$pkg/bin"

    fi



    if [ -f "$pkg/nix-support/setup-hook" ]; then

    if [[ -f "$pkg/nix-support/setup-hook" ]]; then

        local oldOpts="$(shopt -po nounset)"

        set +u

        source "$pkg/nix-support/setup-hook"
@@ -359,67 +510,55 @@ activatePackage() { 
    fi

}



declare -a nativePkgs crossPkgs

if [ -z "${crossConfig:-}" ]; then

    # Not cross-compiling - both buildInputs (and variants like propagatedBuildInputs)

    # are handled identically to nativeBuildInputs

    for i in ${nativeBuildInputs:-} ${buildInputs:-} \

             ${defaultNativeBuildInputs:-} ${defaultBuildInputs:-} \

             ${propagatedNativeBuildInputs:-} ${propagatedBuildInputs:-}; do

        findInputs "$i" nativePkgs propagated-native-build-inputs propagated-build-inputs

_activatePkgs() {

    local -i hostOffset targetOffset

    local pkg



    for hostOffset in "${allPlatOffsets[@]}"; do

        local pkgsVar="${pkgAccumVarVars[$hostOffset + 1]}"

        for targetOffset in "${allPlatOffsets[@]}"; do

            (( "$hostOffset" <= "$targetOffset" )) || continue

            local pkgsRef="${pkgsVar}[$targetOffset - $hostOffset]"

            local pkgsSlice="${!pkgsRef}[@]"

            for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; do

                activatePackage "$pkg" "$hostOffset" "$targetOffset"

            done

else

    for i in ${nativeBuildInputs:-} ${defaultNativeBuildInputs:-} ${propagatedNativeBuildInputs:-}; do

        findInputs "$i" nativePkgs propagated-native-build-inputs

        done

    for i in ${buildInputs:-} ${defaultBuildInputs:-} ${propagatedBuildInputs:-}; do

        findInputs "$i" crossPkgs propagated-build-inputs

    done

fi



for i in ${nativePkgs+"${nativePkgs[@]}"} ${crossPkgs+"${crossPkgs[@]}"}; do

    activatePackage "$i"

    done

}



# Run the package setup hooks and build _PATH

_activatePkgs



# Set the relevant environment variables to point to the build inputs

# found above.

#

# These `depOffset`s tell the env hook what sort of dependency

# (ignoring propagatedness) is being passed to the env hook. In a real

# language, we'd append a closure with this information to the

# relevant env hook array, but bash doesn't have closures, so it's

# easier to just pass this in.



_addToNativeEnv() {

    local pkg="$1"

    if [[ -n "${crossConfig:-}" ]]; then

        local -i depOffset=-1

    else

        local -i depOffset=0

    fi



    # Run the package-specific hooks set by the setup-hook scripts.

    runHook envHook "$pkg"

}



# Old bash empty array hack

for i in ${nativePkgs+"${nativePkgs[@]}"}; do

    _addToNativeEnv "$i"

# These `depOffset`s, beyond indexing the arrays, also tell the env

# hook what sort of dependency (ignoring propagatedness) is being

# passed to the env hook. In a real language, we'd append a closure

# with this information to the relevant env hook array, but bash

# doesn't have closures, so it's easier to just pass this in.

_addToEnv() {

    local -i depHostOffset depTargetOffset

    local pkg



    for depHostOffset in "${allPlatOffsets[@]}"; do

        local hookVar="${pkgHookVarVars[$depHostOffset + 1]}"

        local pkgsVar="${pkgAccumVarVars[$depHostOffset + 1]}"

        for depTargetOffset in "${allPlatOffsets[@]}"; do

            (( "$depHostOffset" <= "$depTargetOffset" )) || continue

            local hookRef="${hookVar}[$depTargetOffset - $depHostOffset]"

            local pkgsRef="${pkgsVar}[$depTargetOffset - $depHostOffset]"

            local pkgsSlice="${!pkgsRef}[@]"

            for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; do

                runHook "${!hookRef}" "$pkg"

            done

        done

    done



_addToCrossEnv() {

    local pkg="$1"

    local -i depOffset=0



    # Run the package-specific hooks set by the setup-hook scripts.

    runHook crossEnvHook "$pkg"

}



# Old bash empty array hack

for i in ${crossPkgs+"${crossPkgs[@]}"}; do

    _addToCrossEnv "$i"

done

# Run the package-specific hooks set by the setup-hook scripts.

_addToEnv





_addRpathPrefix "$out"
@@ -882,6 +1021,7 @@ installPhase() { 
# propagated-build-inputs.

fixupPhase() {

    # Make sure everything is writable so "strip" et al. work.

    local output

    for output in $outputs; do

        if [ -e "${!output}" ]; then chmod -R u+w "${!output}"; fi

    done
@@ -895,19 +1035,35 @@ fixupPhase() { 
    done





    # Propagate build inputs and setup hook into the development output.

    # Propagate dependencies & setup hook into the development output.

    declare -ra flatVars=(

        # Build

        depsBuildBuildPropagated

        propagatedNativeBuildInputs

        depsBuildTargetPropagated

        # Host

        depsHostHostPropagated

        propagatedBuildInputs

        # Target

        depsTargetTargetPropagated

    )

    declare -ra flatFiles=(

        "${propagatedBuildDepFiles[@]}"

        "${propagatedHostDepFiles[@]}"

        "${propagatedTargetDepFiles[@]}"

    )



    if [ -n "${propagatedBuildInputs:-}" ]; then

        mkdir -p "${!outputDev}/nix-support"

        # shellcheck disable=SC2086

        printWords $propagatedBuildInputs > "${!outputDev}/nix-support/propagated-build-inputs"

    fi

    local propagatedInputsIndex

    for propagatedInputsIndex in "${!flatVars[@]}"; do

        local propagatedInputsSlice="${flatVars[$propagatedInputsIndex]}[@]"

        local propagatedInputsFile="${flatFiles[$propagatedInputsIndex]}"



        [[ "${!propagatedInputsSlice}" ]] || continue



    if [ -n "${propagatedNativeBuildInputs:-}" ]; then

        mkdir -p "${!outputDev}/nix-support"

        # shellcheck disable=SC2086

        printWords $propagatedNativeBuildInputs > "${!outputDev}/nix-support/propagated-native-build-inputs"

    fi

        printWords ${!propagatedInputsSlice} > "${!outputDev}/nix-support/$propagatedInputsFile"

    done





    if [ -n "${setupHook:-}" ]; then