Commit 3616cfb8 authored by Alois Wohlschlager's avatar Alois Wohlschlager Committed by Yureka
Browse files

replaceDependencies: add support for ca-derivations 

Unlike regular input-addressed or fixed-output derivations, floating and
deferred derivations do not have their store path available at evaluation time,
so their outPath is a placeholder. The following changes are needed for
replaceDependencies to continue working:
* Detect the placeholder and retrieve the store path using another IFD hack
  when collecting the rewrite plan.
* Try to obtain the derivation name needed for replaceDirectDependencies from
  the derivation arguments if a placeholder is detected.
* Move the length mismatch detection to build time, since the placeholder has a
  fixed length which is unrelated to the store path.
parent 59ca239d

nixos/tests/replace-dependencies/default.nix

+1 −0
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ import ../make-test-python.nix ( 
    nodes.machine =

      { ... }:

      {

        nix.settings.experimental-features = [ "ca-derivations" ];

        system.extraDependencies = [ pkgs.stdenvNoCC ];

      };

nixos/tests/replace-dependencies/guest.nix

+17 −0
Original line number Diff line number Diff line
@@ -21,12 +21,17 @@ let 
  oldDependency = writeShellScriptBin "dependency" ''

    echo "got old dependency"

  '';

  oldDependency-ca = oldDependency.overrideAttrs { __contentAddressed = true; };

  newDependency = writeShellScriptBin "dependency" ''

    echo "got new dependency"

  '';

  newDependency-ca = newDependency.overrideAttrs { __contentAddressed = true; };

  basic = writeShellScriptBin "test" ''

    ${oldDependency}/bin/dependency

  '';

  basic-ca = writeShellScriptBin "test" ''

    ${oldDependency-ca}/bin/dependency

  '';

  transitive = writeShellScriptBin "test" ''

    ${basic}/bin/test

  '';
@@ -58,6 +63,18 @@ in 
    inherit oldDependency newDependency;

  }) "got new dependency";



  replacedependency-basic-old-ca = mkCheckOutput "replacedependency-basic" (replaceDependency {

    drv = basic-ca;

    oldDependency = oldDependency-ca;

    inherit newDependency;

  }) "got new dependency";



  replacedependency-basic-new-ca = mkCheckOutput "replacedependency-basic" (replaceDependency {

    drv = basic;

    inherit oldDependency;

    newDependency = newDependency-ca;

  }) "got new dependency";



  replacedependency-transitive = mkCheckOutput "replacedependency-transitive" (replaceDependency {

    drv = transitive;

    inherit oldDependency newDependency;

pkgs/build-support/replace-dependencies.nix

+36 −18
Original line number Diff line number Diff line
@@ -43,14 +43,14 @@ let 
  inherit (builtins) unsafeDiscardStringContext appendContext;

  inherit (lib)

    trace

    stringLength

    listToAttrs

    isStorePath

    readFile

    attrValues

    mapAttrs

    filter

    hasAttr

    mapAttrsToList

    all

    ;

  inherit (lib.attrsets) mergeAttrsList;
@@ -89,17 +89,38 @@ let 
        ''

      ).outPath;



  knownDerivations = [ drv ] ++ map ({ newDependency, ... }: newDependency) replacements;

  targetDerivations = [ drv ] ++ map ({ newDependency, ... }: newDependency) replacements;

  realisation =

    drv:

    if isStorePath drv then

      # Input-addressed and fixed-output derivations have their realisation as outPath.

      toContextlessString drv

    else

      # Floating and deferred derivations have a placeholder outPath.

      # The realisation can only be obtained by performing an actual build.

      unsafeDiscardStringContext (

        readFile (

          runCommandLocal "realisation"

            {

              env = {

                inherit drv;

              };

            }

            ''

              echo -n "$drv" > $out

            ''

        )

      );

  referencesMemo = listToAttrs (

    map (drv: {

      name = toContextlessString drv;

      name = realisation drv;

      value = referencesOf drv;

    }) knownDerivations

    }) targetDerivations

  );

  relevantReferences = mergeAttrsList (attrValues referencesMemo);

  # Make sure a derivation is returned even when no replacements are actually applied.

  # Yes, even in the stupid edge case where the root derivation itself is replaced.

  storePathOrKnownDerivationMemo =

  storePathOrKnownTargetDerivationMemo =

    mapAttrs (

      drv: _references:

      # builtins.storePath does not work in pure evaluation mode, even though it is not impure.
@@ -109,9 +130,9 @@ let 
    ) relevantReferences

    // listToAttrs (

      map (drv: {

        name = toContextlessString drv;

        name = realisation drv;

        value = drv;

      }) knownDerivations

      }) targetDerivations

    );



  relevantReplacements = filter (
@@ -121,7 +142,7 @@ let 
        # Attempting to replace a dependency by itself is completely useless, and would only lead to infinite recursion.

        # Hence it must not be attempted to apply this replacement in any case.

        false

    else if !hasAttr (toContextlessString oldDependency) referencesMemo.${toContextlessString drv} then

    else if !hasAttr (realisation oldDependency) referencesMemo.${realisation drv} then

      warn "replaceDependencies: ${drv} does not depend on ${oldDependency}"

        # Handle the corner case where one of the other replacements introduces the dependency.

        # It would be more correct to not show the warning in this case, but the added complexity is probably not worth it.
@@ -146,7 +167,7 @@ let 
        );

      in

      replaceDirectDependencies {

        drv = storePathOrKnownDerivationMemo.${drv};

        drv = storePathOrKnownTargetDerivationMemo.${drv};

        replacements = mapAttrsToList (name: value: {

          oldDependency = name;

          newDependency = value;
@@ -155,21 +176,18 @@ let 
    ) relevantReferences

    // listToAttrs (

      map (drv: {

        name = toContextlessString drv;

        value = storePathOrKnownDerivationMemo.${toContextlessString drv};

        name = realisation drv;

        value = storePathOrKnownTargetDerivationMemo.${realisation drv};

      }) cutoffPackages

    )

    // listToAttrs (

      map (

        { oldDependency, newDependency }:

        {

          name = toContextlessString oldDependency;

          value = rewriteMemo.${toContextlessString newDependency};

          name = realisation oldDependency;

          value = rewriteMemo.${realisation newDependency};

        }

      ) relevantReplacements

    );

in

assert all (

  { oldDependency, newDependency }: stringLength oldDependency == stringLength newDependency

) replacements;

rewriteMemo.${toContextlessString drv}
 
rewriteMemo.${realisation drv}

pkgs/build-support/replace-direct-dependencies.nix

+60 −13
Original line number Diff line number Diff line
@@ -6,20 +6,67 @@ 


# Replace some direct dependencies of drv, not recursing into the dependency tree.

# You likely want to use replaceDependencies instead, unless you plan to implement your own recursion mechanism.

{ drv, replacements ? [ ] }:

let inherit (lib) all stringLength substring concatStringsSep;

in assert all ({ oldDependency, newDependency }:

  stringLength oldDependency == stringLength newDependency) replacements;

{

  drv,

  replacements ? [ ],

}:

let

  inherit (lib)

    isStorePath

    substring

    stringLength

    optionalString

    escapeShellArgs

    concatMap

    ;

in

if replacements == [ ] then

  drv

else

  let drvName = substring 33 (stringLength (baseNameOf drv)) (baseNameOf drv);

  in runCommandLocal drvName { nixStore = "${nix}/bin/nix-store"; } ''

    $nixStore --dump ${drv} | sed 's|${

      baseNameOf drv

    }|'$(basename $out)'|g' | sed -e ${

      concatStringsSep " -e " (map ({ oldDependency, newDependency }:

        "'s|${baseNameOf oldDependency}|${baseNameOf newDependency}|g'")

        replacements)

    } | $nixStore --restore $out

  let

    drvName =

      if isStorePath drv then

        # Reconstruct the name from the actual store path if available.

        substring 33 (stringLength (baseNameOf drv)) (baseNameOf drv)

      else if drv ? drvAttrs.name then

        # Try to get the name from the derivation arguments otherwise (for floating or deferred derivations).

        drv.drvAttrs.name

        + (

          let

            outputName = drv.outputName or "out";

          in

          optionalString (outputName != "out") "-${outputName}"

        )

      else

        throw "cannot reconstruct the derivation name from ${drv}";

  in

  runCommandLocal drvName { nativeBuildInputs = [ nix.out ]; } ''

    createRewriteScript() {

        while [ $# -ne 0 ]; do

            oldBasename="$(basename "$1")"

            newBasename="$(basename "$2")"

            shift 2

            if [ ''${#oldBasename} -ne ''${#newBasename} ]; then

                echo "cannot rewrite $oldBasename to $newBasename: length does not match" >&2

                exit 1

            fi

            echo "s|$oldBasename|$newBasename|g" >> rewrite.sed

        done

    }

    createRewriteScript ${

      escapeShellArgs (

        [

          drv

          (placeholder "out")

        ]

        ++ concatMap (

          { oldDependency, newDependency }:

          [

            oldDependency

            newDependency

          ]

        ) replacements

      )

    }

    nix-store --dump ${drv} | sed -f rewrite.sed | nix-store --restore $out

  ''