Commit af3a3f64 authored by Alois Wohlschlager's avatar Alois Wohlschlager Committed by Yureka
Browse files

replaceDependencies: evolve from replaceDependency 

Rewrite replaceDependency so that it can apply multiple replacements in
one go. This includes correctly handling the case where one of the
replacements itself needs to have another replacement applied as well.
This rewritten function is now aptly called replaceDependencies.

For compatibility, replaceDependency is retained as a simple wrapper
over replaceDependencies. It will cause a rebuild because the unpatched
dependency is now referenced by derivation instead of by storePath, but
the functionality is equivalent.

Fixes: https://github.com/NixOS/nixpkgs/issues/199162
parent 2232cae4

pkgs/build-support/replace-dependencies.nix

0 → 100644
+186 −0
Original line number Diff line number Diff line 
{

  runCommandLocal,

  nix,

  lib,

}:



# Replace some dependencies in the requisites tree of drv, propagating the change all the way up the tree, even within other replacements, without a full rebuild.

# This can be useful, for example, to patch a security hole in libc and still use your system safely without rebuilding the world.

# This should be a short term solution, as soon as a rebuild can be done the properly rebuilt derivation should be used.

# Each old dependency and the corresponding new dependency MUST have the same-length name, and ideally should have close-to-identical directory layout.

#

# Example: safeFirefox = replaceDependencies {

#   drv = firefox;

#   replacements = [

#     {

#       oldDependency = glibc;

#       newDependency = glibc.overrideAttrs (oldAttrs: {

#         patches = oldAttrs.patches ++ [ ./fix-glibc-hole.patch ];

#       });

#     }

#     {

#       oldDependency = libwebp;

#       newDependency = libwebp.overrideAttrs (oldAttrs: {

#         patches = oldAttrs.patches ++ [ ./fix-libwebp-hole.patch ];

#       });

#     }

#   ];

# };

# This will first rebuild glibc and libwebp with your security patches.

# Then it copies over firefox (and all of its dependencies) without rebuilding further.

# In particular, the glibc dependency of libwebp will be replaced by the patched version as well.

#

# In rare cases, it is possible for the replacement process to cause breakage (for example due to checksum mismatch).

# The cutoffPackages argument can be used to exempt the problematic packages from the replacement process.

{

  drv,

  replacements,

  cutoffPackages ? [ ],

  verbose ? true,

}:



let

  inherit (builtins) unsafeDiscardStringContext appendContext;

  inherit (lib)

    trace

    substring

    stringLength

    concatStringsSep

    mapAttrsToList

    listToAttrs

    attrValues

    mapAttrs

    filter

    hasAttr

    all

    ;

  inherit (lib.attrsets) mergeAttrsList;



  toContextlessString = x: unsafeDiscardStringContext (toString x);

  warn = if verbose then trace else (x: y: y);



  referencesOf =

    drv:

    import

      (runCommandLocal "references.nix"

        {

          exportReferencesGraph = [

            "graph"

            drv

          ];

        }

        ''

          (echo {

          while read path

          do

              echo "  \"$path\" = ["

              read count

              read count

              while [ "0" != "$count" ]

              do

                  read ref_path

                  if [ "$ref_path" != "$path" ]

                  then

                      echo "    \"$ref_path\""

                  fi

                  count=$(($count - 1))

              done

              echo "  ];"

          done < graph

          echo }) > $out

        ''

      ).outPath;

  rewriteHashes =

    drv: rewrites:

    if rewrites == { } then

      drv

    else

      let

        drvName = substring 33 (stringLength (baseNameOf drv)) (baseNameOf drv);

      in

      runCommandLocal drvName { nixStore = "${nix}/bin/nix-store"; } ''

        $nixStore --dump ${drv} | sed 's|${baseNameOf drv}|'$(basename $out)'|g' | sed -e ${

          concatStringsSep " -e " (

            mapAttrsToList (name: value: "'s|${baseNameOf name}|${baseNameOf value}|g'") rewrites

          )

        } | $nixStore --restore $out

      '';



  knownDerivations = [ drv ] ++ map ({ newDependency, ... }: newDependency) replacements;

  referencesMemo = listToAttrs (

    map (drv: {

      name = toContextlessString drv;

      value = referencesOf drv;

    }) knownDerivations

  );

  relevantReferences = mergeAttrsList (attrValues referencesMemo);

  # Make sure a derivation is returned even when no replacements are actually applied.

  # Yes, even in the stupid edge case where the root derivation itself is replaced.

  storePathOrKnownDerivationMemo =

    mapAttrs (

      drv: _references:

      # builtins.storePath does not work in pure evaluation mode, even though it is not impure.

      # This reimplementation in Nix works as long as the path is already allowed in the evaluation state.

      # This is always the case here, because all paths come from the closure of the original derivation.

      appendContext drv { ${drv}.path = true; }

    ) relevantReferences

    // listToAttrs (

      map (drv: {

        name = toContextlessString drv;

        value = drv;

      }) knownDerivations

    );



  relevantReplacements = filter (

    { oldDependency, newDependency }:

    if toString oldDependency == toString newDependency then

      warn "replaceDependencies: attempting to replace dependency ${oldDependency} of ${drv} with itself"

        # Attempting to replace a dependency by itself is completely useless, and would only lead to infinite recursion.

        # Hence it must not be attempted to apply this replacement in any case.

        false

    else if !hasAttr (toContextlessString oldDependency) referencesMemo.${toContextlessString drv} then

      warn "replaceDependencies: ${drv} does not depend on ${oldDependency}"

        # Handle the corner case where one of the other replacements introduces the dependency.

        # It would be more correct to not show the warning in this case, but the added complexity is probably not worth it.

        true

    else

      true

  ) replacements;



  rewriteMemo =

    # Mind the order of how the three attrsets are merged here.

    # The order of precedence needs to be "explicitly specified replacements" > "rewrite exclusion (cutoffPackages)" > "rewrite".

    # So the attrset merge order is the opposite.

    mapAttrs (

      drv: references:

      let

        rewrittenReferences = filter (dep: dep != drv && toString rewriteMemo.${dep} != dep) references;

        rewrites = listToAttrs (

          map (reference: {

            name = reference;

            value = rewriteMemo.${reference};

          }) rewrittenReferences

        );

      in

      rewriteHashes storePathOrKnownDerivationMemo.${drv} rewrites

    ) relevantReferences

    // listToAttrs (

      map (drv: {

        name = toContextlessString drv;

        value = storePathOrKnownDerivationMemo.${toContextlessString drv};

      }) cutoffPackages

    )

    // listToAttrs (

      map (

        { oldDependency, newDependency }:

        {

          name = toContextlessString oldDependency;

          value = rewriteMemo.${toContextlessString newDependency};

        }

      ) relevantReplacements

    );

in

assert all (

  { oldDependency, newDependency }: stringLength oldDependency == stringLength newDependency

) replacements;

rewriteMemo.${toContextlessString drv}

pkgs/build-support/replace-dependency.nix

deleted100644 → 0
+0 −94
Original line number Diff line number Diff line 
{ runCommandLocal, nix, lib }:



# Replace a single dependency in the requisites tree of drv, propagating

# the change all the way up the tree, without a full rebuild. This can be

# useful, for example, to patch a security hole in libc and still use your

# system safely without rebuilding the world. This should be a short term

# solution, as soon as a rebuild can be done the properly rebuild derivation

# should be used. The old dependency and new dependency MUST have the same-length

# name, and ideally should have close-to-identical directory layout.

#

# Example: safeFirefox = replaceDependency {

#   drv = firefox;

#   oldDependency = glibc;

#   newDependency = overrideDerivation glibc (attrs: {

#     patches  = attrs.patches ++ [ ./fix-glibc-hole.patch ];

#   });

# };

# This will rebuild glibc with your security patch, then copy over firefox

# (and all of its dependencies) without rebuilding further.

{ drv, oldDependency, newDependency, verbose ? true }:



let

  inherit (lib)

    any

    attrNames

    concatStringsSep

    elem

    filter

    filterAttrs

    listToAttrs

    mapAttrsToList

    stringLength

    substring

    ;



  warn = if verbose then builtins.trace else (x: y: y);

  references = import (runCommandLocal "references.nix" { exportReferencesGraph = [ "graph" drv ]; } ''

    (echo {

    while read path

    do

        echo "  \"$path\" = ["

        read count

        read count

        while [ "0" != "$count" ]

        do

            read ref_path

            if [ "$ref_path" != "$path" ]

            then

                echo "    (builtins.storePath (/. + \"$ref_path\"))"

            fi

            count=$(($count - 1))

        done

        echo "  ];"

    done < graph

    echo }) > $out

  '').outPath;



  discard = builtins.unsafeDiscardStringContext;



  oldStorepath = builtins.storePath (discard (toString oldDependency));



  referencesOf = drv: references.${discard (toString drv)};



  dependsOnOldMemo = listToAttrs (map

    (drv: { name = discard (toString drv);

            value = elem oldStorepath (referencesOf drv) ||

                    any dependsOnOld (referencesOf drv);

          }) (attrNames references));



  dependsOnOld = drv: dependsOnOldMemo.${discard (toString drv)};



  drvName = drv:

    discard (substring 33 (stringLength (builtins.baseNameOf drv)) (builtins.baseNameOf drv));



  rewriteHashes = drv: hashes: runCommandLocal (drvName drv) { nixStore = "${nix.out}/bin/nix-store"; } ''

    $nixStore --dump ${drv} | sed 's|${baseNameOf drv}|'$(basename $out)'|g' | sed -e ${

      concatStringsSep " -e " (mapAttrsToList (name: value:

        "'s|${baseNameOf name}|${baseNameOf value}|g'"

      ) hashes)

    } | $nixStore --restore $out

  '';



  rewrittenDeps = listToAttrs [ {name = discard (toString oldDependency); value = newDependency;} ];



  rewriteMemo = listToAttrs (map

    (drv: { name = discard (toString drv);

            value = rewriteHashes (builtins.storePath drv)

              (filterAttrs (n: v: elem (builtins.storePath (discard (toString n))) (referencesOf drv)) rewriteMemo);

          })

    (filter dependsOnOld (attrNames references))) // rewrittenDeps;



  drvHash = discard (toString drv);

in assert (stringLength (drvName (toString oldDependency)) == stringLength (drvName (toString newDependency)));

rewriteMemo.${drvHash} or (warn "replace-dependency.nix: Derivation ${drvHash} does not depend on ${discard (toString oldDependency)}" drv)

pkgs/top-level/all-packages.nix

+10 −1
Original line number Diff line number Diff line
@@ -1306,7 +1306,16 @@ with pkgs; 














  substituteAllFiles = callPackage ../build-support/substitute-files/substitute-all-files.nix { };



























  replaceDependency = callPackage ../build-support/replace-dependency.nix { };













  replaceDependencies = callPackage ../build-support/replace-dependencies.nix { };

























  replaceDependency = { drv, oldDependency, newDependency, verbose ? true }: replaceDependencies {













    inherit drv verbose;













    replacements = [{













      inherit oldDependency newDependency;













    }];













    # When newDependency depends on drv, instead of causing infinite recursion, keep it as is.













    cutoffPackages = [ newDependency ];













  };





























  replaceVars = callPackage ../build-support/replace-vars { };