Loading lib/fetchers.nix +173 −1 Original line number Diff line number Diff line # snippets that can be shared by multiple fetchers (pkgs/build-support) { lib }: { let commonH = hashTypes: rec { hashNames = [ "hash" ] ++ hashTypes; hashSet = lib.genAttrs hashNames (lib.const {}); }; fakeH = { hash = lib.fakeHash; sha256 = lib.fakeSha256; sha512 = lib.fakeSha512; }; in rec { proxyImpureEnvVars = [ # We borrow these environment variables from the caller to allow Loading @@ -14,4 +25,165 @@ "NIX_SSL_CERT_FILE" ]; /** Converts an attrset containing one of `hash`, `sha256` or `sha512`, into one containing `outputHash{,Algo}` as accepted by `mkDerivation`. An appropriate “fake hash” is substituted when the hash value is `""`, as is the [convention for fetchers](#sec-pkgs-fetchers-updating-source-hashes-fakehash-method). All other attributes in the set remain as-is. # Example ```nix normalizeHash { } { hash = ""; foo = "bar"; } => { outputHash = lib.fakeHash; outputHashAlgo = null; foo = "bar"; } ``` ```nix normalizeHash { } { sha256 = lib.fakeSha256; } => { outputHash = lib.fakeSha256; outputHashAlgo = "sha256"; } ``` ```nix normalizeHash { } { sha512 = lib.fakeSha512; } => { outputHash = lib.fakeSha512; outputHashAlgo = "sha512"; } ``` # Type ``` normalizeHash :: { hashTypes :: List String, required :: Bool } -> AttrSet -> AttrSet ``` # Arguments hashTypes : the set of attribute names accepted as hash inputs, in addition to `hash` required : whether to throw if no hash was present in the input; otherwise returns the original input, unmodified */ normalizeHash = { hashTypes ? [ "sha256" ], required ? true, }: let inherit (lib) concatMapStringsSep head tail throwIf; inherit (lib.attrsets) attrsToList intersectAttrs removeAttrs optionalAttrs; inherit (commonH hashTypes) hashNames hashSet; in args: if args ? "outputHash" then args else let # The argument hash, as a {name, value} pair h = # All hashes passed in arguments (possibly 0 or >1) as a list of {name, value} pairs let hashesAsNVPairs = attrsToList (intersectAttrs hashSet args); in if hashesAsNVPairs == [] then throwIf required "fetcher called without `hash`" null else if tail hashesAsNVPairs != [] then throw "fetcher called with mutually-incompatible arguments: ${concatMapStringsSep ", " (a: a.name) hashesAsNVPairs}" else head hashesAsNVPairs ; in removeAttrs args hashNames // (optionalAttrs (h != null) { outputHashAlgo = if h.name == "hash" then null else h.name; outputHash = if h.value == "" then fakeH.${h.name} or (throw "no “fake hash” defined for ${h.name}") else h.value; }) ; /** Wraps a function which accepts `outputHash{,Algo}` into one which accepts `hash` or `sha{256,512}` # Example ```nix withNormalizedHash { hashTypes = [ "sha256" "sha512" ]; } ( { outputHash, outputHashAlgo, ... }: ... ) ``` is a function which accepts one of `hash`, `sha256`, or `sha512` (or the original's `outputHash` and `outputHashAlgo`). Its `functionArgs` metadata only lists `hash` as a parameter, optional iff. `outputHash` was an optional parameter of the original function. `sha256`, `sha512`, `outputHash`, or `outputHashAlgo` are not mentioned in the `functionArgs` metadata. # Type ``` withNormalizedHash :: { hashTypes :: List String } -> (AttrSet -> T) -> (AttrSet -> T) ``` # Arguments hashTypes : the set of attribute names accepted as hash inputs, in addition to `hash` : they must correspond to a valid value for `outputHashAlgo`, currently one of: `md5`, `sha1`, `sha256`, or `sha512`. f : the function to be wrapped ::: {.note} In nixpkgs, `mkDerivation` rejects MD5 `outputHash`es, and SHA-1 is being deprecated. As such, there is no reason to add `md5` to `hashTypes`, and `sha1` should only ever be included for backwards compatibility. ::: # Output `withNormalizedHash { inherit hashTypes; } f` is functionally equivalent to ```nix args: f (normalizeHash { inherit hashTypes; required = !(lib.functionArgs f).outputHash; } args) ``` However, `withNormalizedHash` preserves `functionArgs` metadata insofar as possible, and is implemented somewhat more efficiently. */ withNormalizedHash = { hashTypes ? [ "sha256" ] }: fetcher: let inherit (lib.attrsets) genAttrs intersectAttrs removeAttrs; inherit (lib.trivial) const functionArgs setFunctionArgs; inherit (commonH hashTypes) hashSet; fArgs = functionArgs fetcher; normalize = normalizeHash { inherit hashTypes; required = !fArgs.outputHash; }; in # The o.g. fetcher must *only* accept outputHash and outputHashAlgo assert fArgs ? outputHash && fArgs ? outputHashAlgo; assert intersectAttrs fArgs hashSet == {}; setFunctionArgs (args: fetcher (normalize args)) (removeAttrs fArgs [ "outputHash" "outputHashAlgo" ] // { hash = fArgs.outputHash; }); } lib/tests/fetchers.nix 0 → 100644 +165 −0 Original line number Diff line number Diff line let lib = import ./..; inherit (lib) fakeHash fakeSha256 fakeSha512 flip functionArgs runTests ; inherit (lib.fetchers) normalizeHash withNormalizedHash; testingThrow = expr: { expr = with builtins; tryEval (seq expr "didn't throw"); expected = { success = false; value = false; }; }; # hashes of empty sri256 = "sha256-d6xi4mKdjkX2JFicDIv5niSzpyI0m/Hnm8GGAIU04kY="; sri512 = "sha512-AXFyVo7jiZ5we10fxZ5E9qfPjSfqkizY2apCzORKFVYZaNhCIVbooY+J4cYST00ztLf0EjivIBPPdtIYFUMfzQ=="; unionOfDisjoints = lib.foldl lib.attrsets.unionOfDisjoint { }; genTests = n: f: { "test${n}AlreadyNormalized" = { expr = f { } { outputHash = ""; outputHashAlgo = "md42"; }; expected = { outputHash = ""; outputHashAlgo = "md42"; }; }; "test${n}EmptySha256" = { expr = f { } { sha256 = ""; }; expected = { outputHash = fakeSha256; outputHashAlgo = "sha256"; }; }; "test${n}EmptySha512" = { expr = f { hashTypes = [ "sha512" ]; } { sha512 = ""; }; expected = { outputHash = fakeSha512; outputHashAlgo = "sha512"; }; }; "test${n}EmptyHash" = { expr = f { } { hash = ""; }; expected = { outputHash = fakeHash; outputHashAlgo = null; }; }; "test${n}Sri256" = { expr = f { } { hash = sri256; }; expected = { outputHash = sri256; outputHashAlgo = null; }; }; "test${n}Sri512" = { expr = f { } { hash = sri512; }; expected = { outputHash = sri512; outputHashAlgo = null; }; }; "test${n}PreservesAttrs" = { expr = f { } { hash = "aaaa"; destination = "Earth"; }; expected = { outputHash = "aaaa"; outputHashAlgo = null; destination = "Earth"; }; }; "test${n}RejectsSha1ByDefault" = testingThrow (f { } { sha1 = ""; }); "test${n}RejectsSha512ByDefault" = testingThrow (f { } { sha512 = ""; }); "test${n}ThrowsOnMissing" = testingThrow (f { } { gibi = false; }); }; in runTests (unionOfDisjoints [ (genTests "NormalizeHash" normalizeHash) (genTests "WithNormalized" ( flip withNormalizedHash ({ outputHash, outputHashAlgo, ... }@args: args) )) { testNormalizeNotRequiredEquivalent = { expr = normalizeHash { required = false; } { hash = ""; prof = "shadoko"; }; expected = normalizeHash { } { hash = ""; prof = "shadoko"; }; }; testNormalizeNotRequiredPassthru = { expr = normalizeHash { required = false; } { "ga bu" = "zo meu"; }; expected."ga bu" = "zo meu"; }; testOptionalArg = { expr = withNormalizedHash { } ( { outputHash ? "", outputHashAlgo ? null, ... }@args: args ) { author = "Jacques Rouxel"; }; expected.author = "Jacques Rouxel"; }; testOptionalArgMetadata = { expr = functionArgs ( withNormalizedHash { } ( { outputHash ? "", outputHashAlgo ? null, }: { } ) ); expected.hash = true; }; testPreservesArgsMetadata = { expr = functionArgs ( withNormalizedHash { } ( { outputHash, outputHashAlgo, pumping ? true, }: { } ) ); expected = { hash = false; pumping = true; }; }; testRejectsMissingHashArg = testingThrow (withNormalizedHash { } ({ outputHashAlgo }: { })); testRejectsMissingAlgoArg = testingThrow (withNormalizedHash { } ({ outputHash }: { })); } ]) lib/tests/test-with-nix.nix +1 −0 Original line number Diff line number Diff line Loading @@ -17,6 +17,7 @@ pkgs.runCommand "nixpkgs-lib-tests-nix-${nix.version}" { buildInputs = [ (import ./check-eval.nix) (import ./fetchers.nix) (import ./maintainers.nix { inherit pkgs; lib = import ../.; Loading pkgs/build-support/build-bazel-package/default.nix +5 −3 Original line number Diff line number Diff line Loading @@ -64,7 +64,7 @@ let ; }; fBuildAttrs = fArgs // buildAttrs; fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ]; fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "hash" "sha256" ]; bazelCmd = { cmd, additionalFlags, targets, targetRunFlags ? [ ] }: lib.optionalString (targets != [ ]) '' # See footnote called [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables] Loading Loading @@ -197,8 +197,10 @@ stdenv.mkDerivation (fBuildAttrs // { dontFixup = true; allowedRequisites = []; outputHashAlgo = "sha256"; outputHash = fetchAttrs.sha256; inherit (lib.fetchers.normalizeHash { hashTypes = [ "sha256" ]; } fetchAttrs) outputHash outputHashAlgo ; }); nativeBuildInputs = fBuildAttrs.nativeBuildInputs or [] ++ [ (bazel.override { enableNixHacks = true; }) ]; Loading pkgs/build-support/fetchgit/default.nix +6 −13 Original line number Diff line number Diff line Loading @@ -10,8 +10,9 @@ appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${short}"; in "${if matched == null then base else builtins.head matched}${appendShort}"; in lib.makeOverridable ( { url, rev ? "HEAD", sha256 ? "", hash ? "", leaveDotGit ? deepClone lib.makeOverridable (lib.fetchers.withNormalizedHash { } ( { url, rev ? "HEAD", leaveDotGit ? deepClone , outputHash ? lib.fakeHash, outputHashAlgo ? null , fetchSubmodules ? true, deepClone ? false , branchName ? null , sparseCheckout ? [] Loading Loading @@ -56,9 +57,7 @@ lib.makeOverridable ( assert deepClone -> leaveDotGit; assert nonConeMode -> (sparseCheckout != []); if hash != "" && sha256 != "" then throw "Only one of sha256 or hash can be set" else if builtins.isString sparseCheckout then if builtins.isString sparseCheckout then # Changed to throw on 2023-06-04 throw "Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more." else Loading @@ -70,14 +69,8 @@ stdenvNoCC.mkDerivation { nativeBuildInputs = [ git cacert ] ++ lib.optionals fetchLFS [ git-lfs ]; outputHashAlgo = if hash != "" then null else "sha256"; inherit outputHash outputHashAlgo; outputHashMode = "recursive"; outputHash = if hash != "" then hash else if sha256 != "" then sha256 else lib.fakeSha256; # git-sparse-checkout(1) says: # > When the --stdin option is provided, the directories or patterns are read Loading Loading @@ -105,4 +98,4 @@ stdenvNoCC.mkDerivation { gitRepoUrl = url; }; } ) )) Loading
lib/fetchers.nix +173 −1 Original line number Diff line number Diff line # snippets that can be shared by multiple fetchers (pkgs/build-support) { lib }: { let commonH = hashTypes: rec { hashNames = [ "hash" ] ++ hashTypes; hashSet = lib.genAttrs hashNames (lib.const {}); }; fakeH = { hash = lib.fakeHash; sha256 = lib.fakeSha256; sha512 = lib.fakeSha512; }; in rec { proxyImpureEnvVars = [ # We borrow these environment variables from the caller to allow Loading @@ -14,4 +25,165 @@ "NIX_SSL_CERT_FILE" ]; /** Converts an attrset containing one of `hash`, `sha256` or `sha512`, into one containing `outputHash{,Algo}` as accepted by `mkDerivation`. An appropriate “fake hash” is substituted when the hash value is `""`, as is the [convention for fetchers](#sec-pkgs-fetchers-updating-source-hashes-fakehash-method). All other attributes in the set remain as-is. # Example ```nix normalizeHash { } { hash = ""; foo = "bar"; } => { outputHash = lib.fakeHash; outputHashAlgo = null; foo = "bar"; } ``` ```nix normalizeHash { } { sha256 = lib.fakeSha256; } => { outputHash = lib.fakeSha256; outputHashAlgo = "sha256"; } ``` ```nix normalizeHash { } { sha512 = lib.fakeSha512; } => { outputHash = lib.fakeSha512; outputHashAlgo = "sha512"; } ``` # Type ``` normalizeHash :: { hashTypes :: List String, required :: Bool } -> AttrSet -> AttrSet ``` # Arguments hashTypes : the set of attribute names accepted as hash inputs, in addition to `hash` required : whether to throw if no hash was present in the input; otherwise returns the original input, unmodified */ normalizeHash = { hashTypes ? [ "sha256" ], required ? true, }: let inherit (lib) concatMapStringsSep head tail throwIf; inherit (lib.attrsets) attrsToList intersectAttrs removeAttrs optionalAttrs; inherit (commonH hashTypes) hashNames hashSet; in args: if args ? "outputHash" then args else let # The argument hash, as a {name, value} pair h = # All hashes passed in arguments (possibly 0 or >1) as a list of {name, value} pairs let hashesAsNVPairs = attrsToList (intersectAttrs hashSet args); in if hashesAsNVPairs == [] then throwIf required "fetcher called without `hash`" null else if tail hashesAsNVPairs != [] then throw "fetcher called with mutually-incompatible arguments: ${concatMapStringsSep ", " (a: a.name) hashesAsNVPairs}" else head hashesAsNVPairs ; in removeAttrs args hashNames // (optionalAttrs (h != null) { outputHashAlgo = if h.name == "hash" then null else h.name; outputHash = if h.value == "" then fakeH.${h.name} or (throw "no “fake hash” defined for ${h.name}") else h.value; }) ; /** Wraps a function which accepts `outputHash{,Algo}` into one which accepts `hash` or `sha{256,512}` # Example ```nix withNormalizedHash { hashTypes = [ "sha256" "sha512" ]; } ( { outputHash, outputHashAlgo, ... }: ... ) ``` is a function which accepts one of `hash`, `sha256`, or `sha512` (or the original's `outputHash` and `outputHashAlgo`). Its `functionArgs` metadata only lists `hash` as a parameter, optional iff. `outputHash` was an optional parameter of the original function. `sha256`, `sha512`, `outputHash`, or `outputHashAlgo` are not mentioned in the `functionArgs` metadata. # Type ``` withNormalizedHash :: { hashTypes :: List String } -> (AttrSet -> T) -> (AttrSet -> T) ``` # Arguments hashTypes : the set of attribute names accepted as hash inputs, in addition to `hash` : they must correspond to a valid value for `outputHashAlgo`, currently one of: `md5`, `sha1`, `sha256`, or `sha512`. f : the function to be wrapped ::: {.note} In nixpkgs, `mkDerivation` rejects MD5 `outputHash`es, and SHA-1 is being deprecated. As such, there is no reason to add `md5` to `hashTypes`, and `sha1` should only ever be included for backwards compatibility. ::: # Output `withNormalizedHash { inherit hashTypes; } f` is functionally equivalent to ```nix args: f (normalizeHash { inherit hashTypes; required = !(lib.functionArgs f).outputHash; } args) ``` However, `withNormalizedHash` preserves `functionArgs` metadata insofar as possible, and is implemented somewhat more efficiently. */ withNormalizedHash = { hashTypes ? [ "sha256" ] }: fetcher: let inherit (lib.attrsets) genAttrs intersectAttrs removeAttrs; inherit (lib.trivial) const functionArgs setFunctionArgs; inherit (commonH hashTypes) hashSet; fArgs = functionArgs fetcher; normalize = normalizeHash { inherit hashTypes; required = !fArgs.outputHash; }; in # The o.g. fetcher must *only* accept outputHash and outputHashAlgo assert fArgs ? outputHash && fArgs ? outputHashAlgo; assert intersectAttrs fArgs hashSet == {}; setFunctionArgs (args: fetcher (normalize args)) (removeAttrs fArgs [ "outputHash" "outputHashAlgo" ] // { hash = fArgs.outputHash; }); }
lib/tests/fetchers.nix 0 → 100644 +165 −0 Original line number Diff line number Diff line let lib = import ./..; inherit (lib) fakeHash fakeSha256 fakeSha512 flip functionArgs runTests ; inherit (lib.fetchers) normalizeHash withNormalizedHash; testingThrow = expr: { expr = with builtins; tryEval (seq expr "didn't throw"); expected = { success = false; value = false; }; }; # hashes of empty sri256 = "sha256-d6xi4mKdjkX2JFicDIv5niSzpyI0m/Hnm8GGAIU04kY="; sri512 = "sha512-AXFyVo7jiZ5we10fxZ5E9qfPjSfqkizY2apCzORKFVYZaNhCIVbooY+J4cYST00ztLf0EjivIBPPdtIYFUMfzQ=="; unionOfDisjoints = lib.foldl lib.attrsets.unionOfDisjoint { }; genTests = n: f: { "test${n}AlreadyNormalized" = { expr = f { } { outputHash = ""; outputHashAlgo = "md42"; }; expected = { outputHash = ""; outputHashAlgo = "md42"; }; }; "test${n}EmptySha256" = { expr = f { } { sha256 = ""; }; expected = { outputHash = fakeSha256; outputHashAlgo = "sha256"; }; }; "test${n}EmptySha512" = { expr = f { hashTypes = [ "sha512" ]; } { sha512 = ""; }; expected = { outputHash = fakeSha512; outputHashAlgo = "sha512"; }; }; "test${n}EmptyHash" = { expr = f { } { hash = ""; }; expected = { outputHash = fakeHash; outputHashAlgo = null; }; }; "test${n}Sri256" = { expr = f { } { hash = sri256; }; expected = { outputHash = sri256; outputHashAlgo = null; }; }; "test${n}Sri512" = { expr = f { } { hash = sri512; }; expected = { outputHash = sri512; outputHashAlgo = null; }; }; "test${n}PreservesAttrs" = { expr = f { } { hash = "aaaa"; destination = "Earth"; }; expected = { outputHash = "aaaa"; outputHashAlgo = null; destination = "Earth"; }; }; "test${n}RejectsSha1ByDefault" = testingThrow (f { } { sha1 = ""; }); "test${n}RejectsSha512ByDefault" = testingThrow (f { } { sha512 = ""; }); "test${n}ThrowsOnMissing" = testingThrow (f { } { gibi = false; }); }; in runTests (unionOfDisjoints [ (genTests "NormalizeHash" normalizeHash) (genTests "WithNormalized" ( flip withNormalizedHash ({ outputHash, outputHashAlgo, ... }@args: args) )) { testNormalizeNotRequiredEquivalent = { expr = normalizeHash { required = false; } { hash = ""; prof = "shadoko"; }; expected = normalizeHash { } { hash = ""; prof = "shadoko"; }; }; testNormalizeNotRequiredPassthru = { expr = normalizeHash { required = false; } { "ga bu" = "zo meu"; }; expected."ga bu" = "zo meu"; }; testOptionalArg = { expr = withNormalizedHash { } ( { outputHash ? "", outputHashAlgo ? null, ... }@args: args ) { author = "Jacques Rouxel"; }; expected.author = "Jacques Rouxel"; }; testOptionalArgMetadata = { expr = functionArgs ( withNormalizedHash { } ( { outputHash ? "", outputHashAlgo ? null, }: { } ) ); expected.hash = true; }; testPreservesArgsMetadata = { expr = functionArgs ( withNormalizedHash { } ( { outputHash, outputHashAlgo, pumping ? true, }: { } ) ); expected = { hash = false; pumping = true; }; }; testRejectsMissingHashArg = testingThrow (withNormalizedHash { } ({ outputHashAlgo }: { })); testRejectsMissingAlgoArg = testingThrow (withNormalizedHash { } ({ outputHash }: { })); } ])
lib/tests/test-with-nix.nix +1 −0 Original line number Diff line number Diff line Loading @@ -17,6 +17,7 @@ pkgs.runCommand "nixpkgs-lib-tests-nix-${nix.version}" { buildInputs = [ (import ./check-eval.nix) (import ./fetchers.nix) (import ./maintainers.nix { inherit pkgs; lib = import ../.; Loading
pkgs/build-support/build-bazel-package/default.nix +5 −3 Original line number Diff line number Diff line Loading @@ -64,7 +64,7 @@ let ; }; fBuildAttrs = fArgs // buildAttrs; fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ]; fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "hash" "sha256" ]; bazelCmd = { cmd, additionalFlags, targets, targetRunFlags ? [ ] }: lib.optionalString (targets != [ ]) '' # See footnote called [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables] Loading Loading @@ -197,8 +197,10 @@ stdenv.mkDerivation (fBuildAttrs // { dontFixup = true; allowedRequisites = []; outputHashAlgo = "sha256"; outputHash = fetchAttrs.sha256; inherit (lib.fetchers.normalizeHash { hashTypes = [ "sha256" ]; } fetchAttrs) outputHash outputHashAlgo ; }); nativeBuildInputs = fBuildAttrs.nativeBuildInputs or [] ++ [ (bazel.override { enableNixHacks = true; }) ]; Loading
pkgs/build-support/fetchgit/default.nix +6 −13 Original line number Diff line number Diff line Loading @@ -10,8 +10,9 @@ appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${short}"; in "${if matched == null then base else builtins.head matched}${appendShort}"; in lib.makeOverridable ( { url, rev ? "HEAD", sha256 ? "", hash ? "", leaveDotGit ? deepClone lib.makeOverridable (lib.fetchers.withNormalizedHash { } ( { url, rev ? "HEAD", leaveDotGit ? deepClone , outputHash ? lib.fakeHash, outputHashAlgo ? null , fetchSubmodules ? true, deepClone ? false , branchName ? null , sparseCheckout ? [] Loading Loading @@ -56,9 +57,7 @@ lib.makeOverridable ( assert deepClone -> leaveDotGit; assert nonConeMode -> (sparseCheckout != []); if hash != "" && sha256 != "" then throw "Only one of sha256 or hash can be set" else if builtins.isString sparseCheckout then if builtins.isString sparseCheckout then # Changed to throw on 2023-06-04 throw "Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more." else Loading @@ -70,14 +69,8 @@ stdenvNoCC.mkDerivation { nativeBuildInputs = [ git cacert ] ++ lib.optionals fetchLFS [ git-lfs ]; outputHashAlgo = if hash != "" then null else "sha256"; inherit outputHash outputHashAlgo; outputHashMode = "recursive"; outputHash = if hash != "" then hash else if sha256 != "" then sha256 else lib.fakeSha256; # git-sparse-checkout(1) says: # > When the --stdin option is provided, the directories or patterns are read Loading Loading @@ -105,4 +98,4 @@ stdenvNoCC.mkDerivation { gitRepoUrl = url; }; } ) ))
