Commit b9464c6a authored by Adam Dinwoodie's avatar Adam Dinwoodie
Browse files

tests.fetchtorrent: verify and delete in postFetch 

The Sintel torrent used for testing fetchtorrent is about 124MB, which
is downloaded and cached for every test every time the test inputs
change.  Avoid that by instead validating the file hashes in the
postFetch stage, then deleting the downloaded files, so the only thing
that needs to be stored is an empty directory.
parent e99366c6

pkgs/build-support/fetchtorrent/test-hashes.sha512sum

0 → 100644
+11 −0
Original line number Diff line number Diff line 
1755c55e08aec3872db177464e0bc3f48c89aa40c1ec32605b472d88f6e92963eb08a10aa37bc47e837ae693db18652f40dfbcd6bf26f07278f0566a19c72cd5  poster.jpg

3ec33eae8153b6e2bdfff8157b332e6f0cfbc14d0c30e6a34042deecf0cff01ee19ca54314678b5ad970ae669da8cb91425e96253caf91d95ae01a62917827b7  Sintel.de.srt

ceac1ea59a9224b26946c6683acfaae27e1551c696fb2c632b55151c17f3faa8f930bbd93904cd7a13a896ad3d670b01e2f2bc1a8851060e05fedd85f804ba17  Sintel.en.srt

6efbc68f872c212f6579c5a5d42d6c2a60d775d32be8a7badcd147dbfc07b24cb2cc97e04a796af4002fcd4c8860a7061aed5d59a1280e4352d1bbd701d612a8  Sintel.es.srt

5619ad5bc7ca82e7f1f3b5e9197ab6ef6f5edd7293d396a3c036fef95d5ef592cf9e8d403e8532f4ba71a93fc9556614bfba3473eceae392d2e3175116c06f95  Sintel.fr.srt

fa93c051491b4e11f8e7a643c2138198f795315c942134b94ceace1adda4a5b28dc58b01b507e0da4c26784fc968f47bca1409f985eec2bf80c3e694662ed7c5  Sintel.it.srt

63426110ed7aa60ed3525078fa324cef4276393b4ada55e0a8cf8292500fdd59c2984e299a4e4f65e4f273b8ae2624da22700de4a55054b00e9d987f606c3c7f  Sintel.mp4

5fb04c25fda9dada8c808a9f289cec2cca001d60966b69d61ad663240a869c76dd0aff605122823ccbe199e266331f6a4b11a414bba92999a7041c4b534ef432  Sintel.nl.srt

510f052cbbda0b9fc31679b9a26cc16766df9054fc15b66cff475975725a1e8452e69a28a6f90bb965330a49e49bbf42a90efb13de07eaece7d273ef1ca7978a  Sintel.pl.srt

510dc0eca608bd0fdd9633ea80365c916f8c4f0fd8e2fdf77534f455f03814401d210df47d940be4fd2fb7ab4c3ff5038a548846c5fa9badbe77b1731b3cae29  Sintel.pt.srt

57c014bd3d781a94850f85fd6c9364421e17cb2c4e25e22c0a8578f50c874e3f2eaf5a0b3da615db7dc95efbe2e6be7c7238e2e1ba27671c818b9e34b05ee1d0  Sintel.ru.srt

pkgs/build-support/fetchtorrent/tests.nix

+64 −24
Original line number Diff line number Diff line
@@ -2,10 +2,24 @@ 
  lib,

  testers,

  fetchtorrent,

  emptyDirectory,

  ...

}:



let

  # This meta attribute isn't used anywhere, as the actual derivation

  # realizations are only empty directories.  It's maintained here as a record

  # of the details of the intermediate product that exists briefly while

  # building the test derivations.

  #

  # Ideally we'd use a smaller download, but neither of the Bittorrent backends

  # supported by fetchtorrent appear to support torrents that are only reliably

  # seeded by HTTP sources rather than other people using Bittorrent clients.

  # Sintel was the smallest torrent I could find that had a free license and

  # was reliably seeded by other Bittorrent clients.

  #

  # For more information, see the discussion at

  # https://github.com/NixOS/nixpkgs/pull/432091/files/bd13421b2b70f3f125061018c800439ef2d43e8d#r2264073113

  sintel.meta = {

    description = "An open source short film to show off open source technologies.";

    longDescription = ''
@@ -21,71 +35,97 @@ let 
  http.url = "https://webtorrent.io/torrents/sintel.torrent";

  magnet.url = "magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent";



  flattened.hash = "sha256-EzbmBiTEWOlFUNaV5R4eDeD9EBbp6d93rfby88ACg0s=";

  unflattened.hash = "sha256-lVrlo1AwmFcxwsIsY976VYqb3hAprFH1xWYdmlTuw0U=";

  # Sintel isn't a massive download, but it's not small.  There's also no real

  # value in storing copies of it in Nix caches, which is what happens by

  # default when this test succeeds.  Avoid that by verifying the downloaded

  # files using `sha512sum` in the post-fetch hook, then deleting the files so

  # the actual derivation result is an empty directory.

  #

  # Chain `&&` in the postFetch phase because the transmission backend does not

  # run that phase with `errexit` enabled.

  flattened.postFetch = ''

    pushd "$out" &&

    sha512sum --check --strict ${./test-hashes.sha512sum} &&

    sed 's/.*  //' ${./test-hashes.sha512sum} | xargs rm --verbose &&

    popd

  '';

  unflattened.postFetch = ''

    pushd "$out" &&

    pushd Sintel &&

    sha512sum --check --strict ${./test-hashes.sha512sum} &&

    sed 's/.*  //' ${./test-hashes.sha512sum} | xargs rm --verbose &&

    popd &&

    rm --dir --verbose Sintel &&

    popd

  '';



  # Fixed output derivation hash is identical for all derivations: the empty

  # directory.

  fetchtorrentWithHash =

    args:

    fetchtorrent (

      {

        hash = builtins.convertHash {

          hash = emptyDirectory.outputHash;

          toHashFormat = "sri";

          hashAlgo = emptyDirectory.outputHashAlgo;

        };

      }

      // args

    );

in

# Seems almost but not quite worth using lib.mapCartesianProduct...

builtins.mapAttrs (n: v: testers.invalidateFetcherByDrvHash fetchtorrent v) {

builtins.mapAttrs (n: v: testers.invalidateFetcherByDrvHash fetchtorrentWithHash v) {

  http-link = {

    inherit (http) url;

    inherit (flattened) hash;

    inherit (sintel) meta;

    inherit (flattened) postFetch;

  };

  http-link-transmission = {

    inherit (http) url;

    backend = "transmission";

    inherit (flattened) hash;

    inherit (sintel) meta;

    inherit (flattened) postFetch;

  };

  magnet-link = {

    inherit (magnet) url;

    inherit (flattened) hash;

    inherit (sintel) meta;

    inherit (flattened) postFetch;

  };

  magnet-link-transmission = {

    inherit (magnet) url;

    backend = "transmission";

    inherit (flattened) hash;

    inherit (sintel) meta;

    inherit (flattened) postFetch;

  };

  http-link-rqbit = {

    inherit (http) url;

    backend = "rqbit";

    inherit (flattened) hash;

    inherit (sintel) meta;

    inherit (flattened) postFetch;

  };

  magnet-link-rqbit = {

    inherit (magnet) url;

    backend = "rqbit";

    inherit (flattened) hash;

    inherit (sintel) meta;

    inherit (flattened) postFetch;

  };

  http-link-rqbit-flattened = {

    inherit (http) url;

    backend = "rqbit";

    flatten = true;

    inherit (flattened) hash;

    inherit (sintel) meta;

    inherit (flattened) postFetch;

  };

  magnet-link-rqbit-flattened = {

    inherit (magnet) url;

    backend = "rqbit";

    flatten = true;

    inherit (flattened) hash;

    inherit (sintel) meta;

    inherit (flattened) postFetch;

  };

  http-link-rqbit-unflattened = {

    inherit (http) url;

    backend = "rqbit";

    flatten = false;

    inherit (unflattened) hash;

    inherit (sintel) meta;

    inherit (unflattened) postFetch;

  };

  magnet-link-rqbit-unflattened = {

    inherit (magnet) url;

    backend = "rqbit";

    flatten = false;

    inherit (unflattened) hash;

    inherit (sintel) meta;

    inherit (unflattened) postFetch;

  };

}