Commit b289b43b authored by matthewcroughan's avatar matthewcroughan Committed by Emery Hemingway
Browse files

fetchFromBittorrent: init



Adds a basic FOD Fetcher for Bittorrent that uses Transmission as a client

Co-authored-by: default avatarRobert Hensing <roberth@users.noreply.github.com>
parent 9b74d6b0
Loading
Loading
Loading
Loading
+23 −0
Original line number Diff line number Diff line
@@ -243,3 +243,26 @@ or

***
```
## `fetchFromBittorrent` {#fetchfrombittorrent}

`fetchFromBittorrent` expects two arguments. `url` which can either be a Magnet URI (Magnet Link) such as `magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c` or an HTTP URL pointing to a `.torrent` file. It can also take a `config` argument which will craft a `settings.json` configuration file and give it to `transmission`, the underlying program that is performing the fetch. The available config options for `transmission` can be found [here](https://github.com/transmission/transmission/blob/main/docs/Editing-Configuration-Files.md#options)

```
{ fetchFromBittorrent }:

fetchFromBittorrent {
  config = { peer-limit-global = 100; };
  url = "magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c";
  sha256 = "";
}
```

### Parameters {#fetchfrombittorrent-parameters}

- `url`: Magnet URI (Magnet Link) such as `magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c` or an HTTP URL pointing to a `.torrent` file.

- `backend`: Which bittorrent program to use. Default: `"transmission"`. Valid values are `"rqbit"` or `"transmission"`. These are the two most suitable torrent clients for fetching in a fixed-output derivation at the time of writing, as they can be easily exited after usage. `rqbit` is written in Rust and has a smaller closure size than `transmission`, and the performance and peer discovery properties differs between these clients, requiring experimentation to decide upon which is the best.

- `config`: When using `transmission` as the `backend`, a json configuration can
  be supplied to transmission. Refer to the [upstream documentation](https://github.com/transmission/transmission/blob/main/docs/Editing-Configuration-Files.md) for information on how to configure.
+60 −0
Original line number Diff line number Diff line
{ lib, runCommand, transmission_noSystemd, rqbit, writeShellScript, formats, cacert, rsync }:
let
  urlRegexp = ''.*xt=urn:bt[im]h:([^&]{64}|[^&]{40}).*'';
in
{ url
, name ?
  if (builtins.match urlRegexp url) == null then
    "bittorrent"
  else
    "bittorrent-" + builtins.head (builtins.match urlRegexp url)
, config ? if (backend == "transmission") then { } else throw "json config for configuring fetchFromBitorrent only works with the transmission backend"
, hash
, backend ? "transmission"
, recursiveHash ? true
, postFetch ? ""
, postUnpack ? ""
}:
let
  afterSuccess = writeShellScript "fetch-bittorrent-done.sh" ''
    ${postUnpack}
    # Flatten the directory, so that only the torrent contents are in $out, not
    # the folder name
    shopt -s dotglob
    mv -v $downloadedDirectory/*/* $out
    rm -v -rf $downloadedDirectory
    unset downloadedDirectory
    ${postFetch}
    kill $PPID
  '';
  jsonConfig = (formats.json {}).generate "jsonConfig" config;
in
runCommand name {
  nativeBuildInputs = [ cacert ] ++ (if (backend == "transmission" ) then [ transmission_noSystemd ] else if (backend == "rqbit") then [ rqbit ] else throw "rqbit or transmission are the only available backends for fetchbittorrent");
  outputHashAlgo = if hash != "" then null else "sha256";
  outputHash = hash;
  outputHashMode = if recursiveHash then "recursive" else "flat";

  # url will be written to the derivation, meaning it can be parsed and utilized
  # by external tools, such as tools that may want to seed fetchBittorrent calls
  # in nixpkgs
  inherit url;
}
(if (backend == "transmission") then ''
  export HOME=$TMP
  export downloadedDirectory=$out/downloadedDirectory
  mkdir -p $downloadedDirectory
  mkdir -p $HOME/.config/transmission
  cp ${jsonConfig} $HOME/.config/transmission/settings.json
  function handleChild {
    # This detects failures and logs the contents of the transmission fetch
    find $out
    exit 0
  }
  trap handleChild CHLD
  transmission-cli --port $(shuf -n 1 -i 49152-65535) --portmap --finish ${afterSuccess} --download-dir $downloadedDirectory --config-dir "$HOME"/.config/transmission "$url"
'' else
''
  export HOME=$TMP
  rqbit --disable-dht-persistence --http-api-listen-addr "127.0.0.1:$(shuf -n 1 -i 49152-65535)" download -o $out --exit-on-finish "$url"
'')
+25 −0
Original line number Diff line number Diff line
{ testers, fetchFromBittorrent, ... }:

{
  http-link = testers.invalidateFetcherByDrvHash fetchFromBittorrent {
    url = "https://webtorrent.io/torrents/wired-cd.torrent";
    hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo=";
    backend = "transmission";
  };
  magnet-link = testers.invalidateFetcherByDrvHash fetchFromBittorrent {
    url = "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent";
    hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo=";
    backend = "transmission";
  };
  http-link-rqbit = testers.invalidateFetcherByDrvHash fetchFromBittorrent {
    url = "https://webtorrent.io/torrents/wired-cd.torrent";
    hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo=";
    backend = "rqbit";
  };
  magnet-link-rqbit = testers.invalidateFetcherByDrvHash fetchFromBittorrent {
    url = "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent";
    hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo=";
    backend = "rqbit";
  };
}
+1 −0
Original line number Diff line number Diff line
@@ -104,6 +104,7 @@ with pkgs;
  cc-multilib-clang = callPackage ./cc-wrapper/multilib.nix { stdenv = clangMultiStdenv; };

  fetchurl = callPackages ../build-support/fetchurl/tests.nix { };
  fetchFromBittorrent = callPackages ../build-support/fetchbittorrent/tests.nix { };
  fetchpatch = callPackages ../build-support/fetchpatch/tests.nix { };
  fetchpatch2 = callPackages ../build-support/fetchpatch/tests.nix { fetchpatch = fetchpatch2; };
  fetchDebianPatch = callPackages ../build-support/fetchdebianpatch/tests.nix { };
+2 −0
Original line number Diff line number Diff line
@@ -1116,6 +1116,8 @@ with pkgs;
  fetchs3 = callPackage ../build-support/fetchs3 { };
  fetchFromBittorrent = callPackage ../build-support/fetchbittorrent { };
  fetchsvn = if stdenv.buildPlatform != stdenv.hostPlatform
    # hack around splicing being crummy with things that (correctly) don't eval.
    then buildPackages.fetchsvn