diff --git a/doc/builders/fetchers.chapter.md b/doc/builders/fetchers.chapter.md index 75a261db8dcd..ba7b1b190128 100644 --- a/doc/builders/fetchers.chapter.md +++ b/doc/builders/fetchers.chapter.md @@ -243,3 +243,26 @@ or *** ``` +## `fetchFromBittorrent` {#fetchfrombittorrent} + +`fetchFromBittorrent` expects two arguments. `url` which can either be a Magnet URI (Magnet Link) such as `magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c` or an HTTP URL pointing to a `.torrent` file. It can also take a `config` argument which will craft a `settings.json` configuration file and give it to `transmission`, the underlying program that is performing the fetch. The available config options for `transmission` can be found [here](https://github.com/transmission/transmission/blob/main/docs/Editing-Configuration-Files.md#options) + +``` +{ fetchFromBittorrent }: + +fetchFromBittorrent { + config = { peer-limit-global = 100; }; + url = "magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c"; + sha256 = ""; +} +``` + +### Parameters {#fetchfrombittorrent-parameters} + +- `url`: Magnet URI (Magnet Link) such as `magnet:?xt=urn:btih:dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c` or an HTTP URL pointing to a `.torrent` file. + +- `backend`: Which bittorrent program to use. Default: `"transmission"`. Valid values are `"rqbit"` or `"transmission"`. These are the two most suitable torrent clients for fetching in a fixed-output derivation at the time of writing, as they can be easily exited after usage. `rqbit` is written in Rust and has a smaller closure size than `transmission`, and the performance and peer discovery properties differs between these clients, requiring experimentation to decide upon which is the best. + +- `config`: When using `transmission` as the `backend`, a json configuration can + be supplied to transmission. Refer to the [upstream documentation](https://github.com/transmission/transmission/blob/main/docs/Editing-Configuration-Files.md) for information on how to configure. + diff --git a/pkgs/build-support/fetchbittorrent/default.nix b/pkgs/build-support/fetchbittorrent/default.nix new file mode 100644 index 000000000000..916c7e7d8c94 --- /dev/null +++ b/pkgs/build-support/fetchbittorrent/default.nix @@ -0,0 +1,60 @@ +{ lib, runCommand, transmission_noSystemd, rqbit, writeShellScript, formats, cacert, rsync }: +let + urlRegexp = ''.*xt=urn:bt[im]h:([^&]{64}|[^&]{40}).*''; +in +{ url +, name ? + if (builtins.match urlRegexp url) == null then + "bittorrent" + else + "bittorrent-" + builtins.head (builtins.match urlRegexp url) +, config ? if (backend == "transmission") then { } else throw "json config for configuring fetchFromBitorrent only works with the transmission backend" +, hash +, backend ? "transmission" +, recursiveHash ? true +, postFetch ? "" +, postUnpack ? "" +}: +let + afterSuccess = writeShellScript "fetch-bittorrent-done.sh" '' + ${postUnpack} + # Flatten the directory, so that only the torrent contents are in $out, not + # the folder name + shopt -s dotglob + mv -v $downloadedDirectory/*/* $out + rm -v -rf $downloadedDirectory + unset downloadedDirectory + ${postFetch} + kill $PPID + ''; + jsonConfig = (formats.json {}).generate "jsonConfig" config; +in +runCommand name { + nativeBuildInputs = [ cacert ] ++ (if (backend == "transmission" ) then [ transmission_noSystemd ] else if (backend == "rqbit") then [ rqbit ] else throw "rqbit or transmission are the only available backends for fetchbittorrent"); + outputHashAlgo = if hash != "" then null else "sha256"; + outputHash = hash; + outputHashMode = if recursiveHash then "recursive" else "flat"; + + # url will be written to the derivation, meaning it can be parsed and utilized + # by external tools, such as tools that may want to seed fetchBittorrent calls + # in nixpkgs + inherit url; +} +(if (backend == "transmission") then '' + export HOME=$TMP + export downloadedDirectory=$out/downloadedDirectory + mkdir -p $downloadedDirectory + mkdir -p $HOME/.config/transmission + cp ${jsonConfig} $HOME/.config/transmission/settings.json + function handleChild { + # This detects failures and logs the contents of the transmission fetch + find $out + exit 0 + } + trap handleChild CHLD + transmission-cli --port $(shuf -n 1 -i 49152-65535) --portmap --finish ${afterSuccess} --download-dir $downloadedDirectory --config-dir "$HOME"/.config/transmission "$url" +'' else +'' + export HOME=$TMP + rqbit --disable-dht-persistence --http-api-listen-addr "127.0.0.1:$(shuf -n 1 -i 49152-65535)" download -o $out --exit-on-finish "$url" +'') diff --git a/pkgs/build-support/fetchbittorrent/tests.nix b/pkgs/build-support/fetchbittorrent/tests.nix new file mode 100644 index 000000000000..fce1b3933a72 --- /dev/null +++ b/pkgs/build-support/fetchbittorrent/tests.nix @@ -0,0 +1,25 @@ +{ testers, fetchFromBittorrent, ... }: + +{ + http-link = testers.invalidateFetcherByDrvHash fetchFromBittorrent { + url = "https://webtorrent.io/torrents/wired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "transmission"; + }; + magnet-link = testers.invalidateFetcherByDrvHash fetchFromBittorrent { + url = "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "transmission"; + }; + http-link-rqbit = testers.invalidateFetcherByDrvHash fetchFromBittorrent { + url = "https://webtorrent.io/torrents/wired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "rqbit"; + }; + magnet-link-rqbit = testers.invalidateFetcherByDrvHash fetchFromBittorrent { + url = "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "rqbit"; + }; +} + diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index 2b1768515bab..4cc9ecc0db3e 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -104,6 +104,7 @@ with pkgs; cc-multilib-clang = callPackage ./cc-wrapper/multilib.nix { stdenv = clangMultiStdenv; }; fetchurl = callPackages ../build-support/fetchurl/tests.nix { }; + fetchFromBittorrent = callPackages ../build-support/fetchbittorrent/tests.nix { }; fetchpatch = callPackages ../build-support/fetchpatch/tests.nix { }; fetchpatch2 = callPackages ../build-support/fetchpatch/tests.nix { fetchpatch = fetchpatch2; }; fetchDebianPatch = callPackages ../build-support/fetchdebianpatch/tests.nix { }; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 5ea92e283ab6..213127a4d313 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1116,6 +1116,8 @@ with pkgs; fetchs3 = callPackage ../build-support/fetchs3 { }; + fetchFromBittorrent = callPackage ../build-support/fetchbittorrent { }; + fetchsvn = if stdenv.buildPlatform != stdenv.hostPlatform # hack around splicing being crummy with things that (correctly) don't eval. then buildPackages.fetchsvn