From f6326352218852fc9dcb32a9d0c4ca4ab6cd19d9 Mon Sep 17 00:00:00 2001 From: Adam Dinwoodie Date: Sat, 9 Aug 2025 18:25:00 +0100 Subject: [PATCH 1/3] fetchtorrent: run hooks when using rqbit The postFetch and postUnpack hooks ought to be run in fetchtorrent regardless of the backend in use. Ensure that's the case, rather than only running them if the transmission backend is being used. --- pkgs/build-support/fetchtorrent/default.nix | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/fetchtorrent/default.nix b/pkgs/build-support/fetchtorrent/default.nix index 91bd35012319..d281e71fe4ae 100644 --- a/pkgs/build-support/fetchtorrent/default.nix +++ b/pkgs/build-support/fetchtorrent/default.nix @@ -31,7 +31,7 @@ in meta ? { }, }: let - afterSuccess = writeShellScript "fetch-bittorrent-done.sh" '' + transmissionFinishScript = writeShellScript "fetch-bittorrent-done.sh" '' ${postUnpack} # Flatten the directory, so that only the torrent contents are in $out, not # the folder name @@ -75,17 +75,26 @@ runCommand name mkdir -p $downloadedDirectory mkdir -p $HOME/.config/transmission cp ${jsonConfig} $HOME/.config/transmission/settings.json + port="$(shuf -n 1 -i 49152-65535)" function handleChild { # This detects failures and logs the contents of the transmission fetch find $out exit 0 } trap handleChild CHLD - transmission-cli --port $(shuf -n 1 -i 49152-65535) --portmap --finish ${afterSuccess} --download-dir $downloadedDirectory --config-dir "$HOME"/.config/transmission "$url" + transmission-cli \ + --port "$port" \ + --portmap \ + --finish ${transmissionFinishScript} \ + --download-dir "$downloadedDirectory" \ + --config-dir "$HOME"/.config/transmission \ + "$url" '' else '' export HOME=$TMP rqbit --disable-dht-persistence --http-api-listen-addr "127.0.0.1:$(shuf -n 1 -i 49152-65535)" download -o $out --exit-on-finish "$url" + ${postUnpack} + ${postFetch} '' ) From 613a98461fabecf819afef74155c9934c98456b2 Mon Sep 17 00:00:00 2001 From: Adam Dinwoodie Date: Sat, 9 Aug 2025 18:31:14 +0100 Subject: [PATCH 2/3] fetchtorrent: check JSON config usage correctly The arguments to `fetchtorrent` are supposed to throw an error if the user attempts to specify JSON configuration when they're not using the transmission backend, however a logic error means the error will never be thrown. Move the test to somewhere where the behaviour will actually be checked. --- pkgs/build-support/fetchtorrent/default.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkgs/build-support/fetchtorrent/default.nix b/pkgs/build-support/fetchtorrent/default.nix index d281e71fe4ae..48c781d864f8 100644 --- a/pkgs/build-support/fetchtorrent/default.nix +++ b/pkgs/build-support/fetchtorrent/default.nix @@ -18,11 +18,7 @@ in "bittorrent" else "bittorrent-" + builtins.head (builtins.match urlRegexp url), - config ? - if (backend == "transmission") then - { } - else - throw "json config for configuring fetchFromBitorrent only works with the transmission backend", + config ? { }, hash, backend ? "transmission", recursiveHash ? true, @@ -44,6 +40,9 @@ let ''; jsonConfig = (formats.json { }).generate "jsonConfig" config; in +assert lib.assertMsg (config != { } -> backend == "transmission") '' + json config for configuring fetchtorrent only works with the transmission backend +''; runCommand name { inherit meta; From 588622a0e3ca50d268959a56893d80430ca827a5 Mon Sep 17 00:00:00 2001 From: Adam Dinwoodie Date: Sat, 9 Aug 2025 18:37:04 +0100 Subject: [PATCH 3/3] fetchtorrent: add flatten argument for rqbit Currently, the output from fetchtorrent will be different depending on whether the default "transmission" backend or the "rqbit" backend is used, because "rqbit" changed its behaviour in v6.0.0 to create subdirectories. Restore the old behaviour for the rqbit backend of flattening the directory structure, but add a "flatten" argument to allow users to explicitly request the behaviour without this change. Because fetchtorrent produces fixed-output derivations, it's possible that people won't notice the changes in behaviour here, so add a warning that the behaviour might be unexpected (in either direction!) if the flatten argument isn't specified for the rqbit backend, and -- to avoid needing to support it indefinitely -- a warning that `flatten = false` will be deprecated in future. Update the fetchtorrent tests to check all the relevant combinations, and to mark all tests as now working. Update the release notes to advertise this breaking change. Fixes #432001. --- doc/release-notes/rl-2511.section.md | 2 + pkgs/build-support/fetchtorrent/default.nix | 76 +++++++++++++++++- pkgs/build-support/fetchtorrent/tests.nix | 87 ++++++++++++++------- 3 files changed, 137 insertions(+), 28 deletions(-) diff --git a/doc/release-notes/rl-2511.section.md b/doc/release-notes/rl-2511.section.md index 69e53c6e5260..e5f61294dfd0 100644 --- a/doc/release-notes/rl-2511.section.md +++ b/doc/release-notes/rl-2511.section.md @@ -43,6 +43,8 @@ - `tooling-language-server` has been renamed to `deputy` (both the package and binary), following the rename of the upstream project. +- `fetchtorrent`, when using the "rqbit" backend, erroneously started fetching files into a subdirectory in Nixpkgs 24.11. The original behaviour – which matches the behaviour using the "transmission" backend – has now been restored. Users reliant on the erroneous behaviour can temporarily maintain it by adding `flatten = false` to the `fetchtorrent` arguments; Nix will produce an evaluation warning for anyone using `backend = "rqbit"` without `flatten = true`. + - `webfontkitgenerator` has been renamed to `webfont-bundler`, following the rename of the upstream project. The binary name remains `webfontkitgenerator`. The `webfontkitgenerator` package is an alias to `webfont-bundler`. diff --git a/pkgs/build-support/fetchtorrent/default.nix b/pkgs/build-support/fetchtorrent/default.nix index 48c781d864f8..5848c3e25f00 100644 --- a/pkgs/build-support/fetchtorrent/default.nix +++ b/pkgs/build-support/fetchtorrent/default.nix @@ -22,11 +22,15 @@ in hash, backend ? "transmission", recursiveHash ? true, + flatten ? null, postFetch ? "", postUnpack ? "", meta ? { }, }: let + # Default to flattening if no flatten argument was specified. + flatten' = if flatten == null then true else flatten; + transmissionFinishScript = writeShellScript "fetch-bittorrent-done.sh" '' ${postUnpack} # Flatten the directory, so that only the torrent contents are in $out, not @@ -39,10 +43,52 @@ let kill $PPID ''; jsonConfig = (formats.json { }).generate "jsonConfig" config; + + # https://github.com/NixOS/nixpkgs/issues/432001 + # + # For a while, the transmission backend would put the downloaded torrent in + # the output directory, but whether the rqbit backend would put the output in + # the output directory or a subdirectory depended on the version of rqbit. + # We want to standardise on a single behaviour, but give users of + # fetchtorrent with the rqbit backend some warning that the behaviour might + # be unexpected, particularly since we can't know what behaviour users might + # be expecting at this point, and they probably wouldn't notice a change + # straight away because the results are fixed-output derivations. + # + # This warning was introduced for 25.11, so we can remove handling of the + # `flatten` argument once that release is no longer supported. + warnings = + if backend == "rqbit" && flatten == null then + [ + '' + `fetchtorrent` with the rqbit backend may or may not have the + downloaded files stored in a subdirectory of the output directory. + Verify which behaviour you need, and set the `flatten` argument to + `fetchtorrent` accordingly. + + The `flatten = false` behaviour will still produce a warning, as this + behaviour is deprecated. It is only available with the "rqbit" backend + to provide temporary support for users who are relying on the + previous incorrect behaviour. For a warning-free evaluation, use + `flatten = true`. + '' + ] + else if flatten == false then + [ + '' + `fetchtorrent` with `flatten = false` is deprecated and will be + removed in a future release. + '' + ] + else + [ ]; in assert lib.assertMsg (config != { } -> backend == "transmission") '' json config for configuring fetchtorrent only works with the transmission backend ''; +assert lib.assertMsg (backend == "transmission" -> flatten') '' + `flatten = false` is only supported by the rqbit backend for fetchtorrent +''; runCommand name { inherit meta; @@ -92,8 +138,36 @@ runCommand name else '' export HOME=$TMP - rqbit --disable-dht-persistence --http-api-listen-addr "127.0.0.1:$(shuf -n 1 -i 49152-65535)" download -o $out --exit-on-finish "$url" + '' + + lib.optionalString flatten' '' + downloadedDirectory=$out/downloadedDirectory + mkdir -p $downloadedDirectory + '' + + lib.optionalString (!flatten') '' + downloadedDirectory=$out + '' + + '' + port="$(shuf -n 1 -i 49152-65535)" + + rqbit \ + --disable-dht-persistence \ + --http-api-listen-addr "127.0.0.1:$port" \ + download \ + -o "$downloadedDirectory" \ + --exit-on-finish \ + "$url" + ${postUnpack} + '' + + lib.optionalString flatten' '' + # Flatten the directory, so that only the torrent contents are in $out, + # not the folder name + shopt -s dotglob + mv -v $downloadedDirectory/*/* $out + rm -v -rf $downloadedDirectory + unset downloadedDirectory + '' + + '' ${postFetch} '' ) diff --git a/pkgs/build-support/fetchtorrent/tests.nix b/pkgs/build-support/fetchtorrent/tests.nix index 392d53b8956d..18774ba1d503 100644 --- a/pkgs/build-support/fetchtorrent/tests.nix +++ b/pkgs/build-support/fetchtorrent/tests.nix @@ -18,41 +18,74 @@ let }; # Via https://webtorrent.io/free-torrents - httpUrl = "https://webtorrent.io/torrents/sintel.torrent"; - magnetUrl = "magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent"; + http.url = "https://webtorrent.io/torrents/sintel.torrent"; + magnet.url = "magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent"; - # All routes to download the torrent should produce the same output and - # therefore have the same FOD hash. - hash = "sha256-EzbmBiTEWOlFUNaV5R4eDeD9EBbp6d93rfby88ACg0s="; + flattened.hash = "sha256-EzbmBiTEWOlFUNaV5R4eDeD9EBbp6d93rfby88ACg0s="; + unflattened.hash = "sha256-lVrlo1AwmFcxwsIsY976VYqb3hAprFH1xWYdmlTuw0U="; in - -{ - http-link = testers.invalidateFetcherByDrvHash fetchtorrent { - inherit hash; - url = httpUrl; - backend = "transmission"; +# Seems almost but not quite worth using lib.mapCartesianProduct... +builtins.mapAttrs (n: v: testers.invalidateFetcherByDrvHash fetchtorrent v) { + http-link = { + inherit (http) url; + inherit (flattened) hash; inherit (sintel) meta; }; - magnet-link = testers.invalidateFetcherByDrvHash fetchtorrent { - inherit hash; - url = magnetUrl; + http-link-transmission = { + inherit (http) url; backend = "transmission"; + inherit (flattened) hash; inherit (sintel) meta; }; - http-link-rqbit = testers.invalidateFetcherByDrvHash fetchtorrent { - inherit hash; - url = httpUrl; - backend = "rqbit"; - meta = sintel.meta // { - broken = true; - }; + magnet-link = { + inherit (magnet) url; + inherit (flattened) hash; + inherit (sintel) meta; }; - magnet-link-rqbit = testers.invalidateFetcherByDrvHash fetchtorrent { - inherit hash; - url = magnetUrl; + magnet-link-transmission = { + inherit (magnet) url; + backend = "transmission"; + inherit (flattened) hash; + inherit (sintel) meta; + }; + http-link-rqbit = { + inherit (http) url; backend = "rqbit"; - meta = sintel.meta // { - broken = true; - }; + inherit (flattened) hash; + inherit (sintel) meta; + }; + magnet-link-rqbit = { + inherit (magnet) url; + backend = "rqbit"; + inherit (flattened) hash; + inherit (sintel) meta; + }; + http-link-rqbit-flattened = { + inherit (http) url; + backend = "rqbit"; + flatten = true; + inherit (flattened) hash; + inherit (sintel) meta; + }; + magnet-link-rqbit-flattened = { + inherit (magnet) url; + backend = "rqbit"; + flatten = true; + inherit (flattened) hash; + inherit (sintel) meta; + }; + http-link-rqbit-unflattened = { + inherit (http) url; + backend = "rqbit"; + flatten = false; + inherit (unflattened) hash; + inherit (sintel) meta; + }; + magnet-link-rqbit-unflattened = { + inherit (magnet) url; + backend = "rqbit"; + flatten = false; + inherit (unflattened) hash; + inherit (sintel) meta; }; }