fetchurl: Allow hashed mirror overriding with config.hashedMirrors

Allows having alternate hashed mirrors as fallbacks. Useful in case the
default hashed mirror is not accessible or doesn't have everything
needed.

Co-authored-by: Johan Herland <johan.herland@tweag.io>
Co-authored-by: Yuriy Taraday <yuriy.taraday@tweag.io>
Co-authored-by: Alexander Bantyev <balsoft@balsoft.ru>

doc/release-notes/rl-2511.section.md

@@ -225,6 +225,7 @@
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->
 
 - Added `rewriteURL` attribute to the nixpkgs `config`, to allow for rewriting the URLs downloaded by `fetchurl`.
+- Added `hashedMirrors` attribute to the nixpkgs `config`, to allow for customization of the hashed mirrors used by `fetchurl`.
 
 - Added `gitConfig` and `gitConfigFile` option to the nixpkgs `config`, to allow for setting a default `gitConfigFile` for all `fetchgit` invocations.
 

pkgs/build-support/fetchurl/default.nix

@@ -7,11 +7,14 @@
   curl, # Note that `curl' may be `null', in case of the native stdenvNoCC.
   cacert ? null,
   rewriteURL,
+  hashedMirrors,
 }:
 
 let
 
-  mirrors = import ./mirrors.nix;
+  mirrors = import ./mirrors.nix // {
+    inherit hashedMirrors;
+  };
 
   # Write the list of mirrors to a file that we can reuse between
   # fetchurl instantiations, instead of passing the mirrors to

pkgs/build-support/fetchurl/mirrors.nix

@@ -1,9 +1,6 @@
 {
 
-  # Content-addressable Nix mirrors
-  hashedMirrors = [
-    "https://tarballs.nixos.org"
-  ];
+  hashedMirrors = throw "Use config.hashedMirrors instead of (import ./pkgs/build-support/fetchurl/mirrors.nix).hashedMirrors";
 
   # Mirrors for mirror://site/filename URIs, where "site" is
   # "sourceforge", "gnu", etc.

pkgs/build-support/fetchurl/tests.nix

@@ -22,6 +22,21 @@
         ${jq}/bin/jq -r '.headers.Hello' $out | ${moreutils}/bin/sponge $out
       '';
     };
+
+  # Tests that hashedMirrors works
+  hashedMirrors = testers.invalidateFetcherByDrvHash fetchurl {
+    # Make sure that we can only download from hashed mirrors
+    url = "http://broken";
+    # A file with this hash is definitely on tarballs.nixos.org
+    sha256 = "1j1y3cq6ys30m734axc0brdm2q9n2as4h32jws15r7w5fwr991km";
+
+    # No chance
+    curlOptsList = [
+      "--retry"
+      "0"
+    ];
+  };
+
   # Tests that downloadToTemp works with hashedMirrors
   no-skipPostFetch = testers.invalidateFetcherByDrvHash fetchurl {
     # Make sure that we can only download from hashed mirrors
@@ -40,6 +55,5 @@
     # $downloadedFile, but here we know that because the URL is broken, it will
     # have to fallback to fetching the previously-built derivation from
     # tarballs.nixos.org, which provides pre-built derivation outputs.
-
   };
 }

pkgs/stdenv/darwin/default.nix

@@ -188,7 +188,7 @@ let
           inherit lib;
           stdenvNoCC = prevStage.ccWrapperStdenv or thisStdenv;
           curl = bootstrapTools;
-          inherit (config) rewriteURL;
+          inherit (config) hashedMirrors rewriteURL;
         };
 
         inherit cc;

pkgs/stdenv/freebsd/default.nix

@@ -404,7 +404,7 @@ let
       fetchurlBoot = import ../../build-support/fetchurl {
         inherit lib stdenvNoCC;
         inherit (prevStage) curl;
-        inherit (config) rewriteURL;
+        inherit (config) hashedMirrors rewriteURL;
       };
       stdenv = import ../generic {
         inherit
@@ -502,7 +502,7 @@ in
           inherit lib;
           inherit (self) stdenvNoCC;
           inherit (prevStage) curl;
-          inherit (config) rewriteURL;
+          inherit (config) hashedMirrors rewriteURL;
         };
         gettext = super.gettext.overrideAttrs {
           NIX_CFLAGS_COMPILE = "-DHAVE_ICONV=1"; # we clearly have iconv. what do you want?

pkgs/stdenv/native/default.nix

@@ -192,7 +192,7 @@ in
         inherit lib stdenvNoCC;
         # Curl should be in /usr/bin or so.
         curl = null;
-        inherit (config) rewriteURL;
+        inherit (config) hashedMirrors rewriteURL;
       };
 
     }

pkgs/top-level/all-packages.nix

@@ -591,7 +591,7 @@ with pkgs;
       makeOverridable (import ../build-support/fetchurl) {
         inherit lib stdenvNoCC buildPackages;
         inherit cacert;
-        inherit (config) rewriteURL;
+        inherit (config) hashedMirrors rewriteURL;
         curl = buildPackages.curlMinimal.override (old: rec {
           # break dependency cycles
           fetchurl = stdenv.fetchurlBoot;

pkgs/top-level/config.nix

@@ -288,6 +288,19 @@ let
       '';
     };
 
+    hashedMirrors = mkOption {
+      type = types.listOf types.str;
+      default = [ "https://tarballs.nixos.org" ];
+      description = ''
+        The set of content-addressed/hashed mirror URLs used by [`pkgs.fetchurl`](#sec-pkgs-fetchers-fetchurl).
+        In case `pkgs.fetchurl` can't download from the given URLs,
+        it will try the hashed mirrors based on the expected output hash.
+
+        See [`copy-tarballs.pl`](https://github.com/NixOS/nixpkgs/blob/a2d829eaa7a455eaa3013c45f6431e705702dd46/maintainers/scripts/copy-tarballs.pl)
+        for more details on how hashed mirrors are constructed.
+      '';
+    };
+
     rewriteURL = mkOption {
       type = types.functionTo (types.nullOr types.str);
       description = ''