From 91e5816acd146dc210c4ea2e3a8d468c6cddaa49 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 1 Apr 2025 22:06:02 +0200 Subject: [PATCH 1/9] recon-ng: use replaceVars instead of substituteAll --- pkgs/by-name/re/recon-ng/package.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/re/recon-ng/package.nix b/pkgs/by-name/re/recon-ng/package.nix index 03aed8ec2638..1a08b38e3462 100644 --- a/pkgs/by-name/re/recon-ng/package.nix +++ b/pkgs/by-name/re/recon-ng/package.nix @@ -2,7 +2,7 @@ lib, fetchFromGitHub, python3, - substituteAll, + replaceVars, fetchpatch, }: python3.pkgs.buildPythonApplication rec { @@ -49,8 +49,7 @@ python3.pkgs.buildPythonApplication rec { postPatch = let - setup = substituteAll { - src = ./setup.py; + setup = replaceVars ./setup.py { inherit pname version; }; in From 78a0cd852f4dea005858981de8258a6929fd62bc Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 1 Apr 2025 22:06:22 +0200 Subject: [PATCH 2/9] supercell-wx: use replaceVars instead of substituteAll --- pkgs/by-name/su/supercell-wx/package.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/su/supercell-wx/package.nix b/pkgs/by-name/su/supercell-wx/package.nix index a13e18c80c5c..4ee9dfbe87f1 100644 --- a/pkgs/by-name/su/supercell-wx/package.nix +++ b/pkgs/by-name/su/supercell-wx/package.nix @@ -26,7 +26,7 @@ qt6, tbb_2021_11, tracy, - substituteAll, + replaceVars, python3, }: let @@ -94,9 +94,8 @@ stdenv.mkDerivation (finalAttrs: { patches = [ # These are for Nix compatibility {{{ ./patches/use-find-package.patch # Replace some vendored dependencies with Nix provided versions - (substituteAll { + (replaceVars ./patches/skip-git-versioning.patch { # Skip tagging build with git version, and substitute it with the src revision (still uses current year timestamp) - src = ./patches/skip-git-versioning.patch; rev = finalAttrs.src.rev; }) # Prevents using some Qt scripts that seemed to break the install step. Fixes missing link to some targets. From 4e38e744d70aca7143a22802cd1a3544e1cb308d Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 1 Apr 2025 22:06:46 +0200 Subject: [PATCH 3/9] xdg-desktop-portal: use replaceVars instead of substituteAll --- pkgs/development/libraries/xdg-desktop-portal/default.nix | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkgs/development/libraries/xdg-desktop-portal/default.nix b/pkgs/development/libraries/xdg-desktop-portal/default.nix index 239cbd16be0f..69f97a833a47 100644 --- a/pkgs/development/libraries/xdg-desktop-portal/default.nix +++ b/pkgs/development/libraries/xdg-desktop-portal/default.nix @@ -27,7 +27,7 @@ gst_all_1, libgudev, umockdev, - substituteAll, + replaceVars, enableGeoLocation ? true, enableSystemd ? true, }: @@ -51,14 +51,12 @@ stdenv.mkDerivation (finalAttrs: { patches = [ # The icon validator copied from Flatpak needs to access the gdk-pixbuf loaders # in the Nix store and cannot bind FHS paths since those are not available on NixOS. - (substituteAll { - src = ./fix-icon-validation.patch; + (replaceVars ./fix-icon-validation.patch { inherit (builtins) storeDir; }) # Same for the sound validator, except the gdk-pixbuf part. - (substituteAll { - src = ./fix-sound-validation.patch; + (replaceVars ./fix-sound-validation.patch { inherit (builtins) storeDir; }) From 9c45add790213c6fae7c4d2fccb3c99f33916044 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 23 Apr 2025 23:26:29 +0200 Subject: [PATCH 4/9] rocmPackages.llvm.clang-unwrapped: use replaceVars instead of substituteAll --- pkgs/development/rocm-modules/6/llvm/default.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/development/rocm-modules/6/llvm/default.nix b/pkgs/development/rocm-modules/6/llvm/default.nix index dc2b3561936c..5131fc979df8 100644 --- a/pkgs/development/rocm-modules/6/llvm/default.nix +++ b/pkgs/development/rocm-modules/6/llvm/default.nix @@ -15,7 +15,7 @@ zlib, gcc-unwrapped, glibc, - substituteAll, + replaceVars, libffi, libxml2, removeReferencesTo, @@ -319,8 +319,7 @@ rec { }) # FIXME: Needed due to https://github.com/NixOS/nixpkgs/issues/375431 # Once we can switch to overrideScope this can be removed - (substituteAll { - src = ./../../../compilers/llvm/common/clang/clang-at-least-16-LLVMgold-path.patch; + (replaceVars ./../../../compilers/llvm/common/clang/clang-at-least-16-LLVMgold-path.patch { libllvmLibdir = "${llvm.lib}/lib"; }) ]; From 2b7b39cdd6134cb86b67062f0280d1fcf05f3f9d Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 23 Apr 2025 23:26:47 +0200 Subject: [PATCH 5/9] fim: use replaceVars instead of substituteAll --- pkgs/by-name/fi/fim/package.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/fi/fim/package.nix b/pkgs/by-name/fi/fim/package.nix index 3a452fac4bda..d31448da9c88 100644 --- a/pkgs/by-name/fi/fim/package.nix +++ b/pkgs/by-name/fi/fim/package.nix @@ -4,7 +4,7 @@ autoconf, automake, pkg-config, - substituteAll, + replaceVars, lib, perl, flex, @@ -41,9 +41,11 @@ stdenv.mkDerivation rec { patches = [ # build tools with a build compiler - (substituteAll { - src = ./native-tools.patch; + (replaceVars ./native-tools.patch { cc_for_build = lib.getExe buildPackages.stdenv.cc; + # patch context + FIM_WANT_CUSTOM_HARDCODED_CONSOLEFONT_TRUE = null; + HAVE_RUNNABLE_TESTS_TRUE = null; }) ]; From 3bfa33ed9958418600ec665825339a1f14d480af Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 23 Apr 2025 23:27:00 +0200 Subject: [PATCH 6/9] box2d: use replaceVars instead of substituteAll --- pkgs/by-name/bo/box2d/package.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/bo/box2d/package.nix b/pkgs/by-name/bo/box2d/package.nix index 8b29fa3afa45..27da46b1027e 100644 --- a/pkgs/by-name/bo/box2d/package.nix +++ b/pkgs/by-name/bo/box2d/package.nix @@ -3,7 +3,7 @@ stdenv, fetchFromGitHub, - substituteAll, + replaceVars, # nativeBuildInputs cmake, @@ -41,8 +41,7 @@ stdenv.mkDerivation (finalAttrs: { patches = [ # prevent CMake from trying to download some libraries from the internet - (substituteAll { - src = ./cmake_dont_fetch_enkits.patch; + (replaceVars ./cmake_dont_fetch_enkits.patch { enkits_src = fetchFromGitHub { owner = "dougbinks"; repo = "enkiTS"; From fe7dba685893030b2dec2b002164069ec9b7366a Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 1 Apr 2025 22:11:42 +0200 Subject: [PATCH 7/9] substituteAll: deprecate --- doc/release-notes/rl-2505.section.md | 2 + .../substitute/substitute-all.nix | 39 ++++++++++--------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/doc/release-notes/rl-2505.section.md b/doc/release-notes/rl-2505.section.md index 346319db23ab..1d9cd80d851b 100644 --- a/doc/release-notes/rl-2505.section.md +++ b/doc/release-notes/rl-2505.section.md @@ -112,6 +112,8 @@ - All support for 32‐bit Darwin systems has been dropped. +- `substituteAll` has been deprecated in favor of `replaceVars` and will be removed in the next release. + - Default ICU version updated from 74 to 76 - Apache Kafka was updated to `>= 4.0.0`. Please note that this is the first release which operates diff --git a/pkgs/build-support/substitute/substitute-all.nix b/pkgs/build-support/substitute/substitute-all.nix index 7f8d55449472..1135c822e22d 100644 --- a/pkgs/build-support/substitute/substitute-all.nix +++ b/pkgs/build-support/substitute/substitute-all.nix @@ -6,21 +6,24 @@ let isInvalidArgName = x: builtins.match "^[a-z][a-zA-Z0-9_]*$" x == null; invalidArgs = builtins.filter isInvalidArgName (builtins.attrNames args); in -if invalidArgs == [ ] then - stdenvNoCC.mkDerivation ( - { - name = if args ? name then args.name else baseNameOf (toString args.src); - builder = ./substitute-all.sh; - inherit (args) src; - preferLocalBuild = true; - allowSubstitutes = false; - } - // args - ) -else - throw '' - Argument names for `pkgs.substituteAll` must: - - start with a lower case ASCII letter - - only contain ASCII letters, digits and underscores - Found invalid argument names: ${lib.concatStringsSep ", " invalidArgs}. - '' +# TODO(@wolfgangwalther): Remove substituteAll, the nix function, after 25.05 branch-off. +lib.warn "substituteAll is deprecated and will be removed in 25.11. Use replaceVars instead." ( + if invalidArgs == [ ] then + stdenvNoCC.mkDerivation ( + { + name = if args ? name then args.name else baseNameOf (toString args.src); + builder = ./substitute-all.sh; + inherit (args) src; + preferLocalBuild = true; + allowSubstitutes = false; + } + // args + ) + else + throw '' + Argument names for `pkgs.substituteAll` must: + - start with a lower case ASCII letter + - only contain ASCII letters, digits and underscores + Found invalid argument names: ${lib.concatStringsSep ", " invalidArgs}. + '' +) From ac825aaea5f4092b652d56694b3dfba696d7736c Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 5 Apr 2025 14:39:42 +0200 Subject: [PATCH 8/9] substituteAllFiles: deprecate --- doc/release-notes/rl-2505.section.md | 2 +- .../substitute-files/substitute-all-files.nix | 45 ++++++++++--------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/doc/release-notes/rl-2505.section.md b/doc/release-notes/rl-2505.section.md index 1d9cd80d851b..774954bc3bc9 100644 --- a/doc/release-notes/rl-2505.section.md +++ b/doc/release-notes/rl-2505.section.md @@ -112,7 +112,7 @@ - All support for 32‐bit Darwin systems has been dropped. -- `substituteAll` has been deprecated in favor of `replaceVars` and will be removed in the next release. +- `substituteAll` and `substituteAllFiles` have been deprecated in favor of `replaceVars` and will be removed in the next release. - Default ICU version updated from 74 to 76 diff --git a/pkgs/build-support/substitute-files/substitute-all-files.nix b/pkgs/build-support/substitute-files/substitute-all-files.nix index 66cafe07c058..b8d1ae8a6834 100644 --- a/pkgs/build-support/substitute-files/substitute-all-files.nix +++ b/pkgs/build-support/substitute-files/substitute-all-files.nix @@ -2,27 +2,32 @@ args: -stdenv.mkDerivation ( - { - name = if args ? name then args.name else baseNameOf (toString args.src); - builder = builtins.toFile "builder.sh" '' - set -o pipefail +# TODO(@wolfgangwalther): Remove substituteAllFiles after 25.05 branch-off. +lib.warn + "substituteAllFiles is deprecated and will be removed in 25.11. Use replaceVars for each file instead." + ( + stdenv.mkDerivation ( + { + name = if args ? name then args.name else baseNameOf (toString args.src); + builder = builtins.toFile "builder.sh" '' + set -o pipefail - eval "$preInstall" + eval "$preInstall" - args= + args= - pushd "$src" - echo -ne "${lib.concatStringsSep "\\0" args.files}" | xargs -0 -n1 -I {} -- find {} -type f -print0 | while read -d "" line; do - mkdir -p "$out/$(dirname "$line")" - substituteAll "$line" "$out/$line" - done - popd + pushd "$src" + echo -ne "${lib.concatStringsSep "\\0" args.files}" | xargs -0 -n1 -I {} -- find {} -type f -print0 | while read -d "" line; do + mkdir -p "$out/$(dirname "$line")" + substituteAll "$line" "$out/$line" + done + popd - eval "$postInstall" - ''; - preferLocalBuild = true; - allowSubstitutes = false; - } - // args -) + eval "$postInstall" + ''; + preferLocalBuild = true; + allowSubstitutes = false; + } + // args + ) + ) From d3842f26e8c6c9a8ba92fa406a7005f604662c49 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 5 Apr 2025 15:22:01 +0200 Subject: [PATCH 9/9] doc/build-support: replace substituteAll with replaceVars --- doc/languages-frameworks/gnome.section.md | 4 +- doc/packages/build-support.md | 61 ++++++++++++----------- doc/redirects.json | 28 ++++++----- 3 files changed, 50 insertions(+), 43 deletions(-) diff --git a/doc/languages-frameworks/gnome.section.md b/doc/languages-frameworks/gnome.section.md index 03b030fb7be5..718c296bce80 100644 --- a/doc/languages-frameworks/gnome.section.md +++ b/doc/languages-frameworks/gnome.section.md @@ -214,7 +214,7 @@ stdenv.mkDerivation { You can rely on applications depending on the library setting the necessary environment variables but that is often easy to miss. Instead we recommend to patch the paths in the source code whenever possible. Here are some examples: -- []{#ssec-gnome-common-issues-unwrappable-package-gnome-shell-ext} [Replacing a `GI_TYPELIB_PATH` in GNOME Shell extension](https://github.com/NixOS/nixpkgs/blob/7bb8f05f12ca3cff9da72b56caa2f7472d5732bc/pkgs/desktops/gnome-3/core/gnome-shell-extensions/default.nix#L21-L24) – we are using `substituteAll` to include the path to a typelib into a patch. +- []{#ssec-gnome-common-issues-unwrappable-package-gnome-shell-ext} [Replacing a `GI_TYPELIB_PATH` in GNOME Shell extension](https://github.com/NixOS/nixpkgs/blob/e981466fbb08e6231a1377539ff17fbba3270fda/pkgs/by-name/gn/gnome-shell-extensions/package.nix#L25-L32) – we are using `replaceVars` to include the path to a typelib into a patch. - []{#ssec-gnome-common-issues-unwrappable-package-gsettings} The following examples are hardcoding GSettings schema paths. To get the schema paths we use the functions @@ -222,7 +222,7 @@ You can rely on applications depending on the library setting the necessary envi * `glib.makeSchemaPath` Takes a package output like `$out` and a derivation name. You should use this if the schemas you need to hardcode are in the same derivation. - []{#ssec-gnome-common-issues-unwrappable-package-gsettings-vala} [Hard-coding GSettings schema path in Vala plug-in (dynamically loaded library)](https://github.com/NixOS/nixpkgs/blob/7bb8f05f12ca3cff9da72b56caa2f7472d5732bc/pkgs/desktops/pantheon/apps/elementary-files/default.nix#L78-L86) – here, `substituteAll` cannot be used since the schema comes from the same package preventing us from pass its path to the function, probably due to a [Nix bug](https://github.com/NixOS/nix/issues/1846). + []{#ssec-gnome-common-issues-unwrappable-package-gsettings-vala} [Hard-coding GSettings schema path in Vala plug-in (dynamically loaded library)](https://github.com/NixOS/nixpkgs/blob/7bb8f05f12ca3cff9da72b56caa2f7472d5732bc/pkgs/desktops/pantheon/apps/elementary-files/default.nix#L78-L86) – here, `replaceVars` cannot be used since the schema comes from the same package preventing us from pass its path to the function, probably due to a [Nix bug](https://github.com/NixOS/nix/issues/1846). []{#ssec-gnome-common-issues-unwrappable-package-gsettings-c} [Hard-coding GSettings schema path in C library](https://github.com/NixOS/nixpkgs/blob/29c120c065d03b000224872251bed93932d42412/pkgs/development/libraries/glib-networking/default.nix#L31-L34) – nothing special other than using [Coccinelle patch](https://github.com/NixOS/nixpkgs/pull/67957#issuecomment-527717467) to generate the patch itself. diff --git a/doc/packages/build-support.md b/doc/packages/build-support.md index 80392f4d121e..1e4756d7f258 100644 --- a/doc/packages/build-support.md +++ b/doc/packages/build-support.md @@ -30,13 +30,12 @@ substitute { ``` ::: -## `pkgs.substituteAll` {#pkgs-substituteall} +## `pkgs.replaceVars` {#pkgs-replacevars} -`pkgs.substituteAll` substitutes all instances of `@varName@` (`@`s included) in file `src` with the value of the corresponding environment variable. -As this uses the [`substituteAll`] (#fun-substitute) function, its limitations regarding variable names that will or will not be replaced also apply here. +`pkgs.replaceVars ` replaces all instances of `@varName@` (`@`s included) in file `src` with the respective value in the attribute set `replacements`. -:::{.example #ex-pkgs-substituteAll} -# Usage of `pkgs.substituteAll` +:::{.example #ex-pkgs-replace-vars} +# Usage of `pkgs.replaceVars` If `say-goodbye.sh` contains the following: @@ -51,16 +50,14 @@ the following derivation will make substitutions to `@bash@`, `@hello@`, and `@g ```nix { - substituteAll, + replaceVars, bash, hello, }: -substituteAll { - src = ./say-goodbye.sh; - env = { - inherit bash hello; - greeting = "goodbye"; - }; +replaceVars ./say-goodbye.sh { + inherit bash hello; + greeting = "goodbye"; + unchanged = null; } ``` @@ -72,31 +69,37 @@ such that `$out` will result in something like the following: echo @unchanged@ /nix/store/566f5isbvw014h7knmzmxa5l6hshx43k-hello-2.12.1/bin/hello --greeting goodbye ``` + +Note that, in contrast to the old `substituteAll`, `unchanged = null` must explicitly be set. +Any unreferenced `@...@` pattern in the source file will throw an error. ::: -## `pkgs.substituteAllFiles` {#pkgs-substituteallfiles} +## `pkgs.replaceVarsWith` {#pkgs-replacevarswith} -`pkgs.substituteAllFiles` replaces `@varName@` with the value of the environment variable `varName`. -It expects `src` to be a directory and requires a `files` argument that specifies which files will be subject to replacements; only these files will be placed in `$out`. +`pkgs.replaceVarsWith` works the same way as [pkgs.replaceVars](#pkgs-replacevars), but additionally allows more options. -As it also uses the `substituteAll` function, it is subject to the same limitations on environment variables as discussed in [pkgs.substituteAll](#pkgs-substituteall). +:::{.example #ex-pkgs-replace-vars-with} +# Usage of `pkgs.replaceVarsWith` -:::{.example #ex-pkgs-substitute-all-files} -# Usage of `pkgs.substituteAllFiles` - -If the current directory contains `{foo,bar,baz}.txt` and the following `default.nix` +With the example file `say-goodbye.sh`, consider: ```nix -{ substituteAllFiles }: -substituteAllFiles { - src = ./.; - files = [ - "foo.txt" - "bar.txt" - ]; - hello = "there"; +{ replaceVarsWith }: +replaceVarsWith { + src = ./say-goodbye.sh; + + replacements = { + inherit bash hello; + greeting = "goodbye"; + unchanged = null; + }; + + name = "say-goodbye"; + dir = "bin"; + isExecutable = true; + meta.mainProgram = "say-goodbye"; } ``` -in the resulting derivation, every instance of `@hello@` will be replaced with `there` in `$out/foo.txt` and `$out/bar.txt`; `baz.txt` will not be processed nor will it appear in `$out`. +This will make the resulting file executable, put it in `bin/say-goodbye` and set `meta` attributes respectively. ::: diff --git a/doc/redirects.json b/doc/redirects.json index 627b4137281d..15141bb96426 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -8,6 +8,14 @@ "ex-build-helpers-extendMkDerivation": [ "index.html#ex-build-helpers-extendMkDerivation" ], + "ex-pkgs-replace-vars": [ + "index.html#ex-pkgs-replace-vars", + "index.html#ex-pkgs-substituteAll", + "index.html#ex-pkgs-substitute-all-files" + ], + "ex-pkgs-replace-vars-with": [ + "index.html#ex-pkgs-replace-vars-with" + ], "ex-shfmt": [ "index.html#ex-shfmt" ], @@ -35,6 +43,14 @@ "no-broken-symlinks.sh": [ "index.html#no-broken-symlinks.sh" ], + "pkgs-replacevars": [ + "index.html#pkgs-replacevars", + "index.html#pkgs-substituteall", + "index.html#pkgs-substituteallfiles" + ], + "pkgs-replacevarswith": [ + "index.html#pkgs-replacevarswith" + ], "preface": [ "index.html#preface" ], @@ -4193,18 +4209,6 @@ "ex-pkgs-substitute": [ "index.html#ex-pkgs-substitute" ], - "pkgs-substituteall": [ - "index.html#pkgs-substituteall" - ], - "ex-pkgs-substituteAll": [ - "index.html#ex-pkgs-substituteAll" - ], - "pkgs-substituteallfiles": [ - "index.html#pkgs-substituteallfiles" - ], - "ex-pkgs-substitute-all-files": [ - "index.html#ex-pkgs-substitute-all-files" - ], "part-development": [ "index.html#part-development" ],