stdenv.mkDerivation: cache as much as possible in bootstrapping (#525312)

This commit is contained in:
Philip Taron
2026-06-01 16:43:18 +00:00
committed by GitHub
14 changed files with 243 additions and 201 deletions
@@ -361,7 +361,11 @@ lib.makeScope
gnutar = gnutar-latest;
};
inherit (callPackage ./utils.nix { }) derivationWithMeta writeTextFile writeText;
inherit (callPackage ./utils.nix { inherit hostPlatform; })
derivationWithMeta
writeTextFile
writeText
;
test = kaem.runCommand "minimal-bootstrap-test" { } (
''
echo ${bash.tests.get-version}
@@ -6,7 +6,12 @@
kaem,
mescc-tools-extra,
checkMeta,
hostPlatform,
}:
let
assertValidity = checkMeta.assertValidity hostPlatform;
commonMeta = checkMeta.commonMeta hostPlatform;
in
rec {
maybeContentAddressed = lib.optionalAttrs config.contentAddressedByDefault {
__contentAddressed = true;
@@ -18,8 +23,8 @@ rec {
attrs:
let
passthru = attrs.passthru or { };
validity = checkMeta.assertValidity { inherit meta attrs; };
meta = checkMeta.commonMeta { inherit validity attrs; };
validity = assertValidity { inherit meta attrs; };
meta = commonMeta { inherit validity attrs; };
baseDrv = derivation (
{
inherit (buildPlatform) system;
+4 -1
View File
@@ -13,7 +13,10 @@
let
# N.B. Keep in sync with default arg for stdenv/generic.
defaultMkDerivationFromStdenv =
stdenv: (import ./generic/make-derivation.nix { inherit lib config; } stdenv).mkDerivation;
let
makeDerivationFile = import ./generic/make-derivation.nix lib config;
in
stdenv: (makeDerivationFile stdenv).mkDerivation;
# Low level function to help with overriding `mkDerivationFromStdenv`. One
# gives it the old stdenv arguments and a "continuation" function, and
+3 -6
View File
@@ -27,6 +27,7 @@ assert crossSystem == localSystem;
let
inherit (localSystem) system;
genericStdenv = import ../generic { defaultConfig = config; };
llvmVersion = "21"; # This needs to be updated when the default LLVM version is changed.
sdkMajorVersion = lib.versions.major localSystem.darwinSdkVersion;
@@ -104,15 +105,13 @@ let
bashNonInteractive = prevStage.bashNonInteractive or bootstrapTools;
thisStdenv = import ../generic {
thisStdenv = genericStdenv {
name = "${name}-stdenv-darwin";
buildPlatform = localSystem;
hostPlatform = localSystem;
targetPlatform = localSystem;
inherit config;
extraBuildInputs = [ prevStage.apple-sdk ];
inherit extraNativeBuildInputs;
@@ -975,15 +974,13 @@ assert bootstrapTools.passthru.isFromBootstrapFiles or false; # sanity check
in
{
inherit config overlays;
stdenv = import ../generic {
stdenv = genericStdenv {
name = "stdenv-darwin";
buildPlatform = localSystem;
hostPlatform = localSystem;
targetPlatform = localSystem;
inherit config;
preHook = ''
${commonPreHook}
stripDebugFlags="-S" # llvm-strip does not support "-p" for Mach-O
+4 -4
View File
@@ -21,6 +21,8 @@
assert crossSystem == localSystem;
let
genericStdenv = import ../generic { defaultConfig = config; };
inherit (localSystem) system;
mkExtraBuildCommands0 = cc: ''
rsrc="$out/resource-root"
@@ -383,9 +385,8 @@ let
bsdcp
];
shell = "${prevStage.bashNonInteractive}/bin/bash";
stdenvNoCC = import ../generic {
stdenvNoCC = genericStdenv {
inherit
config
initialPath
shell
fetchurlBoot
@@ -401,9 +402,8 @@ let
inherit (prevStage) curl;
inherit (config) hashedMirrors rewriteURL;
};
stdenv = import ../generic {
stdenv = genericStdenv {
inherit
config
initialPath
shell
fetchurlBoot
+1 -1
View File
@@ -47,7 +47,7 @@ let
checkMeta = testPkgs.callPackage ./check-meta.nix { };
tryEval = expression: builtins.tryEval (builtins.deepSeq expression expression);
actual = tryEval (
checkMeta.assertValidity {
checkMeta.assertValidity pkgs.stdenv.hostPlatform {
meta = pkg.meta;
attrs = pkg;
}
+18 -6
View File
@@ -4,7 +4,6 @@
{
lib,
config,
hostPlatform,
}:
let
@@ -122,6 +121,7 @@ let
# Logical inversion of meta.availableOn for hostPlatform
hasUnsupportedPlatform =
hostPlatform:
let
inherit (hostPlatform) system;
# in almost all cases, meta.platforms is a simple list of strings, and we
@@ -416,6 +416,10 @@ let
# !!! reason strings are hardcoded into OfBorg, make sure to keep them in sync
# Along with a boolean flag for each reason
checkValidity =
hostPlatform:
let
hasUnsupportedPlatform' = hasUnsupportedPlatform hostPlatform;
in
attrs:
if !attrs ? meta then
null
@@ -458,7 +462,7 @@ let
msg = "contains elements not built from source (${showSourceType attrs.meta.sourceProvenance})";
remediation = remediate_allowlist "NonSource" (remediate_predicate "allowNonSourcePredicate" attrs);
}
else if hasUnsupportedPlatform attrs && !allowUnsupportedSystem then
else if hasUnsupportedPlatform' attrs && !allowUnsupportedSystem then
let
toPretty' = toPretty {
allowPrettyValues = true;
@@ -518,9 +522,13 @@ let
# passed to the builder and is not a dependency. But since we
# include it in the result, it *is* available to nix-env for queries.
# Example:
# meta = checkMeta.commonMeta { inherit validity attrs pos references; };
# validity = checkMeta.assertValidity { inherit meta attrs; };
# meta = checkMeta.commonMeta hostPlatform { inherit validity attrs pos references; };
# validity = checkMeta.assertValidity hostPlatform { inherit meta attrs; };
commonMeta =
hostPlatform:
let
hasUnsupportedPlatform' = hasUnsupportedPlatform hostPlatform;
in
{
validity,
attrs,
@@ -659,7 +667,7 @@ let
# Expose the result of the checks for everyone to see.
unfree = hasUnfreeLicense attrs;
broken = isMarkedBroken attrs;
unsupported = hasUnsupportedPlatform attrs;
unsupported = hasUnsupportedPlatform' attrs;
insecure = isMarkedInsecure attrs;
available =
@@ -705,9 +713,13 @@ let
builtins.seq (foldl' giveWarning null warnings) withError;
assertValidity =
hostPlatform:
let
checkValidity' = checkValidity hostPlatform;
in
{ meta, attrs }:
let
invalid = checkValidity attrs;
invalid = checkValidity' attrs;
problems = checkProblems attrs;
in
if isNull invalid then
+39 -20
View File
@@ -1,5 +1,37 @@
{
defaultConfig ? null,
}@args:
let
lib = import ../../../lib;
# By taking defaultConfig early, we can cache the result of calling
# make-derivation.nix with config, which leads to more memoisation between
# bootstrapping stages. We only have to re-call the file with another config
# if stdenv-overridable is actually called with config, otherwise we stick to
# defaultConfig. No stdenvs currently specify a non-default config, but we
# leave it open as a possibility.
makeDerivationFile = import ./make-derivation.nix lib;
makeDerivationFileWithConfig =
assert args ? defaultConfig;
makeDerivationFile args.defaultConfig;
defaultNativeBuildInputs0 = [
../../build-support/setup-hooks/no-broken-symlinks.sh
../../build-support/setup-hooks/audit-tmpdir.sh
../../build-support/setup-hooks/compress-man-pages.sh
../../build-support/setup-hooks/make-symlinks-relative.sh
../../build-support/setup-hooks/move-docs.sh
../../build-support/setup-hooks/move-lib64.sh
../../build-support/setup-hooks/move-sbin.sh
../../build-support/setup-hooks/move-systemd-user-units.sh
../../build-support/setup-hooks/multiple-outputs.sh
../../build-support/setup-hooks/patch-shebangs.sh
../../build-support/setup-hooks/prune-libtool-files.sh
../../build-support/setup-hooks/reproducible-builds.sh
../../build-support/setup-hooks/set-source-date-epoch-to-latest.sh
../../build-support/setup-hooks/strip.sh
];
stdenv-overridable = lib.makeOverridable (
argsStdenv@{
@@ -21,7 +53,7 @@ let
allowedRequisites ? null,
extraAttrs ? { },
overrides ? (self: super: { }),
config,
config ? args.defaultConfig,
disallowedRequisites ? [ ],
# The `fetchurl' to use for downloading curl and its dependencies
@@ -64,29 +96,16 @@ let
# The implementation of `mkDerivation`, parameterized with the final stdenv so we can tie the knot.
# This is convenient to have as a parameter so the stdenv "adapters" work better
mkDerivationFromStdenv ?
stdenv: (import ./make-derivation.nix { inherit lib config; } stdenv).mkDerivation,
let
makeDerivationWithConfig' =
if argsStdenv ? config then makeDerivationFile config else makeDerivationFileWithConfig;
in
stdenv: (makeDerivationWithConfig' stdenv).mkDerivation,
}:
let
defaultNativeBuildInputs =
extraNativeBuildInputs
++ [
../../build-support/setup-hooks/no-broken-symlinks.sh
../../build-support/setup-hooks/audit-tmpdir.sh
../../build-support/setup-hooks/compress-man-pages.sh
../../build-support/setup-hooks/make-symlinks-relative.sh
../../build-support/setup-hooks/move-docs.sh
../../build-support/setup-hooks/move-lib64.sh
../../build-support/setup-hooks/move-sbin.sh
../../build-support/setup-hooks/move-systemd-user-units.sh
../../build-support/setup-hooks/multiple-outputs.sh
../../build-support/setup-hooks/patch-shebangs.sh
../../build-support/setup-hooks/prune-libtool-files.sh
../../build-support/setup-hooks/reproducible-builds.sh
../../build-support/setup-hooks/set-source-date-epoch-to-latest.sh
../../build-support/setup-hooks/strip.sh
]
++ lib.optionals hasCC [ cc ];
extraNativeBuildInputs ++ defaultNativeBuildInputs0 ++ lib.optionals hasCC [ cc ];
defaultBuildInputs = extraBuildInputs;
+148 -148
View File
@@ -6,10 +6,7 @@
#
# See https://github.com/NixOS/nixpkgs/pull/430969 for measurements.
{ lib, config }:
stdenv:
lib:
let
# Lib attributes are inherited to the lexical scope for performance reasons.
inherit (lib)
@@ -52,7 +49,6 @@ let
unsafeDiscardStringContext
unsafeGetAttrPos
warn
warnIf
zipAttrsWith
any
;
@@ -60,9 +56,154 @@ let
inherit (lib.generators) toPretty;
inherit (lib.strings) sanitizeDerivationName;
knownHardeningFlags = [
"bindnow"
"format"
"fortify"
"fortify3"
"strictflexarrays1"
"strictflexarrays3"
"shadowstack"
"nostrictaliasing"
"pacret"
"pic"
"relro"
"stackprotector"
"glibcxxassertions"
"libcxxhardeningfast"
"libcxxhardeningextensive"
"stackclashprotection"
"strictoverflow"
"trivialautovarinit"
"zerocallusedregs"
];
removedOrReplacedAttrNames = [
"checkInputs"
"installCheckInputs"
"nativeCheckInputs"
"nativeInstallCheckInputs"
"__contentAddressed"
"__darwinAllowLocalNetworking"
"__impureHostDeps"
"__propagatedImpureHostDeps"
"sandboxProfile"
"propagatedSandboxProfile"
"disallowedReferences"
"disallowedRequisites"
"allowedReferences"
"allowedRequisites"
"allowedImpureDLLs"
];
referenceCheckingAttrsToRemove = [
"allowedReferences"
"allowedRequisites"
"disallowedReferences"
"disallowedRequisites"
];
argumentAttrsToRemove = [
"meta"
"passthru"
"pos"
"env"
];
attrsToRemoveLast = [
# Fixed-output derivations may not reference other paths, which means that for a fixed-output
# derivation, the corresponding inputDerivation should *not* be fixed-output. To achieve this we
# simply delete the attributes that would make it fixed-output.
"outputHashAlgo"
"outputHash"
"outputHashMode"
# inputDerivation produces the inputs; not the outputs, so any restrictions on what used to be
# the outputs don't serve a purpose anymore.
"allowedReferences"
"allowedRequisites"
"disallowedReferences"
"disallowedRequisites"
"outputChecks"
];
defaultBuilderArgs = [
"-e"
./source-stdenv.sh
./default-builder.sh
];
isSingularDependency = dep: dep == null || isDerivation dep || isString dep || isPath dep;
cachedOutputChecks = {
out = { };
};
debugCachedOutputChecks = {
out = { };
debug = { };
};
# Turn a derivation into its outPath without a string context attached.
# See the comment at the usage site.
unsafeDerivationToUntrackedOutpath =
drv:
if isDerivation drv && (!drv.__contentAddressed or false) then
unsafeDiscardStringContext drv.outPath
else
drv;
makeOutputChecks =
attrs:
# If we use derivations directly here, they end up as build-time dependencies.
# This is especially problematic in the case of disallowed*, since the disallowed
# derivations will be built by nix as build-time dependencies, while those
# derivations might take a very long time to build, or might not even build
# successfully on the platform used.
# We can improve on this situation by instead passing only the outPath,
# without an attached string context, to nix. The out path will be a placeholder
# which will be replaced by the actual out path if the derivation in question
# is part of the final closure (and thus needs to be built). If it is not
# part of the final closure, then the placeholder will be passed along,
# but in that case we know for a fact that the derivation is not part of the closure.
# This means that passing the out path to nix does the right thing in either
# case, both for disallowed and allowed references/requisites, and we won't
# build the derivation if it wouldn't be part of the closure, saving time and resources.
# While the problem is less severe for allowed*, since we want the derivation
# to be built eventually, we would still like to get the error early and without
# having to wait while nix builds a derivation that might not be used.
# See also https://github.com/NixOS/nix/issues/4629
{
${if (attrs ? disallowedReferences) then "disallowedReferences" else null} =
map unsafeDerivationToUntrackedOutpath attrs.disallowedReferences;
${if (attrs ? disallowedRequisites) then "disallowedRequisites" else null} =
map unsafeDerivationToUntrackedOutpath attrs.disallowedRequisites;
${if (attrs ? allowedReferences) then "allowedReferences" else null} =
mapNullable unsafeDerivationToUntrackedOutpath attrs.allowedReferences;
${if (attrs ? allowedRequisites) then "allowedRequisites" else null} =
mapNullable unsafeDerivationToUntrackedOutpath attrs.allowedRequisites;
};
in
config:
let
doCheckByDefault = config.doCheckByDefault or false;
structuredAttrsByDefault = config.structuredAttrsByDefault or false;
inherit (config) enableParallelBuildingByDefault contentAddressedByDefault;
userHook = config.stdenv.userHook or null;
checkMeta = import ./check-meta.nix {
inherit lib config;
};
in
stdenv:
let
inherit (import ../../build-support/lib/cmake.nix { inherit lib stdenv; }) makeCMakeFlags;
inherit (import ../../build-support/lib/meson.nix { inherit lib stdenv; }) makeMesonFlags;
# Nix itself uses the `system` field of a derivation to decide where
# to build it. This is a bit confusing for cross compilation.
commonMeta = checkMeta.commonMeta hostPlatform;
assertValidity = checkMeta.assertValidity hostPlatform;
/**
This function creates a derivation, and returns it in the form of a [package attribute set](https://nix.dev/manual/nix/latest/glossary#package-attribute-set)
that refers to the derivation's outputs.
@@ -83,13 +224,6 @@ let
*/
mkDerivation = fnOrAttrs: makeDerivationExtensible (toFunction fnOrAttrs);
checkMeta = import ./check-meta.nix {
inherit lib config;
# Nix itself uses the `system` field of a derivation to decide where
# to build it. This is a bit confusing for cross compilation.
inherit (stdenv) hostPlatform;
};
# Based off lib.makeExtensible, with modifications:
makeDerivationExtensible =
rattrs:
@@ -176,89 +310,6 @@ let
in
finalPackage;
knownHardeningFlags = [
"bindnow"
"format"
"fortify"
"fortify3"
"strictflexarrays1"
"strictflexarrays3"
"shadowstack"
"nostrictaliasing"
"pacret"
"pic"
"relro"
"stackprotector"
"glibcxxassertions"
"libcxxhardeningfast"
"libcxxhardeningextensive"
"stackclashprotection"
"strictoverflow"
"trivialautovarinit"
"zerocallusedregs"
];
removedOrReplacedAttrNames = [
"checkInputs"
"installCheckInputs"
"nativeCheckInputs"
"nativeInstallCheckInputs"
"__contentAddressed"
"__darwinAllowLocalNetworking"
"__impureHostDeps"
"__propagatedImpureHostDeps"
"sandboxProfile"
"propagatedSandboxProfile"
"disallowedReferences"
"disallowedRequisites"
"allowedReferences"
"allowedRequisites"
"allowedImpureDLLs"
];
referenceCheckingAttrsToRemove = [
"allowedReferences"
"allowedRequisites"
"disallowedReferences"
"disallowedRequisites"
];
argumentAttrsToRemove = [
"meta"
"passthru"
"pos"
"env"
];
attrsToRemoveLast = [
# Fixed-output derivations may not reference other paths, which means that
# for a fixed-output derivation, the corresponding inputDerivation should
# *not* be fixed-output. To achieve this we simply delete the attributes that
# would make it fixed-output.
"outputHashAlgo"
"outputHash"
"outputHashMode"
# inputDerivation produces the inputs; not the outputs, so any
# restrictions on what used to be the outputs don't serve a purpose
# anymore.
"allowedReferences"
"allowedRequisites"
"disallowedReferences"
"disallowedRequisites"
"outputChecks"
];
defaultBuilderArgs = [
"-e"
./source-stdenv.sh
./default-builder.sh
];
doCheckByDefault = config.doCheckByDefault or false;
structuredAttrsByDefault = config.structuredAttrsByDefault or false;
inherit (config) enableParallelBuildingByDefault contentAddressedByDefault;
inherit (stdenv)
hostPlatform
buildPlatform
@@ -303,14 +354,11 @@ let
# TODO(@Ericson2314): Make always true and remove / resolve #178468
defaultStrictDeps = if config.strictDepsByDefault then true else hostPlatform != buildPlatform;
isSingularDependency = dep: dep == null || isDerivation dep || isString dep || isPath dep;
canExecuteHostOnBuild = buildPlatform.canExecute hostPlatform;
defaultHardeningFlags = stdenv.cc.defaultHardeningFlags or knownHardeningFlags;
hostSuffixNecessary = hostPlatform != buildPlatform && stdenvHasCC;
stdenvHostSuffix = "-${hostPlatform.config}";
stdenvStaticMarker = optionalString isStatic "-static";
userHook = config.stdenv.userHook or null;
requiredSystemFeaturesShouldBeSet =
buildPlatform ? gcc.arch
@@ -326,54 +374,6 @@ let
);
gccArchFeature = [ "gccarch-${buildPlatform.gcc.arch}" ];
cachedOutputChecks = {
out = { };
};
debugCachedOutputChecks = {
out = { };
debug = { };
};
# Turn a derivation into its outPath without a string context attached.
# See the comment at the usage site.
unsafeDerivationToUntrackedOutpath =
drv:
if isDerivation drv && (!drv.__contentAddressed or false) then
unsafeDiscardStringContext drv.outPath
else
drv;
makeOutputChecks =
attrs:
# If we use derivations directly here, they end up as build-time dependencies.
# This is especially problematic in the case of disallowed*, since the disallowed
# derivations will be built by nix as build-time dependencies, while those
# derivations might take a very long time to build, or might not even build
# successfully on the platform used.
# We can improve on this situation by instead passing only the outPath,
# without an attached string context, to nix. The out path will be a placeholder
# which will be replaced by the actual out path if the derivation in question
# is part of the final closure (and thus needs to be built). If it is not
# part of the final closure, then the placeholder will be passed along,
# but in that case we know for a fact that the derivation is not part of the closure.
# This means that passing the out path to nix does the right thing in either
# case, both for disallowed and allowed references/requisites, and we won't
# build the derivation if it wouldn't be part of the closure, saving time and resources.
# While the problem is less severe for allowed*, since we want the derivation
# to be built eventually, we would still like to get the error early and without
# having to wait while nix builds a derivation that might not be used.
# See also https://github.com/NixOS/nix/issues/4629
{
${if (attrs ? disallowedReferences) then "disallowedReferences" else null} =
map unsafeDerivationToUntrackedOutpath attrs.disallowedReferences;
${if (attrs ? disallowedRequisites) then "disallowedRequisites" else null} =
map unsafeDerivationToUntrackedOutpath attrs.disallowedRequisites;
${if (attrs ? allowedReferences) then "allowedReferences" else null} =
mapNullable unsafeDerivationToUntrackedOutpath attrs.allowedReferences;
${if (attrs ? allowedRequisites) then "allowedRequisites" else null} =
mapNullable unsafeDerivationToUntrackedOutpath attrs.allowedRequisites;
};
makeDerivationArgument =
# `makeDerivationArgument` is responsible for the `mkDerivation` arguments that
@@ -972,7 +972,7 @@ let
}
);
meta = checkMeta.commonMeta {
meta = commonMeta {
inherit validity attrs pos;
references =
attrs.nativeBuildInputs or [ ]
@@ -980,7 +980,7 @@ let
++ attrs.propagatedNativeBuildInputs or [ ]
++ attrs.propagatedBuildInputs or [ ];
};
validity = checkMeta.assertValidity { inherit meta attrs; };
validity = assertValidity { inherit meta attrs; };
checkedEnv =
let
+5 -4
View File
@@ -121,6 +121,8 @@
assert crossSystem == localSystem;
let
genericStdenv = import ../generic { defaultConfig = config; };
inherit (localSystem) system;
isFromNixpkgs = pkg: !(isFromBootstrapFiles pkg);
@@ -174,12 +176,12 @@ let
}:
let
thisStdenv = import ../generic {
thisStdenv = genericStdenv {
name = "${name}-stdenv-linux";
buildPlatform = localSystem;
hostPlatform = localSystem;
targetPlatform = localSystem;
inherit config extraNativeBuildInputs;
inherit extraNativeBuildInputs;
inherit (stage0) initialPath;
preHook = ''
# Don't patch #!/interpreter because it leads to retained
@@ -696,13 +698,12 @@ in
assert isBuiltByNixpkgsCompiler prevStage.patchelf;
{
inherit config overlays;
stdenv = import ../generic rec {
stdenv = genericStdenv rec {
name = "stdenv-linux";
buildPlatform = localSystem;
hostPlatform = localSystem;
targetPlatform = localSystem;
inherit config;
preHook = commonPreHook;
+1 -1
View File
@@ -23,7 +23,7 @@ if minbootSupported then
system = localSystem;
inherit (config) rewriteURL;
};
checkMeta = callPackage ../generic/check-meta.nix { hostPlatform = localSystem; };
checkMeta = callPackage ../generic/check-meta.nix { };
}
);
compilerPackage =
+3 -2
View File
@@ -10,6 +10,8 @@
assert crossSystem == localSystem;
let
genericStdenv = import ../generic { defaultConfig = config; };
inherit (localSystem) system;
shell =
@@ -105,7 +107,7 @@ let
extraNativeBuildInputs ? [ ],
}:
import ../generic {
genericStdenv {
buildPlatform = localSystem;
hostPlatform = localSystem;
targetPlatform = localSystem;
@@ -145,7 +147,6 @@ let
shell
cc
overrides
config
;
};
+4 -4
View File
@@ -9,15 +9,15 @@
}:
assert crossSystem == localSystem;
let
genericStdenv = import ../generic { defaultConfig = config; };
in
bootStages
++ [
(prevStage: {
inherit config overlays;
stdenv = import ../generic rec {
inherit config;
stdenv = genericStdenv rec {
inherit (prevStage.stdenv) buildPlatform hostPlatform targetPlatform;
preHook = ''
+1 -1
View File
@@ -8297,7 +8297,7 @@ with pkgs;
inherit (stdenv.buildPlatform) system;
inherit (config) rewriteURL;
};
checkMeta = callPackage ../stdenv/generic/check-meta.nix { inherit (stdenv) hostPlatform; };
checkMeta = callPackage ../stdenv/generic/check-meta.nix { };
}
);
minimal-bootstrap-sources =