From 04c39df3258e848613f95e06775ec46e2665fe13 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sat, 1 Feb 2025 23:40:28 +0800 Subject: [PATCH 1/4] buildPython*: abstract out transformDrv --- .../interpreters/python/mk-python-derivation.nix | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 11bb1af773ee..39263a887639 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -271,7 +271,7 @@ let isSetuptoolsDependency = isSetuptoolsDependency' (attrs.pname or null); # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. - self = toPythonModule ( + self = ( stdenv.mkDerivation ( finalAttrs: (cleanAttrs attrs) @@ -451,7 +451,14 @@ let ) ); + # This derivation transformation function must be independent to `attrs` + # for fixed-point arguments support in the future. + transformDrv = + drv: + extendDerivation ( + drv.disabled + -> throw "${lib.removePrefix namePrefix drv.name} not supported for interpreter ${python.executable}" + ) { } (toPythonModule drv); + in -extendDerivation ( - disabled -> throw "${name} not supported for interpreter ${python.executable}" -) { } self +transformDrv self From 2b6cde5354dc21362b30113ebb2f63dd1c384762 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sat, 1 Feb 2025 23:44:23 +0800 Subject: [PATCH 2/4] buildPython*: move argument-dependent let-in variables down below finalAttrs This commit is intentionally unformatted for smoother merging and rebasing experience. --- .../interpreters/python/mk-python-derivation.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 39263a887639..3f919a095df7 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -206,6 +206,11 @@ in }@attrs: let + # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. + self = ( + stdenv.mkDerivation ( + finalAttrs: + let format' = assert (pyproject != null) -> (format == null); if pyproject != null then @@ -270,10 +275,7 @@ let isSetuptoolsDependency = isSetuptoolsDependency' (attrs.pname or null); - # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. - self = ( - stdenv.mkDerivation ( - finalAttrs: + in (cleanAttrs attrs) // { From 6d2625c64b872b468f632a46a5e27173bca35602 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sat, 1 Feb 2025 23:50:56 +0800 Subject: [PATCH 3/4] buildPython*: remove redundant parenthesis This commit is intentionally unformatted for smoother merging and rebasing experience. --- .../interpreters/python/mk-python-derivation.nix | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 3f919a095df7..698cb402e46d 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -207,8 +207,7 @@ in let # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. - self = ( - stdenv.mkDerivation ( + self = stdenv.mkDerivation ( finalAttrs: let format' = @@ -450,8 +449,7 @@ let unittestFlagsArray = attrs.unittestFlagsArray; } ) - ) - ); + ); # This derivation transformation function must be independent to `attrs` # for fixed-point arguments support in the future. From 3678c2ea8d0f7dee2edebecede04a7444522cc1d Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sat, 1 Feb 2025 23:53:08 +0800 Subject: [PATCH 4/4] buildPython*: format expression after restructuring --- .../python/mk-python-derivation.nix | 436 +++++++++--------- 1 file changed, 218 insertions(+), 218 deletions(-) diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 698cb402e46d..8f3dec2dc8bf 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -208,248 +208,248 @@ in let # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. self = stdenv.mkDerivation ( - finalAttrs: - let - format' = - assert (pyproject != null) -> (format == null); - if pyproject != null then - if pyproject then "pyproject" else "other" - else if format != null then - format - else - "setuptools"; - - withDistOutput = withDistOutput' format'; - - validatePythonMatches = + finalAttrs: let - throwMismatch = - attrName: drv: + format' = + assert (pyproject != null) -> (format == null); + if pyproject != null then + if pyproject then "pyproject" else "other" + else if format != null then + format + else + "setuptools"; + + withDistOutput = withDistOutput' format'; + + validatePythonMatches = let - myName = "'${namePrefix}${name}'"; - theirName = "'${drv.name}'"; - optionalLocation = + throwMismatch = + attrName: drv: let - pos = unsafeGetAttrPos (if attrs ? "pname" then "pname" else "name") attrs; + myName = "'${namePrefix}${name}'"; + theirName = "'${drv.name}'"; + optionalLocation = + let + pos = unsafeGetAttrPos (if attrs ? "pname" then "pname" else "name") attrs; + in + optionalString (pos != null) " at ${pos.file}:${toString pos.line}:${toString pos.column}"; in - optionalString (pos != null) " at ${pos.file}:${toString pos.line}:${toString pos.column}"; + throw '' + Python version mismatch in ${myName}: + + The Python derivation ${myName} depends on a Python derivation + named ${theirName}, but the two derivations use different versions + of Python: + + ${leftPadName myName theirName} uses ${python} + ${leftPadName theirName myName} uses ${toString drv.pythonModule} + + Possible solutions: + + * If ${theirName} is a Python library, change the reference to ${theirName} + in the ${attrName} of ${myName} to use a ${theirName} built from the same + version of Python + + * If ${theirName} is used as a tool during the build, move the reference to + ${theirName} in ${myName} from ${attrName} to nativeBuildInputs + + * If ${theirName} provides executables that are called at run time, pass its + bin path to makeWrapperArgs: + + makeWrapperArgs = [ "--prefix PATH : ''${lib.makeBinPath [ ${getName drv} ] }" ]; + + ${optionalLocation} + ''; + + checkDrv = + attrName: drv: + if (isPythonModule drv) && (isMismatchedPython drv) then throwMismatch attrName drv else drv; + in - throw '' - Python version mismatch in ${myName}: + attrName: inputs: map (checkDrv attrName) inputs; - The Python derivation ${myName} depends on a Python derivation - named ${theirName}, but the two derivations use different versions - of Python: + isBootstrapInstallPackage = isBootstrapInstallPackage' (attrs.pname or null); - ${leftPadName myName theirName} uses ${python} - ${leftPadName theirName myName} uses ${toString drv.pythonModule} + isBootstrapPackage = isBootstrapInstallPackage || isBootstrapPackage' (attrs.pname or null); - Possible solutions: - - * If ${theirName} is a Python library, change the reference to ${theirName} - in the ${attrName} of ${myName} to use a ${theirName} built from the same - version of Python - - * If ${theirName} is used as a tool during the build, move the reference to - ${theirName} in ${myName} from ${attrName} to nativeBuildInputs - - * If ${theirName} provides executables that are called at run time, pass its - bin path to makeWrapperArgs: - - makeWrapperArgs = [ "--prefix PATH : ''${lib.makeBinPath [ ${getName drv} ] }" ]; - - ${optionalLocation} - ''; - - checkDrv = - attrName: drv: - if (isPythonModule drv) && (isMismatchedPython drv) then throwMismatch attrName drv else drv; + isSetuptoolsDependency = isSetuptoolsDependency' (attrs.pname or null); in - attrName: inputs: map (checkDrv attrName) inputs; + (cleanAttrs attrs) + // { - isBootstrapInstallPackage = isBootstrapInstallPackage' (attrs.pname or null); + name = namePrefix + name; - isBootstrapPackage = isBootstrapInstallPackage || isBootstrapPackage' (attrs.pname or null); + nativeBuildInputs = + [ + python + wrapPython + ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)? + pythonRemoveTestsDirHook + ] + ++ optionals (catchConflicts && !isBootstrapPackage && !isSetuptoolsDependency) [ + # + # 1. When building a package that is also part of the bootstrap chain, we + # must ignore conflicts after installation, because there will be one with + # the package in the bootstrap. + # + # 2. When a package is a dependency of setuptools, we must ignore conflicts + # because the hook that checks for conflicts uses setuptools. + # + pythonCatchConflictsHook + ] + ++ optionals (attrs ? pythonRelaxDeps || attrs ? pythonRemoveDeps) [ + pythonRelaxDepsHook + ] + ++ optionals removeBinBytecode [ + pythonRemoveBinBytecodeHook + ] + ++ optionals (hasSuffix "zip" (attrs.src.name or "")) [ + unzip + ] + ++ optionals (format' == "setuptools") [ + setuptoolsBuildHook + ] + ++ optionals (format' == "pyproject") [ + ( + if isBootstrapPackage then + pypaBuildHook.override { + inherit (python.pythonOnBuildForHost.pkgs.bootstrap) build; + wheel = null; + } + else + pypaBuildHook + ) + ( + if isBootstrapPackage then + pythonRuntimeDepsCheckHook.override { + inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging; + } + else + pythonRuntimeDepsCheckHook + ) + ] + ++ optionals (format' == "wheel") [ + wheelUnpackHook + ] + ++ optionals (format' == "egg") [ + eggUnpackHook + eggBuildHook + eggInstallHook + ] + ++ optionals (format' != "other") [ + ( + if isBootstrapInstallPackage then + pypaInstallHook.override { + inherit (python.pythonOnBuildForHost.pkgs.bootstrap) installer; + } + else + pypaInstallHook + ) + ] + ++ optionals (stdenv.buildPlatform == stdenv.hostPlatform) [ + # This is a test, however, it should be ran independent of the checkPhase and checkInputs + pythonImportsCheckHook + ] + ++ optionals (python.pythonAtLeast "3.3") [ + # Optionally enforce PEP420 for python3 + pythonNamespacesHook + ] + ++ optionals withDistOutput [ + pythonOutputDistHook + ] + ++ nativeBuildInputs + ++ build-system; - isSetuptoolsDependency = isSetuptoolsDependency' (attrs.pname or null); + buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath); - in - (cleanAttrs attrs) - // { + propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" ( + propagatedBuildInputs + ++ dependencies + ++ [ + # we propagate python even for packages transformed with 'toPythonApplication' + # this pollutes the PATH but avoids rebuilds + # see https://github.com/NixOS/nixpkgs/issues/170887 for more context + python + ] + ); - name = namePrefix + name; + inherit strictDeps; - nativeBuildInputs = - [ - python - wrapPython - ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)? - pythonRemoveTestsDirHook - ] - ++ optionals (catchConflicts && !isBootstrapPackage && !isSetuptoolsDependency) [ - # - # 1. When building a package that is also part of the bootstrap chain, we - # must ignore conflicts after installation, because there will be one with - # the package in the bootstrap. - # - # 2. When a package is a dependency of setuptools, we must ignore conflicts - # because the hook that checks for conflicts uses setuptools. - # - pythonCatchConflictsHook - ] - ++ optionals (attrs ? pythonRelaxDeps || attrs ? pythonRemoveDeps) [ - pythonRelaxDepsHook - ] - ++ optionals removeBinBytecode [ - pythonRemoveBinBytecodeHook - ] - ++ optionals (hasSuffix "zip" (attrs.src.name or "")) [ - unzip - ] - ++ optionals (format' == "setuptools") [ - setuptoolsBuildHook - ] - ++ optionals (format' == "pyproject") [ - ( - if isBootstrapPackage then - pypaBuildHook.override { - inherit (python.pythonOnBuildForHost.pkgs.bootstrap) build; - wheel = null; - } - else - pypaBuildHook - ) - ( - if isBootstrapPackage then - pythonRuntimeDepsCheckHook.override { - inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging; - } - else - pythonRuntimeDepsCheckHook - ) - ] - ++ optionals (format' == "wheel") [ - wheelUnpackHook - ] - ++ optionals (format' == "egg") [ - eggUnpackHook - eggBuildHook - eggInstallHook - ] - ++ optionals (format' != "other") [ - ( - if isBootstrapInstallPackage then - pypaInstallHook.override { - inherit (python.pythonOnBuildForHost.pkgs.bootstrap) installer; - } - else - pypaInstallHook - ) - ] - ++ optionals (stdenv.buildPlatform == stdenv.hostPlatform) [ - # This is a test, however, it should be ran independent of the checkPhase and checkInputs - pythonImportsCheckHook - ] - ++ optionals (python.pythonAtLeast "3.3") [ - # Optionally enforce PEP420 for python3 - pythonNamespacesHook - ] - ++ optionals withDistOutput [ - pythonOutputDistHook - ] - ++ nativeBuildInputs - ++ build-system; + LANG = "${if python.stdenv.hostPlatform.isDarwin then "en_US" else "C"}.UTF-8"; - buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath); + # Python packages don't have a checkPhase, only an installCheckPhase + doCheck = false; + doInstallCheck = attrs.doCheck or true; + nativeInstallCheckInputs = nativeCheckInputs; + installCheckInputs = checkInputs; - propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" ( - propagatedBuildInputs - ++ dependencies - ++ [ - # we propagate python even for packages transformed with 'toPythonApplication' - # this pollutes the PATH but avoids rebuilds - # see https://github.com/NixOS/nixpkgs/issues/170887 for more context - python - ] - ); + postFixup = + optionalString (!dontWrapPythonPrograms) '' + wrapPythonPrograms + '' + + attrs.postFixup or ""; - inherit strictDeps; + # Python packages built through cross-compilation are always for the host platform. + disallowedReferences = optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ + python.pythonOnBuildForHost + ]; - LANG = "${if python.stdenv.hostPlatform.isDarwin then "en_US" else "C"}.UTF-8"; + outputs = outputs ++ optional withDistOutput "dist"; - # Python packages don't have a checkPhase, only an installCheckPhase - doCheck = false; - doInstallCheck = attrs.doCheck or true; - nativeInstallCheckInputs = nativeCheckInputs; - installCheckInputs = checkInputs; + passthru = + { + inherit disabled; + } + // { + updateScript = + let + filename = head (splitString ":" finalAttrs.finalPackage.meta.position); + in + [ + update-python-libraries + filename + ]; + } + // optionalAttrs (dependencies != [ ]) { + inherit dependencies; + } + // optionalAttrs (optional-dependencies != { }) { + inherit optional-dependencies; + } + // optionalAttrs (build-system != [ ]) { + inherit build-system; + } + // attrs.passthru or { }; - postFixup = - optionalString (!dontWrapPythonPrograms) '' - wrapPythonPrograms - '' - + attrs.postFixup or ""; - - # Python packages built through cross-compilation are always for the host platform. - disallowedReferences = optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ - python.pythonOnBuildForHost - ]; - - outputs = outputs ++ optional withDistOutput "dist"; - - passthru = - { - inherit disabled; - } - // { - updateScript = - let - filename = head (splitString ":" finalAttrs.finalPackage.meta.position); - in - [ - update-python-libraries - filename - ]; - } - // optionalAttrs (dependencies != [ ]) { - inherit dependencies; - } - // optionalAttrs (optional-dependencies != { }) { - inherit optional-dependencies; - } - // optionalAttrs (build-system != [ ]) { - inherit build-system; - } - // attrs.passthru or { }; - - meta = { - # default to python's platforms - platforms = python.meta.platforms; - isBuildPythonPackage = python.meta.platforms; - } // meta; + meta = { + # default to python's platforms + platforms = python.meta.platforms; + isBuildPythonPackage = python.meta.platforms; + } // meta; + } + // optionalAttrs (attrs ? checkPhase) { + # If given use the specified checkPhase, otherwise use the setup hook. + # Longer-term we should get rid of `checkPhase` and use `installCheckPhase`. + installCheckPhase = attrs.checkPhase; + } + // optionalAttrs (attrs.doCheck or true) ( + optionalAttrs (disabledTestPaths != [ ]) { + disabledTestPaths = escapeShellArgs disabledTestPaths; } - // optionalAttrs (attrs ? checkPhase) { - # If given use the specified checkPhase, otherwise use the setup hook. - # Longer-term we should get rid of `checkPhase` and use `installCheckPhase`. - installCheckPhase = attrs.checkPhase; + // optionalAttrs (attrs ? disabledTests) { + # `escapeShellArgs` should be used as well as `disabledTestPaths`, + # but some packages rely on existing raw strings. + disabledTests = attrs.disabledTests; } - // optionalAttrs (attrs.doCheck or true) ( - optionalAttrs (disabledTestPaths != [ ]) { - disabledTestPaths = escapeShellArgs disabledTestPaths; - } - // optionalAttrs (attrs ? disabledTests) { - # `escapeShellArgs` should be used as well as `disabledTestPaths`, - # but some packages rely on existing raw strings. - disabledTests = attrs.disabledTests; - } - // optionalAttrs (attrs ? pytestFlagsArray) { - pytestFlagsArray = attrs.pytestFlagsArray; - } - // optionalAttrs (attrs ? unittestFlagsArray) { - unittestFlagsArray = attrs.unittestFlagsArray; - } - ) - ); + // optionalAttrs (attrs ? pytestFlagsArray) { + pytestFlagsArray = attrs.pytestFlagsArray; + } + // optionalAttrs (attrs ? unittestFlagsArray) { + unittestFlagsArray = attrs.unittestFlagsArray; + } + ) + ); # This derivation transformation function must be independent to `attrs` # for fixed-point arguments support in the future.