From 2270b66d759f0a9c022576ce42fa5a770a754250 Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Wed, 20 Jul 2022 08:45:47 +0200 Subject: [PATCH] pythonPackagesExtensions: override all Python package sets at once Python package sets can be overridden by overriding an interpreter and passing in `packageOverrides = self: super: {...};`. This is fine in case you need a single interpreter, however, it does not help you when you want to override all sets. With this change it is possible to override all sets at once by appending a list of "extensions" to `pythonPackagesExtensions`. From reading the implementation you might wonder why a list is used, and not `lib.composeExtensions`? The reason is the latter requires knowledge of the library function. This approach should be easier for most users as it is similar to how we append to lists of e.g. inputs or patches when overriding a derivation. --- doc/languages-frameworks/python.section.md | 20 +++++++++++++++++++ .../python/cpython/2.7/default.nix | 2 +- .../interpreters/python/cpython/default.nix | 2 +- .../interpreters/python/default.nix | 9 ++++++++- .../interpreters/python/pypy/default.nix | 1 + .../development/interpreters/python/tests.nix | 12 +++++++++++ pkgs/top-level/all-packages.nix | 2 ++ 7 files changed, 45 insertions(+), 3 deletions(-) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 7df8464917cf..7fb8ba2e7c27 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -1664,6 +1664,26 @@ self: super: { } ``` +### How to override a Python package for all Python versions using extensions? {#how-to-override-a-python-package-for-all-python-versions-using-extensions} + +The following overlay overrides the call to `buildPythonPackage` for the +`foo` package for all interpreters by appending a Python extension to the +`pythonPackagesExtensions` list of extensions. + +```nix +final: prev: { + pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ + ( + python-final: python-prev: { + foo = python-prev.foo.overridePythonAttrs (oldAttrs: { + ... + }); + } + ) + ]; +} +``` + ### How to use Intel’s MKL with numpy and scipy? {#how-to-use-intels-mkl-with-numpy-and-scipy} MKL can be configured using an overlay. See the section "[Using overlays to diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix index fd7e684113e7..25446f5fca89 100644 --- a/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -68,7 +68,7 @@ let executable = libPrefix; pythonVersion = with sourceVersion; "${major}.${minor}"; sitePackages = "lib/${libPrefix}/site-packages"; - inherit hasDistutilsCxxPatch; + inherit hasDistutilsCxxPatch pythonAttr; pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr}; pythonOnBuildForHost = pkgsBuildHost.${pythonAttr}; pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr}; diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix index a52935c69dfd..dca7d9bb0db5 100644 --- a/pkgs/development/interpreters/python/cpython/default.nix +++ b/pkgs/development/interpreters/python/cpython/default.nix @@ -91,7 +91,7 @@ let executable = libPrefix; pythonVersion = with sourceVersion; "${major}.${minor}"; sitePackages = "lib/${libPrefix}/site-packages"; - inherit hasDistutilsCxxPatch; + inherit hasDistutilsCxxPatch pythonAttr; pythonOnBuildForBuild = override pkgsBuildBuild.${pythonAttr}; pythonOnBuildForHost = override pkgsBuildHost.${pythonAttr}; pythonOnBuildForTarget = override pkgsBuildTarget.${pythonAttr}; diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix index 5593158e7f36..471f96b51887 100644 --- a/pkgs/development/interpreters/python/default.nix +++ b/pkgs/development/interpreters/python/default.nix @@ -19,9 +19,14 @@ with pkgs; , pythonOnBuildForTarget , pythonOnHostForHost , pythonOnTargetForTarget + , pythonAttr ? null , self # is pythonOnHostForTarget }: let pythonPackages = callPackage + # Function that when called + # - imports python-packages.nix + # - adds spliced package sets to the package set + # - applies overrides from `packageOverrides` and `pythonPackagesOverlays`. ({ pkgs, stdenv, python, overrides }: let pythonPackagesFun = import ../../../top-level/python-packages.nix { inherit stdenv pkgs lib; @@ -74,7 +79,7 @@ with pkgs; extra = _: {}; optionalExtensions = cond: as: if cond then as else []; python2Extension = import ../../../top-level/python2-packages.nix; - extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ [ overrides ]); + extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ pkgs.pythonPackagesExtensions ++ [ overrides ]); aliases = self: super: lib.optionalAttrs config.allowAliases (import ../../../top-level/python-aliases.nix lib self super); in lib.makeScopeWithSplicing pkgs.splicePackages @@ -117,6 +122,8 @@ with pkgs; tests = callPackage ./tests.nix { python = self; }; + + inherit pythonAttr; }; sources = { diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix index 668a4e24b39e..e8c7d3ba65e5 100644 --- a/pkgs/development/interpreters/python/pypy/default.nix +++ b/pkgs/development/interpreters/python/pypy/default.nix @@ -30,6 +30,7 @@ let executable = "pypy${if isPy3k then "3" else ""}"; sitePackages = "site-packages"; hasDistutilsCxxPatch = false; + inherit pythonAttr; pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr}; pythonOnBuildForHost = pkgsBuildHost.${pythonAttr}; diff --git a/pkgs/development/interpreters/python/tests.nix b/pkgs/development/interpreters/python/tests.nix index 49dfe0e4644e..037c8f87b8ba 100644 --- a/pkgs/development/interpreters/python/tests.nix +++ b/pkgs/development/interpreters/python/tests.nix @@ -11,6 +11,7 @@ , substituteAll , lib , callPackage +, pkgs }: let @@ -133,6 +134,17 @@ let # test-overrideScope = let # myPackages = python.pkgs.overrideScope extension; # in assert myPackages.foobar == myPackages.numpy; myPackages.python.withPackages(ps: with ps; [ foobar ]); + } // lib.optionalAttrs (python ? pythonAttr) { + # Test applying overrides using pythonPackagesOverlays. + test-pythonPackagesExtensions = let + pkgs_ = pkgs.extend(final: prev: { + pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ + (python-final: python-prev: { + foo = python-prev.setuptools; + }) + ]; + }); + in pkgs_.${python.pythonAttr}.pkgs.foo; }; condaTests = let diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index e218ce7abb52..996390d070d0 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -14924,6 +14924,8 @@ with pkgs; pythonInterpreters = callPackage ./../development/interpreters/python { }; inherit (pythonInterpreters) python27 python37 python38 python39 python310 python311 python3Minimal pypy27 pypy38 pypy37 rustpython; + # List of extensions with overrides to apply to all Python package sets. + pythonPackagesExtensions = [ ]; # Python package sets. python27Packages = python27.pkgs; python37Packages = python37.pkgs;