From 8dadb44196a07f6b4ab42b16fdb64f2fcc3fc94c Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Fri, 28 Oct 2016 13:31:07 +0200 Subject: [PATCH 1/3] pythonPackages: use fixed-point combinator Use a fixed-point combinator for the Python package set to allow easier overriding of its contents. Earlier implementations were proposed in #16784 and #17428. This commit is by comparison much smaller and changes only what is needed. --- pkgs/top-level/all-packages.nix | 7 ------- pkgs/top-level/python-packages.nix | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 34d24a89ff7e..e0b0656a59e7 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -9897,37 +9897,30 @@ in # regardless. python26Packages = callPackage ./python-packages.nix { python = python26; - self = python26Packages; }; python27Packages = lib.hiPrioSet (recurseIntoAttrs (callPackage ./python-packages.nix { python = python27; - self = python27Packages; })); python33Packages = callPackage ./python-packages.nix { python = python33; - self = python33Packages; }; python34Packages = callPackage ./python-packages.nix { python = python34; - self = python34Packages; }; python35Packages = recurseIntoAttrs (callPackage ./python-packages.nix { python = python35; - self = python35Packages; }); python36Packages = (callPackage ./python-packages.nix { python = python36; - self = python36Packages; }); pypyPackages = callPackage ./python-packages.nix { python = pypy; - self = pypyPackages; }; ### DEVELOPMENT / R MODULES diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index fc5972bfbf0f..ca2e88da94e4 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -1,7 +1,14 @@ -{ pkgs, stdenv, python, self }: +{ pkgs +, stdenv +, python +, overrides ? (self: super: {}) +}: with pkgs.lib; +let + packages = ( self: + let pythonAtLeast = versionAtLeast python.pythonVersion; pythonOlder = versionOlder python.pythonVersion; @@ -31323,4 +31330,7 @@ in { zeitgeist = if isPy3k then throw "zeitgeist not supported for interpreter ${python.executable}" else (pkgs.zeitgeist.override{python2Packages=self;}).py; -} + +}); + +in fix' (extends overrides packages) From 31e32b6d9ee2712966e2b09a68d4f783c164ddd3 Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Sun, 4 Dec 2016 09:51:12 +0100 Subject: [PATCH 2/3] Python interpreters: add pkgs attribute A package set is constructed for a specific interpreter. Therefore, we add the possibility to override the package set to the interpreter. This should make it easier to override the interpreter and the package set at the same time. --- .../python/cpython/2.6/default.nix | 11 ++++++-- .../python/cpython/2.7/default.nix | 10 +++++-- .../python/cpython/3.3/default.nix | 10 +++++-- .../python/cpython/3.4/default.nix | 10 +++++-- .../python/cpython/3.5/default.nix | 10 +++++-- .../python/cpython/3.6/default.nix | 10 +++++-- .../interpreters/python/pypy/2.7/default.nix | 12 ++++++-- pkgs/top-level/all-packages.nix | 28 +++++-------------- 8 files changed, 59 insertions(+), 42 deletions(-) diff --git a/pkgs/development/interpreters/python/cpython/2.6/default.nix b/pkgs/development/interpreters/python/cpython/2.6/default.nix index 64f2b80d09fa..9a4c2d5b3981 100644 --- a/pkgs/development/interpreters/python/cpython/2.6/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.6/default.nix @@ -1,6 +1,8 @@ { stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, includeModules ? false , sqlite, tcl, tk, xlibsWrapper, openssl, readline, db, ncurses, gdbm, self, callPackage -, python26Packages }: +# For the Python package set +, pkgs, packageOverrides ? (self: super: {}) +}: assert zlibSupport -> zlib != null; @@ -100,13 +102,16 @@ let ${ optionalString includeModules "$out/bin/python ./setup.py build_ext"} ''; - passthru = rec { + passthru = let + pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;}; + in rec { inherit libPrefix; inherit zlibSupport; isPy2 = true; isPy26 = true; buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python26Packages; }; + withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; + pkgs = pythonPackages; executable = libPrefix; sitePackages = "lib/${libPrefix}/site-packages"; interpreter = "${self}/bin/${executable}"; diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix index a6eeee25be96..4a25382997c3 100644 --- a/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -10,12 +10,13 @@ , zlib , callPackage , self -, python27Packages , gettext , db , expat , libffi , CF, configd, coreutils +# For the Python package set +, pkgs, packageOverrides ? (self: super: {}) }: assert x11Support -> tcl != null @@ -180,11 +181,14 @@ in stdenv.mkDerivation { rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev ''; - passthru = rec { + passthru = let + pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;}; + in rec { inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch; executable = libPrefix; buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python27Packages; }; + withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; + pkgs = pythonPackages; isPy2 = true; isPy27 = true; interpreter = "${self}/bin/${executable}"; diff --git a/pkgs/development/interpreters/python/cpython/3.3/default.nix b/pkgs/development/interpreters/python/cpython/3.3/default.nix index b25e2ffd0cb1..6a543a8a0ee0 100644 --- a/pkgs/development/interpreters/python/cpython/3.3/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.3/default.nix @@ -10,8 +10,9 @@ , zlib , callPackage , self -, python33Packages , CF, configd +# For the Python package set +, pkgs, packageOverrides ? (self: super: {}) }: assert x11Support -> tcl != null @@ -102,11 +103,14 @@ in stdenv.mkDerivation { ''; - passthru = rec { + passthru = let + pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;}; + in rec { inherit libPrefix sitePackages x11Support; executable = "${libPrefix}m"; buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python33Packages; }; + withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; + pkgs = pythonPackages; isPy3 = true; isPy33 = true; is_py3k = true; # deprecated diff --git a/pkgs/development/interpreters/python/cpython/3.4/default.nix b/pkgs/development/interpreters/python/cpython/3.4/default.nix index 43edce8a44c2..623fa5d74f64 100644 --- a/pkgs/development/interpreters/python/cpython/3.4/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.4/default.nix @@ -10,8 +10,9 @@ , zlib , callPackage , self -, python34Packages , CF, configd +# For the Python package set +, pkgs, packageOverrides ? (self: super: {}) }: assert x11Support -> tcl != null @@ -111,11 +112,14 @@ in stdenv.mkDerivation { ''; - passthru = rec { + passthru = let + pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;}; + in rec { inherit libPrefix sitePackages x11Support; executable = "${libPrefix}m"; buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python34Packages; }; + withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; + pkgs = pythonPackages; isPy3 = true; isPy34 = true; is_py3k = true; # deprecated diff --git a/pkgs/development/interpreters/python/cpython/3.5/default.nix b/pkgs/development/interpreters/python/cpython/3.5/default.nix index dd2cce707efe..7172e429f236 100644 --- a/pkgs/development/interpreters/python/cpython/3.5/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.5/default.nix @@ -10,8 +10,9 @@ , zlib , callPackage , self -, python35Packages , CF, configd +# For the Python package set +, pkgs, packageOverrides ? (self: super: {}) }: assert x11Support -> tcl != null @@ -110,11 +111,14 @@ in stdenv.mkDerivation { rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython* ''; - passthru = rec { + passthru = let + pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;}; + in rec { inherit libPrefix sitePackages x11Support; executable = "${libPrefix}m"; buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python35Packages; }; + withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; + pkgs = pythonPackages; isPy3 = true; isPy35 = true; interpreter = "${self}/bin/${executable}"; diff --git a/pkgs/development/interpreters/python/cpython/3.6/default.nix b/pkgs/development/interpreters/python/cpython/3.6/default.nix index d5960ccde99f..f65989f5bb57 100644 --- a/pkgs/development/interpreters/python/cpython/3.6/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.6/default.nix @@ -11,8 +11,9 @@ , zlib , callPackage , self -, python36Packages , CF, configd +# For the Python package set +, pkgs, packageOverrides ? (self: super: {}) }: assert x11Support -> tcl != null @@ -99,11 +100,14 @@ in stdenv.mkDerivation { ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc" ''; - passthru = rec { + passthru = let + pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;}; + in rec { inherit libPrefix sitePackages x11Support; executable = "${libPrefix}m"; buildEnv = callPackage ../../wrapper.nix { python = self; }; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python36Packages; }; + withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; + pkgs = pythonPackages; isPy3 = true; isPy35 = true; is_py3k = true; # deprecated diff --git a/pkgs/development/interpreters/python/pypy/2.7/default.nix b/pkgs/development/interpreters/python/pypy/2.7/default.nix index bce6d19d58d3..1fd9eaee4303 100644 --- a/pkgs/development/interpreters/python/pypy/2.7/default.nix +++ b/pkgs/development/interpreters/python/pypy/2.7/default.nix @@ -1,6 +1,9 @@ { stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, pkgconfig, libffi , sqlite, openssl, ncurses, python, expat, tcl, tk, xlibsWrapper, libX11 -, makeWrapper, callPackage, self, pypyPackages, gdbm, db }: +, makeWrapper, callPackage, self, gdbm, db +# For the Python package set +, pkgs, packageOverrides ? (self: super: {}) +}: assert zlibSupport -> zlib != null; @@ -120,14 +123,17 @@ let echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py ''; - passthru = rec { + passthru = let + pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;}; + in rec { inherit zlibSupport libPrefix; executable = "pypy"; isPypy = true; buildEnv = callPackage ../../wrapper.nix { python = self; }; interpreter = "${self}/bin/${executable}"; sitePackages = "site-packages"; - withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = pypyPackages; }; + withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; + pkgs = pythonPackages; }; enableParallelBuilding = true; # almost no parallelization without STM diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index e0b0656a59e7..b3ecfb0d67ad 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -9895,33 +9895,19 @@ in # Therefore we do not recurse into attributes here, in contrast to # python27Packages. `nix-env -iA python26Packages.nose` works # regardless. - python26Packages = callPackage ./python-packages.nix { - python = python26; - }; + python26Packages = python26.pkgs; - python27Packages = lib.hiPrioSet (recurseIntoAttrs (callPackage ./python-packages.nix { - python = python27; - })); + python27Packages = lib.hiPrioSet (recurseIntoAttrs python27.pkgs); - python33Packages = callPackage ./python-packages.nix { - python = python33; - }; + python33Packages = python33.pkgs; - python34Packages = callPackage ./python-packages.nix { - python = python34; - }; + python34Packages = python34.pkgs; - python35Packages = recurseIntoAttrs (callPackage ./python-packages.nix { - python = python35; - }); + python35Packages = recurseIntoAttrs python35.pkgs; - python36Packages = (callPackage ./python-packages.nix { - python = python36; - }); + python36Packages = python36.pkgs; - pypyPackages = callPackage ./python-packages.nix { - python = pypy; - }; + pypyPackages = pypy.pkgs; ### DEVELOPMENT / R MODULES From 3d59b82925da364a509673fdb74ba69aa77d58db Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Sun, 4 Dec 2016 11:55:25 +0100 Subject: [PATCH 3/3] DOCS: update Python docs to reflect fixed-point combinator --- doc/languages-frameworks/python.md | 85 ++++++++++++++---------------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/doc/languages-frameworks/python.md b/doc/languages-frameworks/python.md index 121e123c8cfa..fc0100c5f2d5 100644 --- a/doc/languages-frameworks/python.md +++ b/doc/languages-frameworks/python.md @@ -434,6 +434,7 @@ Each interpreter has the following attributes: - `withPackages`. Simpler interface to `buildEnv`. See section *python.withPackages function* for usage and documentation. - `sitePackages`. Alias for `lib/${libPrefix}/site-packages`. - `executable`. Name of the interpreter executable, e.g. `python3.4`. +- `pkgs`. Set of Python packages for that specific interpreter. The package set can be modified by overriding the interpreter and passing `packageOverrides`. ### Building packages and applications @@ -699,59 +700,55 @@ should also be done when packaging `A`. ### How to override a Python package? -Recursively updating a package can be done with `pkgs.overridePackages` as explained in the Nixpkgs manual. -Python attribute sets are created for each interpreter version. We will therefore override the attribute set for the interpreter version we're interested. -In the following example we change the name of the package `pandas` to `foo`. -``` -newpkgs = pkgs.overridePackages(self: super: rec { - python35Packages = (super.python35Packages.override { self = python35Packages;}) - // { pandas = super.python35Packages.pandas.override {name = "foo";}; - }; -}); -``` -This can be tested with -``` +We can override the interpreter and pass `packageOverrides`. +In the following example we rename the `pandas` package and build it. +```nix with import {}; -(let +let + python = let + packageOverrides = self: super: { + pandas = super.pandas.override {name="foo";}; + }; + in pkgs.python35.override {inherit packageOverrides;}; -newpkgs = pkgs.overridePackages(self: super: rec { - python35Packages = (super.python35Packages.override { self = python35Packages;}) - // { pandas = super.python35Packages.pandas.override {name = "foo";}; - }; -}); -in newpkgs.python35.withPackages (ps: [ps.blaze]) -).env -``` -A typical use case is to switch to another version of a certain package. For example, in the Nixpkgs repository we have multiple versions of `django` and `scipy`. -In the following example we use a different version of `scipy`. All packages in `newpkgs` will now use the updated `scipy` version. +in python.pkgs.pandas ``` +Using `nix-build` on this expression will build the package `pandas` +but with the new name `foo`. + +All packages in the package set will use the renamed package. +A typical use case is to switch to another version of a certain package. +For example, in the Nixpkgs repository we have multiple versions of `django` and `scipy`. +In the following example we use a different version of `scipy` and create an environment that uses it. +All packages in the Python package set will now use the updated `scipy` version. + +```nix with import {}; -(let - -newpkgs = pkgs.overridePackages(self: super: rec { - python35Packages = super.python35Packages.override { - self = python35Packages // { scipy = python35Packages.scipy_0_17;}; +( +let + packageOverrides = self: super: { + scipy = super.scipy_0_17; }; -}); -in newpkgs.python35.withPackages (ps: [ps.blaze]) +in (pkgs.python35.override {inherit packageOverrides;}).withPackages (ps: [ps.blaze]) ).env ``` -The requested package `blaze` depends upon `pandas` which itself depends on `scipy`. +The requested package `blaze` depends on `pandas` which itself depends on `scipy`. -A similar example but now using `django` +If you want the whole of Nixpkgs to use your modifications, then you can use `pkgs.overridePackages` +as explained in this manual. In the following example we build a `inkscape` using a different version of `numpy`. ``` -with import {}; - -(let - -newpkgs = pkgs.overridePackages(self: super: rec { - python27Packages = (super.python27Packages.override {self = python27Packages;}) - // { django = super.python27Packages.django_1_9; }; -}); -in newpkgs.python27.withPackages (ps: [ps.django_guardian ]) -).env +let + pkgs = import {}; + newpkgs = pkgs.overridePackages ( pkgsself: pkgssuper: { + python27 = let + packageOverrides = self: super: { + numpy = super.numpy_1_10; + }; + in pkgssuper.python27.override {inherit packageOverrides;}; + } ); +in newpkgs.inkscape ``` ### `python setup.py bdist_wheel` cannot create .whl @@ -772,9 +769,9 @@ or the current time: nix-shell --run "SOURCE_DATE_EPOCH=$(date +%s) python3 setup.py bdist_wheel" ``` or unset: -""" +``` nix-shell --run "unset SOURCE_DATE_EPOCH; python3 setup.py bdist_wheel" -""" +``` ### `install_data` / `data_files` problems