gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mk-python-derivation: Add dependencies & optional-dependencies arguments

Browse Source 
Since https://github.com/NixOS/nixpkgs/pull/161835 we've had the
concept of `passthru.optional-dependencies` for Python optional deps.

Having to explicitly put optional-dependencies in the passthru attrset
is a bit strange API-wise, even though it semantically makes sense.

This change unifies the handling of non-optional & optional Python
dependencies using the names established from PEP-621 (standardized pyproject.toml project metadata).
This commit is contained in:
adisbladis
2023-12-02 17:11:10 +13:00
parent c81dee1ff8
commit b9138b7c07
2 changed files with 60 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
doc/languages-frameworks/python.section.md
View File

@@ -120,7 +120,7 @@ buildPythonPackage rec {
setuptools-scm setuptools-scm
]; ];
propagatedBuildInputs = [ dependencies = [
attrs attrs
py py
setuptools setuptools
@@ -214,9 +214,14 @@ because their behaviour is different:
* `nativeCheckInputs ? []`: Dependencies needed for running the [`checkPhase`](#ssec-check-phase). These * `nativeCheckInputs ? []`: Dependencies needed for running the [`checkPhase`](#ssec-check-phase). These
are added to [`nativeBuildInputs`](#var-stdenv-nativeBuildInputs) when [`doCheck = true`](#var-stdenv-doCheck). Items listed in are added to [`nativeBuildInputs`](#var-stdenv-nativeBuildInputs) when [`doCheck = true`](#var-stdenv-doCheck). Items listed in
`tests_require` go here. `tests_require` go here.
* `propagatedBuildInputs ? []`: Aside from propagating dependencies, * `dependencies ? []`: Aside from propagating dependencies,
`buildPythonPackage` also injects code into and wraps executables with the `buildPythonPackage` also injects code into and wraps executables with the
paths included in this list. Items listed in `install_requires` go here. paths included in this list. Items listed in `install_requires` go here.
* `optional-dependencies ? { }`: Optional feature flagged dependencies. Items listed in `extras_requires` go here.
Aside from propagating dependencies,
`buildPythonPackage` also injects code into and wraps executables with the
paths included in this list. Items listed in `extras_requires` go here.
##### Overriding Python packages {#overriding-python-packages} ##### Overriding Python packages {#overriding-python-packages}
@@ -303,9 +308,9 @@ python3Packages.buildPythonApplication rec {
setuptools setuptools
]; ];
propagatedBuildInputs = with python3Packages; [ dependencies = [
tornado python3Packages.tornado
python-daemon python3Packages.python-daemon
]; ];
meta = with lib; { meta = with lib; {
@@ -977,13 +982,15 @@ that we introduced with the `let` expression.
#### Handling dependencies {#handling-dependencies} #### Handling dependencies {#handling-dependencies}
Our example, `toolz`, does not have any dependencies on other Python packages or Our example, `toolz`, does not have any dependencies on other Python packages or system libraries.
system libraries. According to the manual, [`buildPythonPackage`](#buildpythonpackage-function) uses the [`buildPythonPackage`](#buildpythonpackage-function) uses the the following arguments in the following circumstances:
arguments [`buildInputs`](#var-stdenv-buildInputs) and [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs) to specify dependencies. If
something is exclusively a build-time dependency, then the dependency should be - `dependencies` - For Python runtime dependencies.
included in [`buildInputs`](#var-stdenv-buildInputs), but if it is (also) a runtime dependency, then it - `build-system` - For Python build-time requirements.
should be added to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs). Test dependencies are considered - [`buildInputs`](#var-stdenv-buildInputs) - For non-Python build-time requirements.
build-time dependencies and passed to [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs). - [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) - For test dependencies
Dependencies can belong to multiple arguments, for example if something is both a build time requirement & a runtime dependency.
The following example shows which arguments are given to [`buildPythonPackage`](#buildpythonpackage-function) in The following example shows which arguments are given to [`buildPythonPackage`](#buildpythonpackage-function) in
order to build [`datashape`](https://github.com/blaze/datashape). order to build [`datashape`](https://github.com/blaze/datashape).
@@ -1018,7 +1025,7 @@ buildPythonPackage rec {
wheel wheel
]; ];
propagatedBuildInputs = [ dependencies = [
multipledispatch multipledispatch
numpy numpy
python-dateutil python-dateutil
@@ -1041,7 +1048,7 @@ buildPythonPackage rec {
We can see several runtime dependencies, `numpy`, `multipledispatch`, and We can see several runtime dependencies, `numpy`, `multipledispatch`, and
`python-dateutil`. Furthermore, we have [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) with `pytest`. `python-dateutil`. Furthermore, we have [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) with `pytest`.
`pytest` is a test runner and is only used during the [`checkPhase`](#ssec-check-phase) and is `pytest` is a test runner and is only used during the [`checkPhase`](#ssec-check-phase) and is
therefore not added to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs). therefore not added to `dependencies`.
In the previous case we had only dependencies on other Python packages to consider. In the previous case we had only dependencies on other Python packages to consider.
Occasionally you have also system libraries to consider. E.g., `lxml` provides Occasionally you have also system libraries to consider. E.g., `lxml` provides
@@ -1136,7 +1143,7 @@ buildPythonPackage rec {
fftwLongDouble fftwLongDouble
]; ];
propagatedBuildInputs = [ dependencies = [
numpy numpy
scipy scipy
]; ];
@@ -1459,9 +1466,7 @@ mode is activated.
In the following example, we create a simple environment that has a Python 3.11 In the following example, we create a simple environment that has a Python 3.11
version of our package in it, as well as its dependencies and other packages we version of our package in it, as well as its dependencies and other packages we
like to have in the environment, all specified with [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs). like to have in the environment, all specified with `dependencies`.
Indeed, we can just add any package we like to have in our environment to
[`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs).
```nix ```nix
with import <nixpkgs> {}; with import <nixpkgs> {};
@@ -1470,9 +1475,11 @@ with python311Packages;
buildPythonPackage rec { buildPythonPackage rec {
name = "mypackage"; name = "mypackage";
src = ./path/to/package/source; src = ./path/to/package/source;
propagatedBuildInputs = [ dependencies = [
pytest pytest
numpy numpy
];
propagatedBuildInputs = [
pkgs.libsndfile pkgs.libsndfile
]; ];
} }
@@ -1903,8 +1910,8 @@ configure alternatives](#sec-overlays-alternatives-blas-lapack)".
In a `setup.py` or `setup.cfg` it is common to declare dependencies: In a `setup.py` or `setup.cfg` it is common to declare dependencies:
* `setup_requires` corresponds to [`nativeBuildInputs`](#var-stdenv-nativeBuildInputs) * `setup_requires` corresponds to `build-system`
* `install_requires` corresponds to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs) * `install_requires` corresponds to `dependencies`
* `tests_require` corresponds to [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) * `tests_require` corresponds to [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs)
### How to enable interpreter optimizations? {#optimizations} ### How to enable interpreter optimizations? {#optimizations}
@@ -1928,12 +1935,10 @@ in mypython
Some packages define optional dependencies for additional features. With Some packages define optional dependencies for additional features. With
`setuptools` this is called `extras_require` and `flit` calls it `setuptools` this is called `extras_require` and `flit` calls it
`extras-require`, while PEP 621 calls these `optional-dependencies`. A `extras-require`, while PEP 621 calls these `optional-dependencies`.
method for supporting this is by declaring the extras of a package in its
`passthru`, e.g. in case of the package `dask`
```nix ```nix
passthru.optional-dependencies = { optional-dependencies = {
complete = [ distributed ]; complete = [ distributed ];
}; };
``` ```
@@ -1941,11 +1946,13 @@ passthru.optional-dependencies = {
and letting the package requiring the extra add the list to its dependencies and letting the package requiring the extra add the list to its dependencies
```nix ```nix
propagatedBuildInputs = [ dependencies = [
... ...
] ++ dask.optional-dependencies.complete; ] ++ dask.optional-dependencies.complete;
``` ```
This method is using `passthru`, meaning that changing `optional-dependencies` of a package won't cause it to rebuild.
Note this method is preferred over adding parameters to builders, as that can Note this method is preferred over adding parameters to builders, as that can
result in packages depending on different variants and thereby causing result in packages depending on different variants and thereby causing
collisions. collisions.

40
pkgs/development/interpreters/python/mk-python-derivation.nix
View File

@@ -45,6 +45,11 @@
# C can import package A propagated by B # C can import package A propagated by B
, propagatedBuildInputs ? [] , propagatedBuildInputs ? []
# Python module dependencies.
# These are named after PEP-621.
, dependencies ? []
, optional-dependencies ? {}
# DEPRECATED: use propagatedBuildInputs # DEPRECATED: use propagatedBuildInputs
, pythonPath ? [] , pythonPath ? []
@@ -97,8 +102,6 @@
, meta ? {} , meta ? {}
, passthru ? {}
, doCheck ? config.doCheckByDefault or false , doCheck ? config.doCheckByDefault or false
, disabledTestPaths ? [] , disabledTestPaths ? []
@@ -193,10 +196,25 @@ let
"setuptools" "wheel" "setuptools" "wheel"
]; ];
passthru =
attrs.passthru or { }
// {
updateScript = let
filename = builtins.head (lib.splitString ":" self.meta.position);
in attrs.passthru.updateScript or [ update-python-libraries filename ];
}
// lib.optionalAttrs (dependencies != []) {
inherit dependencies;
}
// lib.optionalAttrs (optional-dependencies != {}) {
inherit optional-dependencies;
};
# Keep extra attributes from `attrs`, e.g., `patchPhase', etc. # Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [ self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [
"disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "pyproject" "format" "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "pyproject" "format"
"disabledTestPaths" "outputs" "stdenv" "disabledTestPaths" "outputs" "stdenv"
"dependencies" "optional-dependencies"
]) // { ]) // {
name = namePrefix + name_; name = namePrefix + name_;
@@ -260,7 +278,7 @@ let
buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath); buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath);
propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ [ propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ dependencies ++ [
# we propagate python even for packages transformed with 'toPythonApplication' # we propagate python even for packages transformed with 'toPythonApplication'
# this pollutes the PATH but avoids rebuilds # this pollutes the PATH but avoids rebuilds
# see https://github.com/NixOS/nixpkgs/issues/170887 for more context # see https://github.com/NixOS/nixpkgs/issues/170887 for more context
@@ -292,6 +310,8 @@ let
outputs = outputs ++ lib.optional withDistOutput "dist"; outputs = outputs ++ lib.optional withDistOutput "dist";
inherit passthru;
meta = { meta = {
# default to python's platforms # default to python's platforms
platforms = python.meta.platforms; platforms = python.meta.platforms;
@@ -305,13 +325,7 @@ let
disabledTestPaths = lib.escapeShellArgs disabledTestPaths; disabledTestPaths = lib.escapeShellArgs disabledTestPaths;
})); }));
passthru.updateScript = let in lib.extendDerivation
filename = builtins.head (lib.splitString ":" self.meta.position); (disabled -> throw "${name} not supported for interpreter ${python.executable}")
in attrs.passthru.updateScript or [ update-python-libraries filename ]; passthru
in self
if disabled then
throw "${name} not supported for interpreter ${python.executable}"
else
self.overrideAttrs (attrs: {
passthru = lib.recursiveUpdate passthru attrs.passthru;
})