From e67331103e23dc2a2ee44dbf0e093bcdda417462 Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Fri, 25 Feb 2022 13:36:54 +0100 Subject: [PATCH 1/3] python3.pkgs.panel: fix package by switching to prebuilt wheel This package is difficult to upgrade because of the nodejs modules needed. --- .../python-modules/panel/default.nix | 37 +- .../python-modules/panel/node/default.nix | 17 - .../python-modules/panel/node/node-env.nix | 573 ------------------ .../panel/node/node-packages.nix | 562 ----------------- .../python-modules/panel/node/package.json | 27 - 5 files changed, 10 insertions(+), 1206 deletions(-) delete mode 100644 pkgs/development/python-modules/panel/node/default.nix delete mode 100644 pkgs/development/python-modules/panel/node/node-env.nix delete mode 100644 pkgs/development/python-modules/panel/node/node-packages.nix delete mode 100644 pkgs/development/python-modules/panel/node/package.json diff --git a/pkgs/development/python-modules/panel/default.nix b/pkgs/development/python-modules/panel/default.nix index 8673c637eda6..c5da10a30b07 100644 --- a/pkgs/development/python-modules/panel/default.nix +++ b/pkgs/development/python-modules/panel/default.nix @@ -9,39 +9,22 @@ , pyct , testpath , tqdm -, callPackage +, nodejs }: -let - node = callPackage ./node { }; -in buildPythonPackage rec { pname = "panel"; version = "0.12.6"; - # Don't forget to also update the node packages - # 1. retrieve the package.json file - # 2. nix shell nixpkgs#nodePackages.node2nix - # 3. node2nix - src = fetchPypi { - inherit pname version; - sha256 = "97e158e8eb941f88d71929407f9455c903b5e18d89969db8ce8af66036f46b53"; - }; + format = "wheel"; - # Since 0.10.0 panel attempts to fetch from the web. - # We avoid this: - # - we use node2nix to fetch assets - # - we disable bundling (which also tries to fetch assets) - # Downside of disabling bundling is that in an airgapped environment - # one may miss assets. - # https://github.com/holoviz/panel/issues/1819 - preBuild = '' - substituteInPlace setup.py --replace "bundle_resources()" "" - pushd panel - ln -s ${node.nodeDependencies}/lib/node_modules - export PATH="${node.nodeDependencies}/bin:$PATH" - popd - ''; + # We fetch a wheel because while we can fetch the node + # artifacts using npm, the bundling invoked in setup.py + # tries to fetch even more artifacts + src = fetchPypi { + inherit pname version format; + hash = "sha256-ARAbBM0QYZlZqV51lMRoEZEQH1jlHRhlon3nfTi7dnM="; + }; propagatedBuildInputs = [ bleach @@ -58,7 +41,7 @@ buildPythonPackage rec { doCheck = false; passthru = { - inherit node; # For convenience + inherit nodejs; # For convenience }; meta = with lib; { diff --git a/pkgs/development/python-modules/panel/node/default.nix b/pkgs/development/python-modules/panel/node/default.nix deleted file mode 100644 index 64326c6b2160..000000000000 --- a/pkgs/development/python-modules/panel/node/default.nix +++ /dev/null @@ -1,17 +0,0 @@ -# This file has been generated by node2nix 1.9.0. Do not edit! - -{pkgs ? import { - inherit system; - }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}: - -let - nodeEnv = import ./node-env.nix { - inherit (pkgs) stdenv lib python2 runCommand writeTextFile; - inherit pkgs nodejs; - libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; - }; -in -import ./node-packages.nix { - inherit (pkgs) fetchurl nix-gitignore stdenv lib fetchgit; - inherit nodeEnv; -} diff --git a/pkgs/development/python-modules/panel/node/node-env.nix b/pkgs/development/python-modules/panel/node/node-env.nix deleted file mode 100644 index 21089c4d5459..000000000000 --- a/pkgs/development/python-modules/panel/node/node-env.nix +++ /dev/null @@ -1,573 +0,0 @@ -# This file originates from node2nix - -{lib, stdenv, nodejs, python2, pkgs, libtool, runCommand, writeTextFile}: - -let - # Workaround to cope with utillinux in Nixpkgs 20.09 and util-linux in Nixpkgs master - utillinux = if pkgs ? utillinux then pkgs.utillinux else pkgs.util-linux; - - python = if nodejs ? python then nodejs.python else python2; - - # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise - tarWrapper = runCommand "tarWrapper" {} '' - mkdir -p $out/bin - - cat > $out/bin/tar <> $out/nix-support/hydra-build-products - ''; - }; - - includeDependencies = {dependencies}: - lib.optionalString (dependencies != []) - (lib.concatMapStrings (dependency: - '' - # Bundle the dependencies of the package - mkdir -p node_modules - cd node_modules - - # Only include dependencies if they don't exist. They may also be bundled in the package. - if [ ! -e "${dependency.name}" ] - then - ${composePackage dependency} - fi - - cd .. - '' - ) dependencies); - - # Recursively composes the dependencies of a package - composePackage = { name, packageName, src, dependencies ? [], ... }@args: - builtins.addErrorContext "while evaluating node package '${packageName}'" '' - DIR=$(pwd) - cd $TMPDIR - - unpackFile ${src} - - # Make the base dir in which the target dependency resides first - mkdir -p "$(dirname "$DIR/${packageName}")" - - if [ -f "${src}" ] - then - # Figure out what directory has been unpacked - packageDir="$(find . -maxdepth 1 -type d | tail -1)" - - # Restore write permissions to make building work - find "$packageDir" -type d -exec chmod u+x {} \; - chmod -R u+w "$packageDir" - - # Move the extracted tarball into the output folder - mv "$packageDir" "$DIR/${packageName}" - elif [ -d "${src}" ] - then - # Get a stripped name (without hash) of the source directory. - # On old nixpkgs it's already set internally. - if [ -z "$strippedName" ] - then - strippedName="$(stripHash ${src})" - fi - - # Restore write permissions to make building work - chmod -R u+w "$strippedName" - - # Move the extracted directory into the output folder - mv "$strippedName" "$DIR/${packageName}" - fi - - # Unset the stripped name to not confuse the next unpack step - unset strippedName - - # Include the dependencies of the package - cd "$DIR/${packageName}" - ${includeDependencies { inherit dependencies; }} - cd .. - ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} - ''; - - pinpointDependencies = {dependencies, production}: - let - pinpointDependenciesFromPackageJSON = writeTextFile { - name = "pinpointDependencies.js"; - text = '' - var fs = require('fs'); - var path = require('path'); - - function resolveDependencyVersion(location, name) { - if(location == process.env['NIX_STORE']) { - return null; - } else { - var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json"); - - if(fs.existsSync(dependencyPackageJSON)) { - var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON)); - - if(dependencyPackageObj.name == name) { - return dependencyPackageObj.version; - } - } else { - return resolveDependencyVersion(path.resolve(location, ".."), name); - } - } - } - - function replaceDependencies(dependencies) { - if(typeof dependencies == "object" && dependencies !== null) { - for(var dependency in dependencies) { - var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency); - - if(resolvedVersion === null) { - process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n"); - } else { - dependencies[dependency] = resolvedVersion; - } - } - } - } - - /* Read the package.json configuration */ - var packageObj = JSON.parse(fs.readFileSync('./package.json')); - - /* Pinpoint all dependencies */ - replaceDependencies(packageObj.dependencies); - if(process.argv[2] == "development") { - replaceDependencies(packageObj.devDependencies); - } - replaceDependencies(packageObj.optionalDependencies); - - /* Write the fixed package.json file */ - fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); - ''; - }; - in - '' - node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"} - - ${lib.optionalString (dependencies != []) - '' - if [ -d node_modules ] - then - cd node_modules - ${lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies} - cd .. - fi - ''} - ''; - - # Recursively traverses all dependencies of a package and pinpoints all - # dependencies in the package.json file to the versions that are actually - # being used. - - pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args: - '' - if [ -d "${packageName}" ] - then - cd "${packageName}" - ${pinpointDependencies { inherit dependencies production; }} - cd .. - ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} - fi - ''; - - # Extract the Node.js source code which is used to compile packages with - # native bindings - nodeSources = runCommand "node-sources" {} '' - tar --no-same-owner --no-same-permissions -xf ${nodejs.src} - mv node-* $out - ''; - - # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty) - addIntegrityFieldsScript = writeTextFile { - name = "addintegrityfields.js"; - text = '' - var fs = require('fs'); - var path = require('path'); - - function augmentDependencies(baseDir, dependencies) { - for(var dependencyName in dependencies) { - var dependency = dependencies[dependencyName]; - - // Open package.json and augment metadata fields - var packageJSONDir = path.join(baseDir, "node_modules", dependencyName); - var packageJSONPath = path.join(packageJSONDir, "package.json"); - - if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored - console.log("Adding metadata fields to: "+packageJSONPath); - var packageObj = JSON.parse(fs.readFileSync(packageJSONPath)); - - if(dependency.integrity) { - packageObj["_integrity"] = dependency.integrity; - } else { - packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads. - } - - if(dependency.resolved) { - packageObj["_resolved"] = dependency.resolved; // Adopt the resolved property if one has been provided - } else { - packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories. - } - - if(dependency.from !== undefined) { // Adopt from property if one has been provided - packageObj["_from"] = dependency.from; - } - - fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2)); - } - - // Augment transitive dependencies - if(dependency.dependencies !== undefined) { - augmentDependencies(packageJSONDir, dependency.dependencies); - } - } - } - - if(fs.existsSync("./package-lock.json")) { - var packageLock = JSON.parse(fs.readFileSync("./package-lock.json")); - - if(![1, 2].includes(packageLock.lockfileVersion)) { - process.stderr.write("Sorry, I only understand lock file versions 1 and 2!\n"); - process.exit(1); - } - - if(packageLock.dependencies !== undefined) { - augmentDependencies(".", packageLock.dependencies); - } - } - ''; - }; - - # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes - reconstructPackageLock = writeTextFile { - name = "addintegrityfields.js"; - text = '' - var fs = require('fs'); - var path = require('path'); - - var packageObj = JSON.parse(fs.readFileSync("package.json")); - - var lockObj = { - name: packageObj.name, - version: packageObj.version, - lockfileVersion: 1, - requires: true, - dependencies: {} - }; - - function augmentPackageJSON(filePath, dependencies) { - var packageJSON = path.join(filePath, "package.json"); - if(fs.existsSync(packageJSON)) { - var packageObj = JSON.parse(fs.readFileSync(packageJSON)); - dependencies[packageObj.name] = { - version: packageObj.version, - integrity: "sha1-000000000000000000000000000=", - dependencies: {} - }; - processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies); - } - } - - function processDependencies(dir, dependencies) { - if(fs.existsSync(dir)) { - var files = fs.readdirSync(dir); - - files.forEach(function(entry) { - var filePath = path.join(dir, entry); - var stats = fs.statSync(filePath); - - if(stats.isDirectory()) { - if(entry.substr(0, 1) == "@") { - // When we encounter a namespace folder, augment all packages belonging to the scope - var pkgFiles = fs.readdirSync(filePath); - - pkgFiles.forEach(function(entry) { - if(stats.isDirectory()) { - var pkgFilePath = path.join(filePath, entry); - augmentPackageJSON(pkgFilePath, dependencies); - } - }); - } else { - augmentPackageJSON(filePath, dependencies); - } - } - }); - } - } - - processDependencies("node_modules", lockObj.dependencies); - - fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); - ''; - }; - - prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: - let - forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; - in - '' - # Pinpoint the versions of all dependencies to the ones that are actually being used - echo "pinpointing versions of dependencies..." - source $pinpointDependenciesScriptPath - - # Patch the shebangs of the bundled modules to prevent them from - # calling executables outside the Nix store as much as possible - patchShebangs . - - # Deploy the Node.js package by running npm install. Since the - # dependencies have been provided already by ourselves, it should not - # attempt to install them again, which is good, because we want to make - # it Nix's responsibility. If it needs to install any dependencies - # anyway (e.g. because the dependency parameters are - # incomplete/incorrect), it fails. - # - # The other responsibilities of NPM are kept -- version checks, build - # steps, postprocessing etc. - - export HOME=$TMPDIR - cd "${packageName}" - runHook preRebuild - - ${lib.optionalString bypassCache '' - ${lib.optionalString reconstructLock '' - if [ -f package-lock.json ] - then - echo "WARNING: Reconstruct lock option enabled, but a lock file already exists!" - echo "This will most likely result in version mismatches! We will remove the lock file and regenerate it!" - rm package-lock.json - else - echo "No package-lock.json file found, reconstructing..." - fi - - node ${reconstructPackageLock} - ''} - - node ${addIntegrityFieldsScript} - ''} - - npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild - - if [ "''${dontNpmInstall-}" != "1" ] - then - # NPM tries to download packages even when they already exist if npm-shrinkwrap is used. - rm -f npm-shrinkwrap.json - - npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install - fi - ''; - - # Builds and composes an NPM package including all its dependencies - buildNodePackage = - { name - , packageName - , version - , dependencies ? [] - , buildInputs ? [] - , production ? true - , npmFlags ? "" - , dontNpmInstall ? false - , bypassCache ? false - , reconstructLock ? false - , preRebuild ? "" - , dontStrip ? true - , unpackPhase ? "true" - , buildPhase ? "true" - , meta ? {} - , ... }@args: - - let - extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" "meta" ]; - in - stdenv.mkDerivation ({ - name = "${name}-${version}"; - buildInputs = [ tarWrapper python nodejs ] - ++ lib.optional (stdenv.isLinux) utillinux - ++ lib.optional (stdenv.isDarwin) libtool - ++ buildInputs; - - inherit nodejs; - - inherit dontStrip; # Stripping may fail a build for some package deployments - inherit dontNpmInstall preRebuild unpackPhase buildPhase; - - compositionScript = composePackage args; - pinpointDependenciesScript = pinpointDependenciesOfPackage args; - - passAsFile = [ "compositionScript" "pinpointDependenciesScript" ]; - - installPhase = '' - # Create and enter a root node_modules/ folder - mkdir -p $out/lib/node_modules - cd $out/lib/node_modules - - # Compose the package and all its dependencies - source $compositionScriptPath - - ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} - - # Create symlink to the deployed executable folder, if applicable - if [ -d "$out/lib/node_modules/.bin" ] - then - ln -s $out/lib/node_modules/.bin $out/bin - fi - - # Create symlinks to the deployed manual page folders, if applicable - if [ -d "$out/lib/node_modules/${packageName}/man" ] - then - mkdir -p $out/share - for dir in "$out/lib/node_modules/${packageName}/man/"* - do - mkdir -p $out/share/man/$(basename "$dir") - for page in "$dir"/* - do - ln -s $page $out/share/man/$(basename "$dir") - done - done - fi - - # Run post install hook, if provided - runHook postInstall - ''; - - meta = { - # default to Node.js' platforms - platforms = nodejs.meta.platforms; - } // meta; - } // extraArgs); - - # Builds a node environment (a node_modules folder and a set of binaries) - buildNodeDependencies = - { name - , packageName - , version - , src - , dependencies ? [] - , buildInputs ? [] - , production ? true - , npmFlags ? "" - , dontNpmInstall ? false - , bypassCache ? false - , reconstructLock ? false - , dontStrip ? true - , unpackPhase ? "true" - , buildPhase ? "true" - , ... }@args: - - let - extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ]; - in - stdenv.mkDerivation ({ - name = "node-dependencies-${name}-${version}"; - - buildInputs = [ tarWrapper python nodejs ] - ++ lib.optional (stdenv.isLinux) utillinux - ++ lib.optional (stdenv.isDarwin) libtool - ++ buildInputs; - - inherit dontStrip; # Stripping may fail a build for some package deployments - inherit dontNpmInstall unpackPhase buildPhase; - - includeScript = includeDependencies { inherit dependencies; }; - pinpointDependenciesScript = pinpointDependenciesOfPackage args; - - passAsFile = [ "includeScript" "pinpointDependenciesScript" ]; - - installPhase = '' - mkdir -p $out/${packageName} - cd $out/${packageName} - - source $includeScriptPath - - # Create fake package.json to make the npm commands work properly - cp ${src}/package.json . - chmod 644 package.json - ${lib.optionalString bypassCache '' - if [ -f ${src}/package-lock.json ] - then - cp ${src}/package-lock.json . - fi - ''} - - # Go to the parent folder to make sure that all packages are pinpointed - cd .. - ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} - - ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} - - # Expose the executables that were installed - cd .. - ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} - - mv ${packageName} lib - ln -s $out/lib/node_modules/.bin $out/bin - ''; - } // extraArgs); - - # Builds a development shell - buildNodeShell = - { name - , packageName - , version - , src - , dependencies ? [] - , buildInputs ? [] - , production ? true - , npmFlags ? "" - , dontNpmInstall ? false - , bypassCache ? false - , reconstructLock ? false - , dontStrip ? true - , unpackPhase ? "true" - , buildPhase ? "true" - , ... }@args: - - let - nodeDependencies = buildNodeDependencies args; - in - stdenv.mkDerivation { - name = "node-shell-${name}-${version}"; - - buildInputs = [ python nodejs ] ++ lib.optional (stdenv.isLinux) utillinux ++ buildInputs; - buildCommand = '' - mkdir -p $out/bin - cat > $out/bin/shell < Date: Fri, 25 Feb 2022 14:04:07 +0100 Subject: [PATCH 2/3] python3.pkgs.dask: don't offer an option for extras-require Using such an option within the package set this way will result in different packages depending on different versions of dask and thereby collisions. Instead, we can simply declare in the package offering the extras the packages the extra needs. --- pkgs/development/python-modules/dask/default.nix | 8 ++++---- pkgs/development/python-modules/datashader/default.nix | 2 +- pkgs/top-level/python-packages.nix | 4 +--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pkgs/development/python-modules/dask/default.nix b/pkgs/development/python-modules/dask/default.nix index 132753b1583f..7af0eca747e8 100644 --- a/pkgs/development/python-modules/dask/default.nix +++ b/pkgs/development/python-modules/dask/default.nix @@ -18,7 +18,6 @@ , pythonOlder , pyyaml , toolz -, withExtraComplete ? false }: buildPythonPackage rec { @@ -46,9 +45,6 @@ buildPythonPackage rec { jinja2 bokeh numpy - ] ++ lib.optionals (withExtraComplete) [ - # infinite recursion between distributed and dask - distributed ]; doCheck = true; @@ -104,6 +100,10 @@ buildPythonPackage rec { "dask.diagnostics" ]; + passthru.extras-require = { + complete = [ distributed ]; + }; + meta = with lib; { description = "Minimal task scheduling abstraction"; homepage = "https://dask.org/"; diff --git a/pkgs/development/python-modules/datashader/default.nix b/pkgs/development/python-modules/datashader/default.nix index d4515b064833..152582f6716f 100644 --- a/pkgs/development/python-modules/datashader/default.nix +++ b/pkgs/development/python-modules/datashader/default.nix @@ -46,7 +46,7 @@ buildPythonPackage rec { param pyct scipy - ]; + ] ++ dask.extras-require.complete; checkInputs = [ pytestCheckHook diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index a7f59aea3b45..c6a62cde8f7d 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2044,9 +2044,7 @@ in { datasette-template-sql = callPackage ../development/python-modules/datasette-template-sql { }; - datashader = callPackage ../development/python-modules/datashader { - dask = self.dask.override { withExtraComplete = true; }; - }; + datashader = callPackage ../development/python-modules/datashader { }; datashape = callPackage ../development/python-modules/datashape { }; From 1986bedcffc8f378a8bfe90e8225c9d5fcf69883 Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Fri, 25 Feb 2022 14:09:41 +0100 Subject: [PATCH 3/3] python3Packages: document how to handle extras-require --- doc/languages-frameworks/python.section.md | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 9f9ace513c30..693ea016e0a5 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -979,6 +979,31 @@ with import {}; in python.withPackages(ps: [ps.blaze])).env ``` +#### Optional extra dependencies + +Some packages define optional dependencies for additional features. With +`setuptools` this is called `extras_require` and `flit` calls it `extras-require`. A +method for supporting this is by declaring the extras of a package in its +`passthru`, e.g. in case of the package `dask` + +```nix +passthru.extras-require = { + complete = [ distributed ]; +}; +``` + +and letting the package requiring the extra add the list to its dependencies + +```nix +propagatedBuildInputs = [ + ... +] ++ dask.extras-require.complete; +``` + +Note this method is preferred over adding parameters to builders, as that can +result in packages depending on different variants and thereby causing +collisions. + #### `buildPythonApplication` function {#buildpythonapplication-function} The `buildPythonApplication` function is practically the same as