Python: move python-modules/generic to interpreter folder

pkgs/development/interpreters/python/buildpythonpackage.nix

@@ -0,0 +1,174 @@
+/* This function provides a generic Python package builder.  It is
+   intended to work with packages that use `distutils/setuptools'
+   (http://pypi.python.org/pypi/setuptools/), which represents a large
+   number of Python packages nowadays.  */
+
+{ python, setuptools, unzip, wrapPython, lib, bootstrapped-pip
+, ensureNewerSourcesHook }:
+
+{ name
+
+# by default prefix `name` e.g. "python3.3-${name}"
+, namePrefix ? python.libPrefix + "-"
+
+, buildInputs ? []
+
+# propagate build dependencies so in case we have A -> B -> C,
+# C can import package A propagated by B 
+, propagatedBuildInputs ? []
+
+# passed to "python setup.py build_ext"
+# https://github.com/pypa/pip/issues/881
+, setupPyBuildFlags ? []
+
+# DEPRECATED: use propagatedBuildInputs
+, pythonPath ? []
+
+# used to disable derivation, useful for specific python versions
+, disabled ? false
+
+, meta ? {}
+
+# Execute before shell hook
+, preShellHook ? ""
+
+# Execute after shell hook
+, postShellHook ? ""
+
+# Additional arguments to pass to the makeWrapper function, which wraps
+# generated binaries.
+, makeWrapperArgs ? []
+
+# Additional flags to pass to "pip install".
+, installFlags ? []
+
+# Raise an error if two packages are installed with the same name
+, catchConflicts ? true
+
+, format ? "setup"
+
+, ... } @ attrs:
+
+
+# Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
+if disabled
+then throw "${name} not supported for interpreter ${python.executable}"
+else
+
+let
+  # use setuptools shim (so that setuptools is imported before distutils)
+  # pip does the same thing: https://github.com/pypa/pip/pull/3265
+  setuppy = ./run_setup.py;
+
+  formatspecific =
+    if format == "wheel" then
+      {
+        unpackPhase = ''
+          mkdir dist
+          cp $src dist/"''${src#*-}"
+        '';
+
+        # Wheels are pre-compiled
+        buildPhase = attrs.buildPhase or ":";
+        installCheckPhase = attrs.checkPhase or ":";
+
+        # Wheels don't have any checks to run
+        doInstallCheck = attrs.doCheck or false;
+      }
+    else if format == "setup" then
+      {
+        # propagate python/setuptools to active setup-hook in nix-shell
+        propagatedBuildInputs =
+          propagatedBuildInputs ++ [ python setuptools ];
+
+        # we copy nix_run_setup.py over so it's executed relative to the root of the source
+        # many project make that assumption
+        buildPhase = attrs.buildPhase or ''
+          runHook preBuild
+          cp ${setuppy} nix_run_setup.py
+          ${python.interpreter} nix_run_setup.py ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
+          runHook postBuild
+        '';
+
+        installCheckPhase = attrs.checkPhase or ''
+          runHook preCheck
+          ${python.interpreter} nix_run_setup.py test
+          runHook postCheck
+        '';
+
+        # Python packages that are installed with setuptools
+        # are typically distributed with tests.
+        # With Python it's a common idiom to run the tests
+        # after the software has been installed.
+
+        # For backwards compatibility, let's use an alias
+        doInstallCheck = attrs.doCheck or true;
+      }
+    else
+      throw "Unsupported format ${format}";
+in
+python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled" "doCheck"] // {
+  name = namePrefix + name;
+
+  buildInputs = [ wrapPython bootstrapped-pip ] ++ buildInputs ++ pythonPath
+    ++ [ (ensureNewerSourcesHook { year = "1980"; }) ]
+    ++ (lib.optional (lib.hasSuffix "zip" attrs.src.name or "") unzip);
+
+  pythonPath = pythonPath;
+
+  configurePhase = attrs.configurePhase or ''
+    runHook preConfigure
+
+    # patch python interpreter to write null timestamps when compiling python files
+    # this way python doesn't try to update them when we freeze timestamps in nix store
+    export DETERMINISTIC_BUILD=1
+
+    runHook postConfigure
+  '';
+
+  # Python packages don't have a checkPhase, only an installCheckPhase
+  doCheck = false;
+
+  installPhase = attrs.installPhase or ''
+    runHook preInstall
+
+    mkdir -p "$out/${python.sitePackages}"
+    export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
+
+    pushd dist
+    ${bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags}
+    popd
+
+    runHook postInstall
+  '';
+
+  postFixup = attrs.postFixup or ''
+    wrapPythonPrograms
+  '' + lib.optionalString catchConflicts ''
+    # check if we have two packages with the same name in closure and fail
+    # this shouldn't happen, something went wrong with dependencies specs
+    ${python.interpreter} ${./catch_conflicts.py}
+  '';
+
+  shellHook = attrs.shellHook or ''
+    ${preShellHook}
+    if test -e setup.py; then
+       tmp_path=$(mktemp -d)
+       export PATH="$tmp_path/bin:$PATH"
+       export PYTHONPATH="$tmp_path/${python.sitePackages}:$PYTHONPATH"
+       mkdir -p $tmp_path/${python.sitePackages}
+       ${bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path
+    fi
+    ${postShellHook}
+  '';
+
+  meta = with lib.maintainers; {
+    # default to python's platforms
+    platforms = python.meta.platforms;
+  } // meta // {
+    # add extra maintainer(s) to every package
+    maintainers = (meta.maintainers or []) ++ [ chaoflow domenkozar ];
+    # a marker for release utilities to discover python packages
+    isBuildPythonPackage = python.meta.platforms;
+  };
+} // formatspecific)

pkgs/development/interpreters/python/catch_conflicts.py

@@ -0,0 +1,30 @@
+import pkg_resources
+import collections
+import sys
+
+do_abort = False
+packages = collections.defaultdict(list)
+
+for f in sys.path:
+    for req in pkg_resources.find_distributions(f):
+        if req not in packages[req.project_name]:
+            # some exceptions inside buildPythonPackage
+            if req.project_name in ['setuptools', 'pip', 'wheel']:
+                continue
+            packages[req.project_name].append(req)
+
+
+for name, duplicates in packages.items():
+    if len(duplicates) > 1:
+        do_abort = True
+        print("Found duplicated packages in closure for dependency '{}': ".format(name))
+        for dup in duplicates:
+            print("  " + repr(dup))
+
+if do_abort:
+    print("")
+    print(
+        'Package duplicates found in closure, see above. Usually this '
+        'happens if two packages depend on different version '
+        'of the same dependency.')
+    sys.exit(1)

pkgs/development/interpreters/python/run_setup.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+
+import setuptools
+import tokenize
+
+__file__='setup.py';
+
+exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec'))

pkgs/development/interpreters/python/wrap.sh

@@ -0,0 +1,118 @@
+# Wrapper around wrapPythonProgramsIn, below. The $pythonPath
+# variable is passed in from the buildPythonPackage function.
+wrapPythonPrograms() {
+    wrapPythonProgramsIn $out "$out $pythonPath"
+}
+
+# Builds environment variables like PYTHONPATH and PATH walking through closure
+# of dependencies.
+buildPythonPath() {
+    local pythonPath="$1"
+    local path
+
+    # Create an empty table of python paths (see doc on _addToPythonPath
+    # for how this is used). Build up the program_PATH and program_PYTHONPATH
+    # variables.
+    declare -A pythonPathsSeen=()
+    program_PYTHONPATH=
+    program_PATH=
+    pythonPathsSeen["@python@"]=1
+    addToSearchPath program_PATH @python@/bin
+    for path in $pythonPath; do
+        _addToPythonPath $path
+    done
+}
+
+# Patches a Python script so that it has correct libraries path and executable
+# name.
+patchPythonScript() {
+    local f="$1"
+
+    # The magicalSedExpression will invoke a "$(basename "$f")", so
+    # if you change $f to something else, be sure to also change it
+    # in pkgs/top-level/python-packages.nix!
+    # It also uses $program_PYTHONPATH.
+    sed -i "$f" -re '@magicalSedExpression@'
+}
+
+# Transforms any binaries generated by the setup.py script, replacing them
+# with an executable shell script which will set some environment variables
+# and then call into the original binary (which has been given a .wrapped
+# suffix).
+wrapPythonProgramsIn() {
+    local dir="$1"
+    local pythonPath="$2"
+    local f
+
+    buildPythonPath "$pythonPath"
+
+    # Find all regular files in the output directory that are executable.
+    for f in $(find "$dir" -type f -perm -0100); do
+        # Rewrite "#! .../env python" to "#! /nix/store/.../python".
+        # Strip suffix, like "3" or "2.7m" -- we don't have any choice on which
+        # Python to use besides one with this hook anyway.
+        if head -n1 "$f" | grep -q '#!.*/env.*\(python\|pypy\)'; then
+            sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#! @executable@^"
+        fi
+
+        # catch /python and /.python-wrapped
+        if head -n1 "$f" | grep -q '/\.\?\(python\|pypy\)'; then
+            # dont wrap EGG-INFO scripts since they are called from python
+            if echo "$f" | grep -qv EGG-INFO/scripts; then
+                echo "wrapping \`$f'..."
+                patchPythonScript "$f"
+                # wrapProgram creates the executable shell script described
+                # above. The script will set PYTHONPATH and PATH variables.!
+                # (see pkgs/build-support/setup-hooks/make-wrapper.sh)
+                local -a wrap_args=("$f"
+                                 --prefix PATH ':' "$program_PATH")
+
+                # Add any additional arguments provided by makeWrapperArgs
+                # argument to buildPythonPackage.
+                local -a user_args="($makeWrapperArgs)"
+                local -a wrapProgramArgs=("${wrap_args[@]}" "${user_args[@]}")
+                wrapProgram "${wrapProgramArgs[@]}"
+            fi
+        fi
+    done
+}
+
+# Adds the lib and bin directories to the PYTHONPATH and PATH variables,
+# respectively. Recurses on any paths declared in
+# `propagated-native-build-inputs`, while avoiding duplicating paths by
+# flagging the directories it has visited in `pythonPathsSeen`.
+_addToPythonPath() {
+    local dir="$1"
+    # Stop if we've already visited here.
+    if [ -n "${pythonPathsSeen[$dir]}" ]; then return; fi
+    pythonPathsSeen[$dir]=1
+    # addToSearchPath is defined in stdenv/generic/setup.sh. It will have
+    # the effect of calling `export program_X=$dir/...:$program_X`.
+    addToSearchPath program_PYTHONPATH $dir/lib/@libPrefix@/site-packages
+    addToSearchPath program_PATH $dir/bin
+
+    # Inspect the propagated inputs (if they exist) and recur on them.
+    local prop="$dir/nix-support/propagated-native-build-inputs"
+    if [ -e $prop ]; then
+        local new_path
+        for new_path in $(cat $prop); do
+            _addToPythonPath $new_path
+        done
+    fi
+}
+
+createBuildInputsPth() {
+    local category="$1"
+    local inputs="$2"
+    if [ foo"$inputs" != foo ]; then
+        for x in $inputs; do
+            if $(echo -n $x |grep -q python-recursive-pth-loader); then
+                continue
+            fi
+            if test -d "$x"/lib/@libPrefix@/site-packages; then
+                echo $x/lib/@libPrefix@/site-packages \
+                    >> "$out"/lib/@libPrefix@/site-packages/${name}-nix-python-$category.pth
+            fi
+        done
+    fi
+}