Fix venv creation in Python environments
The way we build python environments is subtly broken. A python
environment should be semantically identical to a vanilla Python
installation in, say, /usr/local. The current implementation, however,
differs in two important ways. The first is that it's impossible to use
python packages from the environment in python virtual environments. The
second is that the nix-generated environment appears to be a venv, but
it's not.
This commit changes the way python environments are built:
* When generating wrappers for python executables, we inherit argv[0]
from the wrapper. This causes python to initialize its configuration
in the environment with all the correct paths.
* We remove the sitecustomize.py file from the base python package.
This file was used tweak the python configuration after it was
incorrectly initialized. That's no longer necessary.
The end result is that python environments no longer appear to be venvs,
and behave more like a vanilla python installation. In addition it's
possible to create a venv using an environment and use packages from
both the environment and the venv.
This commit is contained in:
Colin Putney
@@ -318,8 +318,6 @@ in with passthru; stdenv.mkDerivation ({
|
||||
inherit passthru;
|
||||
|
||||
postFixup = ''
|
||||
# Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7.
|
||||
cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
|
||||
'' + lib.optionalString strip2to3 ''
|
||||
rm -R $out/bin/2to3 $out/lib/python*/lib2to3
|
||||
'' + lib.optionalString stripConfig ''
|
||||
|
||||
@@ -530,8 +530,7 @@ in with passthru; stdenv.mkDerivation (finalAttrs: {
|
||||
# Strip tests
|
||||
rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
|
||||
'' + optionalString includeSiteCustomize ''
|
||||
# Include a sitecustomize.py file
|
||||
cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
|
||||
|
||||
'' + optionalString stripBytecode ''
|
||||
# Determinism: deterministic bytecode
|
||||
# First we delete all old bytecode.
|
||||
|
||||
@@ -126,9 +126,6 @@ in with passthru; stdenv.mkDerivation rec {
|
||||
ln -s $out/${executable}-c/include $out/include/${libPrefix}
|
||||
ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
|
||||
|
||||
# Include a sitecustomize.py file
|
||||
cp ${../sitecustomize.py} $out/${if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"}/sitecustomize.py
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
|
||||
@@ -95,9 +95,6 @@ in with passthru; stdenv.mkDerivation {
|
||||
echo "Removing bytecode"
|
||||
find . -name "__pycache__" -type d -depth -delete
|
||||
|
||||
# Include a sitecustomize.py file
|
||||
cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
|
||||
@@ -96,9 +96,6 @@ in with passthru; stdenv.mkDerivation {
|
||||
echo "Removing bytecode"
|
||||
find . -name "__pycache__" -type d -depth -delete
|
||||
|
||||
# Include a sitecustomize.py file
|
||||
cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
"""
|
||||
This is a Nix-specific module for discovering modules built with Nix.
|
||||
|
||||
The module recursively adds paths that are on `NIX_PYTHONPATH` to `sys.path`. In
|
||||
order to process possible `.pth` files `site.addsitedir` is used.
|
||||
|
||||
The paths listed in `PYTHONPATH` are added to `sys.path` afterwards, but they
|
||||
will be added before the entries we add here and thus take precedence.
|
||||
|
||||
Note the `NIX_PYTHONPATH` environment variable is unset in order to prevent leakage.
|
||||
|
||||
Similarly, this module listens to the environment variable `NIX_PYTHONEXECUTABLE`
|
||||
and sets `sys.executable` to its value.
|
||||
"""
|
||||
import site
|
||||
import sys
|
||||
import os
|
||||
import functools
|
||||
|
||||
paths = os.environ.pop('NIX_PYTHONPATH', None)
|
||||
if paths:
|
||||
functools.reduce(lambda k, p: site.addsitedir(p, k), paths.split(':'), site._init_pathinfo())
|
||||
|
||||
# Check whether we are in a venv or virtualenv.
|
||||
# For Python 3 we check whether our `base_prefix` is different from our current `prefix`.
|
||||
# For Python 2 we check whether the non-standard `real_prefix` is set.
|
||||
# https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv
|
||||
in_venv = (sys.version_info.major == 3 and sys.prefix != sys.base_prefix) or (sys.version_info.major == 2 and hasattr(sys, "real_prefix"))
|
||||
|
||||
if not in_venv:
|
||||
executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None)
|
||||
prefix = os.environ.pop('NIX_PYTHONPREFIX', None)
|
||||
|
||||
if 'PYTHONEXECUTABLE' not in os.environ and executable is not None:
|
||||
sys.executable = executable
|
||||
if prefix is not None:
|
||||
# Sysconfig does not like it when sys.prefix is set to None
|
||||
sys.prefix = sys.exec_prefix = prefix
|
||||
site.PREFIXES.insert(0, prefix)
|
||||
@@ -39,11 +39,21 @@ let
|
||||
is_virtualenv = "False";
|
||||
};
|
||||
} // lib.optionalAttrs (!python.isPyPy) {
|
||||
# Use virtualenv from a Nix env.
|
||||
nixenv-virtualenv = rec {
|
||||
env = runCommand "${python.name}-virtualenv" {} ''
|
||||
${pythonVirtualEnv.interpreter} -m virtualenv venv
|
||||
mv venv $out
|
||||
# Use virtualenv with symlinks from a Nix env.
|
||||
nixenv-virtualenv-links = rec {
|
||||
env = runCommand "${python.name}-virtualenv-links" {} ''
|
||||
${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --symlinks --no-seed $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${python.executable}";
|
||||
is_venv = "False";
|
||||
is_nixenv = "True";
|
||||
is_virtualenv = "True";
|
||||
};
|
||||
} // lib.optionalAttrs (!python.isPyPy) {
|
||||
# Use virtualenv with copies from a Nix env.
|
||||
nixenv-virtualenv-copies = rec {
|
||||
env = runCommand "${python.name}-virtualenv-copies" {} ''
|
||||
${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --copies --no-seed $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${python.executable}";
|
||||
is_venv = "False";
|
||||
@@ -59,27 +69,48 @@ let
|
||||
is_nixenv = "True";
|
||||
is_virtualenv = "False";
|
||||
};
|
||||
} // lib.optionalAttrs (python.isPy3k && (!python.isPyPy)) {
|
||||
# Venv built using plain Python
|
||||
} // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) {
|
||||
# Venv built using links to plain Python
|
||||
# Python 2 does not support venv
|
||||
# TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3.
|
||||
plain-venv = rec {
|
||||
env = runCommand "${python.name}-venv" {} ''
|
||||
${python.interpreter} -m venv $out
|
||||
plain-venv-links = rec {
|
||||
env = runCommand "${python.name}-venv-links" {} ''
|
||||
${python.interpreter} -m venv --system-site-packages --symlinks --without-pip $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${python.executable}";
|
||||
is_venv = "True";
|
||||
is_nixenv = "False";
|
||||
is_virtualenv = "False";
|
||||
};
|
||||
} // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) {
|
||||
# Venv built using copies from plain Python
|
||||
# Python 2 does not support venv
|
||||
# TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3.
|
||||
plain-venv-copies = rec {
|
||||
env = runCommand "${python.name}-venv-copies" {} ''
|
||||
${python.interpreter} -m venv --system-site-packages --copies --without-pip $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${python.executable}";
|
||||
is_venv = "True";
|
||||
is_nixenv = "False";
|
||||
is_virtualenv = "False";
|
||||
};
|
||||
|
||||
} // lib.optionalAttrs (python.pythonAtLeast "3.8") {
|
||||
# Venv built using Python Nix environment (python.buildEnv)
|
||||
# TODO: Cannot create venv from a nix env
|
||||
# Error: Command '['/nix/store/ddc8nqx73pda86ibvhzdmvdsqmwnbjf7-python3-3.7.6-venv/bin/python3.7', '-Im', 'ensurepip', '--upgrade', '--default-pip']' returned non-zero exit status 1.
|
||||
nixenv-venv = rec {
|
||||
env = runCommand "${python.name}-venv" {} ''
|
||||
${pythonEnv.interpreter} -m venv $out
|
||||
nixenv-venv-links = rec {
|
||||
env = runCommand "${python.name}-venv-links" {} ''
|
||||
${pythonEnv.interpreter} -m venv --system-site-packages --symlinks --without-pip $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${pythonEnv.executable}";
|
||||
is_venv = "True";
|
||||
is_nixenv = "True";
|
||||
is_virtualenv = "False";
|
||||
};
|
||||
} // lib.optionalAttrs (python.pythonAtLeast "3.8") {
|
||||
# Venv built using Python Nix environment (python.buildEnv)
|
||||
nixenv-venv-copies = rec {
|
||||
env = runCommand "${python.name}-venv-copies" {} ''
|
||||
${pythonEnv.interpreter} -m venv --system-site-packages --copies --without-pip $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${pythonEnv.executable}";
|
||||
is_venv = "True";
|
||||
@@ -91,11 +122,33 @@ let
|
||||
testfun = name: attrs: runCommand "${python.name}-tests-${name}" ({
|
||||
inherit (python) pythonVersion;
|
||||
} // attrs) ''
|
||||
mkdir $out
|
||||
|
||||
# set up the test files
|
||||
cp -r ${./tests/test_environments} tests
|
||||
chmod -R +w tests
|
||||
substituteAllInPlace tests/test_python.py
|
||||
${attrs.interpreter} -m unittest discover --verbose tests #/test_python.py
|
||||
mkdir $out
|
||||
|
||||
# run the tests by invoking the interpreter via full path
|
||||
echo "absolute path: ${attrs.interpreter}"
|
||||
${attrs.interpreter} -m unittest discover --verbose tests 2>&1 | tee "$out/full.txt"
|
||||
|
||||
# run the tests by invoking the interpreter via $PATH
|
||||
export PATH="$(dirname ${attrs.interpreter}):$PATH"
|
||||
echo "PATH: $(basename ${attrs.interpreter})"
|
||||
"$(basename ${attrs.interpreter})" -m unittest discover --verbose tests 2>&1 | tee "$out/path.txt"
|
||||
|
||||
# make sure we get the right path when invoking through a result link
|
||||
ln -s "${attrs.env}" result
|
||||
relative="result/bin/$(basename ${attrs.interpreter})"
|
||||
expected="$PWD/$relative"
|
||||
actual="$(./$relative -c "import sys; print(sys.executable)" | tee "$out/result.txt")"
|
||||
if [ "$actual" != "$expected" ]; then
|
||||
echo "expected $expected, got $actual"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# if we got this far, the tests passed
|
||||
touch $out/success
|
||||
'';
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ class TestCasePython(unittest.TestCase):
|
||||
|
||||
@unittest.skipIf(IS_PYPY or sys.version_info.major==2, "Python 2 does not have base_prefix")
|
||||
def test_base_prefix(self):
|
||||
if IS_VENV or IS_NIXENV or IS_VIRTUALENV:
|
||||
if IS_VENV or IS_VIRTUALENV:
|
||||
self.assertNotEqual(sys.prefix, sys.base_prefix)
|
||||
else:
|
||||
self.assertEqual(sys.prefix, sys.base_prefix)
|
||||
|
||||
Reference in New Issue
Block a user