From 92f01732131eae253baac0a6910e5c33d90e706f Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 8 Jan 2023 11:20:31 +0000 Subject: [PATCH 1/5] pypy{27,38,39}: refactor derivation - Remove code related to i686 archs (not supported anymore since #209100) - Split non-install steps in installPhase in installCheckPhase/preFixup - Add missing pre/post hooks --- .../interpreters/python/pypy/default.nix | 81 ++++++++++++------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix index e9c2e3d1aa0e..d0e2364e1092 100644 --- a/pkgs/development/interpreters/python/pypy/default.nix +++ b/pkgs/development/interpreters/python/pypy/default.nix @@ -22,7 +22,7 @@ assert zlibSupport -> zlib != null; with lib; let - isPy3k = substring 0 1 pythonVersion == "3"; + isPy3k = (lib.versions.major pythonVersion) == "3"; isPy39OrNewer = versionAtLeast pythonVersion "3.9"; passthru = passthruFun { inherit self sourceVersion pythonVersion packageOverrides; @@ -64,8 +64,6 @@ in with passthru; stdenv.mkDerivation rec { libunwind Security ]; - hardeningDisable = optional stdenv.isi686 "pic"; - # Remove bootstrap python from closure dontPatchShebangs = true; disallowedReferences = [ python ]; @@ -96,14 +94,47 @@ in with passthru; stdenv.mkDerivation rec { substituteInPlace lib_pypy/pypy_tools/build_cffi_imports.py \ --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES" - substituteInPlace "lib-python/${if isPy3k then "3/tkinter/tix.py" else "2.7/lib-tk/Tix.py"}" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'" + substituteInPlace "lib-python/${if isPy3k then "3/tkinter/tix.py" else "2.7/lib-tk/Tix.py"}" \ + --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'" ''; buildPhase = '' + runHook preBuild + ${pythonForPypy.interpreter} rpython/bin/rpython \ --make-jobs="$NIX_BUILD_CORES" \ -Ojit \ --batch pypy/goal/targetpypystandalone.py + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/{bin,include,lib,${executable}-c} + + cp -R {include,lib_pypy,lib-python,${executable}-c} $out/${executable}-c + cp lib${executable}-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/ + ln -s $out/${executable}-c/${executable}-c $out/bin/${executable} + ${optionalString isPy39OrNewer "ln -s $out/bin/${executable}-c $out/bin/pypy3"} + + # other packages expect to find stuff according to libPrefix + ln -s $out/${executable}-c/include $out/include/${libPrefix} + ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix} + + # Include a sitecustomize.py file + cp ${../sitecustomize.py} $out/lib/${libPrefix}/${sitePackages}/sitecustomize.py + + runHook postInstall + ''; + + preFixup = lib.optionalString (stdenv.isDarwin) '' + install_name_tool -change @rpath/lib${executable}-c.dylib $out/lib/lib${executable}-c.dylib $out/bin/${executable} + '' + lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) '' + mkdir -p $out/${executable}-c/pypy/bin + mv $out/bin/${executable} $out/${executable}-c/pypy/bin/${executable} + ln -s $out/${executable}-c/pypy/bin/${executable} $out/bin/${executable} ''; setupHook = python-setup-hook sitePackages; @@ -143,32 +174,22 @@ in with passthru; stdenv.mkDerivation rec { ${pythonForPypy.interpreter} ./pypy/test_all.py --pypy=./${executable}-c -k 'not (${concatStringsSep " or " disabledTests})' lib-python ''; - installPhase = '' - mkdir -p $out/{bin,include,lib,${executable}-c} - - cp -R {include,lib_pypy,lib-python,${executable}-c} $out/${executable}-c - cp lib${executable}-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/ - ln -s $out/${executable}-c/${executable}-c $out/bin/${executable} - ${optionalString isPy39OrNewer "ln -s $out/bin/${executable}-c $out/bin/pypy3"} - - # other packages expect to find stuff according to libPrefix - ln -s $out/${executable}-c/include $out/include/${libPrefix} - ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix} - - ${lib.optionalString stdenv.isDarwin '' - install_name_tool -change @rpath/lib${executable}-c.dylib $out/lib/lib${executable}-c.dylib $out/bin/${executable} - ''} - ${lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) '' - mkdir -p $out/${executable}-c/pypy/bin - mv $out/bin/${executable} $out/${executable}-c/pypy/bin/${executable} - ln -s $out/${executable}-c/pypy/bin/${executable} $out/bin/${executable} - ''} - - # verify cffi modules - $out/bin/${executable} -c ${if isPy3k then "'import tkinter;import sqlite3;import curses;import lzma'" else "'import Tkinter;import sqlite3;import curses'"} - - # Include a sitecustomize.py file - cp ${../sitecustomize.py} $out/lib/${libPrefix}/${sitePackages}/sitecustomize.py + # verify cffi modules + doInstallCheck = true; + installCheckPhase = let + modules = [ + "curses" + "sqlite3" + ] ++ optionals (!isPy3k) [ + "Tkinter" + ] ++ optionals isPy3k [ + "tkinter" + "lzma" + ]; + imports = concatMapStringsSep "; " (x: "import ${x}") modules; + in '' + echo "Testing whether we can import modules" + $out/bin/${executable} -c '${imports}' ''; inherit passthru; From a7ffcced44625f1c2406ce1fc24a4bc4a2add966 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 8 Jan 2023 12:59:25 +0000 Subject: [PATCH 2/5] pypy{27,38,39}: remove global with lib --- .../interpreters/python/pypy/default.nix | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix index d0e2364e1092..4575041b68f2 100644 --- a/pkgs/development/interpreters/python/pypy/default.nix +++ b/pkgs/development/interpreters/python/pypy/default.nix @@ -19,11 +19,9 @@ assert zlibSupport -> zlib != null; -with lib; - let isPy3k = (lib.versions.major pythonVersion) == "3"; - isPy39OrNewer = versionAtLeast pythonVersion "3.9"; + isPy39OrNewer = lib.versionAtLeast pythonVersion "3.9"; passthru = passthruFun { inherit self sourceVersion pythonVersion packageOverrides; implementation = "pypy"; @@ -54,13 +52,13 @@ in with passthru; stdenv.mkDerivation rec { nativeBuildInputs = [ pkg-config ]; buildInputs = [ bzip2 openssl pythonForPypy libffi ncurses expat sqlite tk tcl libX11 gdbm db - ] ++ optionals isPy3k [ + ] ++ lib.optionals isPy3k [ xz - ] ++ optionals (stdenv ? cc && stdenv.cc.libc != null) [ + ] ++ lib.optionals (stdenv ? cc && stdenv.cc.libc != null) [ stdenv.cc.libc - ] ++ optionals zlibSupport [ + ] ++ lib.optionals zlibSupport [ zlib - ] ++ optionals stdenv.isDarwin [ + ] ++ lib.optionals stdenv.isDarwin [ libunwind Security ]; @@ -68,9 +66,9 @@ in with passthru; stdenv.mkDerivation rec { dontPatchShebangs = true; disallowedReferences = [ python ]; - C_INCLUDE_PATH = makeSearchPathOutput "dev" "include" buildInputs; - LIBRARY_PATH = makeLibraryPath buildInputs; - LD_LIBRARY_PATH = makeLibraryPath (filter (x : x.outPath != stdenv.cc.libc.outPath or "") buildInputs); + C_INCLUDE_PATH = lib.makeSearchPathOutput "dev" "include" buildInputs; + LIBRARY_PATH = lib.makeLibraryPath buildInputs; + LD_LIBRARY_PATH = lib.makeLibraryPath (builtins.filter (x : x.outPath != stdenv.cc.libc.outPath or "") buildInputs); patches = [ ./dont_fetch_vendored_deps.patch @@ -117,7 +115,7 @@ in with passthru; stdenv.mkDerivation rec { cp -R {include,lib_pypy,lib-python,${executable}-c} $out/${executable}-c cp lib${executable}-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/ ln -s $out/${executable}-c/${executable}-c $out/bin/${executable} - ${optionalString isPy39OrNewer "ln -s $out/bin/${executable}-c $out/bin/pypy3"} + ${lib.optionalString isPy39OrNewer "ln -s $out/bin/${executable}-c $out/bin/pypy3"} # other packages expect to find stuff according to libPrefix ln -s $out/${executable}-c/include $out/include/${libPrefix} @@ -148,12 +146,12 @@ in with passthru; stdenv.mkDerivation rec { "test_shutil" # disable socket because it has two actual network tests that fail "test_socket" - ] ++ optionals (!isPy3k) [ + ] ++ lib.optionals (!isPy3k) [ # disable test_urllib2net, test_urllib2_localnet, and test_urllibnet because they require networking (example.com) "test_urllib2net" "test_urllibnet" "test_urllib2_localnet" - ] ++ optionals isPy3k [ + ] ++ lib.optionals isPy3k [ # disable asyncio due to https://github.com/NixOS/nix/issues/1238 "test_asyncio" # disable os due to https://github.com/NixOS/nixpkgs/issues/10496 @@ -171,7 +169,7 @@ in with passthru; stdenv.mkDerivation rec { export TERM="xterm"; export HOME="$TMPDIR"; - ${pythonForPypy.interpreter} ./pypy/test_all.py --pypy=./${executable}-c -k 'not (${concatStringsSep " or " disabledTests})' lib-python + ${pythonForPypy.interpreter} ./pypy/test_all.py --pypy=./${executable}-c -k 'not (${lib.concatStringsSep " or " disabledTests})' lib-python ''; # verify cffi modules @@ -180,13 +178,13 @@ in with passthru; stdenv.mkDerivation rec { modules = [ "curses" "sqlite3" - ] ++ optionals (!isPy3k) [ + ] ++ lib.optionals (!isPy3k) [ "Tkinter" - ] ++ optionals isPy3k [ + ] ++ lib.optionals isPy3k [ "tkinter" "lzma" ]; - imports = concatMapStringsSep "; " (x: "import ${x}") modules; + imports = lib.concatMapStringsSep "; " (x: "import ${x}") modules; in '' echo "Testing whether we can import modules" $out/bin/${executable} -c '${imports}' From eddcc1c01c73e8744f77bff999a1d7503bda33d0 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 8 Jan 2023 13:00:49 +0000 Subject: [PATCH 3/5] pythonInterpreters.pypy27_prebuilt: remove global with lib --- .../development/interpreters/python/pypy/prebuilt_2_7.nix | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix index d57d77351c51..a95e0c2f651e 100644 --- a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix +++ b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix @@ -26,8 +26,6 @@ # This version of PyPy is primarily added to speed-up translation of # our PyPy source build when developing that expression. -with lib; - let isPy3k = majorVersion == "3"; passthru = passthruFun { @@ -134,12 +132,12 @@ in with passthru; stdenv.mkDerivation { "ssl" "sys" "curses" - ] ++ optionals (!isPy3k) [ + ] ++ lib.optionals (!isPy3k) [ "Tkinter" - ] ++ optionals isPy3k [ + ] ++ lib.optionals isPy3k [ "tkinter" ]; - imports = concatMapStringsSep "; " (x: "import ${x}") modules; + imports = lib.concatMapStringsSep "; " (x: "import ${x}") modules; in '' echo "Testing whether we can import modules" $out/bin/${executable} -c '${imports}' From c04fc21156a5ef5212faff83807ec2a5c8861239 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 8 Jan 2023 13:01:39 +0000 Subject: [PATCH 4/5] pythonInterpreters.pypy39_prebuilt: remove global with lib --- pkgs/development/interpreters/python/pypy/prebuilt.nix | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkgs/development/interpreters/python/pypy/prebuilt.nix b/pkgs/development/interpreters/python/pypy/prebuilt.nix index af782654fbdb..a25ab8a78221 100644 --- a/pkgs/development/interpreters/python/pypy/prebuilt.nix +++ b/pkgs/development/interpreters/python/pypy/prebuilt.nix @@ -26,8 +26,6 @@ # This version of PyPy is primarily added to speed-up translation of # our PyPy source build when developing that expression. -with lib; - let isPy3k = majorVersion == "3"; passthru = passthruFun rec { @@ -138,12 +136,12 @@ in with passthru; stdenv.mkDerivation { "ssl" "sys" "curses" - ] ++ optionals (!isPy3k) [ + ] ++ lib.optionals (!isPy3k) [ "Tkinter" - ] ++ optionals isPy3k [ + ] ++ lib.optionals isPy3k [ "tkinter" ]; - imports = concatMapStringsSep "; " (x: "import ${x}") modules; + imports = lib.concatMapStringsSep "; " (x: "import ${x}") modules; in '' echo "Testing whether we can import modules" $out/bin/${executable} -c '${imports}' From 4b74c3c3d1cba8d25c6aa2313218959100cd9d8c Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 8 Jan 2023 18:40:41 +0000 Subject: [PATCH 5/5] pypy39: fix pypy3 symlink --- pkgs/development/interpreters/python/pypy/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix index 4575041b68f2..6fad25ee8d63 100644 --- a/pkgs/development/interpreters/python/pypy/default.nix +++ b/pkgs/development/interpreters/python/pypy/default.nix @@ -115,7 +115,7 @@ in with passthru; stdenv.mkDerivation rec { cp -R {include,lib_pypy,lib-python,${executable}-c} $out/${executable}-c cp lib${executable}-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/ ln -s $out/${executable}-c/${executable}-c $out/bin/${executable} - ${lib.optionalString isPy39OrNewer "ln -s $out/bin/${executable}-c $out/bin/pypy3"} + ${lib.optionalString isPy39OrNewer "ln -s $out/bin/${executable} $out/bin/pypy3"} # other packages expect to find stuff according to libPrefix ln -s $out/${executable}-c/include $out/include/${libPrefix}