From 1971cea5492516371f36931c5b2793074d7815cc Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Thu, 24 Oct 2024 01:47:43 +0200 Subject: [PATCH 1/5] python312Packages.pillow-simd: drop Pillow is on 11.0, pillow-simd remains on 9.5.0.post2. We've seen various CVEs since, and since pillow-simd is a leaf package we remove it now. --- .../python-modules/pillow-simd/default.nix | 63 ------------------- pkgs/top-level/python-aliases.nix | 1 + pkgs/top-level/python-packages.nix | 5 -- 3 files changed, 1 insertion(+), 68 deletions(-) delete mode 100644 pkgs/development/python-modules/pillow-simd/default.nix diff --git a/pkgs/development/python-modules/pillow-simd/default.nix b/pkgs/development/python-modules/pillow-simd/default.nix deleted file mode 100644 index 2d3551d171e9..000000000000 --- a/pkgs/development/python-modules/pillow-simd/default.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - lib, - stdenv, - buildPythonPackage, - fetchFromGitHub, - isPyPy, - isPy3k, - olefile, - freetype, - libjpeg, - zlib, - libtiff, - libwebp, - libxcrypt, - tcl, - lcms2, - libxcb, - tk, - libX11, - openjpeg, - libimagequant, - pyroma, - numpy, - defusedxml, - pytestCheckHook, - setuptools, -}@args: - -import ../pillow/generic.nix ( - rec { - pname = "Pillow-SIMD"; - # check for release version on https://pypi.org/project/Pillow-SIMD/#history - # does not match the latest pillow release version! - version = "9.0.0.post1"; - format = "setuptools"; - - disabled = !isPy3k; - - src = fetchFromGitHub { - owner = "uploadcare"; - repo = "pillow-simd"; - rev = "v${version}"; - hash = "sha256-qTZYhgHjVMXqoYl3mG1xVrFaWrPidSY8HlyFQizV27Y="; - }; - - meta = with lib; { - broken = - (stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64) || stdenv.hostPlatform.isDarwin; - homepage = "https://python-pillow.github.io/pillow-perf/"; - description = "Friendly PIL fork - SIMD version"; - longDescription = '' - Pillow-SIMD is "following" Pillow. Pillow-SIMD versions are 100% compatible drop-in replacements for Pillow of the same version. - - SIMD stands for "single instruction, multiple data" and its essence is in performing the same operation on multiple data points simultaneously by using multiple processing elements. Common CPU SIMD instruction sets are MMX, SSE-SSE4, AVX, AVX2, AVX512, NEON. - - Currently, Pillow-SIMD can be compiled with SSE4 (default) or AVX2 support. - ''; - license = licenses.hpnd; - maintainers = [ ]; - }; - } - // args -) diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index b55e051e41a5..634992d9eaee 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -402,6 +402,7 @@ mapAliases ({ pdfminer = pdfminer-six; # added 2022-05-25 pdfx = throw "pdfx has been removed because the upstream repository was archived in 2023"; # Added 2024-10-04 pep257 = pydocstyle; # added 2022-04-12 + pillow-simd = throw "pillow-simd has been removed for lagging behind pillow upstream, which exposes it to various security issues."; # Added 2024-10-24 pixelmatch = "pixelmatch has been removed as it was unmaintained"; # Added 2024-08-18 pkutils = throw "pkutils was removed as it was unused and is not applicable to modern Python build tools"; # added 2024-07-28 poetry = throw "poetry was promoted to a top-level attribute, use poetry-core to build Python packages"; # added 2023-01-09 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index bc41e8b2046f..492c9dba514d 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -10079,11 +10079,6 @@ self: super: with self; { pillow-jpls = callPackage ../development/python-modules/pillow-jpls { }; - pillow-simd = callPackage ../development/python-modules/pillow-simd { - inherit (pkgs) freetype libjpeg zlib libtiff libwebp tcl lcms2 tk; - inherit (pkgs.xorg) libX11; - }; - pims = callPackage ../development/python-modules/pims { }; pinboard = callPackage ../development/python-modules/pinboard { }; From fa3075a22d795d1a55e50e811cecb483aed5b474 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Thu, 24 Oct 2024 03:11:22 +0200 Subject: [PATCH 2/5] lib/licenses: introduce MIT-CMU license --- lib/licenses.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/licenses.nix b/lib/licenses.nix index fb7b6c2e837f..495823c90a35 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -890,6 +890,11 @@ lib.mapAttrs mkLicense ({ fullName = "MIT License"; }; + mit-cmu = { + spdxId = "MIT-CMU"; + fullName = "CMU License"; + }; + mit-feh = { spdxId = "MIT-feh"; fullName = "feh License"; From 477d7f77483665a57d06c0962189dde3a9395034 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Thu, 24 Oct 2024 03:05:57 +0200 Subject: [PATCH 3/5] python312Packages.pillow: 10.4.0 -> 11.0 https://pillow.readthedocs.io/en/stable/releasenotes/11.0.0.html Removes the generalization of the package that has become obsolete with the pillow-simd removal. --- .../python-modules/pillow/default.nix | 174 +++++++++++++----- .../python-modules/pillow/generic.nix | 109 ----------- pkgs/top-level/python-packages.nix | 14 +- 3 files changed, 137 insertions(+), 160 deletions(-) delete mode 100644 pkgs/development/python-modules/pillow/generic.nix diff --git a/pkgs/development/python-modules/pillow/default.nix b/pkgs/development/python-modules/pillow/default.nix index b5257f5101d2..30d741052a09 100644 --- a/pkgs/development/python-modules/pillow/default.nix +++ b/pkgs/development/python-modules/pillow/default.nix @@ -3,26 +3,35 @@ stdenv, buildPythonPackage, pythonOlder, - fetchPypi, - isPyPy, - defusedxml, - olefile, + fetchFromGitHub, + + # build-system + setuptools, + pkg-config, + + # native dependencies freetype, + lcms2, + libimagequant, libjpeg, - zlib, + libraqm, libtiff, libwebp, - libxcrypt, - tcl, - lcms2, - tk, - libX11, libxcb, openjpeg, - libimagequant, + tkinter, + zlib, + + # optional dependencies + defusedxml, + olefile, + typing-extensions, + + # tests numpy, + pytest-cov-stub, pytestCheckHook, - setuptools, + # for passthru.tests imageio, matplotlib, @@ -30,45 +39,112 @@ pydicom, reportlab, sage, -}@args: +}: -import ./generic.nix ( - rec { - pname = "pillow"; - version = "10.4.0"; - format = "pyproject"; +buildPythonPackage rec { + pname = "pillow"; + version = "11.0.0"; + pyproject = true; - disabled = pythonOlder "3.8"; + src = fetchFromGitHub { + owner = "python-pillow"; + repo = "pillow"; + rev = "refs/tags/${version}"; + hash = "sha256-vWNqzA2ZfJcWexXw790RgyYtP8WDtahoQIX16otCRnk="; + }; - src = fetchPypi { - pname = "pillow"; - inherit version; - hash = "sha256-Fmwc1NJDCbMNYfefSpEUt7IxPXRQkSJ3hV/139fNSgY="; - }; + build-system = [ setuptools ]; - passthru.tests = { - inherit - imageio - matplotlib - pilkit - pydicom - reportlab - sage - ; - }; + nativeBuildInputs = [ pkg-config ]; - meta = with lib; { - homepage = "https://python-pillow.org/"; - description = "Friendly PIL fork (Python Imaging Library)"; - longDescription = '' - The Python Imaging Library (PIL) adds image processing - capabilities to your Python interpreter. This library - supports many file formats, and provides powerful image - processing and graphics capabilities. - ''; - license = licenses.hpnd; - maintainers = with maintainers; [ prikhi ]; - }; - } - // args -) + # https://pillow.readthedocs.io/en/latest/installation/building-from-source.html#building-from-source + buildInputs = [ + freetype + lcms2 + libimagequant + libjpeg + libraqm + libtiff + libwebp + libxcb + openjpeg + tkinter + zlib + ]; + + pypaBuildFlags = [ + # Disable platform guessing, which tries various FHS paths + "--config=setting=--disable-platform-guessing" + ]; + + preConfigure = + let + getLibAndInclude = pkg: ''"${pkg.out}/lib", "${lib.getDev pkg}/include"''; + in + '' + # The build process fails to find the pkg-config files for these dependencies + substituteInPlace setup.py \ + --replace-fail 'IMAGEQUANT_ROOT = None' 'IMAGEQUANT_ROOT = ${getLibAndInclude libimagequant}' \ + --replace-fail 'JPEG2K_ROOT = None' 'JPEG2K_ROOT = ${getLibAndInclude openjpeg}' + + # Build with X11 support + export LDFLAGS="$LDFLAGS -L${libxcb}/lib" + export CFLAGS="$CFLAGS -I${libxcb.dev}/include" + ''; + + optional-dependencies = { + fpx = [ olefile ]; + mic = [ olefile ]; + typing = lib.optionals (pythonOlder "3.10") [ typing-extensions ]; + xmp = [ defusedxml ]; + }; + + nativeCheckInputs = [ + pytest-cov-stub + pytestCheckHook + numpy + ] ++ lib.flatten (lib.attrValues optional-dependencies); + + disabledTests = + [ + # Code quality mismathch 9 vs 10 + "test_pyroma" + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + # Disable darwin tests which require executables: `iconutil` and `screencapture` + "test_grab" + "test_grabclipboard" + "test_save" + ]; + + disabledTestPaths = lib.optionals stdenv.isDarwin [ + # Crashes the interpreter + "Tests/test_imagetk.py" + ]; + + passthru.tests = { + inherit + imageio + matplotlib + pilkit + pydicom + reportlab + sage + ; + }; + + meta = with lib; { + homepage = "https://python-pillow.org"; + changelog = "https://pillow.readthedocs.io/en/stable/releasenotes/${version}.html"; + description = "Friendly PIL fork (Python Imaging Library)"; + longDescription = '' + The Python Imaging Library (PIL) adds image processing + capabilities to your Python interpreter. This library + supports many file formats, and provides powerful image + processing and graphics capabilities. + ''; + license = licenses.mit-cmu; + maintainers = with maintainers; [ hexa ]; + }; + +} diff --git a/pkgs/development/python-modules/pillow/generic.nix b/pkgs/development/python-modules/pillow/generic.nix deleted file mode 100644 index 1ecaea6b205d..000000000000 --- a/pkgs/development/python-modules/pillow/generic.nix +++ /dev/null @@ -1,109 +0,0 @@ -{ - pname, - version, - src, - patches ? [ ], - meta, - passthru ? { }, - ... -}@args: - -with args; - -buildPythonPackage rec { - inherit - pname - version - format - src - meta - passthru - patches - ; - - # Disable imagefont tests, because they don't work well with infinality: - # https://github.com/python-pillow/Pillow/issues/1259 - postPatch = '' - rm Tests/test_imagefont.py - ''; - - disabledTests = - [ - # Code quality mismathch 9 vs 10 - "test_pyroma" - - # pillow-simd - "test_roundtrip" - "test_basic" - "test_custom_metadata" - ] - ++ lib.optionals stdenv.hostPlatform.isDarwin [ - # Disable darwin tests which require executables: `iconutil` and `screencapture` - "test_grab" - "test_grabclipboard" - "test_save" - ]; - - propagatedBuildInputs = [ - olefile - ] ++ lib.optionals (lib.versionAtLeast version "8.2.0") [ defusedxml ]; - - nativeCheckInputs = [ - pytestCheckHook - numpy - ]; - - nativeBuildInputs = [ setuptools ]; - - buildInputs = - [ - freetype - libjpeg - openjpeg - libimagequant - zlib - libtiff - libwebp - libxcrypt - tcl - lcms2 - ] - ++ lib.optionals (lib.versionAtLeast version "7.1.0") [ libxcb ] - ++ lib.optionals (isPyPy) [ - tk - libX11 - ]; - - # NOTE: we use LCMS_ROOT as WEBP root since there is not other setting for webp. - # NOTE: The Pillow install script will, by default, add paths like /usr/lib - # and /usr/include to the search paths. This can break things when building - # on a non-NixOS system that has some libraries installed that are not - # installed in Nix (for example, Arch Linux has jpeg2000 but Nix doesn't - # build Pillow with this support). We patch the `disable_platform_guessing` - # setting here, instead of passing the `--disable-platform-guessing` - # command-line option, since the command-line option doesn't work when we run - # tests. - preConfigure = - let - libinclude' = pkg: ''"${pkg.out}/lib", "${pkg.out}/include"''; - libinclude = pkg: ''"${pkg.out}/lib", "${pkg.dev}/include"''; - in - '' - sed -i "setup.py" \ - -e 's|^FREETYPE_ROOT =.*$|FREETYPE_ROOT = ${libinclude freetype}|g ; - s|^JPEG_ROOT =.*$|JPEG_ROOT = ${libinclude libjpeg}|g ; - s|^JPEG2K_ROOT =.*$|JPEG2K_ROOT = ${libinclude openjpeg}|g ; - s|^IMAGEQUANT_ROOT =.*$|IMAGEQUANT_ROOT = ${libinclude' libimagequant}|g ; - s|^ZLIB_ROOT =.*$|ZLIB_ROOT = ${libinclude zlib}|g ; - s|^LCMS_ROOT =.*$|LCMS_ROOT = ${libinclude lcms2}|g ; - s|^TIFF_ROOT =.*$|TIFF_ROOT = ${libinclude libtiff}|g ; - s|^TCL_ROOT=.*$|TCL_ROOT = ${libinclude' tcl}|g ; - s|self\.disable_platform_guessing = None|self.disable_platform_guessing = True|g ;' - export LDFLAGS="$LDFLAGS -L${libwebp}/lib" - export CFLAGS="$CFLAGS -I${libwebp}/include" - '' - + lib.optionalString (lib.versionAtLeast version "7.1.0") '' - export LDFLAGS="$LDFLAGS -L${libxcb}/lib" - export CFLAGS="$CFLAGS -I${libxcb.dev}/include" - ''; -} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 492c9dba514d..7becfdc8c678 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -10071,8 +10071,18 @@ self: super: with self; { pillowfight = callPackage ../development/python-modules/pillowfight { }; pillow = callPackage ../development/python-modules/pillow { - inherit (pkgs) freetype libjpeg zlib libtiff libwebp tcl lcms2 tk; - inherit (pkgs.xorg) libX11 libxcb; + inherit (pkgs) + freetype + lcms2 + libimagequant + libjpeg + libraqm + libtiff + libwebp + openjpeg + zlib + ; + inherit (pkgs.xorg) libxcb; }; pillow-heif = callPackage ../development/python-modules/pillow-heif { }; From 2713abab68939a23b13d2add48a527167eb277da Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Thu, 24 Oct 2024 03:40:49 +0200 Subject: [PATCH 4/5] python312Packages.pillow-heif: 0.18.0 -> 0.20.0 https://github.com/bigcat88/pillow_heif/releases/tag/v0.20.0 --- pkgs/development/python-modules/pillow-heif/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/pillow-heif/default.nix b/pkgs/development/python-modules/pillow-heif/default.nix index be57d452680b..caa81ce9720b 100644 --- a/pkgs/development/python-modules/pillow-heif/default.nix +++ b/pkgs/development/python-modules/pillow-heif/default.nix @@ -28,14 +28,14 @@ buildPythonPackage rec { pname = "pillow-heif"; - version = "0.18.0"; + version = "0.20.0"; pyproject = true; src = fetchFromGitHub { owner = "bigcat88"; repo = "pillow_heif"; rev = "refs/tags/v${version}"; - hash = "sha256-+HQvDf5aovUtZ++BoD22B012N32A+7++O/jbpkIVQws="; + hash = "sha256-a1qCxI+mMuEYsCk2CUYGNKCe+SONuvVizqUvmQKy3sE="; }; postPatch = '' From 464344156d32f320beafc520288df2e1ba6cdff6 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Thu, 24 Oct 2024 05:24:07 +0200 Subject: [PATCH 5/5] python312Packages.imageio: disable failing test --- .../python-modules/imageio/default.nix | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pkgs/development/python-modules/imageio/default.nix b/pkgs/development/python-modules/imageio/default.nix index 0c489b5c30d2..8ce6d5d4f4ec 100644 --- a/pkgs/development/python-modules/imageio/default.nix +++ b/pkgs/development/python-modules/imageio/default.nix @@ -3,7 +3,6 @@ stdenv, buildPythonPackage, fetchFromGitHub, - fetchpatch, isPyPy, substituteAll, @@ -98,15 +97,21 @@ buildPythonPackage rec { "tests/test_swf.py" ]; - disabledTests = lib.optionals stdenv.hostPlatform.isDarwin [ - # Segmentation fault - "test_bayer_write" - # RuntimeError: No valid H.264 encoder was found with the ffmpeg installation - "test_writer_file_properly_closed" - "test_writer_pixelformat_size_verbose" - "test_writer_ffmpeg_params" - "test_reverse_read" - ]; + disabledTests = + [ + # Pillow 11.0.0 compat + # https://github.com/imageio/imageio/issues/1104 + "test_gif" + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + # Segmentation fault + "test_bayer_write" + # RuntimeError: No valid H.264 encoder was found with the ffmpeg installation + "test_writer_file_properly_closed" + "test_writer_pixelformat_size_verbose" + "test_writer_ffmpeg_params" + "test_reverse_read" + ]; meta = { description = "Library for reading and writing a wide range of image, video, scientific, and volumetric data formats";