From 8186ecd283bf2afab1b887ac196b4c7394fcfa60 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 16 Oct 2024 09:33:34 +0200 Subject: [PATCH 1/4] python311Packages.tensorflow-metadata: cleanup --- .../python-modules/tensorflow-metadata/build.patch | 2 +- .../python-modules/tensorflow-metadata/default.nix | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/tensorflow-metadata/build.patch b/pkgs/development/python-modules/tensorflow-metadata/build.patch index 75a774535947..14469d1f60fa 100644 --- a/pkgs/development/python-modules/tensorflow-metadata/build.patch +++ b/pkgs/development/python-modules/tensorflow-metadata/build.patch @@ -1,5 +1,5 @@ diff --git a/setup.py b/setup.py -index 7a09b2f..94c5aa6 100644 +index 30ac370..a05812d 100644 --- a/setup.py +++ b/setup.py @@ -141,8 +141,4 @@ setup( diff --git a/pkgs/development/python-modules/tensorflow-metadata/default.nix b/pkgs/development/python-modules/tensorflow-metadata/default.nix index b961578c8810..c1ccd3c3befd 100644 --- a/pkgs/development/python-modules/tensorflow-metadata/default.nix +++ b/pkgs/development/python-modules/tensorflow-metadata/default.nix @@ -45,11 +45,16 @@ buildPythonPackage rec { # has no tests doCheck = false; - pythonImportsCheck = [ "tensorflow_metadata" ]; + pythonImportsCheck = [ + "tensorflow_metadata" + "tensorflow_metadata.proto.v0" + "google.protobuf.runtime_version" + ]; meta = { description = "Standard representations for metadata that are useful when training machine learning models with TensorFlow"; homepage = "https://github.com/tensorflow/metadata"; + changelog = "https://github.com/tensorflow/metadata/releases/tag/v${version}"; license = lib.licenses.asl20; maintainers = with lib.maintainers; [ ndl ]; }; From 28177221cb1e24acdce4435aa5b45b4706c3f43d Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 12 Mar 2025 17:44:16 +0100 Subject: [PATCH 2/4] python312Packages.array-record: 0.5.0 -> 0.7.1 --- .../python-modules/array-record/default.nix | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/pkgs/development/python-modules/array-record/default.nix b/pkgs/development/python-modules/array-record/default.nix index 83385b535429..e151d2aa071e 100644 --- a/pkgs/development/python-modules/array-record/default.nix +++ b/pkgs/development/python-modules/array-record/default.nix @@ -7,18 +7,14 @@ fetchPypi, absl-py, etils, - importlib-resources, - typing-extensions, - zipp, }: buildPythonPackage rec { pname = "array-record"; - version = "0.5.0"; + version = "0.7.1"; format = "wheel"; - # As of 2023-10-31, PyPI includes wheels for Python 3.9, 3.10, and 3.11. - disabled = pythonOlder "3.9" || pythonAtLeast "3.12"; + disabled = pythonOlder "3.10" || pythonAtLeast "3.13"; src = let @@ -33,28 +29,25 @@ buildPythonPackage rec { platform = "manylinux_2_17_x86_64.manylinux2014_x86_64"; hash = { - cp39 = "sha256-BzMOVue7E1S1+5+XTcPELko81ujc9MbmqLhNsU7pqO0="; - cp310 = "sha256-eUD9pQu9GsbV8MPD1MiF3Ihr+zYioSOo6P15hYIwPYo="; - cp311 = "sha256-rAmkI3EIZPYiXrxFowfDC0Gf3kRw0uX0i6Kx6Zu+hNM="; + cp310 = "sha256-JDaj1iJy1BQ7fHjmCbGQkNqG5rIRuwTwENbanM9a8hg="; + cp311 = "sha256-QVynMK9t0BnEtgdfbJ5T3s7N02i0XD2siUSRxKtrI+M="; + cp312 = "sha256-xJJGm6kLQ2/TzVYTrBtQ1Hqky1odHfbhe/g+PSSYt1c="; } .${pyShortVersion} or (throw "${pname} is missing hash for ${pyShortVersion}"); }; - propagatedBuildInputs = [ + dependencies = [ absl-py etils - importlib-resources - typing-extensions - zipp - ]; + ] ++ etils.optional-dependencies.epath; pythonImportsCheck = [ "array_record" ]; - meta = with lib; { + meta = { description = "ArrayRecord is a new file format derived from Riegeli, achieving a new frontier of IO efficiency"; homepage = "https://github.com/google/array_record"; - license = licenses.asl20; - maintainers = with maintainers; [ GaetanLepage ]; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ GaetanLepage ]; platforms = [ "x86_64-linux" ]; }; } From 16ed352de88c86f090aaf1202f41891bc26129dc Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 5 Jun 2024 11:01:09 +0200 Subject: [PATCH 3/4] python311Packages.tensorflow-datasets: 4.9.7 -> 4.9.8 Changelog: https://github.com/tensorflow/datasets/releases/tag/v4.9.8 Diff: https://github.com/tensorflow/datasets/compare/v4.9.7...v4.9.8 --- .../tensorflow-datasets/corruptions.patch | 22 ----- .../tensorflow-datasets/default.nix | 91 ++++++++++++------- 2 files changed, 58 insertions(+), 55 deletions(-) delete mode 100644 pkgs/development/python-modules/tensorflow-datasets/corruptions.patch diff --git a/pkgs/development/python-modules/tensorflow-datasets/corruptions.patch b/pkgs/development/python-modules/tensorflow-datasets/corruptions.patch deleted file mode 100644 index 71e5da5e11bb..000000000000 --- a/pkgs/development/python-modules/tensorflow-datasets/corruptions.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/tensorflow_datasets/image_classification/corruptions.py b/tensorflow_datasets/image_classification/corruptions.py -index 066c4460..cb9a6667 100644 ---- a/tensorflow_datasets/image_classification/corruptions.py -+++ b/tensorflow_datasets/image_classification/corruptions.py -@@ -35,7 +35,7 @@ FROST_FILENAMES = [] - - - def _imagemagick_bin(): -- return 'imagemagick' # pylint: disable=unreachable -+ return 'convert' # pylint: disable=unreachable - - - # /////////////// Corruption Helpers /////////////// -@@ -675,7 +675,7 @@ def spatter(x, severity=1): - # ker = np.array([[-1,-2,-3],[-2,0,0],[-3,0,1]], dtype=np.float32) - # ker -= np.mean(ker) - ker = np.array([[-2, -1, 0], [-1, 1, 1], [0, 1, 2]]) -- dist = cv2.filter2D(dist, cv2.CVX_8U, ker) -+ dist = cv2.filter2D(dist, cv2.CV_8U, ker) - dist = cv2.blur(dist, (3, 3)).astype(np.float32) - - m = cv2.cvtColor(liquid_layer * dist, cv2.COLOR_GRAY2BGRA) diff --git a/pkgs/development/python-modules/tensorflow-datasets/default.nix b/pkgs/development/python-modules/tensorflow-datasets/default.nix index efe19803d25a..625619e9bd0a 100644 --- a/pkgs/development/python-modules/tensorflow-datasets/default.nix +++ b/pkgs/development/python-modules/tensorflow-datasets/default.nix @@ -1,81 +1,94 @@ { - apache-beam, - array-record, - attrs, - beautifulsoup4, + lib, buildPythonPackage, - click, - datasets, + fetchFromGitHub, + fetchpatch, + + # dependencies + array-record, dill, dm-tree, - fetchFromGitHub, - ffmpeg, future, - imagemagick, + immutabledict, importlib-resources, + numpy, + promise, + protobuf, + psutil, + requests, + simple-parsing, + six, + tensorflow-metadata, + termcolor, + tqdm, + + # tests + apache-beam, + beautifulsoup4, + click, + datasets, + ffmpeg, + imagemagick, jax, jaxlib, jinja2, langdetect, - lib, lxml, matplotlib, + mlcroissant, mwparserfromhell, mwxml, networkx, nltk, - numpy, opencv4, pandas, pillow, - promise, - protobuf, - psutil, pycocotools, pydub, pytest-xdist, pytestCheckHook, - requests, scikit-image, scipy, - six, + sortedcontainers, tensorflow, - tensorflow-metadata, - termcolor, tifffile, - tqdm, zarr, }: buildPythonPackage rec { pname = "tensorflow-datasets"; - version = "4.9.7"; - format = "setuptools"; + version = "4.9.8"; + pyproject = true; src = fetchFromGitHub { owner = "tensorflow"; repo = "datasets"; tag = "v${version}"; - hash = "sha256-pnut5z3rEYIOOhsQT6uVjPdy+SqftKptSQMwxbMXoKA="; + hash = "sha256-nqveZ+8b0f5sGIn6WufKeA37yEsZjzhCIbCfwMZ9JOM="; }; patches = [ - # addresses https://github.com/tensorflow/datasets/issues/3673 - ./corruptions.patch + # mlmlcroissant uses encoding_formats, not encoding_formats. + # Backport https://github.com/tensorflow/datasets/pull/11037 until released. + (fetchpatch { + url = "https://github.com/tensorflow/datasets/commit/92cbcff725a1036569a515cc3356aa8480740451.patch"; + hash = "sha256-2hnMvQP83+eAJllce19aHujcoWQzUz3+LsasWCo4BtM="; + }) ]; - propagatedBuildInputs = [ + dependencies = [ array-record - attrs dill dm-tree future + immutabledict importlib-resources numpy promise protobuf psutil requests + simple-parsing six tensorflow-metadata termcolor @@ -97,6 +110,7 @@ buildPythonPackage rec { langdetect lxml matplotlib + mlcroissant mwparserfromhell mwxml networkx @@ -110,11 +124,18 @@ buildPythonPackage rec { pytestCheckHook scikit-image scipy + sortedcontainers tensorflow tifffile zarr ]; + pytestFlagsArray = [ + # AttributeError: 'NoneType' object has no attribute 'Table' + "--deselect=tensorflow_datasets/core/file_adapters_test.py::test_read_write" + "--deselect=tensorflow_datasets/text/c4_wsrs/c4_wsrs_test.py::C4WSRSTest" + ]; + disabledTestPaths = [ # Sandbox violations: network access, filesystem write attempts outside of build dir, ... "tensorflow_datasets/core/dataset_builder_test.py" @@ -126,6 +147,7 @@ buildPythonPackage rec { "tensorflow_datasets/import_without_tf_test.py" "tensorflow_datasets/proto/build_tf_proto_test.py" "tensorflow_datasets/scripts/cli/build_test.py" + "tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder_test.py" # Requires `pretty_midi` which is not packaged in `nixpkgs`. "tensorflow_datasets/audio/groove.py" @@ -143,14 +165,16 @@ buildPythonPackage rec { # Requires `gcld3` and `pretty_midi` which are not packaged in `nixpkgs`. "tensorflow_datasets/core/lazy_imports_lib_test.py" + # AttributeError: 'NoneType' object has no attribute 'Table' + "tensorflow_datasets/core/dataset_builder_beam_test.py" + "tensorflow_datasets/core/dataset_builders/adhoc_builder_test.py" + "tensorflow_datasets/core/split_builder_test.py" + "tensorflow_datasets/core/writer_test.py" + # Requires `tensorflow_io` which is not packaged in `nixpkgs`. "tensorflow_datasets/core/features/audio_feature_test.py" "tensorflow_datasets/image/lsun_test.py" - # Requires `envlogger` which is not packaged in `nixpkgs`. - "tensorflow_datasets/rlds/locomotion/locomotion_test.py" - "tensorflow_datasets/rlds/robosuite_panda_pick_place_can/robosuite_panda_pick_place_can_test.py" - # Fails with `TypeError: Constant constructor takes either 0 or 2 positional arguments` # deep in TF AutoGraph. Doesn't reproduce in Docker with Ubuntu 22.04 => might be related # to the differences in some of the dependencies? @@ -171,10 +195,11 @@ buildPythonPackage rec { "tensorflow_datasets/text/c4_utils_test.py" ]; - meta = with lib; { + meta = { description = "Library of datasets ready to use with TensorFlow"; homepage = "https://www.tensorflow.org/datasets/overview"; - license = licenses.asl20; - maintainers = with maintainers; [ ndl ]; + changelog = "https://github.com/tensorflow/datasets/releases/tag/v${version}"; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ ndl ]; }; } From e9ce1c21e7fa055d8aa2e096a11bb9b993b81743 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 23 Apr 2025 19:22:34 +0200 Subject: [PATCH 4/4] python312Packages.dm-sonnet: cleanup, fix build --- .../python-modules/dm-sonnet/default.nix | 65 +++++++++++++------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/pkgs/development/python-modules/dm-sonnet/default.nix b/pkgs/development/python-modules/dm-sonnet/default.nix index 06b01a447843..62cae899b80e 100644 --- a/pkgs/development/python-modules/dm-sonnet/default.nix +++ b/pkgs/development/python-modules/dm-sonnet/default.nix @@ -1,43 +1,45 @@ { lib, buildPythonPackage, - click, - dm-tree, - docutils, - etils, fetchFromGitHub, - fetchpatch, + + # build-system + setuptools, + + # dependencies + dm-tree, + etils, numpy, - pythonOlder, tabulate, + wrapt, + + # tests + click, + docutils, + keras, + pytestCheckHook, tensorflow, tensorflow-datasets, - wrapt, + tf-keras, }: buildPythonPackage rec { pname = "dm-sonnet"; version = "2.0.2"; - format = "setuptools"; - - disabled = pythonOlder "3.6"; + pyproject = true; src = fetchFromGitHub { owner = "deepmind"; repo = "sonnet"; - rev = "refs/tags/v${version}"; + tag = "v${version}"; hash = "sha256-WkloUbqSyPG3cbLG8ktsjdluACkCbUZ7t6rYWst8rs8="; }; - patches = [ - (fetchpatch { - name = "replace-np-bool-with-np-bool_.patch"; - url = "https://github.com/deepmind/sonnet/commit/df5d099d4557a9a81a0eb969e5a81ed917bcd612.patch"; - hash = "sha256-s7abl83osD4wa0ZhqgDyjqQ3gagwGYCdQifwFqhNp34="; - }) + build-system = [ + setuptools ]; - propagatedBuildInputs = [ + dependencies = [ dm-tree etils numpy @@ -52,16 +54,37 @@ buildPythonPackage rec { nativeCheckInputs = [ click docutils + keras + pytestCheckHook tensorflow tensorflow-datasets + tf-keras + ]; + + # ImportError: `keras.optimizers.legacy` is not supported in Keras 3 + preCheck = '' + export TF_USE_LEGACY_KERAS=True + ''; + + disabledTests = [ + # AssertionError: 2 != 0 : 2 doctests failed + "test_doctest_sonnet.functional" + + # AssertionError: Not equal to tolerance + "testComputationAgainstNumPy1" + + # tensorflow.python.framework.errors_impl.InvalidArgumentError: cannot compute MatMul as input #1(zero-based) was expected to be a float tensor but is a half tensor [Op:MatMul] + "testComputationAgainstNumPy0" + "testComputationAgainstNumPy1" ]; pythonImportsCheck = [ "sonnet" ]; - meta = with lib; { + meta = { description = "Library for building neural networks in TensorFlow"; homepage = "https://github.com/deepmind/sonnet"; - license = licenses.asl20; - maintainers = with maintainers; [ onny ]; + changelog = "https://github.com/google-deepmind/sonnet/releases/tag/v${version}"; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ onny ]; }; }