python311Packages.tensorflow-datasets: 4.9.7 -> 4.9.8 (#376040)

2025-04-23 18:57:26 +00:00
parent 546a7aa631 e9ce1c21e7
commit 6c679116a4
6 changed files with 119 additions and 95 deletions
@@ -7,18 +7,14 @@
  fetchPypi,
  absl-py,
  etils,
-  importlib-resources,
-  typing-extensions,
-  zipp,
 }:

 buildPythonPackage rec {
  pname = "array-record";
-  version = "0.5.0";
+  version = "0.7.1";
  format = "wheel";

-  # As of 2023-10-31, PyPI includes wheels for Python 3.9, 3.10, and 3.11.
-  disabled = pythonOlder "3.9" || pythonAtLeast "3.12";
+  disabled = pythonOlder "3.10" || pythonAtLeast "3.13";

  src =
    let
@@ -33,28 +29,25 @@ buildPythonPackage rec {
      platform = "manylinux_2_17_x86_64.manylinux2014_x86_64";
      hash =
        {
-          cp39 = "sha256-BzMOVue7E1S1+5+XTcPELko81ujc9MbmqLhNsU7pqO0=";
-          cp310 = "sha256-eUD9pQu9GsbV8MPD1MiF3Ihr+zYioSOo6P15hYIwPYo=";
-          cp311 = "sha256-rAmkI3EIZPYiXrxFowfDC0Gf3kRw0uX0i6Kx6Zu+hNM=";
+          cp310 = "sha256-JDaj1iJy1BQ7fHjmCbGQkNqG5rIRuwTwENbanM9a8hg=";
+          cp311 = "sha256-QVynMK9t0BnEtgdfbJ5T3s7N02i0XD2siUSRxKtrI+M=";
+          cp312 = "sha256-xJJGm6kLQ2/TzVYTrBtQ1Hqky1odHfbhe/g+PSSYt1c=";
        }
        .${pyShortVersion} or (throw "${pname} is missing hash for ${pyShortVersion}");
    };

-  propagatedBuildInputs = [
+  dependencies = [
    absl-py
    etils
-    importlib-resources
-    typing-extensions
-    zipp
-  ];
+  ] ++ etils.optional-dependencies.epath;

  pythonImportsCheck = [ "array_record" ];

-  meta = with lib; {
+  meta = {
    description = "ArrayRecord is a new file format derived from Riegeli, achieving a new frontier of IO efficiency";
    homepage = "https://github.com/google/array_record";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ GaetanLepage ];
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ GaetanLepage ];
    platforms = [ "x86_64-linux" ];
  };
 }
@@ -1,43 +1,45 @@
 {
  lib,
  buildPythonPackage,
-  click,
-  dm-tree,
-  docutils,
-  etils,
  fetchFromGitHub,
-  fetchpatch,
+
+  # build-system
+  setuptools,
+
+  # dependencies
+  dm-tree,
+  etils,
  numpy,
-  pythonOlder,
  tabulate,
+  wrapt,
+
+  # tests
+  click,
+  docutils,
+  keras,
+  pytestCheckHook,
  tensorflow,
  tensorflow-datasets,
-  wrapt,
+  tf-keras,
 }:

 buildPythonPackage rec {
  pname = "dm-sonnet";
  version = "2.0.2";
-  format = "setuptools";
-
-  disabled = pythonOlder "3.6";
+  pyproject = true;

  src = fetchFromGitHub {
    owner = "deepmind";
    repo = "sonnet";
-    rev = "refs/tags/v${version}";
+    tag = "v${version}";
    hash = "sha256-WkloUbqSyPG3cbLG8ktsjdluACkCbUZ7t6rYWst8rs8=";
  };

-  patches = [
-    (fetchpatch {
-      name = "replace-np-bool-with-np-bool_.patch";
-      url = "https://github.com/deepmind/sonnet/commit/df5d099d4557a9a81a0eb969e5a81ed917bcd612.patch";
-      hash = "sha256-s7abl83osD4wa0ZhqgDyjqQ3gagwGYCdQifwFqhNp34=";
-    })
+  build-system = [
+    setuptools
  ];

-  propagatedBuildInputs = [
+  dependencies = [
    dm-tree
    etils
    numpy
@@ -52,16 +54,37 @@ buildPythonPackage rec {
  nativeCheckInputs = [
    click
    docutils
+    keras
+    pytestCheckHook
    tensorflow
    tensorflow-datasets
+    tf-keras
+  ];
+
+  # ImportError: `keras.optimizers.legacy` is not supported in Keras 3
+  preCheck = ''
+    export TF_USE_LEGACY_KERAS=True
+  '';
+
+  disabledTests = [
+    # AssertionError: 2 != 0 : 2 doctests failed
+    "test_doctest_sonnet.functional"
+
+    # AssertionError: Not equal to tolerance
+    "testComputationAgainstNumPy1"
+
+    # tensorflow.python.framework.errors_impl.InvalidArgumentError: cannot compute MatMul as input #1(zero-based) was expected to be a float tensor but is a half tensor [Op:MatMul]
+    "testComputationAgainstNumPy0"
+    "testComputationAgainstNumPy1"
  ];

  pythonImportsCheck = [ "sonnet" ];

-  meta = with lib; {
+  meta = {
    description = "Library for building neural networks in TensorFlow";
    homepage = "https://github.com/deepmind/sonnet";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ onny ];
+    changelog = "https://github.com/google-deepmind/sonnet/releases/tag/v${version}";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ onny ];
  };
 }
@@ -1,22 +0,0 @@
-diff --git a/tensorflow_datasets/image_classification/corruptions.py b/tensorflow_datasets/image_classification/corruptions.py
-index 066c4460..cb9a6667 100644
--- a/tensorflow_datasets/image_classification/corruptions.py
-+++ b/tensorflow_datasets/image_classification/corruptions.py
-@@ -35,7 +35,7 @@ FROST_FILENAMES = []
- 
- 
- def _imagemagick_bin():
-  return 'imagemagick'  # pylint: disable=unreachable
-+  return 'convert'  # pylint: disable=unreachable
- 
- 
- # /////////////// Corruption Helpers ///////////////
-@@ -675,7 +675,7 @@ def spatter(x, severity=1):
-     #     ker = np.array([[-1,-2,-3],[-2,0,0],[-3,0,1]], dtype=np.float32)
-     #     ker -= np.mean(ker)
-     ker = np.array([[-2, -1, 0], [-1, 1, 1], [0, 1, 2]])
-    dist = cv2.filter2D(dist, cv2.CVX_8U, ker)
-+    dist = cv2.filter2D(dist, cv2.CV_8U, ker)
-     dist = cv2.blur(dist, (3, 3)).astype(np.float32)
- 
-     m = cv2.cvtColor(liquid_layer * dist, cv2.COLOR_GRAY2BGRA)
@@ -1,81 +1,94 @@
 {
-  apache-beam,
-  array-record,
-  attrs,
-  beautifulsoup4,
+  lib,
  buildPythonPackage,
-  click,
-  datasets,
+  fetchFromGitHub,
+  fetchpatch,
+
+  # dependencies
+  array-record,
  dill,
  dm-tree,
-  fetchFromGitHub,
-  ffmpeg,
  future,
-  imagemagick,
+  immutabledict,
  importlib-resources,
+  numpy,
+  promise,
+  protobuf,
+  psutil,
+  requests,
+  simple-parsing,
+  six,
+  tensorflow-metadata,
+  termcolor,
+  tqdm,
+
+  # tests
+  apache-beam,
+  beautifulsoup4,
+  click,
+  datasets,
+  ffmpeg,
+  imagemagick,
  jax,
  jaxlib,
  jinja2,
  langdetect,
-  lib,
  lxml,
  matplotlib,
+  mlcroissant,
  mwparserfromhell,
  mwxml,
  networkx,
  nltk,
-  numpy,
  opencv4,
  pandas,
  pillow,
-  promise,
-  protobuf,
-  psutil,
  pycocotools,
  pydub,
  pytest-xdist,
  pytestCheckHook,
-  requests,
  scikit-image,
  scipy,
-  six,
+  sortedcontainers,
  tensorflow,
-  tensorflow-metadata,
-  termcolor,
  tifffile,
-  tqdm,
  zarr,
 }:

 buildPythonPackage rec {
  pname = "tensorflow-datasets";
-  version = "4.9.7";
-  format = "setuptools";
+  version = "4.9.8";
+  pyproject = true;

  src = fetchFromGitHub {
    owner = "tensorflow";
    repo = "datasets";
    tag = "v${version}";
-    hash = "sha256-pnut5z3rEYIOOhsQT6uVjPdy+SqftKptSQMwxbMXoKA=";
+    hash = "sha256-nqveZ+8b0f5sGIn6WufKeA37yEsZjzhCIbCfwMZ9JOM=";
  };

  patches = [
-    # addresses https://github.com/tensorflow/datasets/issues/3673
-    ./corruptions.patch
+    # mlmlcroissant uses encoding_formats, not encoding_formats.
+    # Backport https://github.com/tensorflow/datasets/pull/11037 until released.
+    (fetchpatch {
+      url = "https://github.com/tensorflow/datasets/commit/92cbcff725a1036569a515cc3356aa8480740451.patch";
+      hash = "sha256-2hnMvQP83+eAJllce19aHujcoWQzUz3+LsasWCo4BtM=";
+    })
  ];

-  propagatedBuildInputs = [
+  dependencies = [
    array-record
-    attrs
    dill
    dm-tree
    future
+    immutabledict
    importlib-resources
    numpy
    promise
    protobuf
    psutil
    requests
+    simple-parsing
    six
    tensorflow-metadata
    termcolor
@@ -97,6 +110,7 @@ buildPythonPackage rec {
    langdetect
    lxml
    matplotlib
+    mlcroissant
    mwparserfromhell
    mwxml
    networkx
@@ -110,11 +124,18 @@ buildPythonPackage rec {
    pytestCheckHook
    scikit-image
    scipy
+    sortedcontainers
    tensorflow
    tifffile
    zarr
  ];

+  pytestFlagsArray = [
+    # AttributeError: 'NoneType' object has no attribute 'Table'
+    "--deselect=tensorflow_datasets/core/file_adapters_test.py::test_read_write"
+    "--deselect=tensorflow_datasets/text/c4_wsrs/c4_wsrs_test.py::C4WSRSTest"
+  ];
+
  disabledTestPaths = [
    # Sandbox violations: network access, filesystem write attempts outside of build dir, ...
    "tensorflow_datasets/core/dataset_builder_test.py"
@@ -126,6 +147,7 @@ buildPythonPackage rec {
    "tensorflow_datasets/import_without_tf_test.py"
    "tensorflow_datasets/proto/build_tf_proto_test.py"
    "tensorflow_datasets/scripts/cli/build_test.py"
+    "tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder_test.py"

    # Requires `pretty_midi` which is not packaged in `nixpkgs`.
    "tensorflow_datasets/audio/groove.py"
@@ -143,14 +165,16 @@ buildPythonPackage rec {
    # Requires `gcld3` and `pretty_midi` which are not packaged in `nixpkgs`.
    "tensorflow_datasets/core/lazy_imports_lib_test.py"

+    # AttributeError: 'NoneType' object has no attribute 'Table'
+    "tensorflow_datasets/core/dataset_builder_beam_test.py"
+    "tensorflow_datasets/core/dataset_builders/adhoc_builder_test.py"
+    "tensorflow_datasets/core/split_builder_test.py"
+    "tensorflow_datasets/core/writer_test.py"
+
    # Requires `tensorflow_io` which is not packaged in `nixpkgs`.
    "tensorflow_datasets/core/features/audio_feature_test.py"
    "tensorflow_datasets/image/lsun_test.py"

-    # Requires `envlogger` which is not packaged in `nixpkgs`.
-    "tensorflow_datasets/rlds/locomotion/locomotion_test.py"
-    "tensorflow_datasets/rlds/robosuite_panda_pick_place_can/robosuite_panda_pick_place_can_test.py"
-
    # Fails with `TypeError: Constant constructor takes either 0 or 2 positional arguments`
    # deep in TF AutoGraph. Doesn't reproduce in Docker with Ubuntu 22.04 => might be related
    # to the differences in some of the dependencies?
@@ -171,10 +195,11 @@ buildPythonPackage rec {
    "tensorflow_datasets/text/c4_utils_test.py"
  ];

-  meta = with lib; {
+  meta = {
    description = "Library of datasets ready to use with TensorFlow";
    homepage = "https://www.tensorflow.org/datasets/overview";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ ndl ];
+    changelog = "https://github.com/tensorflow/datasets/releases/tag/v${version}";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ ndl ];
  };
 }
@@ -1,5 +1,5 @@
 diff --git a/setup.py b/setup.py
-index 7a09b2f..94c5aa6 100644
+index 30ac370..a05812d 100644
 --- a/setup.py
 +++ b/setup.py
@@ -141,8 +141,4 @@ setup(
@@ -45,11 +45,16 @@ buildPythonPackage rec {
  # has no tests
  doCheck = false;

-  pythonImportsCheck = [ "tensorflow_metadata" ];
+  pythonImportsCheck = [
+    "tensorflow_metadata"
+    "tensorflow_metadata.proto.v0"
+    "google.protobuf.runtime_version"
+  ];

  meta = {
    description = "Standard representations for metadata that are useful when training machine learning models with TensorFlow";
    homepage = "https://github.com/tensorflow/metadata";
+    changelog = "https://github.com/tensorflow/metadata/releases/tag/v${version}";
    license = lib.licenses.asl20;
    maintainers = with lib.maintainers; [ ndl ];
  };