From 1cb9c4550dc2dee415980f21e276578948071ad0 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 11:45:28 +0100 Subject: [PATCH 01/19] python312Packages.dask-expr: remove (now included in dask) --- .../python-modules/dask-expr/default.nix | 65 ------------------- pkgs/top-level/python-aliases.nix | 1 + pkgs/top-level/python-packages.nix | 2 - 3 files changed, 1 insertion(+), 67 deletions(-) delete mode 100644 pkgs/development/python-modules/dask-expr/default.nix diff --git a/pkgs/development/python-modules/dask-expr/default.nix b/pkgs/development/python-modules/dask-expr/default.nix deleted file mode 100644 index c7bb6c734ee9..000000000000 --- a/pkgs/development/python-modules/dask-expr/default.nix +++ /dev/null @@ -1,65 +0,0 @@ -{ - lib, - buildPythonPackage, - fetchFromGitHub, - - # build-system - setuptools, - versioneer, - - # dependencies - dask, - pandas, - pyarrow, - - # checks - jinja2, - pytestCheckHook, - xarray, -}: - -buildPythonPackage rec { - pname = "dask-expr"; - version = "1.1.21"; - pyproject = true; - - src = fetchFromGitHub { - owner = "dask"; - repo = "dask-expr"; - tag = "v${version}"; - hash = "sha256-t1vPlTxV5JYArg/a7CzPP13NHbstEoCgHRmd8Y9mDfA="; - }; - - postPatch = '' - substituteInPlace pyproject.toml \ - --replace-fail "versioneer[toml]==0.28" "versioneer[toml]" - ''; - - build-system = [ - setuptools - versioneer - ]; - - dependencies = [ - dask - pandas - pyarrow - ]; - - pythonImportsCheck = [ "dask_expr" ]; - - nativeCheckInputs = [ - jinja2 - pytestCheckHook - xarray - ]; - - __darwinAllowLocalNetworking = true; - - meta = { - description = "Rewrite of Dask DataFrame that includes query optimization and generally improved organization"; - homepage = "https://github.com/dask/dask-expr"; - license = lib.licenses.bsd3; - maintainers = with lib.maintainers; [ GaetanLepage ]; - }; -} diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index 434e1b6f3cd9..c87843dcb618 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -142,6 +142,7 @@ mapAliases ({ cx_Freeze = cx-freeze; # added 2023-08-02 cx_oracle = cx-oracle; # added 2024-01-03 d2to1 = throw "d2to1 is archived and no longer works with setuptools v68"; # added 2023-07-30 + dask-expr = throw "dask-expr was removed because its features are available in dask"; # added 2025-02-07 dask-xgboost = throw "dask-xgboost was removed because its features are available in xgboost"; # added 2022-05-24 dash-renderer = throw "dash-renderer has been removed since it is abandoned"; # added 2025-01-01 dateutil = python-dateutil; # added 2021-07-03 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 84f44003ad7a..5a0133d8743d 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2961,8 +2961,6 @@ self: super: with self; { dask-awkward = callPackage ../development/python-modules/dask-awkward { }; - dask-expr = callPackage ../development/python-modules/dask-expr { }; - dask-gateway = callPackage ../development/python-modules/dask-gateway { }; dask-gateway-server = callPackage ../development/python-modules/dask-gateway-server { }; From 24c5862bbec741e99e5b288c7de2177e4f5ae74b Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 11:46:55 +0100 Subject: [PATCH 02/19] python312Packages.dask: 2024.12.1 -> 2025.1.0 Diff: https://github.com/dask/dask/compare/refs/tags/2024.12.1...2025.1.0 Changelog: https://docs.dask.org/en/latest/changelog.html --- .../python-modules/dask/default.nix | 272 ++++++++---------- 1 file changed, 115 insertions(+), 157 deletions(-) diff --git a/pkgs/development/python-modules/dask/default.nix b/pkgs/development/python-modules/dask/default.nix index 93b38b11b1f7..b37ddce56526 100644 --- a/pkgs/development/python-modules/dask/default.nix +++ b/pkgs/development/python-modules/dask/default.nix @@ -1,6 +1,5 @@ { lib, - stdenv, buildPythonPackage, fetchFromGitHub, @@ -27,168 +26,127 @@ jinja2, # tests - arrow-cpp, - dask-expr, hypothesis, pytest-asyncio, + pytest-mock, pytest-rerunfailures, pytest-xdist, pytestCheckHook, + versionCheckHook, }: -let - self = buildPythonPackage rec { - pname = "dask"; - version = "2024.12.1"; - pyproject = true; +buildPythonPackage rec { + pname = "dask"; + version = "2025.1.0"; + pyproject = true; - src = fetchFromGitHub { - owner = "dask"; - repo = "dask"; - tag = version; - hash = "sha256-QqvdldAHW2UYt1NXfk3Aa+oe97e+OpRbF8d6eKV3OJ4="; - }; - - build-system = [ setuptools ]; - - dependencies = [ - click - cloudpickle - fsspec - packaging - partd - pyyaml - importlib-metadata - toolz - ]; - - optional-dependencies = lib.fix (self: { - array = [ numpy ]; - complete = - [ - pyarrow - lz4 - ] - ++ self.array - ++ self.dataframe - ++ self.distributed - ++ self.diagnostics; - dataframe = [ - # dask-expr -> circular dependency with dask-expr - numpy - pandas - ]; - distributed = [ distributed ]; - diagnostics = [ - bokeh - jinja2 - ]; - }); - - nativeCheckInputs = - [ - dask-expr - pytestCheckHook - pytest-rerunfailures - pytest-xdist - # from panda[test] - hypothesis - pytest-asyncio - ] - ++ self.optional-dependencies.array - ++ self.optional-dependencies.dataframe - ++ lib.optionals (!arrow-cpp.meta.broken) [ - # support is sparse on aarch64 - pyarrow - ]; - - dontUseSetuptoolsCheck = true; - - postPatch = '' - # versioneer hack to set version of GitHub package - echo "def get_versions(): return {'dirty': False, 'error': None, 'full-revisionid': None, 'version': '${version}'}" > dask/_version.py - - substituteInPlace setup.py \ - --replace-fail "import versioneer" "" \ - --replace-fail "version=versioneer.get_version()," "version='${version}'," \ - --replace-fail "cmdclass=versioneer.get_cmdclass()," "" - - substituteInPlace pyproject.toml \ - --replace-fail ', "versioneer[toml]==0.29"' "" \ - --replace-fail " --durations=10" "" \ - --replace-fail " --cov-config=pyproject.toml" "" \ - --replace-fail "\"-v" "\" " - ''; - - pytestFlagsArray = [ - # Rerun failed tests up to three times - "--reruns 3" - # Don't run tests that require network access - "-m 'not network'" - ]; - - disabledTests = - lib.optionals stdenv.hostPlatform.isDarwin [ - # Test requires features of python3Packages.psutil that are - # blocked in sandboxed-builds - "test_auto_blocksize_csv" - # AttributeError: 'str' object has no attribute 'decode' - "test_read_dir_nometa" - ] - ++ lib.optionals (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) [ - # concurrent.futures.process.BrokenProcessPool: A process in the process pool terminated abpruptly... - "test_foldby_tree_reduction" - "test_to_bag" - ] - ++ [ - # https://github.com/dask/dask/issues/10347#issuecomment-1589683941 - "test_concat_categorical" - # AttributeError: 'ArrowStringArray' object has no attribute 'tobytes'. Did you mean: 'nbytes'? - "test_dot" - "test_dot_nan" - "test_merge_column_with_nulls" - # FileNotFoundError: [Errno 2] No such file or directory: '/build/tmp301jryv_/createme/0.part' - "test_to_csv_nodir" - "test_to_json_results" - # FutureWarning: Those tests should be working fine when pandas will have been upgraded to 2.1.1 - "test_apply" - "test_apply_infer_columns" - ]; - - __darwinAllowLocalNetworking = true; - - pythonImportsCheck = [ - "dask" - "dask.bag" - "dask.bytes" - "dask.diagnostics" - ]; - - doCheck = false; - - # Enable tests via passthru to avoid cyclic dependency with dask-expr. - passthru.tests = { - check = self.overridePythonAttrs (old: { - doCheck = true; - pythonImportsCheck = [ - # Requires the `dask.optional-dependencies.array` that are only in `nativeCheckInputs` - "dask.array" - # Requires the `dask.optional-dependencies.dataframe` that are only in `nativeCheckInputs` - "dask.dataframe" - "dask.dataframe.io" - "dask.dataframe.tseries" - ] ++ old.pythonImportsCheck; - }); - }; - - meta = { - description = "Minimal task scheduling abstraction"; - mainProgram = "dask"; - homepage = "https://dask.org/"; - changelog = "https://docs.dask.org/en/latest/changelog.html"; - license = lib.licenses.bsd3; - maintainers = with lib.maintainers; [ GaetanLepage ]; - }; + src = fetchFromGitHub { + owner = "dask"; + repo = "dask"; + tag = version; + hash = "sha256-KBqOyf471mNg3L9dYmR7IRSltEtC+VgC+6ptsoKgVmM="; }; -in -self + + postPatch = '' + # versioneer hack to set version of GitHub package + echo "def get_versions(): return {'dirty': False, 'error': None, 'full-revisionid': None, 'version': '${version}'}" > dask/_version.py + + substituteInPlace setup.py \ + --replace-fail "import versioneer" "" \ + --replace-fail "version=versioneer.get_version()," "version='${version}'," \ + --replace-fail "cmdclass=versioneer.get_cmdclass()," "" + + substituteInPlace pyproject.toml \ + --replace-fail ', "versioneer[toml]==0.29"' "" \ + --replace-fail " --durations=10" "" \ + --replace-fail " --cov-config=pyproject.toml" "" \ + --replace-fail "\"-v" "\" " + ''; + + build-system = [ setuptools ]; + + dependencies = [ + click + cloudpickle + fsspec + packaging + partd + pyyaml + importlib-metadata + toolz + ]; + + optional-dependencies = lib.fix (self: { + array = [ numpy ]; + complete = + [ + pyarrow + lz4 + ] + ++ self.array + ++ self.dataframe + ++ self.distributed + ++ self.diagnostics; + dataframe = [ + pandas + pyarrow + ] ++ self.array; + distributed = [ distributed ]; + diagnostics = [ + bokeh + jinja2 + ]; + }); + + nativeCheckInputs = + [ + hypothesis + pytest-asyncio + pytest-mock + pytest-rerunfailures + pytest-xdist + pytestCheckHook + versionCheckHook + pyarrow + ] + ++ optional-dependencies.array + ++ optional-dependencies.dataframe; + versionCheckProgramArg = [ "--version" ]; + + pytestFlagsArray = [ + # Rerun failed tests up to three times + "--reruns 3" + # Don't run tests that require network access + "-m 'not network'" + ]; + + disabledTests = [ + # UserWarning: Insufficient elements for `head`. 10 elements requested, only 5 elements available. Try passing larger `npartitions` to `head`. + "test_set_index_head_nlargest_string" + ]; + + __darwinAllowLocalNetworking = true; + + pythonImportsCheck = [ + "dask" + "dask.bag" + "dask.bytes" + "dask.diagnostics" + + # Requires the `dask.optional-dependencies.array` that are only in `nativeCheckInputs` + "dask.array" + # Requires the `dask.optional-dependencies.dataframe` that are only in `nativeCheckInputs` + "dask.dataframe" + "dask.dataframe.io" + "dask.dataframe.tseries" + ]; + + meta = { + description = "Minimal task scheduling abstraction"; + mainProgram = "dask"; + homepage = "https://dask.org/"; + changelog = "https://docs.dask.org/en/latest/changelog.html"; + license = lib.licenses.bsd3; + maintainers = with lib.maintainers; [ GaetanLepage ]; + }; +} From 43ef67841e9ccc78e7b9d9684b741f9b9c4cab77 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 12:41:10 +0100 Subject: [PATCH 03/19] python312Packages.streamz: cleanup --- .../python-modules/streamz/default.nix | 58 ++++++++----------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/pkgs/development/python-modules/streamz/default.nix b/pkgs/development/python-modules/streamz/default.nix index eea52b8fa8f2..4a4e8855d22a 100644 --- a/pkgs/development/python-modules/streamz/default.nix +++ b/pkgs/development/python-modules/streamz/default.nix @@ -1,23 +1,24 @@ { lib, buildPythonPackage, - pythonOlder, - fetchPypi, + fetchFromGitHub, + + # build-system setuptools, - confluent-kafka, - dask, - dask-expr, - distributed, - flaky, - graphviz, - networkx, - pytest-asyncio, - pytestCheckHook, - requests, + + # dependencies six, toolz, tornado, zict, + + # tests + dask, + distributed, + flaky, + pandas, + pyarrow, + pytestCheckHook, }: buildPythonPackage rec { @@ -25,17 +26,16 @@ buildPythonPackage rec { version = "0.6.4"; pyproject = true; - disabled = pythonOlder "3.6"; - - src = fetchPypi { - inherit pname version; - hash = "sha256-VXfWkEwuxInBQVQJV3IQXgGVRkiBmYfUZCBMbjyWNPM="; + src = fetchFromGitHub { + owner = "python-streamz"; + repo = "streamz"; + tag = version; + hash = "sha256-lSb3gl+TSIzz4BZzxH8zXu74HvzSntOAoVQUUJKIEvA="; }; build-system = [ setuptools ]; dependencies = [ - networkx six toolz tornado @@ -43,15 +43,12 @@ buildPythonPackage rec { ]; nativeCheckInputs = [ - confluent-kafka dask - dask-expr distributed flaky - graphviz - pytest-asyncio + pandas + pyarrow pytestCheckHook - requests ]; pythonImportsCheck = [ "streamz" ]; @@ -63,24 +60,17 @@ buildPythonPackage rec { "test_partition_then_scatter_sync" "test_sync" "test_sync_2" - # Test fail in the sandbox - "test_tcp_async" - "test_tcp" - "test_partition_timeout" + # Tests are flaky - "test_from_iterable" "test_buffer" ]; - disabledTestPaths = [ - # Disable kafka tests - "streamz/tests/test_kafka.py" - ]; + __darwinAllowLocalNetworking = true; - meta = with lib; { + meta = { description = "Pipelines to manage continuous streams of data"; homepage = "https://github.com/python-streamz/streamz"; - license = licenses.bsd3; + license = lib.licenses.bsd3; maintainers = [ ]; }; } From 57778e074c29671facff76729fad59aa3b3af92e Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 12:41:38 +0100 Subject: [PATCH 04/19] python312Packages.streamz: add GaetanLepage as maintainer --- pkgs/development/python-modules/streamz/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/streamz/default.nix b/pkgs/development/python-modules/streamz/default.nix index 4a4e8855d22a..5aab96e7c462 100644 --- a/pkgs/development/python-modules/streamz/default.nix +++ b/pkgs/development/python-modules/streamz/default.nix @@ -71,6 +71,6 @@ buildPythonPackage rec { description = "Pipelines to manage continuous streams of data"; homepage = "https://github.com/python-streamz/streamz"; license = lib.licenses.bsd3; - maintainers = [ ]; + maintainers = with lib.maintainers; [ GaetanLepage ]; }; } From b6cc264c8a30a2d9171732946301d58a7674ac13 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 14:29:01 +0100 Subject: [PATCH 05/19] python312Packages.narwhals: 1.22.0 -> 1.25.2 Diff: https://github.com/narwhals-dev/narwhals/compare/refs/tags/v1.22.0...v1.25.2 Changelog: https://github.com/narwhals-dev/narwhals/releases/tag/v1.25.2 --- .../python-modules/narwhals/default.nix | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/pkgs/development/python-modules/narwhals/default.nix b/pkgs/development/python-modules/narwhals/default.nix index 5aa36f62364c..f96998a32908 100644 --- a/pkgs/development/python-modules/narwhals/default.nix +++ b/pkgs/development/python-modules/narwhals/default.nix @@ -1,34 +1,36 @@ { lib, buildPythonPackage, + fetchFromGitHub, + + # build-system + hatchling, + + # optional-dependencies # cudf, dask, - dask-expr, - duckdb, - fetchFromGitHub, - hatchling, - hypothesis, # modin, pandas, polars, pyarrow, + + # tests + duckdb, + hypothesis, pytest-env, pytestCheckHook, - pythonOlder, }: buildPythonPackage rec { pname = "narwhals"; - version = "1.22.0"; + version = "1.25.2"; pyproject = true; - disabled = pythonOlder "3.8"; - src = fetchFromGitHub { owner = "narwhals-dev"; repo = "narwhals"; tag = "v${version}"; - hash = "sha256-I6nJJiiW1v04YH70OPxXKeO80N52nnCPKRzJLILEWmw="; + hash = "sha256-cp/Ang5RXLpgTdpOHRKz9fGN9D+SgFC5KhF3ukf8G+w="; }; build-system = [ @@ -39,7 +41,6 @@ buildPythonPackage rec { # cudf = [ cudf ]; dask = [ dask - dask-expr ]; # modin = [ modin ]; pandas = [ pandas ]; @@ -64,7 +65,7 @@ buildPythonPackage rec { meta = { description = "Lightweight and extensible compatibility layer between dataframe libraries"; homepage = "https://github.com/narwhals-dev/narwhals"; - changelog = "https://github.com/narwhals-dev/narwhals/releases/tag/${src.tag}"; + changelog = "https://github.com/narwhals-dev/narwhals/releases/tag/v${version}"; license = lib.licenses.mit; maintainers = with lib.maintainers; [ fab ]; }; From f1309365070057256fdd32bc63ece36278ec3b5d Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 14:37:28 +0100 Subject: [PATCH 06/19] python312Packages.dask-glm: add GaetanLepage as maintainer --- pkgs/development/python-modules/dask-glm/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/development/python-modules/dask-glm/default.nix b/pkgs/development/python-modules/dask-glm/default.nix index a4bead52b43b..7fbb0359e2ca 100644 --- a/pkgs/development/python-modules/dask-glm/default.nix +++ b/pkgs/development/python-modules/dask-glm/default.nix @@ -63,5 +63,6 @@ buildPythonPackage rec { homepage = "https://github.com/dask/dask-glm/"; license = licenses.bsd3; maintainers = [ ]; + maintainers = with maintainers; [ GaetanLepage ]; }; } From 8e0c34c2824abbd1fbf9360b535980fbf02fe8bf Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 14:38:52 +0100 Subject: [PATCH 07/19] python312Packages.dask-glm: fix build --- .../python-modules/dask-glm/default.nix | 59 ++++++++++--------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/pkgs/development/python-modules/dask-glm/default.nix b/pkgs/development/python-modules/dask-glm/default.nix index 7fbb0359e2ca..97cfa6100313 100644 --- a/pkgs/development/python-modules/dask-glm/default.nix +++ b/pkgs/development/python-modules/dask-glm/default.nix @@ -1,34 +1,46 @@ { lib, buildPythonPackage, + fetchFromGitHub, + + # build-system + setuptools-scm, + + # dependencies cloudpickle, - dask, distributed, - fetchPypi, multipledispatch, - pytestCheckHook, - pythonOlder, scikit-learn, scipy, - setuptools-scm, sparse, + dask, + + # tests + pytest-xdist, + pytestCheckHook, }: buildPythonPackage rec { pname = "dask-glm"; version = "0.3.2"; - format = "setuptools"; + pyproject = true; - disabled = pythonOlder "3.7"; - - src = fetchPypi { - inherit pname version; - hash = "sha256-yUelZoZmmKAdeZeK5zIzy16DitXq1ghRQ1gsXpMLmko="; + src = fetchFromGitHub { + owner = "dask"; + repo = "dask-glm"; + tag = version; + hash = "sha256-q98QMmw1toashimS16of54cgZgIPqkua3xGD1FZ1nTc="; }; - nativeBuildInputs = [ setuptools-scm ]; + # ValueError: The truth value of an empty array is ambiguous. Use `array.size > 0` to check that an array is not empty. + postPatch = '' + substituteInPlace dask_glm/utils.py \ + --replace-fail "if arr:" "if (arr is not None) and (arr.size > 0):" + ''; - propagatedBuildInputs = [ + build-system = [ setuptools-scm ]; + + dependencies = [ cloudpickle distributed multipledispatch @@ -38,31 +50,24 @@ buildPythonPackage rec { ] ++ dask.optional-dependencies.array; nativeCheckInputs = [ - sparse + pytest-xdist pytestCheckHook ]; pythonImportsCheck = [ "dask_glm" ]; - disabledTestPaths = [ - # Circular dependency with dask-ml - "dask_glm/tests/test_estimators.py" - # Test tries to imort an obsolete method - "dask_glm/tests/test_utils.py" - ]; - disabledTests = [ - # missing fixture with distributed>=2022.8.0 - "test_determinism_distributed" + # ValueError: can be computed for one-element arrays only. + "test_dot_with_sparse" ]; __darwinAllowLocalNetworking = true; - meta = with lib; { + meta = { description = "Generalized Linear Models with Dask"; homepage = "https://github.com/dask/dask-glm/"; - license = licenses.bsd3; - maintainers = [ ]; - maintainers = with maintainers; [ GaetanLepage ]; + changelog = "https://github.com/dask/dask-glm/releases/tag/${version}"; + license = lib.licenses.bsd3; + maintainers = with lib.maintainers; [ GaetanLepage ]; }; } From 1d52622529ec21840a1684142bd320a0f8d7203e Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 15:12:52 +0100 Subject: [PATCH 08/19] python312Packages.sparse: add GaetanLepage as maintainer --- pkgs/development/python-modules/sparse/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/development/python-modules/sparse/default.nix b/pkgs/development/python-modules/sparse/default.nix index c09040d2beaa..215b826b477d 100644 --- a/pkgs/development/python-modules/sparse/default.nix +++ b/pkgs/development/python-modules/sparse/default.nix @@ -59,5 +59,6 @@ buildPythonPackage rec { downloadPage = "https://github.com/pydata/sparse/releases/tag/${version}"; license = licenses.bsd3; maintainers = [ ]; + maintainers = with maintainers; [ GaetanLepage ]; }; } From f4fd33f95e10a48d18374114444e9b5467e1da47 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 15:13:02 +0100 Subject: [PATCH 09/19] python312Packages.sparse: cleanup & mark as broken on aarch64-linux --- .../python-modules/sparse/default.nix | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/pkgs/development/python-modules/sparse/default.nix b/pkgs/development/python-modules/sparse/default.nix index 215b826b477d..81e567f46ec4 100644 --- a/pkgs/development/python-modules/sparse/default.nix +++ b/pkgs/development/python-modules/sparse/default.nix @@ -1,15 +1,21 @@ { lib, buildPythonPackage, - dask, - fetchPypi, - numba, - numpy, - pytest7CheckHook, - pythonOlder, + fetchFromGitHub, + + # build-system setuptools, setuptools-scm, + + # dependencies + numba, + numpy, scipy, + + # tests + dask, + pytest-cov-stub, + pytestCheckHook, }: buildPythonPackage rec { @@ -17,18 +23,13 @@ buildPythonPackage rec { version = "0.15.5"; pyproject = true; - disabled = pythonOlder "3.8"; - - src = fetchPypi { - inherit pname version; - hash = "sha256-THbODJb1zVwxt+eeZQ8AIkJMKxbwXxAEnpxjge5L4mY="; + src = fetchFromGitHub { + owner = "pydata"; + repo = "sparse"; + tag = version; + hash = "sha256-W4rcq7G/bQsT9oTLieOzWNst5LnIAelRMbm+uUPeQgs="; }; - postPatch = '' - substituteInPlace pytest.ini \ - --replace-fail "--cov-report term-missing --cov-report html --cov-report=xml --cov-report=term --cov sparse --cov-config .coveragerc --junitxml=junit/test-results.xml" "" - ''; - build-system = [ setuptools setuptools-scm @@ -42,23 +43,23 @@ buildPythonPackage rec { nativeCheckInputs = [ dask - pytest7CheckHook + pytest-cov-stub + pytestCheckHook ]; pythonImportsCheck = [ "sparse" ]; - pytestFlagsArray = [ - "-W" - "ignore::pytest.PytestRemovedIn8Warning" - ]; - - meta = with lib; { + meta = { description = "Sparse n-dimensional arrays computations"; homepage = "https://sparse.pydata.org/"; changelog = "https://sparse.pydata.org/en/stable/changelog.html"; downloadPage = "https://github.com/pydata/sparse/releases/tag/${version}"; - license = licenses.bsd3; - maintainers = [ ]; - maintainers = with maintainers; [ GaetanLepage ]; + license = lib.licenses.bsd3; + maintainers = with lib.maintainers; [ GaetanLepage ]; + badPlatforms = [ + # Most tests fail with: Fatal Python error: Segmentation fault + # numba/typed/typedlist.py", line 344 in append + "aarch64-linux" + ]; }; } From e6ea345642291962533b3fbccb27dd63ace175c9 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 16:27:40 +0100 Subject: [PATCH 10/19] python312Packages.devito: fix build --- .../python-modules/devito/default.nix | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/pkgs/development/python-modules/devito/default.nix b/pkgs/development/python-modules/devito/default.nix index 4c9ee562011f..a69ae736a22a 100644 --- a/pkgs/development/python-modules/devito/default.nix +++ b/pkgs/development/python-modules/devito/default.nix @@ -1,27 +1,32 @@ { lib, stdenv, - anytree, buildPythonPackage, + fetchFromGitHub, + + # build-system setuptools, + + # dependencies + anytree, cached-property, cgen, click, codepy, distributed, - fetchFromGitHub, - gcc, llvmPackages, - matplotlib, multidict, nbval, psutil, py-cpuinfo, - pytest-xdist, - pytestCheckHook, - pythonOlder, scipy, sympy, + + # tests + gcc, + matplotlib, + pytest-xdist, + pytestCheckHook, }: buildPythonPackage rec { @@ -29,8 +34,6 @@ buildPythonPackage rec { version = "4.8.11"; pyproject = true; - disabled = pythonOlder "3.8"; - src = fetchFromGitHub { owner = "devitocodes"; repo = "devito"; @@ -42,6 +45,9 @@ buildPythonPackage rec { postPatch = '' substituteInPlace requirements-testing.txt \ --replace-fail 'pooch; python_version >= "3.8"' "pooch" + + substituteInPlace tests/test_builtins.py \ + --replace-fail "from scipy.misc import ascent" "from scipy.datasets import ascent" ''; pythonRemoveDeps = [ "pip" ]; @@ -99,6 +105,10 @@ buildPythonPackage rec { "test_stability_mpi" "test_subdomainset_mpi" "test_subdomains_mpi" + + # Download dataset from the internet + "test_gs_2d_float" + "test_gs_2d_int" ] ++ lib.optionals (stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64) [ # FAILED tests/test_unexpansion.py::Test2Pass::test_v0 - assert False From d506ad7f5c0fadd3994c2895ebb68ed4aba61856 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 7 Feb 2025 17:59:30 +0100 Subject: [PATCH 11/19] python312Packages.catboost: mark as broken --- pkgs/development/python-modules/catboost/default.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/catboost/default.nix b/pkgs/development/python-modules/catboost/default.nix index d0fc77b70038..86c1211aead0 100644 --- a/pkgs/development/python-modules/catboost/default.nix +++ b/pkgs/development/python-modules/catboost/default.nix @@ -22,8 +22,8 @@ buildPythonPackage rec { pname version src - meta ; + pyproject = true; sourceRoot = "${src.name}/catboost/python-package"; @@ -55,4 +55,9 @@ buildPythonPackage rec { doCheck = false; pythonImportsCheck = [ "catboost" ]; + + meta = catboost.meta // { + # Incompatible with numpy >= 2.0 + broken = true; + }; } From cdbaefe74329ed3587d686720e5de04919306292 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sat, 8 Feb 2025 11:49:04 +0100 Subject: [PATCH 12/19] python312Packages.dask-ml: 2024.4.4 -> 2025.1.0 Diff: https://github.com/dask/dask-ml/compare/refs/tags/v2024.4.4...v2025.1.0 --- .../python-modules/dask-ml/default.nix | 46 ++++--------------- 1 file changed, 9 insertions(+), 37 deletions(-) diff --git a/pkgs/development/python-modules/dask-ml/default.nix b/pkgs/development/python-modules/dask-ml/default.nix index e3949fde678b..f30adb7ff246 100644 --- a/pkgs/development/python-modules/dask-ml/default.nix +++ b/pkgs/development/python-modules/dask-ml/default.nix @@ -3,14 +3,13 @@ stdenv, buildPythonPackage, fetchFromGitHub, + fetchpatch, # build-system hatch-vcs, hatchling, - setuptools-scm, # dependencies - dask-expr, dask-glm, distributed, multipledispatch, @@ -29,25 +28,23 @@ buildPythonPackage rec { pname = "dask-ml"; - version = "2024.4.4"; + version = "2025.1.0"; pyproject = true; src = fetchFromGitHub { owner = "dask"; repo = "dask-ml"; tag = "v${version}"; - hash = "sha256-ZiBpCk3b4Tk0Hwb4uapJLEx+Nb/qHFROCnkBTNGDzoU="; + hash = "sha256-DHxx0LFuJmGWYuG/WGHj+a5XHAEekBmlHUUb90rl2IY="; }; build-system = [ hatch-vcs hatchling - setuptools-scm ]; dependencies = [ - dask-expr dask-glm distributed multipledispatch @@ -73,39 +70,14 @@ buildPythonPackage rec { pytestCheckHook ]; - disabledTestPaths = - [ - # AttributeError: 'csr_matrix' object has no attribute 'A' - # Fixed in https://github.com/dask/dask-ml/pull/996 - "tests/test_svd.py" - - # Tests fail with dask>=0.11.2 - # RuntimeError: Not enough arguments provided - # Reported in https://github.com/dask/dask-ml/issues/1003 - "tests/model_selection/test_incremental.py" - ] - ++ lib.optionals stdenv.isDarwin [ - # RuntimeError: Not enough arguments provided: missing keys - "tests/model_selection/test_hyperband.py" - "tests/model_selection/test_incremental.py" - "tests/model_selection/test_incremental_warns.py" - "tests/model_selection/test_successive_halving.py" - ]; + disabledTestPaths = [ + # AttributeError: module 'numpy' has no attribute 'product' + "tests/test_svd.py" + ]; disabledTests = [ - # Flaky: `Arrays are not almost equal to 3 decimals` (although values do actually match) - "test_whitening" - - # Tests fail with dask>=0.11.2 - # RuntimeError: Not enough arguments provided - # Reported in https://github.com/dask/dask-ml/issues/1003 - "test_basic" - "test_hyperband_patience" - "test_same_random_state_same_params" - "test_search_patience_infeasible_tol" - "test_sha_max_iter_and_metadata" - "test_warns_decay_rate" - "test_warns_decay_rate_wanted" + # AssertionError: Regex pattern did not match. + "test_unknown_category_transform_array" ]; __darwinAllowLocalNetworking = true; From 64249c9b7d2d80d7983a1efc382b3a53b6a9ae6b Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sat, 8 Feb 2025 21:52:02 +0100 Subject: [PATCH 13/19] python312Packages.dask-histogram: 2024.12.1 -> 2025.2.0 Diff: https://github.com/dask-contrib/dask-histogram/compare/refs/tags/2024.12.1...2025.2.0 Changelog: https://github.com/dask-contrib/dask-histogram/releases/tag/2025.2.0 --- .../development/python-modules/dask-histogram/default.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/dask-histogram/default.nix b/pkgs/development/python-modules/dask-histogram/default.nix index fc6af4c39c6b..98692269069d 100644 --- a/pkgs/development/python-modules/dask-histogram/default.nix +++ b/pkgs/development/python-modules/dask-histogram/default.nix @@ -10,6 +10,7 @@ # dependencies boost-histogram, dask, + dask-awkward, # tests pytestCheckHook, @@ -17,14 +18,14 @@ buildPythonPackage rec { pname = "dask-histogram"; - version = "2024.12.1"; + version = "2025.2.0"; pyproject = true; src = fetchFromGitHub { owner = "dask-contrib"; repo = "dask-histogram"; tag = version; - hash = "sha256-chznfEsMFuIioIlTz6uNpVAkjMMoMXjS62R4luKxVZQ="; + hash = "sha256-5Ec/qzVc/7LDG6SJym1e76BbxmbVjKKQHckwtTs1+6M="; }; build-system = [ @@ -35,6 +36,7 @@ buildPythonPackage rec { dependencies = [ boost-histogram dask + dask-awkward ]; nativeCheckInputs = [ pytestCheckHook ]; @@ -44,7 +46,7 @@ buildPythonPackage rec { meta = { description = "Histograms with task scheduling"; homepage = "https://dask-histogram.readthedocs.io/"; - changelog = "https://github.com/dask-contrib/dask-histogram/releases/tag/${src.tag}"; + changelog = "https://github.com/dask-contrib/dask-histogram/releases/tag/${version}"; license = with lib.licenses; [ bsd3 ]; maintainers = with lib.maintainers; [ veprbl ]; }; From cb32a8ad1cba105da078243a973927f194e4f821 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sat, 8 Feb 2025 21:54:54 +0100 Subject: [PATCH 14/19] python312Packages.dask-awkward: 2024.12.2 -> 2025.2.0 Diff: https://github.com/dask-contrib/dask-awkward/compare/refs/tags/2024.12.2...2025.2.0 Changelog: https://github.com/dask-contrib/dask-awkward/releases/tag/2025.2.0 --- .../python-modules/dask-awkward/default.nix | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkgs/development/python-modules/dask-awkward/default.nix b/pkgs/development/python-modules/dask-awkward/default.nix index a0d8b63fe06a..8031a140ffbf 100644 --- a/pkgs/development/python-modules/dask-awkward/default.nix +++ b/pkgs/development/python-modules/dask-awkward/default.nix @@ -16,7 +16,7 @@ # optional-dependencies pyarrow, - # checks + # tests dask-histogram, distributed, hist, @@ -27,14 +27,14 @@ buildPythonPackage rec { pname = "dask-awkward"; - version = "2024.12.2"; + version = "2025.2.0"; pyproject = true; src = fetchFromGitHub { owner = "dask-contrib"; repo = "dask-awkward"; tag = version; - hash = "sha256-pL1LDW/q78V/c3Bha38k40018MFO+i8X6htYNdcsy7s="; + hash = "sha256-hhAY2cPUOYnP86FGsLvxlMeoEwY+sTrjPMKyuZrO0/M="; }; build-system = [ @@ -53,8 +53,8 @@ buildPythonPackage rec { io = [ pyarrow ]; }; - checkInputs = [ - dask-histogram + nativeCheckInputs = [ + # dask-histogram (circular dependency) distributed hist pandas @@ -86,7 +86,7 @@ buildPythonPackage rec { meta = { description = "Native Dask collection for awkward arrays, and the library to use it"; homepage = "https://github.com/dask-contrib/dask-awkward"; - changelog = "https://github.com/dask-contrib/dask-awkward/releases/tag/${src.tag}"; + changelog = "https://github.com/dask-contrib/dask-awkward/releases/tag/${version}"; license = lib.licenses.bsd3; maintainers = with lib.maintainers; [ veprbl ]; }; From 3af88eb4248b0a96e82a307ab0104e45a5d01d82 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sat, 8 Feb 2025 21:59:59 +0100 Subject: [PATCH 15/19] python312Packages.awkward-cpp: 43 -> 44 --- pkgs/development/python-modules/awkward-cpp/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/awkward-cpp/default.nix b/pkgs/development/python-modules/awkward-cpp/default.nix index 0063cd3f851f..ce96b50ca49b 100644 --- a/pkgs/development/python-modules/awkward-cpp/default.nix +++ b/pkgs/development/python-modules/awkward-cpp/default.nix @@ -11,13 +11,13 @@ buildPythonPackage rec { pname = "awkward-cpp"; - version = "43"; + version = "44"; pyproject = true; src = fetchPypi { pname = "awkward_cpp"; inherit version; - hash = "sha256-we11WW7IKvZm2ghLW/gF2bcsg1MtA10rZeB5WQXSXq0="; + hash = "sha256-jcSZKI1tFrLqILUaJ9UEflGiR7aqz8vLOzAsrW08h9g="; }; build-system = [ From 5de8457c08fdaf5ba12e2d1bab418c5c6c5f5114 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sat, 8 Feb 2025 21:58:04 +0100 Subject: [PATCH 16/19] python312Packages.awkward: 2.7.2 -> 2.7.4 Diff: https://github.com/scikit-hep/awkward/compare/refs/tags/v2.7.2...v2.7.4 Changelog: https://github.com/scikit-hep/awkward/releases/tag/v2.7.4 --- .../python-modules/awkward/default.nix | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/pkgs/development/python-modules/awkward/default.nix b/pkgs/development/python-modules/awkward/default.nix index 8722a72cf635..6124a408e605 100644 --- a/pkgs/development/python-modules/awkward/default.nix +++ b/pkgs/development/python-modules/awkward/default.nix @@ -1,7 +1,7 @@ { lib, + stdenv, buildPythonPackage, - pythonOlder, fetchFromGitHub, # build-system @@ -13,29 +13,27 @@ fsspec, numpy, packaging, - typing-extensions, - importlib-metadata, - # checks + # tests numba, - setuptools, numexpr, pandas, pyarrow, pytest-xdist, pytestCheckHook, + setuptools, }: buildPythonPackage rec { pname = "awkward"; - version = "2.7.2"; + version = "2.7.4"; pyproject = true; src = fetchFromGitHub { owner = "scikit-hep"; repo = "awkward"; tag = "v${version}"; - hash = "sha256-nOKMwAQ5t8tc64bEKz0j8JxxoVQQu39Iu8Zr9cqSx7A="; + hash = "sha256-OXSl+8sfrx+JlLu40wHf+98WVNNwm9uxvsnGXRDztDg="; }; build-system = [ @@ -43,15 +41,12 @@ buildPythonPackage rec { hatchling ]; - dependencies = - [ - awkward-cpp - fsspec - numpy - packaging - ] - ++ lib.optionals (pythonOlder "3.11") [ typing-extensions ] - ++ lib.optionals (pythonOlder "3.12") [ importlib-metadata ]; + dependencies = [ + awkward-cpp + fsspec + numpy + packaging + ]; dontUseCmakeConfigure = true; @@ -60,19 +55,35 @@ buildPythonPackage rec { nativeCheckInputs = [ fsspec numba - setuptools numexpr pandas pyarrow pytest-xdist pytestCheckHook + setuptools ]; - disabledTestPaths = [ - # Need to be run on a GPU platform. - "tests-cuda" + disabledTests = [ + # pyarrow.lib.ArrowInvalid + "test_recordarray" ]; + disabledTestPaths = + [ + # Need to be run on a GPU platform. + "tests-cuda" + ] + ++ lib.optionals (stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64) [ + # Fatal Python error: Segmentation fault at: + # numba/typed/typedlist.py", line 344 in append + "tests/test_0118_numba_cpointers.py" + "tests/test_0397_arrays_as_constants_in_numba.py" + "tests/test_1677_array_builder_in_numba.py" + "tests/test_2055_array_builder_check.py" + "tests/test_2349_growablebuffer_in_numba.py" + "tests/test_2408_layoutbuilder_in_numba.py" + ]; + meta = { description = "Manipulate JSON-like data with NumPy-like idioms"; homepage = "https://github.com/scikit-hep/awkward"; From 8d23cded07fe77e45db1ada00ff3a3bbda2b51ae Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sat, 8 Feb 2025 22:08:22 +0100 Subject: [PATCH 17/19] python312Packages.mplhep: 0.3.55 -> 0.3.56 Diff: https://github.com/scikit-hep/mplhep/compare/refs/tags/v0.3.55...v0.3.56 --- .../python-modules/mplhep/default.nix | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/pkgs/development/python-modules/mplhep/default.nix b/pkgs/development/python-modules/mplhep/default.nix index 604eea8a8489..a5c167aa603f 100644 --- a/pkgs/development/python-modules/mplhep/default.nix +++ b/pkgs/development/python-modules/mplhep/default.nix @@ -2,36 +2,42 @@ lib, buildPythonPackage, fetchFromGitHub, - setuptools, - setuptools-scm, + + # build-system + hatch-vcs, + hatchling, + + # dependencies matplotlib, mplhep-data, numpy, packaging, uhi, + + # tests + hist, + pytest-mock, + pytest-mpl, pytestCheckHook, scipy, - pytest-mpl, - pytest-mock, uproot, - hist, }: buildPythonPackage rec { pname = "mplhep"; - version = "0.3.55"; + version = "0.3.56"; pyproject = true; src = fetchFromGitHub { owner = "scikit-hep"; repo = "mplhep"; tag = "v${version}"; - hash = "sha256-7YkrrH9Bfn3ctjv+H6TXEDE8yS/wnjO7umuHIXeYTDU="; + hash = "sha256-sMJpJUEtIqmu7kCgZp43t9XLy/6nkDgKcxC4nFb+1po="; }; build-system = [ - setuptools - setuptools-scm + hatch-vcs + hatchling ]; dependencies = [ @@ -43,12 +49,12 @@ buildPythonPackage rec { ]; nativeCheckInputs = [ + hist + pytest-mock + pytest-mpl pytestCheckHook scipy - pytest-mpl - pytest-mock uproot - hist ]; disabledTests = [ @@ -62,6 +68,7 @@ buildPythonPackage rec { meta = { description = "Extended histogram plots on top of matplotlib and HEP compatible styling similar to current collaboration requirements (ROOT)"; homepage = "https://github.com/scikit-hep/mplhep"; + changelog = "https://github.com/scikit-hep/mplhep/releases/tag/v${version}"; license = with lib.licenses; [ mit ]; maintainers = with lib.maintainers; [ veprbl ]; }; From 5b73ca5a0c6930c157c36983ed4fd54bc90fd5f7 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sat, 8 Feb 2025 22:15:52 +0100 Subject: [PATCH 18/19] python312Packages.coffea: fix build --- pkgs/development/python-modules/coffea/default.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkgs/development/python-modules/coffea/default.nix b/pkgs/development/python-modules/coffea/default.nix index 5e61b17c5c2d..0710ca0f5dba 100644 --- a/pkgs/development/python-modules/coffea/default.nix +++ b/pkgs/development/python-modules/coffea/default.nix @@ -57,6 +57,10 @@ buildPythonPackage rec { hatch-vcs ]; + pythonRelaxDeps = [ + "dask" + ]; + dependencies = [ aiohttp awkward @@ -106,6 +110,10 @@ buildPythonPackage rec { # AssertionError: bug in Awkward Array: attempt to convert TypeTracerArray into a concrete array "test_apply_to_fileset" "test_lorentz_behavior" + + # ValueError: The array to mask was deleted before it could be masked. + # If you want to construct this mask, you must either keep the array alive or use 'ak.mask' explicitly. + "test_read_nanomc" ]; __darwinAllowLocalNetworking = true; From ff787004396499583498584e78d705f1f820b592 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sun, 9 Feb 2025 12:28:47 +0100 Subject: [PATCH 19/19] python312Packages.dask-image: fix build --- .../python-modules/dask-image/default.nix | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/dask-image/default.nix b/pkgs/development/python-modules/dask-image/default.nix index a297902bc684..599e99853ad8 100644 --- a/pkgs/development/python-modules/dask-image/default.nix +++ b/pkgs/development/python-modules/dask-image/default.nix @@ -1,15 +1,21 @@ { lib, buildPythonPackage, - pythonOlder, fetchFromGitHub, + + # build-system setuptools, setuptools-scm, + + # dependencies dask, numpy, scipy, pandas, pims, + + # tests + pyarrow, pytestCheckHook, scikit-image, }: @@ -19,8 +25,6 @@ buildPythonPackage rec { version = "2024.5.3"; pyproject = true; - disabled = pythonOlder "3.9"; - src = fetchFromGitHub { owner = "dask"; repo = "dask-image"; @@ -28,6 +32,11 @@ buildPythonPackage rec { hash = "sha256-kXCAqJ2Zgo/2Khvo2YcK+n4oGM219GyQ2Hsq9re1Lac="; }; + postPatch = '' + substituteInPlace dask_image/ndinterp/__init__.py \ + --replace-fail "out_bounds.ptp(axis=1)" "np.ptp(out_bounds, axis=1)" + ''; + build-system = [ setuptools setuptools-scm @@ -42,15 +51,27 @@ buildPythonPackage rec { ]; nativeCheckInputs = [ + pyarrow pytestCheckHook scikit-image ]; pythonImportsCheck = [ "dask_image" ]; + disabledTests = [ + # The following tests are from 'tests/test_dask_image/test_ndmeasure/test_find_objects.py' and + # fail because of errrors on numpy slices + # AttributeError: 'str' object has no attribute 'start' + "test_find_objects" + "test_3d_find_objects" + # AssertionError (comparing slices) + "test_find_objects_with_empty_chunks" + ]; + meta = { description = "Distributed image processing"; homepage = "https://github.com/dask/dask-image"; + changelog = "https://github.com/dask/dask-image/releases/tag/v${version}"; license = lib.licenses.bsdOriginal; maintainers = with lib.maintainers; [ GaetanLepage ]; };