From 6d0ac786baa4681a84633faa598329bf842ff21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=A9tan=20Lepage?= <33058747+GaetanLepage@users.noreply.github.com> Date: Tue, 11 Feb 2025 07:37:12 +0100 Subject: [PATCH] Revert "python312Packages.dask: 2024.12.1 -> 2025.1.0" --- .../python-modules/awkward-cpp/default.nix | 4 +- .../python-modules/awkward/default.nix | 51 ++-- .../python-modules/catboost/default.nix | 7 +- .../python-modules/coffea/default.nix | 8 - .../python-modules/dask-awkward/default.nix | 12 +- .../python-modules/dask-expr/default.nix | 65 +++++ .../python-modules/dask-glm/default.nix | 58 ++-- .../python-modules/dask-histogram/default.nix | 8 +- .../python-modules/dask-image/default.nix | 27 +- .../python-modules/dask-ml/default.nix | 46 +++- .../python-modules/dask/default.nix | 252 ++++++++++-------- .../python-modules/devito/default.nix | 30 +-- .../python-modules/mplhep/default.nix | 31 +-- .../python-modules/narwhals/default.nix | 25 +- .../python-modules/sparse/default.nix | 52 ++-- .../python-modules/streamz/default.nix | 60 +++-- pkgs/top-level/python-aliases.nix | 1 - pkgs/top-level/python-packages.nix | 2 + 18 files changed, 406 insertions(+), 333 deletions(-) create mode 100644 pkgs/development/python-modules/dask-expr/default.nix diff --git a/pkgs/development/python-modules/awkward-cpp/default.nix b/pkgs/development/python-modules/awkward-cpp/default.nix index ce96b50ca49b..0063cd3f851f 100644 --- a/pkgs/development/python-modules/awkward-cpp/default.nix +++ b/pkgs/development/python-modules/awkward-cpp/default.nix @@ -11,13 +11,13 @@ buildPythonPackage rec { pname = "awkward-cpp"; - version = "44"; + version = "43"; pyproject = true; src = fetchPypi { pname = "awkward_cpp"; inherit version; - hash = "sha256-jcSZKI1tFrLqILUaJ9UEflGiR7aqz8vLOzAsrW08h9g="; + hash = "sha256-we11WW7IKvZm2ghLW/gF2bcsg1MtA10rZeB5WQXSXq0="; }; build-system = [ diff --git a/pkgs/development/python-modules/awkward/default.nix b/pkgs/development/python-modules/awkward/default.nix index 6124a408e605..8722a72cf635 100644 --- a/pkgs/development/python-modules/awkward/default.nix +++ b/pkgs/development/python-modules/awkward/default.nix @@ -1,7 +1,7 @@ { lib, - stdenv, buildPythonPackage, + pythonOlder, fetchFromGitHub, # build-system @@ -13,27 +13,29 @@ fsspec, numpy, packaging, + typing-extensions, + importlib-metadata, - # tests + # checks numba, + setuptools, numexpr, pandas, pyarrow, pytest-xdist, pytestCheckHook, - setuptools, }: buildPythonPackage rec { pname = "awkward"; - version = "2.7.4"; + version = "2.7.2"; pyproject = true; src = fetchFromGitHub { owner = "scikit-hep"; repo = "awkward"; tag = "v${version}"; - hash = "sha256-OXSl+8sfrx+JlLu40wHf+98WVNNwm9uxvsnGXRDztDg="; + hash = "sha256-nOKMwAQ5t8tc64bEKz0j8JxxoVQQu39Iu8Zr9cqSx7A="; }; build-system = [ @@ -41,12 +43,15 @@ buildPythonPackage rec { hatchling ]; - dependencies = [ - awkward-cpp - fsspec - numpy - packaging - ]; + dependencies = + [ + awkward-cpp + fsspec + numpy + packaging + ] + ++ lib.optionals (pythonOlder "3.11") [ typing-extensions ] + ++ lib.optionals (pythonOlder "3.12") [ importlib-metadata ]; dontUseCmakeConfigure = true; @@ -55,35 +60,19 @@ buildPythonPackage rec { nativeCheckInputs = [ fsspec numba + setuptools numexpr pandas pyarrow pytest-xdist pytestCheckHook - setuptools ]; - disabledTests = [ - # pyarrow.lib.ArrowInvalid - "test_recordarray" + disabledTestPaths = [ + # Need to be run on a GPU platform. + "tests-cuda" ]; - disabledTestPaths = - [ - # Need to be run on a GPU platform. - "tests-cuda" - ] - ++ lib.optionals (stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64) [ - # Fatal Python error: Segmentation fault at: - # numba/typed/typedlist.py", line 344 in append - "tests/test_0118_numba_cpointers.py" - "tests/test_0397_arrays_as_constants_in_numba.py" - "tests/test_1677_array_builder_in_numba.py" - "tests/test_2055_array_builder_check.py" - "tests/test_2349_growablebuffer_in_numba.py" - "tests/test_2408_layoutbuilder_in_numba.py" - ]; - meta = { description = "Manipulate JSON-like data with NumPy-like idioms"; homepage = "https://github.com/scikit-hep/awkward"; diff --git a/pkgs/development/python-modules/catboost/default.nix b/pkgs/development/python-modules/catboost/default.nix index 86c1211aead0..d0fc77b70038 100644 --- a/pkgs/development/python-modules/catboost/default.nix +++ b/pkgs/development/python-modules/catboost/default.nix @@ -22,8 +22,8 @@ buildPythonPackage rec { pname version src + meta ; - pyproject = true; sourceRoot = "${src.name}/catboost/python-package"; @@ -55,9 +55,4 @@ buildPythonPackage rec { doCheck = false; pythonImportsCheck = [ "catboost" ]; - - meta = catboost.meta // { - # Incompatible with numpy >= 2.0 - broken = true; - }; } diff --git a/pkgs/development/python-modules/coffea/default.nix b/pkgs/development/python-modules/coffea/default.nix index 0710ca0f5dba..5e61b17c5c2d 100644 --- a/pkgs/development/python-modules/coffea/default.nix +++ b/pkgs/development/python-modules/coffea/default.nix @@ -57,10 +57,6 @@ buildPythonPackage rec { hatch-vcs ]; - pythonRelaxDeps = [ - "dask" - ]; - dependencies = [ aiohttp awkward @@ -110,10 +106,6 @@ buildPythonPackage rec { # AssertionError: bug in Awkward Array: attempt to convert TypeTracerArray into a concrete array "test_apply_to_fileset" "test_lorentz_behavior" - - # ValueError: The array to mask was deleted before it could be masked. - # If you want to construct this mask, you must either keep the array alive or use 'ak.mask' explicitly. - "test_read_nanomc" ]; __darwinAllowLocalNetworking = true; diff --git a/pkgs/development/python-modules/dask-awkward/default.nix b/pkgs/development/python-modules/dask-awkward/default.nix index 8031a140ffbf..a0d8b63fe06a 100644 --- a/pkgs/development/python-modules/dask-awkward/default.nix +++ b/pkgs/development/python-modules/dask-awkward/default.nix @@ -16,7 +16,7 @@ # optional-dependencies pyarrow, - # tests + # checks dask-histogram, distributed, hist, @@ -27,14 +27,14 @@ buildPythonPackage rec { pname = "dask-awkward"; - version = "2025.2.0"; + version = "2024.12.2"; pyproject = true; src = fetchFromGitHub { owner = "dask-contrib"; repo = "dask-awkward"; tag = version; - hash = "sha256-hhAY2cPUOYnP86FGsLvxlMeoEwY+sTrjPMKyuZrO0/M="; + hash = "sha256-pL1LDW/q78V/c3Bha38k40018MFO+i8X6htYNdcsy7s="; }; build-system = [ @@ -53,8 +53,8 @@ buildPythonPackage rec { io = [ pyarrow ]; }; - nativeCheckInputs = [ - # dask-histogram (circular dependency) + checkInputs = [ + dask-histogram distributed hist pandas @@ -86,7 +86,7 @@ buildPythonPackage rec { meta = { description = "Native Dask collection for awkward arrays, and the library to use it"; homepage = "https://github.com/dask-contrib/dask-awkward"; - changelog = "https://github.com/dask-contrib/dask-awkward/releases/tag/${version}"; + changelog = "https://github.com/dask-contrib/dask-awkward/releases/tag/${src.tag}"; license = lib.licenses.bsd3; maintainers = with lib.maintainers; [ veprbl ]; }; diff --git a/pkgs/development/python-modules/dask-expr/default.nix b/pkgs/development/python-modules/dask-expr/default.nix new file mode 100644 index 000000000000..c7bb6c734ee9 --- /dev/null +++ b/pkgs/development/python-modules/dask-expr/default.nix @@ -0,0 +1,65 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + + # build-system + setuptools, + versioneer, + + # dependencies + dask, + pandas, + pyarrow, + + # checks + jinja2, + pytestCheckHook, + xarray, +}: + +buildPythonPackage rec { + pname = "dask-expr"; + version = "1.1.21"; + pyproject = true; + + src = fetchFromGitHub { + owner = "dask"; + repo = "dask-expr"; + tag = "v${version}"; + hash = "sha256-t1vPlTxV5JYArg/a7CzPP13NHbstEoCgHRmd8Y9mDfA="; + }; + + postPatch = '' + substituteInPlace pyproject.toml \ + --replace-fail "versioneer[toml]==0.28" "versioneer[toml]" + ''; + + build-system = [ + setuptools + versioneer + ]; + + dependencies = [ + dask + pandas + pyarrow + ]; + + pythonImportsCheck = [ "dask_expr" ]; + + nativeCheckInputs = [ + jinja2 + pytestCheckHook + xarray + ]; + + __darwinAllowLocalNetworking = true; + + meta = { + description = "Rewrite of Dask DataFrame that includes query optimization and generally improved organization"; + homepage = "https://github.com/dask/dask-expr"; + license = lib.licenses.bsd3; + maintainers = with lib.maintainers; [ GaetanLepage ]; + }; +} diff --git a/pkgs/development/python-modules/dask-glm/default.nix b/pkgs/development/python-modules/dask-glm/default.nix index 97cfa6100313..a4bead52b43b 100644 --- a/pkgs/development/python-modules/dask-glm/default.nix +++ b/pkgs/development/python-modules/dask-glm/default.nix @@ -1,46 +1,34 @@ { lib, buildPythonPackage, - fetchFromGitHub, - - # build-system - setuptools-scm, - - # dependencies cloudpickle, + dask, distributed, + fetchPypi, multipledispatch, + pytestCheckHook, + pythonOlder, scikit-learn, scipy, + setuptools-scm, sparse, - dask, - - # tests - pytest-xdist, - pytestCheckHook, }: buildPythonPackage rec { pname = "dask-glm"; version = "0.3.2"; - pyproject = true; + format = "setuptools"; - src = fetchFromGitHub { - owner = "dask"; - repo = "dask-glm"; - tag = version; - hash = "sha256-q98QMmw1toashimS16of54cgZgIPqkua3xGD1FZ1nTc="; + disabled = pythonOlder "3.7"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-yUelZoZmmKAdeZeK5zIzy16DitXq1ghRQ1gsXpMLmko="; }; - # ValueError: The truth value of an empty array is ambiguous. Use `array.size > 0` to check that an array is not empty. - postPatch = '' - substituteInPlace dask_glm/utils.py \ - --replace-fail "if arr:" "if (arr is not None) and (arr.size > 0):" - ''; + nativeBuildInputs = [ setuptools-scm ]; - build-system = [ setuptools-scm ]; - - dependencies = [ + propagatedBuildInputs = [ cloudpickle distributed multipledispatch @@ -50,24 +38,30 @@ buildPythonPackage rec { ] ++ dask.optional-dependencies.array; nativeCheckInputs = [ - pytest-xdist + sparse pytestCheckHook ]; pythonImportsCheck = [ "dask_glm" ]; + disabledTestPaths = [ + # Circular dependency with dask-ml + "dask_glm/tests/test_estimators.py" + # Test tries to imort an obsolete method + "dask_glm/tests/test_utils.py" + ]; + disabledTests = [ - # ValueError: can be computed for one-element arrays only. - "test_dot_with_sparse" + # missing fixture with distributed>=2022.8.0 + "test_determinism_distributed" ]; __darwinAllowLocalNetworking = true; - meta = { + meta = with lib; { description = "Generalized Linear Models with Dask"; homepage = "https://github.com/dask/dask-glm/"; - changelog = "https://github.com/dask/dask-glm/releases/tag/${version}"; - license = lib.licenses.bsd3; - maintainers = with lib.maintainers; [ GaetanLepage ]; + license = licenses.bsd3; + maintainers = [ ]; }; } diff --git a/pkgs/development/python-modules/dask-histogram/default.nix b/pkgs/development/python-modules/dask-histogram/default.nix index 98692269069d..fc6af4c39c6b 100644 --- a/pkgs/development/python-modules/dask-histogram/default.nix +++ b/pkgs/development/python-modules/dask-histogram/default.nix @@ -10,7 +10,6 @@ # dependencies boost-histogram, dask, - dask-awkward, # tests pytestCheckHook, @@ -18,14 +17,14 @@ buildPythonPackage rec { pname = "dask-histogram"; - version = "2025.2.0"; + version = "2024.12.1"; pyproject = true; src = fetchFromGitHub { owner = "dask-contrib"; repo = "dask-histogram"; tag = version; - hash = "sha256-5Ec/qzVc/7LDG6SJym1e76BbxmbVjKKQHckwtTs1+6M="; + hash = "sha256-chznfEsMFuIioIlTz6uNpVAkjMMoMXjS62R4luKxVZQ="; }; build-system = [ @@ -36,7 +35,6 @@ buildPythonPackage rec { dependencies = [ boost-histogram dask - dask-awkward ]; nativeCheckInputs = [ pytestCheckHook ]; @@ -46,7 +44,7 @@ buildPythonPackage rec { meta = { description = "Histograms with task scheduling"; homepage = "https://dask-histogram.readthedocs.io/"; - changelog = "https://github.com/dask-contrib/dask-histogram/releases/tag/${version}"; + changelog = "https://github.com/dask-contrib/dask-histogram/releases/tag/${src.tag}"; license = with lib.licenses; [ bsd3 ]; maintainers = with lib.maintainers; [ veprbl ]; }; diff --git a/pkgs/development/python-modules/dask-image/default.nix b/pkgs/development/python-modules/dask-image/default.nix index 599e99853ad8..a297902bc684 100644 --- a/pkgs/development/python-modules/dask-image/default.nix +++ b/pkgs/development/python-modules/dask-image/default.nix @@ -1,21 +1,15 @@ { lib, buildPythonPackage, + pythonOlder, fetchFromGitHub, - - # build-system setuptools, setuptools-scm, - - # dependencies dask, numpy, scipy, pandas, pims, - - # tests - pyarrow, pytestCheckHook, scikit-image, }: @@ -25,6 +19,8 @@ buildPythonPackage rec { version = "2024.5.3"; pyproject = true; + disabled = pythonOlder "3.9"; + src = fetchFromGitHub { owner = "dask"; repo = "dask-image"; @@ -32,11 +28,6 @@ buildPythonPackage rec { hash = "sha256-kXCAqJ2Zgo/2Khvo2YcK+n4oGM219GyQ2Hsq9re1Lac="; }; - postPatch = '' - substituteInPlace dask_image/ndinterp/__init__.py \ - --replace-fail "out_bounds.ptp(axis=1)" "np.ptp(out_bounds, axis=1)" - ''; - build-system = [ setuptools setuptools-scm @@ -51,27 +42,15 @@ buildPythonPackage rec { ]; nativeCheckInputs = [ - pyarrow pytestCheckHook scikit-image ]; pythonImportsCheck = [ "dask_image" ]; - disabledTests = [ - # The following tests are from 'tests/test_dask_image/test_ndmeasure/test_find_objects.py' and - # fail because of errrors on numpy slices - # AttributeError: 'str' object has no attribute 'start' - "test_find_objects" - "test_3d_find_objects" - # AssertionError (comparing slices) - "test_find_objects_with_empty_chunks" - ]; - meta = { description = "Distributed image processing"; homepage = "https://github.com/dask/dask-image"; - changelog = "https://github.com/dask/dask-image/releases/tag/v${version}"; license = lib.licenses.bsdOriginal; maintainers = with lib.maintainers; [ GaetanLepage ]; }; diff --git a/pkgs/development/python-modules/dask-ml/default.nix b/pkgs/development/python-modules/dask-ml/default.nix index f30adb7ff246..e3949fde678b 100644 --- a/pkgs/development/python-modules/dask-ml/default.nix +++ b/pkgs/development/python-modules/dask-ml/default.nix @@ -3,13 +3,14 @@ stdenv, buildPythonPackage, fetchFromGitHub, - fetchpatch, # build-system hatch-vcs, hatchling, + setuptools-scm, # dependencies + dask-expr, dask-glm, distributed, multipledispatch, @@ -28,23 +29,25 @@ buildPythonPackage rec { pname = "dask-ml"; - version = "2025.1.0"; + version = "2024.4.4"; pyproject = true; src = fetchFromGitHub { owner = "dask"; repo = "dask-ml"; tag = "v${version}"; - hash = "sha256-DHxx0LFuJmGWYuG/WGHj+a5XHAEekBmlHUUb90rl2IY="; + hash = "sha256-ZiBpCk3b4Tk0Hwb4uapJLEx+Nb/qHFROCnkBTNGDzoU="; }; build-system = [ hatch-vcs hatchling + setuptools-scm ]; dependencies = [ + dask-expr dask-glm distributed multipledispatch @@ -70,14 +73,39 @@ buildPythonPackage rec { pytestCheckHook ]; - disabledTestPaths = [ - # AttributeError: module 'numpy' has no attribute 'product' - "tests/test_svd.py" - ]; + disabledTestPaths = + [ + # AttributeError: 'csr_matrix' object has no attribute 'A' + # Fixed in https://github.com/dask/dask-ml/pull/996 + "tests/test_svd.py" + + # Tests fail with dask>=0.11.2 + # RuntimeError: Not enough arguments provided + # Reported in https://github.com/dask/dask-ml/issues/1003 + "tests/model_selection/test_incremental.py" + ] + ++ lib.optionals stdenv.isDarwin [ + # RuntimeError: Not enough arguments provided: missing keys + "tests/model_selection/test_hyperband.py" + "tests/model_selection/test_incremental.py" + "tests/model_selection/test_incremental_warns.py" + "tests/model_selection/test_successive_halving.py" + ]; disabledTests = [ - # AssertionError: Regex pattern did not match. - "test_unknown_category_transform_array" + # Flaky: `Arrays are not almost equal to 3 decimals` (although values do actually match) + "test_whitening" + + # Tests fail with dask>=0.11.2 + # RuntimeError: Not enough arguments provided + # Reported in https://github.com/dask/dask-ml/issues/1003 + "test_basic" + "test_hyperband_patience" + "test_same_random_state_same_params" + "test_search_patience_infeasible_tol" + "test_sha_max_iter_and_metadata" + "test_warns_decay_rate" + "test_warns_decay_rate_wanted" ]; __darwinAllowLocalNetworking = true; diff --git a/pkgs/development/python-modules/dask/default.nix b/pkgs/development/python-modules/dask/default.nix index b37ddce56526..93b38b11b1f7 100644 --- a/pkgs/development/python-modules/dask/default.nix +++ b/pkgs/development/python-modules/dask/default.nix @@ -1,5 +1,6 @@ { lib, + stdenv, buildPythonPackage, fetchFromGitHub, @@ -26,127 +27,168 @@ jinja2, # tests + arrow-cpp, + dask-expr, hypothesis, pytest-asyncio, - pytest-mock, pytest-rerunfailures, pytest-xdist, pytestCheckHook, - versionCheckHook, }: -buildPythonPackage rec { - pname = "dask"; - version = "2025.1.0"; - pyproject = true; +let + self = buildPythonPackage rec { + pname = "dask"; + version = "2024.12.1"; + pyproject = true; - src = fetchFromGitHub { - owner = "dask"; - repo = "dask"; - tag = version; - hash = "sha256-KBqOyf471mNg3L9dYmR7IRSltEtC+VgC+6ptsoKgVmM="; - }; + src = fetchFromGitHub { + owner = "dask"; + repo = "dask"; + tag = version; + hash = "sha256-QqvdldAHW2UYt1NXfk3Aa+oe97e+OpRbF8d6eKV3OJ4="; + }; - postPatch = '' - # versioneer hack to set version of GitHub package - echo "def get_versions(): return {'dirty': False, 'error': None, 'full-revisionid': None, 'version': '${version}'}" > dask/_version.py + build-system = [ setuptools ]; - substituteInPlace setup.py \ - --replace-fail "import versioneer" "" \ - --replace-fail "version=versioneer.get_version()," "version='${version}'," \ - --replace-fail "cmdclass=versioneer.get_cmdclass()," "" - - substituteInPlace pyproject.toml \ - --replace-fail ', "versioneer[toml]==0.29"' "" \ - --replace-fail " --durations=10" "" \ - --replace-fail " --cov-config=pyproject.toml" "" \ - --replace-fail "\"-v" "\" " - ''; - - build-system = [ setuptools ]; - - dependencies = [ - click - cloudpickle - fsspec - packaging - partd - pyyaml - importlib-metadata - toolz - ]; - - optional-dependencies = lib.fix (self: { - array = [ numpy ]; - complete = - [ - pyarrow - lz4 - ] - ++ self.array - ++ self.dataframe - ++ self.distributed - ++ self.diagnostics; - dataframe = [ - pandas - pyarrow - ] ++ self.array; - distributed = [ distributed ]; - diagnostics = [ - bokeh - jinja2 + dependencies = [ + click + cloudpickle + fsspec + packaging + partd + pyyaml + importlib-metadata + toolz ]; - }); - nativeCheckInputs = - [ - hypothesis - pytest-asyncio - pytest-mock - pytest-rerunfailures - pytest-xdist - pytestCheckHook - versionCheckHook - pyarrow - ] - ++ optional-dependencies.array - ++ optional-dependencies.dataframe; - versionCheckProgramArg = [ "--version" ]; + optional-dependencies = lib.fix (self: { + array = [ numpy ]; + complete = + [ + pyarrow + lz4 + ] + ++ self.array + ++ self.dataframe + ++ self.distributed + ++ self.diagnostics; + dataframe = [ + # dask-expr -> circular dependency with dask-expr + numpy + pandas + ]; + distributed = [ distributed ]; + diagnostics = [ + bokeh + jinja2 + ]; + }); - pytestFlagsArray = [ - # Rerun failed tests up to three times - "--reruns 3" - # Don't run tests that require network access - "-m 'not network'" - ]; + nativeCheckInputs = + [ + dask-expr + pytestCheckHook + pytest-rerunfailures + pytest-xdist + # from panda[test] + hypothesis + pytest-asyncio + ] + ++ self.optional-dependencies.array + ++ self.optional-dependencies.dataframe + ++ lib.optionals (!arrow-cpp.meta.broken) [ + # support is sparse on aarch64 + pyarrow + ]; - disabledTests = [ - # UserWarning: Insufficient elements for `head`. 10 elements requested, only 5 elements available. Try passing larger `npartitions` to `head`. - "test_set_index_head_nlargest_string" - ]; + dontUseSetuptoolsCheck = true; - __darwinAllowLocalNetworking = true; + postPatch = '' + # versioneer hack to set version of GitHub package + echo "def get_versions(): return {'dirty': False, 'error': None, 'full-revisionid': None, 'version': '${version}'}" > dask/_version.py - pythonImportsCheck = [ - "dask" - "dask.bag" - "dask.bytes" - "dask.diagnostics" + substituteInPlace setup.py \ + --replace-fail "import versioneer" "" \ + --replace-fail "version=versioneer.get_version()," "version='${version}'," \ + --replace-fail "cmdclass=versioneer.get_cmdclass()," "" - # Requires the `dask.optional-dependencies.array` that are only in `nativeCheckInputs` - "dask.array" - # Requires the `dask.optional-dependencies.dataframe` that are only in `nativeCheckInputs` - "dask.dataframe" - "dask.dataframe.io" - "dask.dataframe.tseries" - ]; + substituteInPlace pyproject.toml \ + --replace-fail ', "versioneer[toml]==0.29"' "" \ + --replace-fail " --durations=10" "" \ + --replace-fail " --cov-config=pyproject.toml" "" \ + --replace-fail "\"-v" "\" " + ''; - meta = { - description = "Minimal task scheduling abstraction"; - mainProgram = "dask"; - homepage = "https://dask.org/"; - changelog = "https://docs.dask.org/en/latest/changelog.html"; - license = lib.licenses.bsd3; - maintainers = with lib.maintainers; [ GaetanLepage ]; + pytestFlagsArray = [ + # Rerun failed tests up to three times + "--reruns 3" + # Don't run tests that require network access + "-m 'not network'" + ]; + + disabledTests = + lib.optionals stdenv.hostPlatform.isDarwin [ + # Test requires features of python3Packages.psutil that are + # blocked in sandboxed-builds + "test_auto_blocksize_csv" + # AttributeError: 'str' object has no attribute 'decode' + "test_read_dir_nometa" + ] + ++ lib.optionals (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) [ + # concurrent.futures.process.BrokenProcessPool: A process in the process pool terminated abpruptly... + "test_foldby_tree_reduction" + "test_to_bag" + ] + ++ [ + # https://github.com/dask/dask/issues/10347#issuecomment-1589683941 + "test_concat_categorical" + # AttributeError: 'ArrowStringArray' object has no attribute 'tobytes'. Did you mean: 'nbytes'? + "test_dot" + "test_dot_nan" + "test_merge_column_with_nulls" + # FileNotFoundError: [Errno 2] No such file or directory: '/build/tmp301jryv_/createme/0.part' + "test_to_csv_nodir" + "test_to_json_results" + # FutureWarning: Those tests should be working fine when pandas will have been upgraded to 2.1.1 + "test_apply" + "test_apply_infer_columns" + ]; + + __darwinAllowLocalNetworking = true; + + pythonImportsCheck = [ + "dask" + "dask.bag" + "dask.bytes" + "dask.diagnostics" + ]; + + doCheck = false; + + # Enable tests via passthru to avoid cyclic dependency with dask-expr. + passthru.tests = { + check = self.overridePythonAttrs (old: { + doCheck = true; + pythonImportsCheck = [ + # Requires the `dask.optional-dependencies.array` that are only in `nativeCheckInputs` + "dask.array" + # Requires the `dask.optional-dependencies.dataframe` that are only in `nativeCheckInputs` + "dask.dataframe" + "dask.dataframe.io" + "dask.dataframe.tseries" + ] ++ old.pythonImportsCheck; + }); + }; + + meta = { + description = "Minimal task scheduling abstraction"; + mainProgram = "dask"; + homepage = "https://dask.org/"; + changelog = "https://docs.dask.org/en/latest/changelog.html"; + license = lib.licenses.bsd3; + maintainers = with lib.maintainers; [ GaetanLepage ]; + }; }; -} +in +self diff --git a/pkgs/development/python-modules/devito/default.nix b/pkgs/development/python-modules/devito/default.nix index a69ae736a22a..4c9ee562011f 100644 --- a/pkgs/development/python-modules/devito/default.nix +++ b/pkgs/development/python-modules/devito/default.nix @@ -1,32 +1,27 @@ { lib, stdenv, - buildPythonPackage, - fetchFromGitHub, - - # build-system - setuptools, - - # dependencies anytree, + buildPythonPackage, + setuptools, cached-property, cgen, click, codepy, distributed, + fetchFromGitHub, + gcc, llvmPackages, + matplotlib, multidict, nbval, psutil, py-cpuinfo, - scipy, - sympy, - - # tests - gcc, - matplotlib, pytest-xdist, pytestCheckHook, + pythonOlder, + scipy, + sympy, }: buildPythonPackage rec { @@ -34,6 +29,8 @@ buildPythonPackage rec { version = "4.8.11"; pyproject = true; + disabled = pythonOlder "3.8"; + src = fetchFromGitHub { owner = "devitocodes"; repo = "devito"; @@ -45,9 +42,6 @@ buildPythonPackage rec { postPatch = '' substituteInPlace requirements-testing.txt \ --replace-fail 'pooch; python_version >= "3.8"' "pooch" - - substituteInPlace tests/test_builtins.py \ - --replace-fail "from scipy.misc import ascent" "from scipy.datasets import ascent" ''; pythonRemoveDeps = [ "pip" ]; @@ -105,10 +99,6 @@ buildPythonPackage rec { "test_stability_mpi" "test_subdomainset_mpi" "test_subdomains_mpi" - - # Download dataset from the internet - "test_gs_2d_float" - "test_gs_2d_int" ] ++ lib.optionals (stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64) [ # FAILED tests/test_unexpansion.py::Test2Pass::test_v0 - assert False diff --git a/pkgs/development/python-modules/mplhep/default.nix b/pkgs/development/python-modules/mplhep/default.nix index a5c167aa603f..604eea8a8489 100644 --- a/pkgs/development/python-modules/mplhep/default.nix +++ b/pkgs/development/python-modules/mplhep/default.nix @@ -2,42 +2,36 @@ lib, buildPythonPackage, fetchFromGitHub, - - # build-system - hatch-vcs, - hatchling, - - # dependencies + setuptools, + setuptools-scm, matplotlib, mplhep-data, numpy, packaging, uhi, - - # tests - hist, - pytest-mock, - pytest-mpl, pytestCheckHook, scipy, + pytest-mpl, + pytest-mock, uproot, + hist, }: buildPythonPackage rec { pname = "mplhep"; - version = "0.3.56"; + version = "0.3.55"; pyproject = true; src = fetchFromGitHub { owner = "scikit-hep"; repo = "mplhep"; tag = "v${version}"; - hash = "sha256-sMJpJUEtIqmu7kCgZp43t9XLy/6nkDgKcxC4nFb+1po="; + hash = "sha256-7YkrrH9Bfn3ctjv+H6TXEDE8yS/wnjO7umuHIXeYTDU="; }; build-system = [ - hatch-vcs - hatchling + setuptools + setuptools-scm ]; dependencies = [ @@ -49,12 +43,12 @@ buildPythonPackage rec { ]; nativeCheckInputs = [ - hist - pytest-mock - pytest-mpl pytestCheckHook scipy + pytest-mpl + pytest-mock uproot + hist ]; disabledTests = [ @@ -68,7 +62,6 @@ buildPythonPackage rec { meta = { description = "Extended histogram plots on top of matplotlib and HEP compatible styling similar to current collaboration requirements (ROOT)"; homepage = "https://github.com/scikit-hep/mplhep"; - changelog = "https://github.com/scikit-hep/mplhep/releases/tag/v${version}"; license = with lib.licenses; [ mit ]; maintainers = with lib.maintainers; [ veprbl ]; }; diff --git a/pkgs/development/python-modules/narwhals/default.nix b/pkgs/development/python-modules/narwhals/default.nix index f96998a32908..5aa36f62364c 100644 --- a/pkgs/development/python-modules/narwhals/default.nix +++ b/pkgs/development/python-modules/narwhals/default.nix @@ -1,36 +1,34 @@ { lib, buildPythonPackage, - fetchFromGitHub, - - # build-system - hatchling, - - # optional-dependencies # cudf, dask, + dask-expr, + duckdb, + fetchFromGitHub, + hatchling, + hypothesis, # modin, pandas, polars, pyarrow, - - # tests - duckdb, - hypothesis, pytest-env, pytestCheckHook, + pythonOlder, }: buildPythonPackage rec { pname = "narwhals"; - version = "1.25.2"; + version = "1.22.0"; pyproject = true; + disabled = pythonOlder "3.8"; + src = fetchFromGitHub { owner = "narwhals-dev"; repo = "narwhals"; tag = "v${version}"; - hash = "sha256-cp/Ang5RXLpgTdpOHRKz9fGN9D+SgFC5KhF3ukf8G+w="; + hash = "sha256-I6nJJiiW1v04YH70OPxXKeO80N52nnCPKRzJLILEWmw="; }; build-system = [ @@ -41,6 +39,7 @@ buildPythonPackage rec { # cudf = [ cudf ]; dask = [ dask + dask-expr ]; # modin = [ modin ]; pandas = [ pandas ]; @@ -65,7 +64,7 @@ buildPythonPackage rec { meta = { description = "Lightweight and extensible compatibility layer between dataframe libraries"; homepage = "https://github.com/narwhals-dev/narwhals"; - changelog = "https://github.com/narwhals-dev/narwhals/releases/tag/v${version}"; + changelog = "https://github.com/narwhals-dev/narwhals/releases/tag/${src.tag}"; license = lib.licenses.mit; maintainers = with lib.maintainers; [ fab ]; }; diff --git a/pkgs/development/python-modules/sparse/default.nix b/pkgs/development/python-modules/sparse/default.nix index 81e567f46ec4..c09040d2beaa 100644 --- a/pkgs/development/python-modules/sparse/default.nix +++ b/pkgs/development/python-modules/sparse/default.nix @@ -1,21 +1,15 @@ { lib, buildPythonPackage, - fetchFromGitHub, - - # build-system - setuptools, - setuptools-scm, - - # dependencies + dask, + fetchPypi, numba, numpy, + pytest7CheckHook, + pythonOlder, + setuptools, + setuptools-scm, scipy, - - # tests - dask, - pytest-cov-stub, - pytestCheckHook, }: buildPythonPackage rec { @@ -23,13 +17,18 @@ buildPythonPackage rec { version = "0.15.5"; pyproject = true; - src = fetchFromGitHub { - owner = "pydata"; - repo = "sparse"; - tag = version; - hash = "sha256-W4rcq7G/bQsT9oTLieOzWNst5LnIAelRMbm+uUPeQgs="; + disabled = pythonOlder "3.8"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-THbODJb1zVwxt+eeZQ8AIkJMKxbwXxAEnpxjge5L4mY="; }; + postPatch = '' + substituteInPlace pytest.ini \ + --replace-fail "--cov-report term-missing --cov-report html --cov-report=xml --cov-report=term --cov sparse --cov-config .coveragerc --junitxml=junit/test-results.xml" "" + ''; + build-system = [ setuptools setuptools-scm @@ -43,23 +42,22 @@ buildPythonPackage rec { nativeCheckInputs = [ dask - pytest-cov-stub - pytestCheckHook + pytest7CheckHook ]; pythonImportsCheck = [ "sparse" ]; - meta = { + pytestFlagsArray = [ + "-W" + "ignore::pytest.PytestRemovedIn8Warning" + ]; + + meta = with lib; { description = "Sparse n-dimensional arrays computations"; homepage = "https://sparse.pydata.org/"; changelog = "https://sparse.pydata.org/en/stable/changelog.html"; downloadPage = "https://github.com/pydata/sparse/releases/tag/${version}"; - license = lib.licenses.bsd3; - maintainers = with lib.maintainers; [ GaetanLepage ]; - badPlatforms = [ - # Most tests fail with: Fatal Python error: Segmentation fault - # numba/typed/typedlist.py", line 344 in append - "aarch64-linux" - ]; + license = licenses.bsd3; + maintainers = [ ]; }; } diff --git a/pkgs/development/python-modules/streamz/default.nix b/pkgs/development/python-modules/streamz/default.nix index 5aab96e7c462..eea52b8fa8f2 100644 --- a/pkgs/development/python-modules/streamz/default.nix +++ b/pkgs/development/python-modules/streamz/default.nix @@ -1,24 +1,23 @@ { lib, buildPythonPackage, - fetchFromGitHub, - - # build-system + pythonOlder, + fetchPypi, setuptools, - - # dependencies + confluent-kafka, + dask, + dask-expr, + distributed, + flaky, + graphviz, + networkx, + pytest-asyncio, + pytestCheckHook, + requests, six, toolz, tornado, zict, - - # tests - dask, - distributed, - flaky, - pandas, - pyarrow, - pytestCheckHook, }: buildPythonPackage rec { @@ -26,16 +25,17 @@ buildPythonPackage rec { version = "0.6.4"; pyproject = true; - src = fetchFromGitHub { - owner = "python-streamz"; - repo = "streamz"; - tag = version; - hash = "sha256-lSb3gl+TSIzz4BZzxH8zXu74HvzSntOAoVQUUJKIEvA="; + disabled = pythonOlder "3.6"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-VXfWkEwuxInBQVQJV3IQXgGVRkiBmYfUZCBMbjyWNPM="; }; build-system = [ setuptools ]; dependencies = [ + networkx six toolz tornado @@ -43,12 +43,15 @@ buildPythonPackage rec { ]; nativeCheckInputs = [ + confluent-kafka dask + dask-expr distributed flaky - pandas - pyarrow + graphviz + pytest-asyncio pytestCheckHook + requests ]; pythonImportsCheck = [ "streamz" ]; @@ -60,17 +63,24 @@ buildPythonPackage rec { "test_partition_then_scatter_sync" "test_sync" "test_sync_2" - + # Test fail in the sandbox + "test_tcp_async" + "test_tcp" + "test_partition_timeout" # Tests are flaky + "test_from_iterable" "test_buffer" ]; - __darwinAllowLocalNetworking = true; + disabledTestPaths = [ + # Disable kafka tests + "streamz/tests/test_kafka.py" + ]; - meta = { + meta = with lib; { description = "Pipelines to manage continuous streams of data"; homepage = "https://github.com/python-streamz/streamz"; - license = lib.licenses.bsd3; - maintainers = with lib.maintainers; [ GaetanLepage ]; + license = licenses.bsd3; + maintainers = [ ]; }; } diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index c87843dcb618..434e1b6f3cd9 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -142,7 +142,6 @@ mapAliases ({ cx_Freeze = cx-freeze; # added 2023-08-02 cx_oracle = cx-oracle; # added 2024-01-03 d2to1 = throw "d2to1 is archived and no longer works with setuptools v68"; # added 2023-07-30 - dask-expr = throw "dask-expr was removed because its features are available in dask"; # added 2025-02-07 dask-xgboost = throw "dask-xgboost was removed because its features are available in xgboost"; # added 2022-05-24 dash-renderer = throw "dash-renderer has been removed since it is abandoned"; # added 2025-01-01 dateutil = python-dateutil; # added 2021-07-03 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 687c342fd174..013e7f91fb05 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2961,6 +2961,8 @@ self: super: with self; { dask-awkward = callPackage ../development/python-modules/dask-awkward { }; + dask-expr = callPackage ../development/python-modules/dask-expr { }; + dask-gateway = callPackage ../development/python-modules/dask-gateway { }; dask-gateway-server = callPackage ../development/python-modules/dask-gateway-server { };