From d98065f2dd9d419ed5f563ced53f389ee166460b Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Sat, 19 Jul 2025 10:24:58 +0200 Subject: [PATCH 1/2] arrow-cpp: 20.0.0 -> 22.0.0, python3Packages.pyarrow: 20.0.0 -> 22.0.0 --- pkgs/by-name/ar/arrow-cpp/package.nix | 28 +++---- .../python-modules/pyarrow/default.nix | 78 +++++++++++-------- 2 files changed, 59 insertions(+), 47 deletions(-) diff --git a/pkgs/by-name/ar/arrow-cpp/package.nix b/pkgs/by-name/ar/arrow-cpp/package.nix index 4ef558b4bf1e..2a5e0d03d7a4 100644 --- a/pkgs/by-name/ar/arrow-cpp/package.nix +++ b/pkgs/by-name/ar/arrow-cpp/package.nix @@ -37,7 +37,7 @@ openssl, perl, pkg-config, - protobuf_31, + protobuf, python3, rapidjson, re2, @@ -66,19 +66,19 @@ let name = "arrow-testing"; owner = "apache"; repo = "arrow-testing"; - rev = "d2a13712303498963395318a4eb42872e66aead7"; - hash = "sha256-c8FL37kG0uo7o0Zp71WjCl7FD5BnVgqUCCXXX9gI0lg="; + rev = "9a02925d1ba80bd493b6d4da6e8a777588d57ac4"; + hash = "sha256-dEFCkeQpQrU61uCwJp/XB2umbQHjXtzado36BGChoc0="; }; parquet-testing = fetchFromGitHub { name = "parquet-testing"; owner = "apache"; repo = "parquet-testing"; - rev = "18d17540097fca7c40be3d42c167e6bfad90763c"; - hash = "sha256-gKEQc2RKpVp39RmuZbIeIXAwiAXDHGnLXF6VQuJtnRA="; + rev = "a3d96a65e11e2bbca7d22a894e8313ede90a33a3"; + hash = "sha256-Xd6o3RT6Q0tPutV77J0P1x3F6U3RHdCBOKGUKtkQCKk="; }; - version = "20.0.0"; + version = "22.0.0"; in stdenv.mkDerivation (finalAttrs: { pname = "arrow-cpp"; @@ -88,7 +88,7 @@ stdenv.mkDerivation (finalAttrs: { owner = "apache"; repo = "arrow"; rev = "apache-arrow-${version}"; - hash = "sha256-JFPdKraCU+xRkBTAHyY4QGnBVlOjQ1P5+gq9uxyqJtk="; + hash = "sha256-i4Smt43oi4sddUt3qH7ePjensBSfPW+w/ExLVcVNKic="; }; sourceRoot = "${finalAttrs.src.name}/cpp"; @@ -107,21 +107,21 @@ stdenv.mkDerivation (finalAttrs: { ARROW_MIMALLOC_URL = fetchFromGitHub { owner = "microsoft"; repo = "mimalloc"; - rev = "v2.0.6"; - hash = "sha256-u2ITXABBN/dwU+mCIbL3tN1f4c17aBuSdNTV+Adtohc="; + tag = "v3.1.5"; + hash = "sha256-fk6nfyBFS1G0sJwUJVgTC1+aKd0We/JjsIYTO+IOfyg="; }; ARROW_XSIMD_URL = fetchFromGitHub { owner = "xtensor-stack"; repo = "xsimd"; - rev = "13.0.0"; + tag = "13.0.0"; hash = "sha256-qElJYW5QDj3s59L3NgZj5zkhnUMzIP2mBa1sPks3/CE="; }; ARROW_SUBSTRAIT_URL = fetchFromGitHub { owner = "substrait-io"; repo = "substrait"; - rev = "v0.44.0"; + tag = "v0.44.0"; hash = "sha256-V739IFTGPtbGPlxcOi8sAaYSDhNUEpITvN9IqdPReug="; }; @@ -138,6 +138,7 @@ stdenv.mkDerivation (finalAttrs: { boost brotli bzip2 + curl flatbuffers gflags glog @@ -145,7 +146,7 @@ stdenv.mkDerivation (finalAttrs: { libbacktrace lz4 nlohmann_json # alternative JSON parser to rapidjson - protobuf_31 # substrait requires protobuf + protobuf # substrait requires protobuf rapidjson re2 snappy @@ -157,7 +158,7 @@ stdenv.mkDerivation (finalAttrs: { ++ lib.optionals enableFlight [ grpc openssl - protobuf_31 + protobuf sqlite ] ++ lib.optionals enableS3 [ @@ -166,7 +167,6 @@ stdenv.mkDerivation (finalAttrs: { ] ++ lib.optionals enableGcs [ crc32c - curl google-cloud-cpp grpc nlohmann_json diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index 136dd474a527..45b315b33a1b 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -97,55 +97,67 @@ buildPythonPackage rec { find "$PWD/pyarrow/src/arrow" -type f -name '*.h' -exec cp {} "$pyarrow_include/arrow/python" \; ''; - pytestFlagsArray = [ - # A couple of tests are missing fixture imports, luckily pytest offers a - # clean solution. - "--fixtures pyarrow/tests/conftest.py" - # Deselect a single test because pyarrow prints a 2-line error message where - # only a single line is expected. The additional line of output comes from - # the glog library which is an optional dependency of arrow-cpp that is - # enabled in nixpkgs. - # Upstream Issue: https://issues.apache.org/jira/browse/ARROW-11393 - "--deselect=pyarrow/tests/test_memory.py::test_env_var" - # these tests require access to s3 via the internet - "--deselect=pyarrow/tests/test_fs.py::test_resolve_s3_region" - "--deselect=pyarrow/tests/test_fs.py::test_s3_real_aws" - "--deselect=pyarrow/tests/test_fs.py::test_s3_real_aws_region_selection" - "--deselect=pyarrow/tests/test_fs.py::test_s3_options" + disabledTestPaths = [ + # These tests require access to s3 via the internet. + "pyarrow/tests/test_fs.py::test_resolve_s3_region" + "pyarrow/tests/test_fs.py::test_s3_finalize" + "pyarrow/tests/test_fs.py::test_s3_finalize_region_resolver" + "pyarrow/tests/test_fs.py::test_s3_real_aws" + "pyarrow/tests/test_fs.py::test_s3_real_aws_region_selection" + "pyarrow/tests/test_fs.py::test_s3_options" # Flaky test - "--deselect=pyarrow/tests/test_flight.py::test_roundtrip_errors" - "--deselect=pyarrow/tests/test_pandas.py::test_threaded_pandas_import" - # Flaky test, works locally but not on Hydra - "--deselect=pyarrow/tests/test_csv.py::TestThreadedCSVTableRead::test_cancellation" - # expects arrow-cpp headers to be bundled - "--deselect=pyarrow/tests/test_cpp_internals.py::test_pyarrow_include" + "pyarrow/tests/test_flight.py::test_roundtrip_errors" + "pyarrow/tests/test_pandas.py::test_threaded_pandas_import" + # Flaky test, works locally but not on Hydra. + "pyarrow/tests/test_csv.py::TestThreadedCSVTableRead::test_cancellation" + # expects arrow-cpp headers to be bundled. + "pyarrow/tests/test_cpp_internals.py::test_pyarrow_include" + # Searches for TZDATA in /usr. + "pyarrow/tests/test_orc.py::test_example_using_json" + # AssertionError: assert 'Europe/Monaco' == 'Europe/Paris' + "pyarrow/tests/test_types.py::test_dateutil_tzinfo_to_string" + # These fail with xxx_fixture not found. + # xxx = unary_func, unary_agg_func, varargs_agg_func + "pyarrow/tests/test_substrait.py::test_udf_via_substrait" + "pyarrow/tests/test_substrait.py::test_scalar_aggregate_udf_basic" + "pyarrow/tests/test_substrait.py::test_hash_aggregate_udf_basic" + "pyarrow/tests/test_udf.py::test_hash_agg_basic" + "pyarrow/tests/test_udf.py::test_hash_agg_empty" + "pyarrow/tests/test_udf.py::test_input_lifetime" + "pyarrow/tests/test_udf.py::test_scalar_agg_basic" + "pyarrow/tests/test_udf.py::test_scalar_agg_empty" + "pyarrow/tests/test_udf.py::test_scalar_agg_varargs" + "pyarrow/tests/test_udf.py::test_scalar_input" + "pyarrow/tests/test_udf.py::test_scalar_udf_context" + "pyarrow/tests/test_udf.py::test_udf_array_unary" ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ - # Requires loopback networking - "--deselect=pyarrow/tests/test_ipc.py::test_socket_" - "--deselect=pyarrow/tests/test_flight.py::test_never_sends_data" - "--deselect=pyarrow/tests/test_flight.py::test_large_descriptor" - "--deselect=pyarrow/tests/test_flight.py::test_large_metadata_client" - "--deselect=pyarrow/tests/test_flight.py::test_none_action_side_effect" - # fails to compile - "--deselect=pyarrow/tests/test_cython.py::test_cython_api" + # Requires loopback networking. + "pyarrow/tests/test_ipc.py::test_socket_" + "pyarrow/tests/test_flight.py::test_never_sends_data" + "pyarrow/tests/test_flight.py::test_large_descriptor" + "pyarrow/tests/test_flight.py::test_large_metadata_client" + "pyarrow/tests/test_flight.py::test_none_action_side_effect" + # Fails to compile. + "pyarrow/tests/test_cython.py::test_cython_api" ] ++ lib.optionals (pythonAtLeast "3.11") [ # Repr output is printing number instead of enum name so these tests fail - "--deselect=pyarrow/tests/test_fs.py::test_get_file_info" + "pyarrow/tests/test_fs.py::test_get_file_info" ] ++ lib.optionals stdenv.hostPlatform.isLinux [ - # this test requires local networking - "--deselect=pyarrow/tests/test_fs.py::test_filesystem_from_uri_gcs" + # This test requires local networking. + "pyarrow/tests/test_fs.py::test_filesystem_from_uri_gcs" ]; disabledTests = [ "GcsFileSystem" ]; preCheck = '' + export PARQUET_TEST_DATA="${arrow-cpp.PARQUET_TEST_DATA}" shopt -s extglob rm -r pyarrow/!(conftest.py|tests) mv pyarrow/conftest.py pyarrow/tests/parent_conftest.py - substituteInPlace pyarrow/tests/conftest.py --replace ..conftest .parent_conftest + substituteInPlace pyarrow/tests/conftest.py --replace-fail ..conftest .parent_conftest '' + lib.optionalString stdenv.hostPlatform.isDarwin '' # OSError: [Errno 24] Too many open files From 2f57d5c636b4d470f1f8e18a6b13db763292a1d8 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sun, 16 Nov 2025 12:16:58 -0500 Subject: [PATCH 2/2] python3Packages.streamlit: allow to build against pyarrow 22+ --- pkgs/development/python-modules/streamlit/default.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkgs/development/python-modules/streamlit/default.nix b/pkgs/development/python-modules/streamlit/default.nix index fc155688f46a..853503a47413 100644 --- a/pkgs/development/python-modules/streamlit/default.nix +++ b/pkgs/development/python-modules/streamlit/default.nix @@ -7,6 +7,7 @@ cachetools, click, fetchPypi, + fetchpatch2, gitpython, numpy, packaging, @@ -38,6 +39,16 @@ buildPythonPackage rec { hash = "sha256-HnQqnAtpj0Zsb1v1jTM77aWh++jeZgdDl2eRtcFEbvY="; }; + patches = [ + # Allow pyarrow 22 + (fetchpatch2 { + url = "https://github.com/streamlit/streamlit/commit/b9e1b875a948a0aa6e972edc6e86a4f89706e08c.diff?full_index=1"; + stripLen = 1; + excludes = [ "tests/streamlit/data_test_cases.py" ]; + hash = "sha256-qZau1XlP8Kf2hPtyFphJN4UEjbp0ZZYngFdRwXTVt3g="; + }) + ]; + build-system = [ setuptools ];