arrow-cpp: 20.0.0 -> 22.0.0, python3Packages.pyarrow: 20.0.0 -> 22.0.0 (#426611)

pkgs/by-name/ar/arrow-cpp/package.nix

@@ -37,7 +37,7 @@
   openssl,
   perl,
   pkg-config,
-  protobuf_31,
+  protobuf,
   python3,
   rapidjson,
   re2,
@@ -66,19 +66,19 @@ let
     name = "arrow-testing";
     owner = "apache";
     repo = "arrow-testing";
-    rev = "d2a13712303498963395318a4eb42872e66aead7";
-    hash = "sha256-c8FL37kG0uo7o0Zp71WjCl7FD5BnVgqUCCXXX9gI0lg=";
+    rev = "9a02925d1ba80bd493b6d4da6e8a777588d57ac4";
+    hash = "sha256-dEFCkeQpQrU61uCwJp/XB2umbQHjXtzado36BGChoc0=";
   };
 
   parquet-testing = fetchFromGitHub {
     name = "parquet-testing";
     owner = "apache";
     repo = "parquet-testing";
-    rev = "18d17540097fca7c40be3d42c167e6bfad90763c";
-    hash = "sha256-gKEQc2RKpVp39RmuZbIeIXAwiAXDHGnLXF6VQuJtnRA=";
+    rev = "a3d96a65e11e2bbca7d22a894e8313ede90a33a3";
+    hash = "sha256-Xd6o3RT6Q0tPutV77J0P1x3F6U3RHdCBOKGUKtkQCKk=";
   };
 
-  version = "20.0.0";
+  version = "22.0.0";
 in
 stdenv.mkDerivation (finalAttrs: {
   pname = "arrow-cpp";
@@ -88,7 +88,7 @@ stdenv.mkDerivation (finalAttrs: {
     owner = "apache";
     repo = "arrow";
     rev = "apache-arrow-${version}";
-    hash = "sha256-JFPdKraCU+xRkBTAHyY4QGnBVlOjQ1P5+gq9uxyqJtk=";
+    hash = "sha256-i4Smt43oi4sddUt3qH7ePjensBSfPW+w/ExLVcVNKic=";
   };
 
   sourceRoot = "${finalAttrs.src.name}/cpp";
@@ -107,21 +107,21 @@ stdenv.mkDerivation (finalAttrs: {
   ARROW_MIMALLOC_URL = fetchFromGitHub {
     owner = "microsoft";
     repo = "mimalloc";
-    rev = "v2.0.6";
-    hash = "sha256-u2ITXABBN/dwU+mCIbL3tN1f4c17aBuSdNTV+Adtohc=";
+    tag = "v3.1.5";
+    hash = "sha256-fk6nfyBFS1G0sJwUJVgTC1+aKd0We/JjsIYTO+IOfyg=";
   };
 
   ARROW_XSIMD_URL = fetchFromGitHub {
     owner = "xtensor-stack";
     repo = "xsimd";
-    rev = "13.0.0";
+    tag = "13.0.0";
     hash = "sha256-qElJYW5QDj3s59L3NgZj5zkhnUMzIP2mBa1sPks3/CE=";
   };
 
   ARROW_SUBSTRAIT_URL = fetchFromGitHub {
     owner = "substrait-io";
     repo = "substrait";
-    rev = "v0.44.0";
+    tag = "v0.44.0";
     hash = "sha256-V739IFTGPtbGPlxcOi8sAaYSDhNUEpITvN9IqdPReug=";
   };
 
@@ -138,6 +138,7 @@ stdenv.mkDerivation (finalAttrs: {
     boost
     brotli
     bzip2
+    curl
     flatbuffers
     gflags
     glog
@@ -145,7 +146,7 @@ stdenv.mkDerivation (finalAttrs: {
     libbacktrace
     lz4
     nlohmann_json # alternative JSON parser to rapidjson
-    protobuf_31 # substrait requires protobuf
+    protobuf # substrait requires protobuf
     rapidjson
     re2
     snappy
@@ -157,7 +158,7 @@ stdenv.mkDerivation (finalAttrs: {
   ++ lib.optionals enableFlight [
     grpc
     openssl
-    protobuf_31
+    protobuf
     sqlite
   ]
   ++ lib.optionals enableS3 [
@@ -166,7 +167,6 @@ stdenv.mkDerivation (finalAttrs: {
   ]
   ++ lib.optionals enableGcs [
     crc32c
-    curl
     google-cloud-cpp
     grpc
     nlohmann_json

pkgs/development/python-modules/pyarrow/default.nix

@@ -97,55 +97,67 @@ buildPythonPackage rec {
     find "$PWD/pyarrow/src/arrow" -type f -name '*.h' -exec cp {} "$pyarrow_include/arrow/python" \;
   '';
 
-  pytestFlagsArray = [
-    # A couple of tests are missing fixture imports, luckily pytest offers a
-    # clean solution.
-    "--fixtures pyarrow/tests/conftest.py"
-    # Deselect a single test because pyarrow prints a 2-line error message where
-    # only a single line is expected. The additional line of output comes from
-    # the glog library which is an optional dependency of arrow-cpp that is
-    # enabled in nixpkgs.
-    # Upstream Issue: https://issues.apache.org/jira/browse/ARROW-11393
-    "--deselect=pyarrow/tests/test_memory.py::test_env_var"
-    # these tests require access to s3 via the internet
-    "--deselect=pyarrow/tests/test_fs.py::test_resolve_s3_region"
-    "--deselect=pyarrow/tests/test_fs.py::test_s3_real_aws"
-    "--deselect=pyarrow/tests/test_fs.py::test_s3_real_aws_region_selection"
-    "--deselect=pyarrow/tests/test_fs.py::test_s3_options"
+  disabledTestPaths = [
+    # These tests require access to s3 via the internet.
+    "pyarrow/tests/test_fs.py::test_resolve_s3_region"
+    "pyarrow/tests/test_fs.py::test_s3_finalize"
+    "pyarrow/tests/test_fs.py::test_s3_finalize_region_resolver"
+    "pyarrow/tests/test_fs.py::test_s3_real_aws"
+    "pyarrow/tests/test_fs.py::test_s3_real_aws_region_selection"
+    "pyarrow/tests/test_fs.py::test_s3_options"
     # Flaky test
-    "--deselect=pyarrow/tests/test_flight.py::test_roundtrip_errors"
-    "--deselect=pyarrow/tests/test_pandas.py::test_threaded_pandas_import"
-    # Flaky test, works locally but not on Hydra
-    "--deselect=pyarrow/tests/test_csv.py::TestThreadedCSVTableRead::test_cancellation"
-    # expects arrow-cpp headers to be bundled
-    "--deselect=pyarrow/tests/test_cpp_internals.py::test_pyarrow_include"
+    "pyarrow/tests/test_flight.py::test_roundtrip_errors"
+    "pyarrow/tests/test_pandas.py::test_threaded_pandas_import"
+    # Flaky test, works locally but not on Hydra.
+    "pyarrow/tests/test_csv.py::TestThreadedCSVTableRead::test_cancellation"
+    # expects arrow-cpp headers to be bundled.
+    "pyarrow/tests/test_cpp_internals.py::test_pyarrow_include"
+    # Searches for TZDATA in /usr.
+    "pyarrow/tests/test_orc.py::test_example_using_json"
+    # AssertionError: assert 'Europe/Monaco' == 'Europe/Paris'
+    "pyarrow/tests/test_types.py::test_dateutil_tzinfo_to_string"
+    # These fail with xxx_fixture not found.
+    # xxx = unary_func, unary_agg_func, varargs_agg_func
+    "pyarrow/tests/test_substrait.py::test_udf_via_substrait"
+    "pyarrow/tests/test_substrait.py::test_scalar_aggregate_udf_basic"
+    "pyarrow/tests/test_substrait.py::test_hash_aggregate_udf_basic"
+    "pyarrow/tests/test_udf.py::test_hash_agg_basic"
+    "pyarrow/tests/test_udf.py::test_hash_agg_empty"
+    "pyarrow/tests/test_udf.py::test_input_lifetime"
+    "pyarrow/tests/test_udf.py::test_scalar_agg_basic"
+    "pyarrow/tests/test_udf.py::test_scalar_agg_empty"
+    "pyarrow/tests/test_udf.py::test_scalar_agg_varargs"
+    "pyarrow/tests/test_udf.py::test_scalar_input"
+    "pyarrow/tests/test_udf.py::test_scalar_udf_context"
+    "pyarrow/tests/test_udf.py::test_udf_array_unary"
   ]
   ++ lib.optionals stdenv.hostPlatform.isDarwin [
-    # Requires loopback networking
-    "--deselect=pyarrow/tests/test_ipc.py::test_socket_"
-    "--deselect=pyarrow/tests/test_flight.py::test_never_sends_data"
-    "--deselect=pyarrow/tests/test_flight.py::test_large_descriptor"
-    "--deselect=pyarrow/tests/test_flight.py::test_large_metadata_client"
-    "--deselect=pyarrow/tests/test_flight.py::test_none_action_side_effect"
-    # fails to compile
-    "--deselect=pyarrow/tests/test_cython.py::test_cython_api"
+    # Requires loopback networking.
+    "pyarrow/tests/test_ipc.py::test_socket_"
+    "pyarrow/tests/test_flight.py::test_never_sends_data"
+    "pyarrow/tests/test_flight.py::test_large_descriptor"
+    "pyarrow/tests/test_flight.py::test_large_metadata_client"
+    "pyarrow/tests/test_flight.py::test_none_action_side_effect"
+    # Fails to compile.
+    "pyarrow/tests/test_cython.py::test_cython_api"
   ]
   ++ lib.optionals (pythonAtLeast "3.11") [
     # Repr output is printing number instead of enum name so these tests fail
-    "--deselect=pyarrow/tests/test_fs.py::test_get_file_info"
+    "pyarrow/tests/test_fs.py::test_get_file_info"
   ]
   ++ lib.optionals stdenv.hostPlatform.isLinux [
-    # this test requires local networking
-    "--deselect=pyarrow/tests/test_fs.py::test_filesystem_from_uri_gcs"
+    # This test requires local networking.
+    "pyarrow/tests/test_fs.py::test_filesystem_from_uri_gcs"
   ];
 
   disabledTests = [ "GcsFileSystem" ];
 
   preCheck = ''
+    export PARQUET_TEST_DATA="${arrow-cpp.PARQUET_TEST_DATA}"
     shopt -s extglob
     rm -r pyarrow/!(conftest.py|tests)
     mv pyarrow/conftest.py pyarrow/tests/parent_conftest.py
-    substituteInPlace pyarrow/tests/conftest.py --replace ..conftest .parent_conftest
+    substituteInPlace pyarrow/tests/conftest.py --replace-fail ..conftest .parent_conftest
   ''
   + lib.optionalString stdenv.hostPlatform.isDarwin ''
     # OSError: [Errno 24] Too many open files

pkgs/development/python-modules/streamlit/default.nix

@@ -7,6 +7,7 @@
   cachetools,
   click,
   fetchPypi,
+  fetchpatch2,
   gitpython,
   numpy,
   packaging,
@@ -38,6 +39,16 @@ buildPythonPackage rec {
     hash = "sha256-HnQqnAtpj0Zsb1v1jTM77aWh++jeZgdDl2eRtcFEbvY=";
   };
 
+  patches = [
+    # Allow pyarrow 22
+    (fetchpatch2 {
+      url = "https://github.com/streamlit/streamlit/commit/b9e1b875a948a0aa6e972edc6e86a4f89706e08c.diff?full_index=1";
+      stripLen = 1;
+      excludes = [ "tests/streamlit/data_test_cases.py" ];
+      hash = "sha256-qZau1XlP8Kf2hPtyFphJN4UEjbp0ZZYngFdRwXTVt3g=";
+    })
+  ];
+
   build-system = [
     setuptools
   ];