Merge pull request #185124 from cpcloud/arrow-9

pkgs/development/libraries/arrow-cpp/default.nix

@@ -18,7 +18,6 @@
 , google-cloud-cpp
 , grpc
 , gtest
-, jemalloc
 , libbacktrace
 , lz4
 , minio
@@ -56,55 +55,58 @@ let
   arrow-testing = fetchFromGitHub {
     owner = "apache";
     repo = "arrow-testing";
-    rev = "634739c664433cec366b4b9a81d1e1044a8c5eda";
-    hash = "sha256-r1WVgJJsI7v485L6Qb+5i7kFO4Tvxyk1T0JBb4og6pg=";
+    rev = "5bab2f264a23f5af68f69ea93d24ef1e8e77fc88";
+    hash = "sha256-Pxx8ohUpXb5u1995IvXmxQMqWiDJ+7LAll/AjQP7ph8=";
   };
 
   parquet-testing = fetchFromGitHub {
     owner = "apache";
     repo = "parquet-testing";
-    rev = "acd375eb86a81cd856476fca0f52ba6036a067ff";
-    hash = "sha256-z/kmi+4dBO/dsVkJA4NgUoxl0pXi8RWIGvI8MGu/gcc=";
+    rev = "aafd3fc9df431c2625a514fb46626e5614f1d199";
+    hash = "sha256-cO5t/mgsbBhbSefx8EMGTyxmgTjhZ8mFujkFQ3p/JS0=";
   };
 
 in
 stdenv.mkDerivation rec {
   pname = "arrow-cpp";
-  version = "8.0.0";
+  version = "9.0.0";
 
   src = fetchurl {
     url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz";
-    hash = "sha256-rZoFcFEXyYnBFrrprHBJL+AVBQ4bgPsOOP3ktdhjqqM=";
+    hash = "sha256-qaAz8KNJAomZj0WGgNGVec8HkRcXumWv3my4AHD3qbU=";
   };
   sourceRoot = "apache-arrow-${version}/cpp";
 
   # versions are all taken from
-  # https://github.com/apache/arrow/blob/apache-arrow-8.0.0/cpp/thirdparty/versions.txt
+  # https://github.com/apache/arrow/blob/apache-arrow-${version}/cpp/thirdparty/versions.txt
 
+  # jemalloc: arrow uses a custom prefix to prevent default allocator symbol
+  # collisions as well as custom build flags
   ${if enableJemalloc then "ARROW_JEMALLOC_URL" else null} = fetchurl {
-    url = "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2";
-    hash = "sha256-NDMOXOJ2CZ4uiVDZM121qHVomkxqVnUe87HYxTf4h/Y=";
+    url = "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2";
+    hash = "sha256-LbgtHnEZ3z5xt2QCGbbf6EeJvAU3mDw7esT3GJrs/qo=";
   };
 
+  # mimalloc: arrow uses custom build flags for mimalloc
   ARROW_MIMALLOC_URL = fetchFromGitHub {
     owner = "microsoft";
     repo = "mimalloc";
-    rev = "v1.7.3";
-    hash = "sha256-Ca877VitpWyKmZNHavqgewk/P+tyd2xHDNVqveKh87M=";
+    rev = "v2.0.6";
+    hash = "sha256-u2ITXABBN/dwU+mCIbL3tN1f4c17aBuSdNTV+Adtohc=";
   };
 
   ARROW_XSIMD_URL = fetchFromGitHub {
     owner = "xtensor-stack";
     repo = "xsimd";
-    rev = "7d1778c3b38d63db7cec7145d939f40bc5d859d1";
-    hash = "sha256-89AysBUVnTdWyMPazeJegnQ6WEH90Ns7qQInZLMSXY4=";
+    rev = "8.1.0";
+    hash = "sha256-Aqs6XJkGjAjGAp0PprabSM4m+32M/UXpSHppCHdzaZk=";
   };
 
   ARROW_SUBSTRAIT_URL = fetchFromGitHub {
     owner = "substrait-io";
     repo = "substrait";
-    rev = "e1b4c04a1b518912f4c4065b16a1b2c0ac8e14cf";
-    hash = "sha256-56FSjDngsROSHLjMv+OYAIYqphEu3GzgIMHbgh/ZQw0=";
+    rev = "v0.6.0";
+    hash = "sha256-hxCBomL4Qg9cHLRg9ZiO9k+JVOZXn6f4ikPtK+V9tno=";
   };
 
   patches = [
@@ -159,7 +161,6 @@ stdenv.mkDerivation rec {
   '';
 
   cmakeFlags = [
-    "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON"
     "-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}"
     "-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}"
     "-DARROW_BUILD_TESTS=ON"
@@ -168,7 +169,7 @@ stdenv.mkDerivation rec {
     "-DARROW_EXTRA_ERROR_CONTEXT=ON"
     "-DARROW_VERBOSE_THIRDPARTY_BUILD=ON"
     "-DARROW_DEPENDENCY_SOURCE=SYSTEM"
-    "-DThrift_SOURCE=AUTO" # search for Thrift using pkg-config (ThriftConfig.cmake requires OpenSSL and libevent)
+    "-Dxsimd_SOURCE=AUTO"
     "-DARROW_DEPENDENCY_USE_SHARED=${if enableShared then "ON" else "OFF"}"
     "-DARROW_COMPUTE=ON"
     "-DARROW_CSV=ON"
@@ -229,6 +230,7 @@ stdenv.mkDerivation rec {
       ];
     in
     lib.optionalString doInstallCheck "-${builtins.concatStringsSep ":" filteredTests}";
+  __darwinAllowLocalNetworking = true;
   installCheckInputs = [ perl which sqlite ] ++ lib.optional enableS3 minio;
   installCheckPhase =
     let

pkgs/development/python-modules/apache-beam/default.nix

@@ -55,7 +55,9 @@ buildPythonPackage rec {
   postPatch = ''
     substituteInPlace setup.py \
       --replace "dill>=0.3.1.1,<0.3.2" "dill" \
-      --replace "pyarrow>=0.15.1,<8.0.0" "pyarrow"
+      --replace "pyarrow>=0.15.1,<8.0.0" "pyarrow" \
+      --replace "numpy>=1.14.3,<1.23.0" "numpy" \
+      --replace "pymongo>=3.8.0,<4.0.0" "pymongo"
   '';
 
   sourceRoot = "source/sdks/python";
@@ -90,6 +92,8 @@ buildPythonPackage rec {
     typing-extensions
   ];
 
+  enableParallelBuilding = true;
+
   pythonImportsCheck = [
     "apache_beam"
   ];

pkgs/development/python-modules/dask-gateway/default.nix

@@ -1,6 +1,6 @@
 { lib
 , buildPythonPackage
-, fetchPypi
+, fetchFromGitHub
 , aiohttp
 , dask
 , distributed
@@ -10,12 +10,17 @@ buildPythonPackage rec {
   pname = "dask-gateway";
   # update dask-gateway lock step with dask-gateway-server
   version = "2022.6.1";
+  format = "pyproject";
 
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "sha256-i0OFXjvDg+D4Sdyg6rluP0k6/Ecr+VZn+RiIEQONQX0=";
+  src = fetchFromGitHub {
+    owner = "dask";
+    repo = "dask-gateway";
+    rev = "refs/tags/${version}";
+    hash = "sha256-PsagZdEPpeuZH9hFL98xB5z6zOdd4Cx/skGQ0eOYkCA=";
   };
 
+  sourceRoot = "source/dask-gateway";
+
   propagatedBuildInputs = [
     aiohttp
     dask
@@ -31,6 +36,6 @@ buildPythonPackage rec {
     description = "A client library for interacting with a dask-gateway server";
     homepage = "https://gateway.dask.org/";
     license = licenses.bsd3;
-    maintainers = [ maintainers.costrouc ];
+    maintainers = with maintainers; [ costrouc ];
   };
 }

pkgs/development/python-modules/db-dtypes/default.nix

@@ -23,6 +23,11 @@ buildPythonPackage rec {
     hash = "sha256-LLKhYLzGUQRx4ciWv1TilYvTOO0sj6rdkPlJLPZ8VXA=";
   };
 
+  postPatch = ''
+    substituteInPlace setup.py \
+      --replace 'pyarrow>=3.0.0, <9.0dev' 'pyarrow>=3.0.0, <10.0dev'
+  '';
+
   propagatedBuildInputs = [
     numpy
     packaging

pkgs/development/python-modules/google-cloud-bigquery/default.nix

@@ -31,7 +31,7 @@ buildPythonPackage rec {
 
   postPatch = ''
     substituteInPlace setup.py \
-      --replace 'pyarrow >= 3.0.0, < 8.0dev' 'pyarrow >= 3.0.0, < 9.0dev'
+      --replace 'pyarrow >= 3.0.0, < 9.0dev' 'pyarrow >= 3.0.0, < 10.0dev'
   '';
 
   propagatedBuildInputs = [
@@ -84,6 +84,7 @@ buildPythonPackage rec {
     "test__initiate_resumable_upload"
     "test__initiate_resumable_upload_mtls"
     "test__initiate_resumable_upload_with_retry"
+    "test_table_clones"
   ];
 
   disabledTestPaths = [

pkgs/development/python-modules/pyarrow/default.nix

@@ -79,6 +79,8 @@ buildPythonPackage rec {
 
   dontUseCmakeConfigure = true;
 
+  __darwinAllowLocalNetworking = true;
+
   preBuild = ''
     export PYARROW_PARALLEL=$NIX_BUILD_CORES
   '';
@@ -111,8 +113,10 @@ buildPythonPackage rec {
 
   preCheck = ''
     shopt -s extglob
-    rm -r pyarrow/!(tests)
-  '' + lib.optionalString stdenv.isDarwin  ''
+    rm -r pyarrow/!(conftest.py|tests)
+    mv pyarrow/conftest.py pyarrow/tests/parent_conftest.py
+    substituteInPlace pyarrow/tests/conftest.py --replace ..conftest .parent_conftest
+  '' + lib.optionalString stdenv.isDarwin ''
     # OSError: [Errno 24] Too many open files
     ulimit -n 1024
   '';