arrow-cpp: 17.0.0 -> 18.0.0 (#351868)

pkgs/by-name/ar/arrow-cpp/package.nix

@@ -35,6 +35,7 @@
   nlohmann_json,
   openssl,
   perl,
+  pkg-config,
   protobuf,
   python3,
   rapidjson,
@@ -49,8 +50,8 @@
   zstd,
   testers,
   enableShared ? !stdenv.hostPlatform.isStatic,
-  enableFlight ? true,
-  enableJemalloc ? !stdenv.hostPlatform.isDarwin,
+  enableFlight ? stdenv.buildPlatform == stdenv.hostPlatform,
+  enableJemalloc ? !stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isAarch64,
   enableS3 ? true,
   enableGcs ? !stdenv.hostPlatform.isDarwin,
 }:
@@ -65,28 +66,29 @@ let
     name = "arrow-testing";
     owner = "apache";
     repo = "arrow-testing";
-    rev = "735ae7128d571398dd798d7ff004adebeb342883";
-    hash = "sha256-67KwnSt+EeEDvk+9kxR51tErL2wJqEPRITKb/dN+HMQ=";
+    rev = "4d209492d514c2d3cb2d392681b9aa00e6d8da1c";
+    hash = "sha256-IkiCbuy0bWyClPZ4ZEdkEP7jFYLhM7RCuNLd6Lazd4o=";
   };
 
   parquet-testing = fetchFromGitHub {
     name = "parquet-testing";
     owner = "apache";
     repo = "parquet-testing";
-    rev = "74278bc4a1122d74945969e6dec405abd1533ec3";
-    hash = "sha256-WbpndtAviph6+I/F2bevuMI9DkfSv4SMPgMaP98k6Qo=";
+    rev = "a7f1d288e693dbb08e3199851c4eb2140ff8dff2";
+    hash = "sha256-zLWJOWcW7OYL32OwBm9VFtHbmG+ibhteRfHlKr9G3CQ=";
   };
 
+  version = "18.0.0";
 in
 stdenv.mkDerivation (finalAttrs: {
   pname = "arrow-cpp";
-  version = "17.0.0";
+  inherit version;
 
   src = fetchFromGitHub {
     owner = "apache";
     repo = "arrow";
-    rev = "apache-arrow-17.0.0";
-    hash = "sha256-ZQqi1RFb4Ey0A0UVCThuIxM7DoFfkLwaeRAc2z8u9so=";
+    rev = "apache-arrow-${version}";
+    hash = "sha256-V2lOYOUJwXSvPPk2G17uc1eZO88EATHKwwDnEroBrPw=";
   };
 
   sourceRoot = "${finalAttrs.src.name}/cpp";
@@ -125,6 +127,7 @@ stdenv.mkDerivation (finalAttrs: {
 
   nativeBuildInputs = [
     cmake
+    pkg-config
     ninja
     autoconf # for vendored jemalloc
     flatbuffers
@@ -171,7 +174,7 @@ stdenv.mkDerivation (finalAttrs: {
   preConfigure = ''
     patchShebangs build-support/
     substituteInPlace "src/arrow/vendored/datetime/tz.cpp" \
-      --replace 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";'
+      --replace-fail 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";'
   '';
 
   cmakeFlags =
@@ -179,7 +182,7 @@ stdenv.mkDerivation (finalAttrs: {
       "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON"
       "-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}"
       "-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}"
-      "-DARROW_BUILD_TESTS=ON"
+      "-DARROW_BUILD_TESTS=${if enableShared then "ON" else "OFF"}"
       "-DARROW_BUILD_INTEGRATION=ON"
       "-DARROW_BUILD_UTILITIES=ON"
       "-DARROW_EXTRA_ERROR_CONTEXT=ON"

pkgs/development/libraries/gdal/default.nix

@@ -94,6 +94,11 @@ stdenv.mkDerivation (finalAttrs: {
       url = "https://github.com/OSGeo/gdal/commit/40c3212fe4ba93e5176df4cd8ae5e29e06bb6027.patch";
       sha256 = "sha256-D55iT6E/YdpSyfN7KUDTh1gdmIDLHXW4VC5d6D9B7ls=";
     })
+    (fetchpatch {
+      name = "arrow-18.patch";
+      url = "https://github.com/OSGeo/gdal/commit/9a8c5c031404bbc81445291bad128bc13766cafa.patch";
+      sha256 = "sha256-tF46DmF7ZReqY8ACTTPXohWLsRn8lVxhKF1s+r254KM=";
+    })
   ];
 
   nativeBuildInputs = [

pkgs/development/python-modules/databricks-sql-connector/default.nix

@@ -16,7 +16,6 @@
   thrift,
   requests,
   urllib3,
-  fetchpatch,
 }:
 
 buildPythonPackage rec {
@@ -33,16 +32,9 @@ buildPythonPackage rec {
     hash = "sha256-8q5qWN+i2mGbzXvkop/G6mjZegzZ/6kr1Fl7FaHwLYA=";
   };
 
-  patches = [
-    (fetchpatch {
-      name = "fix-pandas.patch";
-      url = "https://patch-diff.githubusercontent.com/raw/databricks/databricks-sql-python/pull/416.patch";
-      sha256 = "sha256-sNCp8xSSmKP2yNzDK4wyWC5Hoe574AeHnKTeNcIxaek=";
-    })
-  ];
-
   pythonRelaxDeps = [
     "pyarrow"
+    "thrift"
   ];
 
   nativeBuildInputs = [

pkgs/development/python-modules/datafusion/default.nix

@@ -8,6 +8,7 @@
   libiconv,
   numpy,
   protobuf,
+  protoc,
   pyarrow,
   Security,
   SystemConfiguration,
@@ -55,6 +56,7 @@ buildPythonPackage rec {
   nativeBuildInputs = with rustPlatform; [
     cargoSetupHook
     maturinBuildHook
+    protoc
   ];
 
   buildInputs =

pkgs/development/python-modules/ibis-framework/default.nix

@@ -57,7 +57,6 @@ let
   testBackends = [
     "duckdb"
     "sqlite"
-    "datafusion"
   ];
 
   ibisTestingData = fetchFromGitHub {
@@ -92,9 +91,20 @@ buildPythonPackage rec {
       url = "https://github.com/ibis-project/ibis/commit/a54eceabac1d6592e9f6ab0ca7749e37a748c2ad.patch";
       hash = "sha256-j5BPYVqnEF9GQV5N3/VhFUCdsEwAIOQC0KfZ5LNBSRg=";
     })
+
+    # remove after the 10.0 release
+    (fetchpatch {
+      name = "ibis-framework-arrow-18.patch";
+      url = "https://github.com/ibis-project/ibis/commit/5dc549b22c2eca29a11a31fb29deef7c1466a204.patch";
+      hash = "sha256-4i/g2uixdlkbE6x659wzZJ91FZpzwOVkF6ZeXkiCP3I=";
+      excludes = [
+        "poetry.lock"
+        "requirements-dev.txt"
+      ];
+    })
   ];
 
-  nativeBuildInputs = [
+  build-system = [
     poetry-core
     poetry-dynamic-versioning
   ];
@@ -102,7 +112,7 @@ buildPythonPackage rec {
   dontBypassPoetryDynamicVersioning = true;
   env.POETRY_DYNAMIC_VERSIONING_BYPASS = lib.head (lib.strings.splitString "-" version);
 
-  propagatedBuildInputs = [
+  dependencies = [
     atpublic
     parsy
     python-dateutil
@@ -128,12 +138,9 @@ buildPythonPackage rec {
     pytest-xdist
   ] ++ lib.concatMap (name: optional-dependencies.${name}) testBackends;
 
-  dontUsePytestXdist = true;
-
   pytestFlagsArray = [
     "-m"
-    # tpcds and tpch are slow, so disable them
-    "'not tpcds and not tpch and (${lib.concatStringsSep " or " testBackends} or core)'"
+    "'${lib.concatStringsSep " or " testBackends} or core'"
   ];
 
   disabledTests = [

pkgs/tools/filesystems/ceph/default.nix

@@ -4,7 +4,7 @@
 , fetchurl
 , fetchFromGitHub
 , fetchPypi
-, fetchpatch
+, fetchpatch2
 
 # Build time
 , autoconf
@@ -314,6 +314,16 @@ in rec {
     pname = "ceph";
     inherit src version;
 
+    patches = [
+      (fetchpatch2 {
+        name = "ceph-s3select-arrow-18-compat.patch";
+        url = "https://github.com/ceph/s3select/commit/f333ec82e6e8a3f7eb9ba1041d1442b2c7cd0f05.patch";
+        hash = "sha256-21fi5tMIs/JmuhwPYMWtampv/aqAe+EoPAXZLJlOvgo=";
+        stripLen = 1;
+        extraPrefix = "src/s3select/";
+      })
+    ];
+
     nativeBuildInputs = [
       autoconf # `autoreconf` is called, e.g. for `qatlib_ext`
       automake # `aclocal` is called, e.g. for `qatlib_ext`

pkgs/top-level/python-packages.nix

@@ -2920,6 +2920,7 @@ self: super: with self; {
 
   datafusion = callPackage ../development/python-modules/datafusion {
     inherit (pkgs.darwin.apple_sdk.frameworks) Security SystemConfiguration;
+    protoc = pkgs.protobuf;
   };
 
   datalad = callPackage ../development/python-modules/datalad { };