From 1ea6a63088673f307b94c59c13a1827575d1e075 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 28 Oct 2024 07:46:41 -0400 Subject: [PATCH 01/10] arrow-cpp: 17.0.0 -> 18.0.0 --- pkgs/by-name/ar/arrow-cpp/package.nix | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkgs/by-name/ar/arrow-cpp/package.nix b/pkgs/by-name/ar/arrow-cpp/package.nix index 04a5678f38f1..453b068adde1 100644 --- a/pkgs/by-name/ar/arrow-cpp/package.nix +++ b/pkgs/by-name/ar/arrow-cpp/package.nix @@ -65,28 +65,29 @@ let name = "arrow-testing"; owner = "apache"; repo = "arrow-testing"; - rev = "735ae7128d571398dd798d7ff004adebeb342883"; - hash = "sha256-67KwnSt+EeEDvk+9kxR51tErL2wJqEPRITKb/dN+HMQ="; + rev = "4d209492d514c2d3cb2d392681b9aa00e6d8da1c"; + hash = "sha256-IkiCbuy0bWyClPZ4ZEdkEP7jFYLhM7RCuNLd6Lazd4o="; }; parquet-testing = fetchFromGitHub { name = "parquet-testing"; owner = "apache"; repo = "parquet-testing"; - rev = "74278bc4a1122d74945969e6dec405abd1533ec3"; - hash = "sha256-WbpndtAviph6+I/F2bevuMI9DkfSv4SMPgMaP98k6Qo="; + rev = "a7f1d288e693dbb08e3199851c4eb2140ff8dff2"; + hash = "sha256-zLWJOWcW7OYL32OwBm9VFtHbmG+ibhteRfHlKr9G3CQ="; }; + version = "18.0.0"; in stdenv.mkDerivation (finalAttrs: { pname = "arrow-cpp"; - version = "17.0.0"; + inherit version; src = fetchFromGitHub { owner = "apache"; repo = "arrow"; - rev = "apache-arrow-17.0.0"; - hash = "sha256-ZQqi1RFb4Ey0A0UVCThuIxM7DoFfkLwaeRAc2z8u9so="; + rev = "apache-arrow-${version}"; + hash = "sha256-V2lOYOUJwXSvPPk2G17uc1eZO88EATHKwwDnEroBrPw="; }; sourceRoot = "${finalAttrs.src.name}/cpp"; From c9ae493b654cb16ce6fad1e842569e9ef40e1ea5 Mon Sep 17 00:00:00 2001 From: Roland Coeurjoly Date: Sun, 25 Aug 2024 17:51:44 +0200 Subject: [PATCH 02/10] arrow-cpp: substitute --replace with --replace-fail --- pkgs/by-name/ar/arrow-cpp/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/by-name/ar/arrow-cpp/package.nix b/pkgs/by-name/ar/arrow-cpp/package.nix index 453b068adde1..7bf375d00994 100644 --- a/pkgs/by-name/ar/arrow-cpp/package.nix +++ b/pkgs/by-name/ar/arrow-cpp/package.nix @@ -172,7 +172,7 @@ stdenv.mkDerivation (finalAttrs: { preConfigure = '' patchShebangs build-support/ substituteInPlace "src/arrow/vendored/datetime/tz.cpp" \ - --replace 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";' + --replace-fail 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";' ''; cmakeFlags = From eeff2e3d9b15cf525690c901e7f3de73c68b3468 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Tue, 29 Oct 2024 06:11:23 -0400 Subject: [PATCH 03/10] ceph: patch to pick up upstream arrow 18 s3select fix --- pkgs/tools/filesystems/ceph/default.nix | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix index 25075f612df8..004d0bbca94b 100644 --- a/pkgs/tools/filesystems/ceph/default.nix +++ b/pkgs/tools/filesystems/ceph/default.nix @@ -4,7 +4,7 @@ , fetchurl , fetchFromGitHub , fetchPypi -, fetchpatch +, fetchpatch2 # Build time , autoconf @@ -314,6 +314,16 @@ in rec { pname = "ceph"; inherit src version; + patches = [ + (fetchpatch2 { + name = "ceph-s3select-arrow-18-compat.patch"; + url = "https://github.com/ceph/s3select/commit/f333ec82e6e8a3f7eb9ba1041d1442b2c7cd0f05.patch"; + hash = "sha256-21fi5tMIs/JmuhwPYMWtampv/aqAe+EoPAXZLJlOvgo="; + stripLen = 1; + extraPrefix = "src/s3select/"; + }) + ]; + nativeBuildInputs = [ autoconf # `autoreconf` is called, e.g. for `qatlib_ext` automake # `aclocal` is called, e.g. for `qatlib_ext` From 8b47fb90fe88b5379f07cf0a74d56cfd48643cc2 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:30:14 -0400 Subject: [PATCH 04/10] gdal: apply upstream patch for arrow 18 --- pkgs/development/libraries/gdal/default.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/development/libraries/gdal/default.nix b/pkgs/development/libraries/gdal/default.nix index 841e0808d433..7743f1e9e98d 100644 --- a/pkgs/development/libraries/gdal/default.nix +++ b/pkgs/development/libraries/gdal/default.nix @@ -94,6 +94,11 @@ stdenv.mkDerivation (finalAttrs: { url = "https://github.com/OSGeo/gdal/commit/40c3212fe4ba93e5176df4cd8ae5e29e06bb6027.patch"; sha256 = "sha256-D55iT6E/YdpSyfN7KUDTh1gdmIDLHXW4VC5d6D9B7ls="; }) + (fetchpatch { + name = "arrow-18.patch"; + url = "https://github.com/OSGeo/gdal/commit/9a8c5c031404bbc81445291bad128bc13766cafa.patch"; + sha256 = "sha256-tF46DmF7ZReqY8ACTTPXohWLsRn8lVxhKF1s+r254KM="; + }) ]; nativeBuildInputs = [ From 3bd5960a6cd57bbd3ea527dc8a5bc14088926f93 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:49:59 -0400 Subject: [PATCH 05/10] python3.pkgs.datafusion: add protoc --- pkgs/development/python-modules/datafusion/default.nix | 2 ++ pkgs/top-level/python-packages.nix | 1 + 2 files changed, 3 insertions(+) diff --git a/pkgs/development/python-modules/datafusion/default.nix b/pkgs/development/python-modules/datafusion/default.nix index 96ac76d17530..9badd981aed4 100644 --- a/pkgs/development/python-modules/datafusion/default.nix +++ b/pkgs/development/python-modules/datafusion/default.nix @@ -8,6 +8,7 @@ libiconv, numpy, protobuf, + protoc, pyarrow, Security, SystemConfiguration, @@ -55,6 +56,7 @@ buildPythonPackage rec { nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook + protoc ]; buildInputs = diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 1cc47777528b..be108878449c 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2920,6 +2920,7 @@ self: super: with self; { datafusion = callPackage ../development/python-modules/datafusion { inherit (pkgs.darwin.apple_sdk.frameworks) Security SystemConfiguration; + protoc = pkgs.protobuf; }; datalad = callPackage ../development/python-modules/datalad { }; From 6c0def8319256fe880bcf0dac2483ca78d7eaac7 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 06:36:03 -0400 Subject: [PATCH 06/10] python3.pkgs.ibis-framework: patch tests for arrow 18 --- .../python-modules/ibis-framework/default.nix | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/pkgs/development/python-modules/ibis-framework/default.nix b/pkgs/development/python-modules/ibis-framework/default.nix index 080168ec2287..5a4ab305b1b7 100644 --- a/pkgs/development/python-modules/ibis-framework/default.nix +++ b/pkgs/development/python-modules/ibis-framework/default.nix @@ -54,11 +54,7 @@ typing-extensions, }: let - testBackends = [ - "duckdb" - "sqlite" - "datafusion" - ]; + testBackends = [ "duckdb" "sqlite" ]; ibisTestingData = fetchFromGitHub { name = "ibis-testing-data"; @@ -92,9 +88,17 @@ buildPythonPackage rec { url = "https://github.com/ibis-project/ibis/commit/a54eceabac1d6592e9f6ab0ca7749e37a748c2ad.patch"; hash = "sha256-j5BPYVqnEF9GQV5N3/VhFUCdsEwAIOQC0KfZ5LNBSRg="; }) + + # remove after the 10.0 release + (fetchpatch { + name = "ibis-framework-arrow-18.patch"; + url = "https://github.com/ibis-project/ibis/commit/5dc549b22c2eca29a11a31fb29deef7c1466a204.patch"; + hash = "sha256-4i/g2uixdlkbE6x659wzZJ91FZpzwOVkF6ZeXkiCP3I="; + excludes = [ "poetry.lock" "requirements-dev.txt" ]; + }) ]; - nativeBuildInputs = [ + build-system = [ poetry-core poetry-dynamic-versioning ]; @@ -102,7 +106,7 @@ buildPythonPackage rec { dontBypassPoetryDynamicVersioning = true; env.POETRY_DYNAMIC_VERSIONING_BYPASS = lib.head (lib.strings.splitString "-" version); - propagatedBuildInputs = [ + dependencies = [ atpublic parsy python-dateutil @@ -128,13 +132,7 @@ buildPythonPackage rec { pytest-xdist ] ++ lib.concatMap (name: optional-dependencies.${name}) testBackends; - dontUsePytestXdist = true; - - pytestFlagsArray = [ - "-m" - # tpcds and tpch are slow, so disable them - "'not tpcds and not tpch and (${lib.concatStringsSep " or " testBackends} or core)'" - ]; + pytestFlagsArray = [ "-m" "'${lib.concatStringsSep " or " testBackends} or core'" ]; disabledTests = [ # tries to download duckdb extensions From 982f812ae9e6d22e8e002e08e921163454d902ec Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 07:53:27 -0400 Subject: [PATCH 07/10] chore: style --- .../python-modules/ibis-framework/default.nix | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/ibis-framework/default.nix b/pkgs/development/python-modules/ibis-framework/default.nix index 5a4ab305b1b7..ac33fe0bc15d 100644 --- a/pkgs/development/python-modules/ibis-framework/default.nix +++ b/pkgs/development/python-modules/ibis-framework/default.nix @@ -54,7 +54,10 @@ typing-extensions, }: let - testBackends = [ "duckdb" "sqlite" ]; + testBackends = [ + "duckdb" + "sqlite" + ]; ibisTestingData = fetchFromGitHub { name = "ibis-testing-data"; @@ -94,7 +97,10 @@ buildPythonPackage rec { name = "ibis-framework-arrow-18.patch"; url = "https://github.com/ibis-project/ibis/commit/5dc549b22c2eca29a11a31fb29deef7c1466a204.patch"; hash = "sha256-4i/g2uixdlkbE6x659wzZJ91FZpzwOVkF6ZeXkiCP3I="; - excludes = [ "poetry.lock" "requirements-dev.txt" ]; + excludes = [ + "poetry.lock" + "requirements-dev.txt" + ]; }) ]; @@ -132,7 +138,10 @@ buildPythonPackage rec { pytest-xdist ] ++ lib.concatMap (name: optional-dependencies.${name}) testBackends; - pytestFlagsArray = [ "-m" "'${lib.concatStringsSep " or " testBackends} or core'" ]; + pytestFlagsArray = [ + "-m" + "'${lib.concatStringsSep " or " testBackends} or core'" + ]; disabledTests = [ # tries to download duckdb extensions From cb2ef38a117796ee140dd292189b30f9b1d29336 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 31 Oct 2024 07:19:04 -0400 Subject: [PATCH 08/10] arrow-cpp: fix `pkgsStatic` --- pkgs/by-name/ar/arrow-cpp/package.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/ar/arrow-cpp/package.nix b/pkgs/by-name/ar/arrow-cpp/package.nix index 7bf375d00994..b97653f2db07 100644 --- a/pkgs/by-name/ar/arrow-cpp/package.nix +++ b/pkgs/by-name/ar/arrow-cpp/package.nix @@ -35,6 +35,7 @@ nlohmann_json, openssl, perl, + pkg-config, protobuf, python3, rapidjson, @@ -49,7 +50,7 @@ zstd, testers, enableShared ? !stdenv.hostPlatform.isStatic, - enableFlight ? true, + enableFlight ? stdenv.buildPlatform == stdenv.hostPlatform, enableJemalloc ? !stdenv.hostPlatform.isDarwin, enableS3 ? true, enableGcs ? !stdenv.hostPlatform.isDarwin, @@ -126,6 +127,7 @@ stdenv.mkDerivation (finalAttrs: { nativeBuildInputs = [ cmake + pkg-config ninja autoconf # for vendored jemalloc flatbuffers @@ -180,7 +182,7 @@ stdenv.mkDerivation (finalAttrs: { "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON" "-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}" "-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}" - "-DARROW_BUILD_TESTS=ON" + "-DARROW_BUILD_TESTS=${if enableShared then "ON" else "OFF"}" "-DARROW_BUILD_INTEGRATION=ON" "-DARROW_BUILD_UTILITIES=ON" "-DARROW_EXTRA_ERROR_CONTEXT=ON" From 29069e4fef740b657525c9dfa979fc4e8c7e15e4 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Tue, 5 Nov 2024 07:19:21 -0500 Subject: [PATCH 09/10] arrow-cpp: disable jemalloc for arm hosts by default --- pkgs/by-name/ar/arrow-cpp/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/by-name/ar/arrow-cpp/package.nix b/pkgs/by-name/ar/arrow-cpp/package.nix index b97653f2db07..81b230d09179 100644 --- a/pkgs/by-name/ar/arrow-cpp/package.nix +++ b/pkgs/by-name/ar/arrow-cpp/package.nix @@ -51,7 +51,7 @@ testers, enableShared ? !stdenv.hostPlatform.isStatic, enableFlight ? stdenv.buildPlatform == stdenv.hostPlatform, - enableJemalloc ? !stdenv.hostPlatform.isDarwin, + enableJemalloc ? !stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isAarch64, enableS3 ? true, enableGcs ? !stdenv.hostPlatform.isDarwin, }: From a6388366ac19a42b72faa90304ffb6af5c1da555 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:17:17 -0500 Subject: [PATCH 10/10] python3.pkgs.databricks-sql-connector: fix build by removing patch and relaxing thrift dependency --- .../databricks-sql-connector/default.nix | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pkgs/development/python-modules/databricks-sql-connector/default.nix b/pkgs/development/python-modules/databricks-sql-connector/default.nix index 923cfa7a8cb2..2c3243a11576 100644 --- a/pkgs/development/python-modules/databricks-sql-connector/default.nix +++ b/pkgs/development/python-modules/databricks-sql-connector/default.nix @@ -16,7 +16,6 @@ thrift, requests, urllib3, - fetchpatch, }: buildPythonPackage rec { @@ -33,16 +32,9 @@ buildPythonPackage rec { hash = "sha256-8q5qWN+i2mGbzXvkop/G6mjZegzZ/6kr1Fl7FaHwLYA="; }; - patches = [ - (fetchpatch { - name = "fix-pandas.patch"; - url = "https://patch-diff.githubusercontent.com/raw/databricks/databricks-sql-python/pull/416.patch"; - sha256 = "sha256-sNCp8xSSmKP2yNzDK4wyWC5Hoe574AeHnKTeNcIxaek="; - }) - ]; - pythonRelaxDeps = [ "pyarrow" + "thrift" ]; nativeBuildInputs = [