diff --git a/pkgs/by-name/ar/arrow-cpp/package.nix b/pkgs/by-name/ar/arrow-cpp/package.nix index 04a5678f38f1..81b230d09179 100644 --- a/pkgs/by-name/ar/arrow-cpp/package.nix +++ b/pkgs/by-name/ar/arrow-cpp/package.nix @@ -35,6 +35,7 @@ nlohmann_json, openssl, perl, + pkg-config, protobuf, python3, rapidjson, @@ -49,8 +50,8 @@ zstd, testers, enableShared ? !stdenv.hostPlatform.isStatic, - enableFlight ? true, - enableJemalloc ? !stdenv.hostPlatform.isDarwin, + enableFlight ? stdenv.buildPlatform == stdenv.hostPlatform, + enableJemalloc ? !stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isAarch64, enableS3 ? true, enableGcs ? !stdenv.hostPlatform.isDarwin, }: @@ -65,28 +66,29 @@ let name = "arrow-testing"; owner = "apache"; repo = "arrow-testing"; - rev = "735ae7128d571398dd798d7ff004adebeb342883"; - hash = "sha256-67KwnSt+EeEDvk+9kxR51tErL2wJqEPRITKb/dN+HMQ="; + rev = "4d209492d514c2d3cb2d392681b9aa00e6d8da1c"; + hash = "sha256-IkiCbuy0bWyClPZ4ZEdkEP7jFYLhM7RCuNLd6Lazd4o="; }; parquet-testing = fetchFromGitHub { name = "parquet-testing"; owner = "apache"; repo = "parquet-testing"; - rev = "74278bc4a1122d74945969e6dec405abd1533ec3"; - hash = "sha256-WbpndtAviph6+I/F2bevuMI9DkfSv4SMPgMaP98k6Qo="; + rev = "a7f1d288e693dbb08e3199851c4eb2140ff8dff2"; + hash = "sha256-zLWJOWcW7OYL32OwBm9VFtHbmG+ibhteRfHlKr9G3CQ="; }; + version = "18.0.0"; in stdenv.mkDerivation (finalAttrs: { pname = "arrow-cpp"; - version = "17.0.0"; + inherit version; src = fetchFromGitHub { owner = "apache"; repo = "arrow"; - rev = "apache-arrow-17.0.0"; - hash = "sha256-ZQqi1RFb4Ey0A0UVCThuIxM7DoFfkLwaeRAc2z8u9so="; + rev = "apache-arrow-${version}"; + hash = "sha256-V2lOYOUJwXSvPPk2G17uc1eZO88EATHKwwDnEroBrPw="; }; sourceRoot = "${finalAttrs.src.name}/cpp"; @@ -125,6 +127,7 @@ stdenv.mkDerivation (finalAttrs: { nativeBuildInputs = [ cmake + pkg-config ninja autoconf # for vendored jemalloc flatbuffers @@ -171,7 +174,7 @@ stdenv.mkDerivation (finalAttrs: { preConfigure = '' patchShebangs build-support/ substituteInPlace "src/arrow/vendored/datetime/tz.cpp" \ - --replace 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";' + --replace-fail 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";' ''; cmakeFlags = @@ -179,7 +182,7 @@ stdenv.mkDerivation (finalAttrs: { "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON" "-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}" "-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}" - "-DARROW_BUILD_TESTS=ON" + "-DARROW_BUILD_TESTS=${if enableShared then "ON" else "OFF"}" "-DARROW_BUILD_INTEGRATION=ON" "-DARROW_BUILD_UTILITIES=ON" "-DARROW_EXTRA_ERROR_CONTEXT=ON" diff --git a/pkgs/development/libraries/gdal/default.nix b/pkgs/development/libraries/gdal/default.nix index 841e0808d433..7743f1e9e98d 100644 --- a/pkgs/development/libraries/gdal/default.nix +++ b/pkgs/development/libraries/gdal/default.nix @@ -94,6 +94,11 @@ stdenv.mkDerivation (finalAttrs: { url = "https://github.com/OSGeo/gdal/commit/40c3212fe4ba93e5176df4cd8ae5e29e06bb6027.patch"; sha256 = "sha256-D55iT6E/YdpSyfN7KUDTh1gdmIDLHXW4VC5d6D9B7ls="; }) + (fetchpatch { + name = "arrow-18.patch"; + url = "https://github.com/OSGeo/gdal/commit/9a8c5c031404bbc81445291bad128bc13766cafa.patch"; + sha256 = "sha256-tF46DmF7ZReqY8ACTTPXohWLsRn8lVxhKF1s+r254KM="; + }) ]; nativeBuildInputs = [ diff --git a/pkgs/development/python-modules/databricks-sql-connector/default.nix b/pkgs/development/python-modules/databricks-sql-connector/default.nix index 923cfa7a8cb2..2c3243a11576 100644 --- a/pkgs/development/python-modules/databricks-sql-connector/default.nix +++ b/pkgs/development/python-modules/databricks-sql-connector/default.nix @@ -16,7 +16,6 @@ thrift, requests, urllib3, - fetchpatch, }: buildPythonPackage rec { @@ -33,16 +32,9 @@ buildPythonPackage rec { hash = "sha256-8q5qWN+i2mGbzXvkop/G6mjZegzZ/6kr1Fl7FaHwLYA="; }; - patches = [ - (fetchpatch { - name = "fix-pandas.patch"; - url = "https://patch-diff.githubusercontent.com/raw/databricks/databricks-sql-python/pull/416.patch"; - sha256 = "sha256-sNCp8xSSmKP2yNzDK4wyWC5Hoe574AeHnKTeNcIxaek="; - }) - ]; - pythonRelaxDeps = [ "pyarrow" + "thrift" ]; nativeBuildInputs = [ diff --git a/pkgs/development/python-modules/datafusion/default.nix b/pkgs/development/python-modules/datafusion/default.nix index 96ac76d17530..9badd981aed4 100644 --- a/pkgs/development/python-modules/datafusion/default.nix +++ b/pkgs/development/python-modules/datafusion/default.nix @@ -8,6 +8,7 @@ libiconv, numpy, protobuf, + protoc, pyarrow, Security, SystemConfiguration, @@ -55,6 +56,7 @@ buildPythonPackage rec { nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook + protoc ]; buildInputs = diff --git a/pkgs/development/python-modules/ibis-framework/default.nix b/pkgs/development/python-modules/ibis-framework/default.nix index 080168ec2287..ac33fe0bc15d 100644 --- a/pkgs/development/python-modules/ibis-framework/default.nix +++ b/pkgs/development/python-modules/ibis-framework/default.nix @@ -57,7 +57,6 @@ let testBackends = [ "duckdb" "sqlite" - "datafusion" ]; ibisTestingData = fetchFromGitHub { @@ -92,9 +91,20 @@ buildPythonPackage rec { url = "https://github.com/ibis-project/ibis/commit/a54eceabac1d6592e9f6ab0ca7749e37a748c2ad.patch"; hash = "sha256-j5BPYVqnEF9GQV5N3/VhFUCdsEwAIOQC0KfZ5LNBSRg="; }) + + # remove after the 10.0 release + (fetchpatch { + name = "ibis-framework-arrow-18.patch"; + url = "https://github.com/ibis-project/ibis/commit/5dc549b22c2eca29a11a31fb29deef7c1466a204.patch"; + hash = "sha256-4i/g2uixdlkbE6x659wzZJ91FZpzwOVkF6ZeXkiCP3I="; + excludes = [ + "poetry.lock" + "requirements-dev.txt" + ]; + }) ]; - nativeBuildInputs = [ + build-system = [ poetry-core poetry-dynamic-versioning ]; @@ -102,7 +112,7 @@ buildPythonPackage rec { dontBypassPoetryDynamicVersioning = true; env.POETRY_DYNAMIC_VERSIONING_BYPASS = lib.head (lib.strings.splitString "-" version); - propagatedBuildInputs = [ + dependencies = [ atpublic parsy python-dateutil @@ -128,12 +138,9 @@ buildPythonPackage rec { pytest-xdist ] ++ lib.concatMap (name: optional-dependencies.${name}) testBackends; - dontUsePytestXdist = true; - pytestFlagsArray = [ "-m" - # tpcds and tpch are slow, so disable them - "'not tpcds and not tpch and (${lib.concatStringsSep " or " testBackends} or core)'" + "'${lib.concatStringsSep " or " testBackends} or core'" ]; disabledTests = [ diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix index 25075f612df8..004d0bbca94b 100644 --- a/pkgs/tools/filesystems/ceph/default.nix +++ b/pkgs/tools/filesystems/ceph/default.nix @@ -4,7 +4,7 @@ , fetchurl , fetchFromGitHub , fetchPypi -, fetchpatch +, fetchpatch2 # Build time , autoconf @@ -314,6 +314,16 @@ in rec { pname = "ceph"; inherit src version; + patches = [ + (fetchpatch2 { + name = "ceph-s3select-arrow-18-compat.patch"; + url = "https://github.com/ceph/s3select/commit/f333ec82e6e8a3f7eb9ba1041d1442b2c7cd0f05.patch"; + hash = "sha256-21fi5tMIs/JmuhwPYMWtampv/aqAe+EoPAXZLJlOvgo="; + stripLen = 1; + extraPrefix = "src/s3select/"; + }) + ]; + nativeBuildInputs = [ autoconf # `autoreconf` is called, e.g. for `qatlib_ext` automake # `aclocal` is called, e.g. for `qatlib_ext` diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 453d627ff5ea..23e45510b182 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2920,6 +2920,7 @@ self: super: with self; { datafusion = callPackage ../development/python-modules/datafusion { inherit (pkgs.darwin.apple_sdk.frameworks) Security SystemConfiguration; + protoc = pkgs.protobuf; }; datalad = callPackage ../development/python-modules/datalad { };