From aa91bca6f17a42df33988dc68ebf91f289b82230 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Tue, 9 Jul 2024 22:10:41 +0200 Subject: [PATCH 1/3] arrow-cpp: reformat --- pkgs/by-name/ar/arrow-cpp/package.nix | 332 ++++++++++++++------------ 1 file changed, 179 insertions(+), 153 deletions(-) diff --git a/pkgs/by-name/ar/arrow-cpp/package.nix b/pkgs/by-name/ar/arrow-cpp/package.nix index 534c5f243057..14e6d4968e39 100644 --- a/pkgs/by-name/ar/arrow-cpp/package.nix +++ b/pkgs/by-name/ar/arrow-cpp/package.nix @@ -1,11 +1,12 @@ -{ stdenv -, lib -, fetchurl -, fetchFromGitHub -, fixDarwinDylibNames -, autoconf -, aws-sdk-cpp -, aws-sdk-cpp-arrow ? aws-sdk-cpp.override { +{ + stdenv, + lib, + fetchurl, + fetchFromGitHub, + fixDarwinDylibNames, + autoconf, + aws-sdk-cpp, + aws-sdk-cpp-arrow ? aws-sdk-cpp.override { apis = [ "cognito-identity" "config" @@ -14,50 +15,51 @@ "sts" "transfer" ]; - } -, boost -, brotli -, bzip2 -, c-ares -, cmake -, crc32c -, curl -, flatbuffers -, gflags -, glog -, google-cloud-cpp -, grpc -, gtest -, libbacktrace -, lz4 -, minio -, ninja -, nlohmann_json -, openssl -, perl -, protobuf -, python3 -, rapidjson -, re2 -, snappy -, sqlite -, thrift -, tzdata -, utf8proc -, which -, zlib -, zstd -, testers -, enableShared ? !stdenv.hostPlatform.isStatic -, enableFlight ? true -, enableJemalloc ? !stdenv.isDarwin -, enableS3 ? true -, enableGcs ? !stdenv.isDarwin + }, + boost, + brotli, + bzip2, + c-ares, + cmake, + crc32c, + curl, + flatbuffers, + gflags, + glog, + google-cloud-cpp, + grpc, + gtest, + libbacktrace, + lz4, + minio, + ninja, + nlohmann_json, + openssl, + perl, + protobuf, + python3, + rapidjson, + re2, + snappy, + sqlite, + thrift, + tzdata, + utf8proc, + which, + zlib, + zstd, + testers, + enableShared ? !stdenv.hostPlatform.isStatic, + enableFlight ? true, + enableJemalloc ? !stdenv.isDarwin, + enableS3 ? true, + enableGcs ? !stdenv.isDarwin, }: -assert lib.asserts.assertMsg - ((enableS3 && stdenv.isDarwin) -> (lib.versionOlder boost.version "1.69" || lib.versionAtLeast boost.version "1.70")) - "S3 on Darwin requires Boost != 1.69"; +assert lib.asserts.assertMsg ( + (enableS3 && stdenv.isDarwin) + -> (lib.versionOlder boost.version "1.69" || lib.versionAtLeast boost.version "1.70") +) "S3 on Darwin requires Boost != 1.69"; let arrow-testing = fetchFromGitHub { @@ -126,38 +128,44 @@ stdenv.mkDerivation (finalAttrs: { autoconf # for vendored jemalloc flatbuffers ] ++ lib.optional stdenv.isDarwin fixDarwinDylibNames; - buildInputs = [ - boost - brotli - bzip2 - flatbuffers - gflags - glog - gtest - libbacktrace - lz4 - nlohmann_json # alternative JSON parser to rapidjson - protobuf # substrait requires protobuf - rapidjson - re2 - snappy - thrift - utf8proc - zlib - zstd - ] ++ lib.optionals enableFlight [ - grpc - openssl - protobuf - sqlite - ] ++ lib.optionals enableS3 [ aws-sdk-cpp-arrow openssl ] - ++ lib.optionals enableGcs [ - crc32c - curl - google-cloud-cpp - grpc - nlohmann_json - ]; + buildInputs = + [ + boost + brotli + bzip2 + flatbuffers + gflags + glog + gtest + libbacktrace + lz4 + nlohmann_json # alternative JSON parser to rapidjson + protobuf # substrait requires protobuf + rapidjson + re2 + snappy + thrift + utf8proc + zlib + zstd + ] + ++ lib.optionals enableFlight [ + grpc + openssl + protobuf + sqlite + ] + ++ lib.optionals enableS3 [ + aws-sdk-cpp-arrow + openssl + ] + ++ lib.optionals enableGcs [ + crc32c + curl + google-cloud-cpp + grpc + nlohmann_json + ]; preConfigure = '' patchShebangs build-support/ @@ -165,53 +173,57 @@ stdenv.mkDerivation (finalAttrs: { --replace 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";' ''; - cmakeFlags = [ - "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON" - "-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}" - "-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}" - "-DARROW_BUILD_TESTS=ON" - "-DARROW_BUILD_INTEGRATION=ON" - "-DARROW_BUILD_UTILITIES=ON" - "-DARROW_EXTRA_ERROR_CONTEXT=ON" - "-DARROW_VERBOSE_THIRDPARTY_BUILD=ON" - "-DARROW_DEPENDENCY_SOURCE=SYSTEM" - "-Dxsimd_SOURCE=AUTO" - "-DARROW_DEPENDENCY_USE_SHARED=${if enableShared then "ON" else "OFF"}" - "-DARROW_COMPUTE=ON" - "-DARROW_CSV=ON" - "-DARROW_DATASET=ON" - "-DARROW_FILESYSTEM=ON" - "-DARROW_FLIGHT_SQL=${if enableFlight then "ON" else "OFF"}" - "-DARROW_HDFS=ON" - "-DARROW_IPC=ON" - "-DARROW_JEMALLOC=${if enableJemalloc then "ON" else "OFF"}" - "-DARROW_JSON=ON" - "-DARROW_USE_GLOG=ON" - "-DARROW_WITH_BACKTRACE=ON" - "-DARROW_WITH_BROTLI=ON" - "-DARROW_WITH_BZ2=ON" - "-DARROW_WITH_LZ4=ON" - "-DARROW_WITH_NLOHMANN_JSON=ON" - "-DARROW_WITH_SNAPPY=ON" - "-DARROW_WITH_UTF8PROC=ON" - "-DARROW_WITH_ZLIB=ON" - "-DARROW_WITH_ZSTD=ON" - "-DARROW_MIMALLOC=ON" - "-DARROW_SUBSTRAIT=ON" - "-DARROW_FLIGHT=${if enableFlight then "ON" else "OFF"}" - "-DARROW_FLIGHT_TESTING=${if enableFlight then "ON" else "OFF"}" - "-DARROW_S3=${if enableS3 then "ON" else "OFF"}" - "-DARROW_GCS=${if enableGcs then "ON" else "OFF"}" - # Parquet options: - "-DARROW_PARQUET=ON" - "-DPARQUET_BUILD_EXECUTABLES=ON" - "-DPARQUET_REQUIRE_ENCRYPTION=ON" - ] ++ lib.optionals (!enableShared) [ - "-DARROW_TEST_LINKAGE=static" - ] ++ lib.optionals stdenv.isDarwin [ - "-DCMAKE_INSTALL_RPATH=@loader_path/../lib" # needed for tools executables - ] ++ lib.optionals (!stdenv.isx86_64) [ "-DARROW_USE_SIMD=OFF" ] - ++ lib.optionals enableS3 [ "-DAWSSDK_CORE_HEADER_FILE=${aws-sdk-cpp-arrow}/include/aws/core/Aws.h" ]; + cmakeFlags = + [ + "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON" + "-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}" + "-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}" + "-DARROW_BUILD_TESTS=ON" + "-DARROW_BUILD_INTEGRATION=ON" + "-DARROW_BUILD_UTILITIES=ON" + "-DARROW_EXTRA_ERROR_CONTEXT=ON" + "-DARROW_VERBOSE_THIRDPARTY_BUILD=ON" + "-DARROW_DEPENDENCY_SOURCE=SYSTEM" + "-Dxsimd_SOURCE=AUTO" + "-DARROW_DEPENDENCY_USE_SHARED=${if enableShared then "ON" else "OFF"}" + "-DARROW_COMPUTE=ON" + "-DARROW_CSV=ON" + "-DARROW_DATASET=ON" + "-DARROW_FILESYSTEM=ON" + "-DARROW_FLIGHT_SQL=${if enableFlight then "ON" else "OFF"}" + "-DARROW_HDFS=ON" + "-DARROW_IPC=ON" + "-DARROW_JEMALLOC=${if enableJemalloc then "ON" else "OFF"}" + "-DARROW_JSON=ON" + "-DARROW_USE_GLOG=ON" + "-DARROW_WITH_BACKTRACE=ON" + "-DARROW_WITH_BROTLI=ON" + "-DARROW_WITH_BZ2=ON" + "-DARROW_WITH_LZ4=ON" + "-DARROW_WITH_NLOHMANN_JSON=ON" + "-DARROW_WITH_SNAPPY=ON" + "-DARROW_WITH_UTF8PROC=ON" + "-DARROW_WITH_ZLIB=ON" + "-DARROW_WITH_ZSTD=ON" + "-DARROW_MIMALLOC=ON" + "-DARROW_SUBSTRAIT=ON" + "-DARROW_FLIGHT=${if enableFlight then "ON" else "OFF"}" + "-DARROW_FLIGHT_TESTING=${if enableFlight then "ON" else "OFF"}" + "-DARROW_S3=${if enableS3 then "ON" else "OFF"}" + "-DARROW_GCS=${if enableGcs then "ON" else "OFF"}" + # Parquet options: + "-DARROW_PARQUET=ON" + "-DPARQUET_BUILD_EXECUTABLES=ON" + "-DPARQUET_REQUIRE_ENCRYPTION=ON" + ] + ++ lib.optionals (!enableShared) [ "-DARROW_TEST_LINKAGE=static" ] + ++ lib.optionals stdenv.isDarwin [ + "-DCMAKE_INSTALL_RPATH=@loader_path/../lib" # needed for tools executables + ] + ++ lib.optionals (!stdenv.isx86_64) [ "-DARROW_USE_SIMD=OFF" ] + ++ lib.optionals enableS3 [ + "-DAWSSDK_CORE_HEADER_FILE=${aws-sdk-cpp-arrow}/include/aws/core/Aws.h" + ]; doInstallCheck = true; ARROW_TEST_DATA = lib.optionalString finalAttrs.doInstallCheck "${arrow-testing}/data"; @@ -219,33 +231,38 @@ stdenv.mkDerivation (finalAttrs: { GTEST_FILTER = let # Upstream Issue: https://issues.apache.org/jira/browse/ARROW-11398 - filteredTests = lib.optionals stdenv.hostPlatform.isAarch64 [ - "TestFilterKernelWithNumeric/3.CompareArrayAndFilterRandomNumeric" - "TestFilterKernelWithNumeric/7.CompareArrayAndFilterRandomNumeric" - "TestCompareKernel.PrimitiveRandomTests" - ] ++ lib.optionals enableS3 [ - "S3OptionsTest.FromUri" - "S3RegionResolutionTest.NonExistentBucket" - "S3RegionResolutionTest.PublicBucket" - "S3RegionResolutionTest.RestrictedBucket" - "TestMinioServer.Connect" - "TestS3FS.*" - "TestS3FSGeneric.*" - ] ++ lib.optionals stdenv.isDarwin [ - # TODO: revisit at 12.0.0 or when - # https://github.com/apache/arrow/commit/295c6644ca6b67c95a662410b2c7faea0920c989 - # is available, see - # https://github.com/apache/arrow/pull/15288#discussion_r1071244661 - "ExecPlanExecution.StressSourceSinkStopped" - ]; + filteredTests = + lib.optionals stdenv.hostPlatform.isAarch64 [ + "TestFilterKernelWithNumeric/3.CompareArrayAndFilterRandomNumeric" + "TestFilterKernelWithNumeric/7.CompareArrayAndFilterRandomNumeric" + "TestCompareKernel.PrimitiveRandomTests" + ] + ++ lib.optionals enableS3 [ + "S3OptionsTest.FromUri" + "S3RegionResolutionTest.NonExistentBucket" + "S3RegionResolutionTest.PublicBucket" + "S3RegionResolutionTest.RestrictedBucket" + "TestMinioServer.Connect" + "TestS3FS.*" + "TestS3FSGeneric.*" + ] + ++ lib.optionals stdenv.isDarwin [ + # TODO: revisit at 12.0.0 or when + # https://github.com/apache/arrow/commit/295c6644ca6b67c95a662410b2c7faea0920c989 + # is available, see + # https://github.com/apache/arrow/pull/15288#discussion_r1071244661 + "ExecPlanExecution.StressSourceSinkStopped" + ]; in lib.optionalString finalAttrs.doInstallCheck "-${lib.concatStringsSep ":" filteredTests}"; __darwinAllowLocalNetworking = true; - nativeInstallCheckInputs = [ perl which sqlite ] - ++ lib.optionals enableS3 [ minio ] - ++ lib.optionals enableFlight [ python3 ]; + nativeInstallCheckInputs = [ + perl + which + sqlite + ] ++ lib.optionals enableS3 [ minio ] ++ lib.optionals enableFlight [ python3 ]; installCheckPhase = let @@ -270,7 +287,11 @@ stdenv.mkDerivation (finalAttrs: { homepage = "https://arrow.apache.org/docs/cpp/"; license = licenses.asl20; platforms = platforms.unix; - maintainers = with maintainers; [ tobim veprbl cpcloud ]; + maintainers = with maintainers; [ + tobim + veprbl + cpcloud + ]; pkgConfigModules = [ "arrow" "arrow-acero" @@ -288,7 +309,12 @@ stdenv.mkDerivation (finalAttrs: { ]; }; passthru = { - inherit enableFlight enableJemalloc enableS3 enableGcs; + inherit + enableFlight + enableJemalloc + enableS3 + enableGcs + ; tests.pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; }; }) From e03058e6fe33039fb0d6a9fadd314d6359384d01 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Tue, 9 Jul 2024 22:10:58 +0200 Subject: [PATCH 2/3] arrow-cpp: remove unused input --- pkgs/by-name/ar/arrow-cpp/package.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/pkgs/by-name/ar/arrow-cpp/package.nix b/pkgs/by-name/ar/arrow-cpp/package.nix index 14e6d4968e39..4cec34a0f7f5 100644 --- a/pkgs/by-name/ar/arrow-cpp/package.nix +++ b/pkgs/by-name/ar/arrow-cpp/package.nix @@ -19,7 +19,6 @@ boost, brotli, bzip2, - c-ares, cmake, crc32c, curl, From 3666c4592b9ffd42490996111cbbf3909e9e8f72 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Tue, 9 Jul 2024 22:47:36 +0200 Subject: [PATCH 3/3] arrow-cpp: 16.0.0 -> 17.0.0 --- pkgs/by-name/ar/arrow-cpp/package.nix | 20 ++++++++++--------- .../python-modules/pyarrow/default.nix | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pkgs/by-name/ar/arrow-cpp/package.nix b/pkgs/by-name/ar/arrow-cpp/package.nix index 4cec34a0f7f5..1cc21b5b56dc 100644 --- a/pkgs/by-name/ar/arrow-cpp/package.nix +++ b/pkgs/by-name/ar/arrow-cpp/package.nix @@ -65,8 +65,8 @@ let name = "arrow-testing"; owner = "apache"; repo = "arrow-testing"; - rev = "25d16511e8d42c2744a1d94d90169e3a36e92631"; - hash = "sha256-fXeWM/8jBfJY7KL6PVfRbzB8i4sp6PHsnMSHCX5kzfI="; + rev = "735ae7128d571398dd798d7ff004adebeb342883"; + hash = "sha256-67KwnSt+EeEDvk+9kxR51tErL2wJqEPRITKb/dN+HMQ="; }; parquet-testing = fetchFromGitHub { @@ -80,14 +80,16 @@ let in stdenv.mkDerivation (finalAttrs: { pname = "arrow-cpp"; - version = "16.0.0"; + version = "17.0.0"; - src = fetchurl { - url = "mirror://apache/arrow/arrow-${finalAttrs.version}/apache-arrow-${finalAttrs.version}.tar.gz"; - hash = "sha256-n0BRrpRzyXmR2a+AHi+UrjRVBncZyn+QuBN/nppwC40="; + src = fetchFromGitHub { + owner = "apache"; + repo = "arrow"; + rev = "apache-arrow-17.0.0"; + hash = "sha256-ZQqi1RFb4Ey0A0UVCThuIxM7DoFfkLwaeRAc2z8u9so="; }; - sourceRoot = "apache-arrow-${finalAttrs.version}/cpp"; + sourceRoot = "source/cpp"; # versions are all taken from # https://github.com/apache/arrow/blob/apache-arrow-${version}/cpp/thirdparty/versions.txt @@ -110,8 +112,8 @@ stdenv.mkDerivation (finalAttrs: { ARROW_XSIMD_URL = fetchFromGitHub { owner = "xtensor-stack"; repo = "xsimd"; - rev = "9.0.1"; - hash = "sha256-onALN6agtrHWigtFlCeefD9CiRZI4Y690XTzy2UDnrk="; + rev = "13.0.0"; + hash = "sha256-qElJYW5QDj3s59L3NgZj5zkhnUMzIP2mBa1sPks3/CE="; }; ARROW_SUBSTRAIT_URL = fetchFromGitHub { diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index d275e44f649d..5c90ed002745 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -33,7 +33,7 @@ buildPythonPackage rec { disabled = pythonOlder "3.7"; - sourceRoot = "apache-arrow-${version}/python"; + sourceRoot = "source/python"; nativeBuildInputs = [ cmake