gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #328203 from tobim/pkgs/arrow-cpp-17

Browse Source 
arrow-cpp: 16.0.0 -> 17.0.0
This commit is contained in:
Dmitry Kalinkin
2024-07-20 14:42:04 -04:00
committed by GitHub
parent 8339d08c2a 3666c4592b
commit 603ea74efc
2 changed files with 190 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkgs/by-name/ar/arrow-cpp/package.nix
+189 -162
View File
@@ -1,11 +1,12 @@
{ stdenv
, lib
, fetchurl
, fetchFromGitHub
, fixDarwinDylibNames
, autoconf
, aws-sdk-cpp
, aws-sdk-cpp-arrow ? aws-sdk-cpp.override {
{
stdenv,
lib,
fetchurl,
fetchFromGitHub,
fixDarwinDylibNames,
autoconf,
aws-sdk-cpp,
aws-sdk-cpp-arrow ? aws-sdk-cpp.override {
apis = [
"cognito-identity"
"config"
@@ -14,58 +15,58 @@
"sts"
"transfer"
];
}
, boost
, brotli
, bzip2
, c-ares
, cmake
, crc32c
, curl
, flatbuffers
, gflags
, glog
, google-cloud-cpp
, grpc
, gtest
, libbacktrace
, lz4
, minio
, ninja
, nlohmann_json
, openssl
, perl
, protobuf
, python3
, rapidjson
, re2
, snappy
, sqlite
, thrift
, tzdata
, utf8proc
, which
, zlib
, zstd
, testers
, enableShared ? !stdenv.hostPlatform.isStatic
, enableFlight ? true
, enableJemalloc ? !stdenv.isDarwin
, enableS3 ? true
, enableGcs ? !stdenv.isDarwin
},
boost,
brotli,
bzip2,
cmake,
crc32c,
curl,
flatbuffers,
gflags,
glog,
google-cloud-cpp,
grpc,
gtest,
libbacktrace,
lz4,
minio,
ninja,
nlohmann_json,
openssl,
perl,
protobuf,
python3,
rapidjson,
re2,
snappy,
sqlite,
thrift,
tzdata,
utf8proc,
which,
zlib,
zstd,
testers,
enableShared ? !stdenv.hostPlatform.isStatic,
enableFlight ? true,
enableJemalloc ? !stdenv.isDarwin,
enableS3 ? true,
enableGcs ? !stdenv.isDarwin,
}:
assert lib.asserts.assertMsg
((enableS3 && stdenv.isDarwin) -> (lib.versionOlder boost.version "1.69" || lib.versionAtLeast boost.version "1.70"))
"S3 on Darwin requires Boost != 1.69";
assert lib.asserts.assertMsg (
(enableS3 && stdenv.isDarwin)
-> (lib.versionOlder boost.version "1.69" || lib.versionAtLeast boost.version "1.70")
) "S3 on Darwin requires Boost != 1.69";
let
arrow-testing = fetchFromGitHub {
name = "arrow-testing";
owner = "apache";
repo = "arrow-testing";
rev = "25d16511e8d42c2744a1d94d90169e3a36e92631";
hash = "sha256-fXeWM/8jBfJY7KL6PVfRbzB8i4sp6PHsnMSHCX5kzfI=";
rev = "735ae7128d571398dd798d7ff004adebeb342883";
hash = "sha256-67KwnSt+EeEDvk+9kxR51tErL2wJqEPRITKb/dN+HMQ=";
};
parquet-testing = fetchFromGitHub {
@@ -79,14 +80,16 @@ let
in
stdenv.mkDerivation (finalAttrs: {
pname = "arrow-cpp";
version = "16.0.0";
version = "17.0.0";
src = fetchurl {
url = "mirror://apache/arrow/arrow-${finalAttrs.version}/apache-arrow-${finalAttrs.version}.tar.gz";
hash = "sha256-n0BRrpRzyXmR2a+AHi+UrjRVBncZyn+QuBN/nppwC40=";
src = fetchFromGitHub {
owner = "apache";
repo = "arrow";
rev = "apache-arrow-17.0.0";
hash = "sha256-ZQqi1RFb4Ey0A0UVCThuIxM7DoFfkLwaeRAc2z8u9so=";
};
sourceRoot = "apache-arrow-${finalAttrs.version}/cpp";
sourceRoot = "source/cpp";
# versions are all taken from
# https://github.com/apache/arrow/blob/apache-arrow-${version}/cpp/thirdparty/versions.txt
@@ -109,8 +112,8 @@ stdenv.mkDerivation (finalAttrs: {
ARROW_XSIMD_URL = fetchFromGitHub {
owner = "xtensor-stack";
repo = "xsimd";
rev = "9.0.1";
hash = "sha256-onALN6agtrHWigtFlCeefD9CiRZI4Y690XTzy2UDnrk=";
rev = "13.0.0";
hash = "sha256-qElJYW5QDj3s59L3NgZj5zkhnUMzIP2mBa1sPks3/CE=";
};
ARROW_SUBSTRAIT_URL = fetchFromGitHub {
@@ -126,38 +129,44 @@ stdenv.mkDerivation (finalAttrs: {
autoconf # for vendored jemalloc
flatbuffers
] ++ lib.optional stdenv.isDarwin fixDarwinDylibNames;
buildInputs = [
boost
brotli
bzip2
flatbuffers
gflags
glog
gtest
libbacktrace
lz4
nlohmann_json # alternative JSON parser to rapidjson
protobuf # substrait requires protobuf
rapidjson
re2
snappy
thrift
utf8proc
zlib
zstd
] ++ lib.optionals enableFlight [
grpc
openssl
protobuf
sqlite
] ++ lib.optionals enableS3 [ aws-sdk-cpp-arrow openssl ]
++ lib.optionals enableGcs [
crc32c
curl
google-cloud-cpp
grpc
nlohmann_json
];
buildInputs =
[
boost
brotli
bzip2
flatbuffers
gflags
glog
gtest
libbacktrace
lz4
nlohmann_json # alternative JSON parser to rapidjson
protobuf # substrait requires protobuf
rapidjson
re2
snappy
thrift
utf8proc
zlib
zstd
]
++ lib.optionals enableFlight [
grpc
openssl
protobuf
sqlite
]
++ lib.optionals enableS3 [
aws-sdk-cpp-arrow
openssl
]
++ lib.optionals enableGcs [
crc32c
curl
google-cloud-cpp
grpc
nlohmann_json
];
preConfigure = ''
patchShebangs build-support/
@@ -165,53 +174,57 @@ stdenv.mkDerivation (finalAttrs: {
--replace 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";'
'';
cmakeFlags = [
"-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON"
"-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}"
"-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}"
"-DARROW_BUILD_TESTS=ON"
"-DARROW_BUILD_INTEGRATION=ON"
"-DARROW_BUILD_UTILITIES=ON"
"-DARROW_EXTRA_ERROR_CONTEXT=ON"
"-DARROW_VERBOSE_THIRDPARTY_BUILD=ON"
"-DARROW_DEPENDENCY_SOURCE=SYSTEM"
"-Dxsimd_SOURCE=AUTO"
"-DARROW_DEPENDENCY_USE_SHARED=${if enableShared then "ON" else "OFF"}"
"-DARROW_COMPUTE=ON"
"-DARROW_CSV=ON"
"-DARROW_DATASET=ON"
"-DARROW_FILESYSTEM=ON"
"-DARROW_FLIGHT_SQL=${if enableFlight then "ON" else "OFF"}"
"-DARROW_HDFS=ON"
"-DARROW_IPC=ON"
"-DARROW_JEMALLOC=${if enableJemalloc then "ON" else "OFF"}"
"-DARROW_JSON=ON"
"-DARROW_USE_GLOG=ON"
"-DARROW_WITH_BACKTRACE=ON"
"-DARROW_WITH_BROTLI=ON"
"-DARROW_WITH_BZ2=ON"
"-DARROW_WITH_LZ4=ON"
"-DARROW_WITH_NLOHMANN_JSON=ON"
"-DARROW_WITH_SNAPPY=ON"
"-DARROW_WITH_UTF8PROC=ON"
"-DARROW_WITH_ZLIB=ON"
"-DARROW_WITH_ZSTD=ON"
"-DARROW_MIMALLOC=ON"
"-DARROW_SUBSTRAIT=ON"
"-DARROW_FLIGHT=${if enableFlight then "ON" else "OFF"}"
"-DARROW_FLIGHT_TESTING=${if enableFlight then "ON" else "OFF"}"
"-DARROW_S3=${if enableS3 then "ON" else "OFF"}"
"-DARROW_GCS=${if enableGcs then "ON" else "OFF"}"
# Parquet options:
"-DARROW_PARQUET=ON"
"-DPARQUET_BUILD_EXECUTABLES=ON"
"-DPARQUET_REQUIRE_ENCRYPTION=ON"
] ++ lib.optionals (!enableShared) [
"-DARROW_TEST_LINKAGE=static"
] ++ lib.optionals stdenv.isDarwin [
"-DCMAKE_INSTALL_RPATH=@loader_path/../lib" # needed for tools executables
] ++ lib.optionals (!stdenv.isx86_64) [ "-DARROW_USE_SIMD=OFF" ]
++ lib.optionals enableS3 [ "-DAWSSDK_CORE_HEADER_FILE=${aws-sdk-cpp-arrow}/include/aws/core/Aws.h" ];
cmakeFlags =
[
"-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON"
"-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}"
"-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}"
"-DARROW_BUILD_TESTS=ON"
"-DARROW_BUILD_INTEGRATION=ON"
"-DARROW_BUILD_UTILITIES=ON"
"-DARROW_EXTRA_ERROR_CONTEXT=ON"
"-DARROW_VERBOSE_THIRDPARTY_BUILD=ON"
"-DARROW_DEPENDENCY_SOURCE=SYSTEM"
"-Dxsimd_SOURCE=AUTO"
"-DARROW_DEPENDENCY_USE_SHARED=${if enableShared then "ON" else "OFF"}"
"-DARROW_COMPUTE=ON"
"-DARROW_CSV=ON"
"-DARROW_DATASET=ON"
"-DARROW_FILESYSTEM=ON"
"-DARROW_FLIGHT_SQL=${if enableFlight then "ON" else "OFF"}"
"-DARROW_HDFS=ON"
"-DARROW_IPC=ON"
"-DARROW_JEMALLOC=${if enableJemalloc then "ON" else "OFF"}"
"-DARROW_JSON=ON"
"-DARROW_USE_GLOG=ON"
"-DARROW_WITH_BACKTRACE=ON"
"-DARROW_WITH_BROTLI=ON"
"-DARROW_WITH_BZ2=ON"
"-DARROW_WITH_LZ4=ON"
"-DARROW_WITH_NLOHMANN_JSON=ON"
"-DARROW_WITH_SNAPPY=ON"
"-DARROW_WITH_UTF8PROC=ON"
"-DARROW_WITH_ZLIB=ON"
"-DARROW_WITH_ZSTD=ON"
"-DARROW_MIMALLOC=ON"
"-DARROW_SUBSTRAIT=ON"
"-DARROW_FLIGHT=${if enableFlight then "ON" else "OFF"}"
"-DARROW_FLIGHT_TESTING=${if enableFlight then "ON" else "OFF"}"
"-DARROW_S3=${if enableS3 then "ON" else "OFF"}"
"-DARROW_GCS=${if enableGcs then "ON" else "OFF"}"
# Parquet options:
"-DARROW_PARQUET=ON"
"-DPARQUET_BUILD_EXECUTABLES=ON"
"-DPARQUET_REQUIRE_ENCRYPTION=ON"
]
++ lib.optionals (!enableShared) [ "-DARROW_TEST_LINKAGE=static" ]
++ lib.optionals stdenv.isDarwin [
"-DCMAKE_INSTALL_RPATH=@loader_path/../lib" # needed for tools executables
]
++ lib.optionals (!stdenv.isx86_64) [ "-DARROW_USE_SIMD=OFF" ]
++ lib.optionals enableS3 [
"-DAWSSDK_CORE_HEADER_FILE=${aws-sdk-cpp-arrow}/include/aws/core/Aws.h"
];
doInstallCheck = true;
ARROW_TEST_DATA = lib.optionalString finalAttrs.doInstallCheck "${arrow-testing}/data";
@@ -219,33 +232,38 @@ stdenv.mkDerivation (finalAttrs: {
GTEST_FILTER =
let
# Upstream Issue: https://issues.apache.org/jira/browse/ARROW-11398
filteredTests = lib.optionals stdenv.hostPlatform.isAarch64 [
"TestFilterKernelWithNumeric/3.CompareArrayAndFilterRandomNumeric"
"TestFilterKernelWithNumeric/7.CompareArrayAndFilterRandomNumeric"
"TestCompareKernel.PrimitiveRandomTests"
] ++ lib.optionals enableS3 [
"S3OptionsTest.FromUri"
"S3RegionResolutionTest.NonExistentBucket"
"S3RegionResolutionTest.PublicBucket"
"S3RegionResolutionTest.RestrictedBucket"
"TestMinioServer.Connect"
"TestS3FS.*"
"TestS3FSGeneric.*"
] ++ lib.optionals stdenv.isDarwin [
# TODO: revisit at 12.0.0 or when
# https://github.com/apache/arrow/commit/295c6644ca6b67c95a662410b2c7faea0920c989
# is available, see
# https://github.com/apache/arrow/pull/15288#discussion_r1071244661
"ExecPlanExecution.StressSourceSinkStopped"
];
filteredTests =
lib.optionals stdenv.hostPlatform.isAarch64 [
"TestFilterKernelWithNumeric/3.CompareArrayAndFilterRandomNumeric"
"TestFilterKernelWithNumeric/7.CompareArrayAndFilterRandomNumeric"
"TestCompareKernel.PrimitiveRandomTests"
]
++ lib.optionals enableS3 [
"S3OptionsTest.FromUri"
"S3RegionResolutionTest.NonExistentBucket"
"S3RegionResolutionTest.PublicBucket"
"S3RegionResolutionTest.RestrictedBucket"
"TestMinioServer.Connect"
"TestS3FS.*"
"TestS3FSGeneric.*"
]
++ lib.optionals stdenv.isDarwin [
# TODO: revisit at 12.0.0 or when
# https://github.com/apache/arrow/commit/295c6644ca6b67c95a662410b2c7faea0920c989
# is available, see
# https://github.com/apache/arrow/pull/15288#discussion_r1071244661
"ExecPlanExecution.StressSourceSinkStopped"
];
in
lib.optionalString finalAttrs.doInstallCheck "-${lib.concatStringsSep ":" filteredTests}";
__darwinAllowLocalNetworking = true;
nativeInstallCheckInputs = [ perl which sqlite ]
++ lib.optionals enableS3 [ minio ]
++ lib.optionals enableFlight [ python3 ];
nativeInstallCheckInputs = [
perl
which
sqlite
] ++ lib.optionals enableS3 [ minio ] ++ lib.optionals enableFlight [ python3 ];
installCheckPhase =
let
@@ -270,7 +288,11 @@ stdenv.mkDerivation (finalAttrs: {
homepage = "https://arrow.apache.org/docs/cpp/";
license = licenses.asl20;
platforms = platforms.unix;
maintainers = with maintainers; [ tobim veprbl cpcloud ];
maintainers = with maintainers; [
tobim
veprbl
cpcloud
];
pkgConfigModules = [
"arrow"
"arrow-acero"
@@ -288,7 +310,12 @@ stdenv.mkDerivation (finalAttrs: {
];
};
passthru = {
inherit enableFlight enableJemalloc enableS3 enableGcs;
inherit
enableFlight
enableJemalloc
enableS3
enableGcs
;
tests.pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage;
};
})
pkgs/development/python-modules/pyarrow/default.nix
+1 -1
View File
@@ -33,7 +33,7 @@ buildPythonPackage rec {
disabled = pythonOlder "3.7";
sourceRoot = "apache-arrow-${version}/python";
sourceRoot = "source/python";
nativeBuildInputs = [
cmake