Martin Weinelt
367 lines
10 KiB
Nix
367 lines
10 KiB
Nix
{
|
|
stdenv,
|
|
lib,
|
|
fetchurl,
|
|
fetchFromGitHub,
|
|
fixDarwinDylibNames,
|
|
apache-orc,
|
|
autoconf,
|
|
aws-sdk-cpp,
|
|
aws-sdk-cpp-arrow ? aws-sdk-cpp.override {
|
|
apis = [
|
|
"cognito-identity"
|
|
"config"
|
|
"identity-management"
|
|
"s3"
|
|
"sts"
|
|
"transfer"
|
|
];
|
|
},
|
|
azure-sdk-for-cpp,
|
|
azurite,
|
|
boost,
|
|
brotli,
|
|
bzip2,
|
|
cmake,
|
|
crc32c,
|
|
curl,
|
|
flatbuffers,
|
|
gflags,
|
|
glog,
|
|
google-cloud-cpp,
|
|
grpc,
|
|
gtest,
|
|
libbacktrace,
|
|
lz4,
|
|
ninja,
|
|
nlohmann_json,
|
|
openssl,
|
|
perl,
|
|
pkg-config,
|
|
protobuf,
|
|
python3,
|
|
rapidjson,
|
|
re2,
|
|
snappy,
|
|
sqlite,
|
|
thrift,
|
|
tzdata,
|
|
utf8proc,
|
|
which,
|
|
zlib,
|
|
zstd,
|
|
testers,
|
|
enableShared ? !stdenv.hostPlatform.isStatic,
|
|
enableFlight ? stdenv.buildPlatform == stdenv.hostPlatform,
|
|
# Disable also on RiscV
|
|
# configure: error: cannot determine number of significant virtual address bits
|
|
enableJemalloc ?
|
|
!stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isAarch64 && !stdenv.hostPlatform.isRiscV64,
|
|
enableS3 ? true,
|
|
# google-cloud-cpp fails to build on RiscV
|
|
enableGcs ? !stdenv.hostPlatform.isRiscV64,
|
|
enableAzure ? true,
|
|
}:
|
|
|
|
let
|
|
arrow-testing = fetchFromGitHub {
|
|
name = "arrow-testing";
|
|
owner = "apache";
|
|
repo = "arrow-testing";
|
|
rev = "19dda67f485ffb3ffa92f4c6fa083576ef052d58";
|
|
hash = "sha256-mna6I/a5ZxMLdWN0QfCsgsre6yMeuSv4syX5ePGLhfg=";
|
|
};
|
|
|
|
parquet-testing = fetchFromGitHub {
|
|
name = "parquet-testing";
|
|
owner = "apache";
|
|
repo = "parquet-testing";
|
|
rev = "a3d96a65e11e2bbca7d22a894e8313ede90a33a3";
|
|
hash = "sha256-Xd6o3RT6Q0tPutV77J0P1x3F6U3RHdCBOKGUKtkQCKk=";
|
|
};
|
|
|
|
version = "24.0.0";
|
|
in
|
|
stdenv.mkDerivation (finalAttrs: {
|
|
pname = "arrow-cpp";
|
|
inherit version;
|
|
|
|
src = fetchFromGitHub {
|
|
owner = "apache";
|
|
repo = "arrow";
|
|
rev = "apache-arrow-${version}";
|
|
hash = "sha256-qTdkzZegANNvtO7nbqXVC8hc7BexvmeFF/0l5VzRb8g=";
|
|
};
|
|
|
|
sourceRoot = "${finalAttrs.src.name}/cpp";
|
|
|
|
# versions are all taken from
|
|
# https://github.com/apache/arrow/blob/apache-arrow-${version}/cpp/thirdparty/versions.txt
|
|
|
|
env =
|
|
lib.optionalAttrs enableJemalloc {
|
|
# jemalloc: arrow uses a custom prefix to prevent default allocator symbol
|
|
# collisions as well as custom build flags
|
|
ARROW_JEMALLOC_URL = fetchurl {
|
|
url = "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2";
|
|
hash = "sha256-LbgtHnEZ3z5xt2QCGbbf6EeJvAU3mDw7esT3GJrs/qo=";
|
|
};
|
|
}
|
|
// {
|
|
# mimalloc: arrow uses custom build flags for mimalloc
|
|
ARROW_MIMALLOC_URL = fetchFromGitHub {
|
|
owner = "microsoft";
|
|
repo = "mimalloc";
|
|
tag = "v3.1.5";
|
|
hash = "sha256-fk6nfyBFS1G0sJwUJVgTC1+aKd0We/JjsIYTO+IOfyg=";
|
|
};
|
|
|
|
ARROW_XSIMD_URL = fetchFromGitHub {
|
|
owner = "xtensor-stack";
|
|
repo = "xsimd";
|
|
tag = "14.0.0";
|
|
hash = "sha256-ijNoHb6xC+OHJbUB4j1PRsoHMzjrnOHVoDRe/nKguDo=";
|
|
};
|
|
|
|
ARROW_SUBSTRAIT_URL = fetchFromGitHub {
|
|
owner = "substrait-io";
|
|
repo = "substrait";
|
|
tag = "v0.44.0";
|
|
hash = "sha256-V739IFTGPtbGPlxcOi8sAaYSDhNUEpITvN9IqdPReug=";
|
|
};
|
|
|
|
# apache-orc looks for things in caps
|
|
LZ4_HOME = lz4;
|
|
PROTOBUF_HOME = protobuf;
|
|
SNAPPY_HOME = snappy.dev;
|
|
ZSTD_HOME = zstd.dev;
|
|
ARROW_TEST_DATA = "${arrow-testing}/data";
|
|
PARQUET_TEST_DATA = "${parquet-testing}/data";
|
|
GTEST_FILTER =
|
|
let
|
|
# Upstream Issue: https://issues.apache.org/jira/browse/ARROW-11398
|
|
filteredTests =
|
|
lib.optionals stdenv.hostPlatform.isAarch64 [
|
|
"TestFilterKernelWithNumeric/3.CompareArrayAndFilterRandomNumeric"
|
|
"TestFilterKernelWithNumeric/7.CompareArrayAndFilterRandomNumeric"
|
|
"TestCompareKernel.PrimitiveRandomTests"
|
|
]
|
|
++ lib.optionals enableS3 [
|
|
"S3OptionsTest.FromUri"
|
|
"S3RegionResolutionTest.NonExistentBucket"
|
|
"S3RegionResolutionTest.PublicBucket"
|
|
"S3RegionResolutionTest.RestrictedBucket"
|
|
"TestMinioServer.Connect"
|
|
"TestS3FS.*"
|
|
"TestS3FSGeneric.*"
|
|
"TestS3FSHTTPS.*" # Needs Minio
|
|
]
|
|
++ lib.optionals (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isx86_64) [
|
|
# https://github.com/apache/arrow/issues/41505
|
|
"TestAzuriteGeneric.Empty"
|
|
];
|
|
in
|
|
"-${lib.concatStringsSep ":" filteredTests}";
|
|
};
|
|
|
|
nativeBuildInputs = [
|
|
cmake
|
|
pkg-config
|
|
ninja
|
|
autoconf # for vendored jemalloc
|
|
flatbuffers
|
|
]
|
|
++ lib.optional stdenv.hostPlatform.isDarwin fixDarwinDylibNames;
|
|
buildInputs = [
|
|
apache-orc
|
|
boost
|
|
brotli
|
|
bzip2
|
|
curl
|
|
flatbuffers
|
|
gflags
|
|
glog
|
|
gtest
|
|
libbacktrace
|
|
lz4
|
|
nlohmann_json # alternative JSON parser to rapidjson
|
|
protobuf # substrait requires protobuf
|
|
rapidjson
|
|
re2
|
|
snappy
|
|
thrift
|
|
utf8proc
|
|
zlib
|
|
zstd
|
|
]
|
|
++ lib.optionals enableFlight [
|
|
grpc
|
|
openssl
|
|
protobuf
|
|
sqlite
|
|
]
|
|
++ lib.optionals enableS3 [
|
|
aws-sdk-cpp-arrow
|
|
openssl
|
|
]
|
|
++ lib.optionals enableGcs [
|
|
crc32c
|
|
google-cloud-cpp
|
|
grpc
|
|
nlohmann_json
|
|
]
|
|
++ lib.optionals enableAzure [
|
|
azure-sdk-for-cpp.identity
|
|
azure-sdk-for-cpp.storage-blobs
|
|
azure-sdk-for-cpp.storage-files-datalake
|
|
];
|
|
|
|
# fails tests on glibc with this enabled
|
|
hardeningDisable = [ "glibcxxassertions" ];
|
|
|
|
preConfigure = ''
|
|
patchShebangs build-support/
|
|
substituteInPlace "src/arrow/vendored/datetime/tz.cpp" \
|
|
--replace-fail 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";'
|
|
'';
|
|
|
|
cmakeFlags = [
|
|
(lib.cmakeBool "CMAKE_FIND_PACKAGE_PREFER_CONFIG" true)
|
|
(lib.cmakeBool "ARROW_BUILD_SHARED" enableShared)
|
|
(lib.cmakeBool "ARROW_BUILD_STATIC" (!enableShared))
|
|
(lib.cmakeBool "ARROW_BUILD_TESTS" enableShared)
|
|
(lib.cmakeBool "ARROW_BUILD_INTEGRATION" true)
|
|
(lib.cmakeBool "ARROW_BUILD_UTILITIES" true)
|
|
(lib.cmakeBool "ARROW_EXTRA_ERROR_CONTEXT" true)
|
|
(lib.cmakeBool "ARROW_VERBOSE_THIRDPARTY_BUILD" true)
|
|
(lib.cmakeFeature "ARROW_DEPENDENCY_SOURCE" "SYSTEM")
|
|
(lib.cmakeFeature "xsimd_SOURCE" "AUTO")
|
|
(lib.cmakeBool "ARROW_DEPENDENCY_USE_SHARED" enableShared)
|
|
(lib.cmakeBool "ARROW_COMPUTE" true)
|
|
(lib.cmakeBool "ARROW_CSV" true)
|
|
(lib.cmakeBool "ARROW_DATASET" true)
|
|
(lib.cmakeBool "ARROW_FILESYSTEM" true)
|
|
(lib.cmakeBool "ARROW_FLIGHT_SQL" enableFlight)
|
|
(lib.cmakeBool "ARROW_HDFS" true)
|
|
(lib.cmakeBool "ARROW_IPC" true)
|
|
(lib.cmakeBool "ARROW_JEMALLOC" enableJemalloc)
|
|
(lib.cmakeBool "ARROW_JSON" true)
|
|
(lib.cmakeBool "ARROW_USE_GLOG" true)
|
|
(lib.cmakeBool "ARROW_WITH_BACKTRACE" true)
|
|
(lib.cmakeBool "ARROW_WITH_BROTLI" true)
|
|
(lib.cmakeBool "ARROW_WITH_BZ2" true)
|
|
(lib.cmakeBool "ARROW_WITH_LZ4" true)
|
|
(lib.cmakeBool "ARROW_WITH_NLOHMANN_JSON" true)
|
|
(lib.cmakeBool "ARROW_WITH_SNAPPY" true)
|
|
(lib.cmakeBool "ARROW_WITH_UTF8PROC" true)
|
|
(lib.cmakeBool "ARROW_WITH_ZLIB" true)
|
|
(lib.cmakeBool "ARROW_WITH_ZSTD" true)
|
|
(lib.cmakeBool "ARROW_MIMALLOC" true)
|
|
(lib.cmakeBool "ARROW_SUBSTRAIT" true)
|
|
(lib.cmakeBool "ARROW_FLIGHT" enableFlight)
|
|
(lib.cmakeBool "ARROW_FLIGHT_TESTING" enableFlight)
|
|
(lib.cmakeBool "ARROW_S3" enableS3)
|
|
(lib.cmakeBool "ARROW_GCS" enableGcs)
|
|
(lib.cmakeBool "ARROW_AZURE" enableAzure)
|
|
(lib.cmakeBool "ARROW_ORC" true)
|
|
# Parquet options:
|
|
(lib.cmakeBool "ARROW_PARQUET" true)
|
|
(lib.cmakeBool "PARQUET_BUILD_EXECUTABLES" true)
|
|
(lib.cmakeBool "PARQUET_REQUIRE_ENCRYPTION" true)
|
|
]
|
|
++ lib.optionals (!enableShared) [
|
|
(lib.cmakeFeature "ARROW_TEST_LINKAGE" "static")
|
|
]
|
|
++ lib.optionals stdenv.hostPlatform.isDarwin [
|
|
# needed for tools executables
|
|
(lib.cmakeFeature "CMAKE_INSTALL_RPATH" "@loader_path/../lib")
|
|
]
|
|
++ lib.optionals (!stdenv.hostPlatform.isx86_64) [
|
|
(lib.cmakeBool "ARROW_USE_SIMD" false)
|
|
]
|
|
++ lib.optionals enableS3 [
|
|
(lib.cmakeFeature "AWSSDK_CORE_HEADER_FILE" "${aws-sdk-cpp-arrow}/include/aws/core/Aws.h")
|
|
];
|
|
|
|
doInstallCheck = true;
|
|
|
|
__darwinAllowLocalNetworking = true;
|
|
|
|
nativeInstallCheckInputs = [
|
|
perl
|
|
which
|
|
sqlite
|
|
]
|
|
++ lib.optionals enableFlight [ python3 ]
|
|
++ lib.optionals enableAzure [ azurite ];
|
|
|
|
installCheckPhase =
|
|
let
|
|
disabledTests = [
|
|
# flaky
|
|
"arrow-flight-test"
|
|
# requires networking
|
|
"arrow-gcsfs-test"
|
|
"arrow-flight-integration-test"
|
|
# File already exists in database: orc_proto.proto
|
|
"arrow-orc-adapter-test"
|
|
# missing test fixture
|
|
"parquet-encryption-test"
|
|
]
|
|
++ lib.optionals stdenv.hostPlatform.isDarwin [
|
|
# https://github.com/NixOS/nixpkgs/issues/460687
|
|
# Failing with "run-test.sh: line 88: 63682 Abort trap: 6"
|
|
"arrow-flight-internals-test"
|
|
"arrow-flight-sql-test"
|
|
];
|
|
in
|
|
''
|
|
runHook preInstallCheck
|
|
|
|
ctest -L unittest --exclude-regex '^(${lib.concatStringsSep "|" disabledTests})$'
|
|
|
|
runHook postInstallCheck
|
|
'';
|
|
|
|
__structuredAttrs = true;
|
|
|
|
meta = {
|
|
# https://hydra.nixos.org/job/nixpkgs/unstable/arrow-cpp.x86_64-darwin/all
|
|
broken = stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isx86_64;
|
|
description = "Cross-language development platform for in-memory data";
|
|
homepage = "https://arrow.apache.org/docs/cpp/";
|
|
changelog = "https://arrow.apache.org/release/${finalAttrs.version}.html";
|
|
license = lib.licenses.asl20;
|
|
platforms = lib.platforms.unix;
|
|
maintainers = with lib.maintainers; [
|
|
tobim
|
|
veprbl
|
|
cpcloud
|
|
];
|
|
pkgConfigModules = [
|
|
"arrow"
|
|
"arrow-acero"
|
|
"arrow-compute"
|
|
"arrow-csv"
|
|
"arrow-dataset"
|
|
"arrow-filesystem"
|
|
"arrow-flight"
|
|
"arrow-flight-sql"
|
|
"arrow-flight-testing"
|
|
"arrow-json"
|
|
"arrow-substrait"
|
|
"arrow-testing"
|
|
"parquet"
|
|
];
|
|
};
|
|
passthru = {
|
|
inherit
|
|
enableFlight
|
|
enableJemalloc
|
|
enableS3
|
|
enableGcs
|
|
;
|
|
tests.pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage;
|
|
};
|
|
})
|