diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix index ea0f67ce8c7f..274a35c6ccfb 100644 --- a/pkgs/development/libraries/arrow-cpp/default.nix +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -162,6 +162,7 @@ stdenv.mkDerivation rec { "-DARROW_ENGINE=ON" "-DARROW_FILESYSTEM=ON" "-DARROW_FLIGHT_SQL=${if enableFlight then "ON" else "OFF"}" + "-DARROW_HDFS=ON" "-DARROW_IPC=ON" "-DARROW_JEMALLOC=${if enableJemalloc then "ON" else "OFF"}" "-DARROW_JSON=ON" diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index c47cdaa02b70..cfe63a34d85a 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -1,4 +1,24 @@ -{ lib, stdenv, buildPythonPackage, python, isPy3k, arrow-cpp, cmake, cython, hypothesis, numpy, pandas, pytestCheckHook, pytest-lazy-fixture, pkg-config, setuptools-scm, six }: +{ lib +, stdenv +, buildPythonPackage +, python +, isPy3k +, arrow-cpp +, cffi +, cloudpickle +, cmake +, cython +, fsspec +, hypothesis +, numpy +, pandas +, pytestCheckHook +, pytest-lazy-fixture +, pkg-config +, scipy +, setuptools-scm +, six +}: let zero_or_one = cond: if cond then 1 else 0; @@ -15,21 +35,23 @@ buildPythonPackage rec { sourceRoot = "apache-arrow-${version}/python"; nativeBuildInputs = [ cmake cython pkg-config setuptools-scm ]; - propagatedBuildInputs = [ numpy six ]; - checkInputs = [ hypothesis pandas pytestCheckHook pytest-lazy-fixture ]; + propagatedBuildInputs = [ numpy six cloudpickle scipy fsspec cffi ]; + checkInputs = [ + hypothesis + pandas + pytestCheckHook + pytest-lazy-fixture + ]; PYARROW_BUILD_TYPE = "release"; PYARROW_WITH_DATASET = zero_or_one true; PYARROW_WITH_FLIGHT = zero_or_one _arrow-cpp.enableFlight; PYARROW_WITH_PARQUET = zero_or_one true; + PYARROW_WITH_HDFS = zero_or_one true; PYARROW_CMAKE_OPTIONS = [ "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib" - - # This doesn't use setup hook to call cmake so we need to workaround #54606 - # ourselves - "-DCMAKE_POLICY_DEFAULT_CMP0025=NEW" ]; ARROW_HOME = _arrow-cpp; @@ -51,9 +73,6 @@ buildPythonPackage rec { # enabled in nixpkgs. # Upstream Issue: https://issues.apache.org/jira/browse/ARROW-11393 "--deselect=pyarrow/tests/test_memory.py::test_env_var" - # Deselect a parquet dataset test because it erroneously fails to find the - # pyarrow._dataset module. - "--deselect=pyarrow/tests/parquet/test_dataset.py::test_parquet_dataset_deprecated_properties" ] ++ lib.optionals stdenv.isDarwin [ # Requires loopback networking "--deselect=pyarrow/tests/test_ipc.py::test_socket_" @@ -61,12 +80,21 @@ buildPythonPackage rec { dontUseSetuptoolsCheck = true; preCheck = '' - mv pyarrow/tests tests - rm -rf pyarrow - mkdir pyarrow - mv tests pyarrow/tests + shopt -s extglob + rm -r pyarrow/!(tests) ''; + pythonImportsCheck = map (module: "pyarrow.${module}") [ + "compute" + "csv" + "dataset" + "flight" + "fs" + "hdfs" + "json" + "parquet" + ]; + meta = with lib; { description = "A cross-language development platform for in-memory data"; homepage = "https://arrow.apache.org/";