From f44d17da59feb985006683fcdfe2d72d8ae73e99 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 17 Feb 2022 07:11:42 -0500 Subject: [PATCH 1/8] arrow-cpp: enable hdfs support --- pkgs/development/libraries/arrow-cpp/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix index ea0f67ce8c7f..274a35c6ccfb 100644 --- a/pkgs/development/libraries/arrow-cpp/default.nix +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -162,6 +162,7 @@ stdenv.mkDerivation rec { "-DARROW_ENGINE=ON" "-DARROW_FILESYSTEM=ON" "-DARROW_FLIGHT_SQL=${if enableFlight then "ON" else "OFF"}" + "-DARROW_HDFS=ON" "-DARROW_IPC=ON" "-DARROW_JEMALLOC=${if enableJemalloc then "ON" else "OFF"}" "-DARROW_JSON=ON" From 430037f8493b50b93554ebaf841569088d4d85c9 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 17 Feb 2022 07:11:54 -0500 Subject: [PATCH 2/8] python3Packages.pyarrow: enable hdfs support --- pkgs/development/python-modules/pyarrow/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index c47cdaa02b70..4aa5c83ed681 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -23,6 +23,7 @@ buildPythonPackage rec { PYARROW_WITH_DATASET = zero_or_one true; PYARROW_WITH_FLIGHT = zero_or_one _arrow-cpp.enableFlight; PYARROW_WITH_PARQUET = zero_or_one true; + PYARROW_WITH_HDFS = zero_or_one true; PYARROW_CMAKE_OPTIONS = [ "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib" From 64dc56cf3604d0f97542e71b07646ed31069a9eb Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 17 Feb 2022 07:12:13 -0500 Subject: [PATCH 3/8] python3Packages.pyarrow: add imports check --- pkgs/development/python-modules/pyarrow/default.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index 4aa5c83ed681..d71fdf96a259 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -68,6 +68,17 @@ buildPythonPackage rec { mv tests pyarrow/tests ''; + pythonImportsCheck = map (module: "pyarrow.${module}") [ + "compute" + "csv" + "dataset" + "flight" + "fs" + "hdfs" + "json" + "parquet" + ]; + meta = with lib; { description = "A cross-language development platform for in-memory data"; homepage = "https://arrow.apache.org/"; From 0d6ee4eb63003ae78be115797883b94c4371c5d9 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 17 Feb 2022 07:12:33 -0500 Subject: [PATCH 4/8] python3Packages.pyarrow: format imports and check inputs --- .../python-modules/pyarrow/default.nix | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index d71fdf96a259..e6df68632b2c 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -1,4 +1,20 @@ -{ lib, stdenv, buildPythonPackage, python, isPy3k, arrow-cpp, cmake, cython, hypothesis, numpy, pandas, pytestCheckHook, pytest-lazy-fixture, pkg-config, setuptools-scm, six }: +{ lib +, stdenv +, buildPythonPackage +, python +, isPy3k +, arrow-cpp +, cmake +, cython +, hypothesis +, numpy +, pandas +, pytestCheckHook +, pytest-lazy-fixture +, pkg-config +, setuptools-scm +, six +}: let zero_or_one = cond: if cond then 1 else 0; @@ -16,7 +32,12 @@ buildPythonPackage rec { nativeBuildInputs = [ cmake cython pkg-config setuptools-scm ]; propagatedBuildInputs = [ numpy six ]; - checkInputs = [ hypothesis pandas pytestCheckHook pytest-lazy-fixture ]; + checkInputs = [ + hypothesis + pandas + pytestCheckHook + pytest-lazy-fixture + ]; PYARROW_BUILD_TYPE = "release"; @@ -87,3 +108,5 @@ buildPythonPackage rec { maintainers = with maintainers; [ veprbl cpcloud ]; }; } + + From f171e5c22d719a0741b3ac1b502bf63e52d494c8 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 17 Feb 2022 07:13:46 -0500 Subject: [PATCH 5/8] python3Packages.pyarrow: add cloudpickle, cffi, scipy and fsspec to propagatedBuildInputs --- pkgs/development/python-modules/pyarrow/default.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index e6df68632b2c..a6b6ad316167 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -4,14 +4,18 @@ , python , isPy3k , arrow-cpp +, cffi +, cloudpickle , cmake , cython +, fsspec , hypothesis , numpy , pandas , pytestCheckHook , pytest-lazy-fixture , pkg-config +, scipy , setuptools-scm , six }: @@ -31,7 +35,7 @@ buildPythonPackage rec { sourceRoot = "apache-arrow-${version}/python"; nativeBuildInputs = [ cmake cython pkg-config setuptools-scm ]; - propagatedBuildInputs = [ numpy six ]; + propagatedBuildInputs = [ numpy six cloudpickle scipy fsspec cffi ]; checkInputs = [ hypothesis pandas @@ -108,5 +112,3 @@ buildPythonPackage rec { maintainers = with maintainers; [ veprbl cpcloud ]; }; } - - From 074ded162ccf783ae889ae41663e61bf4af26072 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Fri, 18 Feb 2022 05:39:36 -0500 Subject: [PATCH 6/8] python3Packages.pyarrow: enable test that is no longer broken --- pkgs/development/python-modules/pyarrow/default.nix | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index a6b6ad316167..502840048ce4 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -77,9 +77,6 @@ buildPythonPackage rec { # enabled in nixpkgs. # Upstream Issue: https://issues.apache.org/jira/browse/ARROW-11393 "--deselect=pyarrow/tests/test_memory.py::test_env_var" - # Deselect a parquet dataset test because it erroneously fails to find the - # pyarrow._dataset module. - "--deselect=pyarrow/tests/parquet/test_dataset.py::test_parquet_dataset_deprecated_properties" ] ++ lib.optionals stdenv.isDarwin [ # Requires loopback networking "--deselect=pyarrow/tests/test_ipc.py::test_socket_" From ce6b8be7d74e0e4f7c99a91ac661b424e84e5ce3 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Fri, 18 Feb 2022 05:52:56 -0500 Subject: [PATCH 7/8] python3Packages.pyarrow: use extglob to reduce preCheck code --- pkgs/development/python-modules/pyarrow/default.nix | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index 502840048ce4..d48fedc6431b 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -84,10 +84,8 @@ buildPythonPackage rec { dontUseSetuptoolsCheck = true; preCheck = '' - mv pyarrow/tests tests - rm -rf pyarrow - mkdir pyarrow - mv tests pyarrow/tests + shopt -s extglob + rm -r pyarrow/!(tests) ''; pythonImportsCheck = map (module: "pyarrow.${module}") [ From ed3872c352d83b71d42832cd1f35c706426cef52 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Fri, 18 Feb 2022 06:52:38 -0500 Subject: [PATCH 8/8] python3Packages.pyarrow: remove unnecessary cmake policy flag --- pkgs/development/python-modules/pyarrow/default.nix | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index d48fedc6431b..cfe63a34d85a 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -52,10 +52,6 @@ buildPythonPackage rec { PYARROW_CMAKE_OPTIONS = [ "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib" - - # This doesn't use setup hook to call cmake so we need to workaround #54606 - # ourselves - "-DCMAKE_POLICY_DEFAULT_CMP0025=NEW" ]; ARROW_HOME = _arrow-cpp;