From a94a26338c244a03008a3b6c0e150b581538727d Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Thu, 31 Aug 2023 21:17:17 +0200 Subject: [PATCH 1/6] arrow-cpp: 12.0.1 -> 13.0.0 --- .../arrow-cpp/cmake-find-protobuf.patch | 103 ++++++++++++++++-- .../libraries/arrow-cpp/darwin.patch | 11 -- .../libraries/arrow-cpp/default.nix | 7 +- 3 files changed, 98 insertions(+), 23 deletions(-) delete mode 100644 pkgs/development/libraries/arrow-cpp/darwin.patch diff --git a/pkgs/development/libraries/arrow-cpp/cmake-find-protobuf.patch b/pkgs/development/libraries/arrow-cpp/cmake-find-protobuf.patch index 5655c151e2f6..32c0a4cab584 100644 --- a/pkgs/development/libraries/arrow-cpp/cmake-find-protobuf.patch +++ b/pkgs/development/libraries/arrow-cpp/cmake-find-protobuf.patch @@ -1,18 +1,105 @@ -diff --git a/cmake_modules/FindProtobufAlt.cmake b/cmake_modules/FindProtobufAlt.cmake -index d29f757ae..61c6e16e1 100644 +diff --git a/cmake_modules/FindProtobufAlt.cmake b/cmake_modules/FindprotobufAlt.cmake +similarity index 60% +rename from cmake_modules/FindProtobufAlt.cmake +rename to cmake_modules/FindprotobufAlt.cmake +index 15fe1b4f2..19bdd6291 100644 --- a/cmake_modules/FindProtobufAlt.cmake -+++ b/cmake_modules/FindProtobufAlt.cmake -@@ -22,11 +22,8 @@ else() ++++ b/cmake_modules/FindprotobufAlt.cmake +@@ -16,24 +16,24 @@ + # under the License. + + if(ARROW_PROTOBUF_USE_SHARED) +- set(Protobuf_USE_STATIC_LIBS OFF) ++ set(protobuf_USE_STATIC_LIBS OFF) + else() +- set(Protobuf_USE_STATIC_LIBS ON) ++ set(protobuf_USE_STATIC_LIBS ON) endif() set(find_package_args) -if(ProtobufAlt_FIND_VERSION) - list(APPEND find_package_args ${ProtobufAlt_FIND_VERSION}) --endif() - if(ProtobufAlt_FIND_QUIETLY) ++if(protobufAlt_FIND_VERSION) ++ list(APPEND find_package_args ${protobufAlt_FIND_VERSION}) + endif() +-if(ProtobufAlt_FIND_QUIETLY) ++if(protobufAlt_FIND_QUIETLY) list(APPEND find_package_args QUIET) endif() -find_package(Protobuf ${find_package_args}) -set(ProtobufAlt_FOUND ${Protobuf_FOUND}) -+find_package(protobuf ${find_package_args}) -+set(ProtobufAlt_FOUND ${protobuf_FOUND}) +-if(ProtobufAlt_FOUND) +- set(ProtobufAlt_VERSION ${Protobuf_VERSION}) +- set(ProtobufAlt_VERSION_MAJOR ${Protobuf_VERSION_MAJOR}) +- set(ProtobufAlt_VERSION_MINOR ${Protobuf_VERSION_MINOR}) +- set(ProtobufAlt_VERSION_PATCH ${Protobuf_VERSION_PATCH}) +- set(ProtobufAlt_VERSION_TWEEK ${Protobuf_VERSION_TWEEK}) ++find_package(protobuf ${find_package_args} CONFIG) ++set(protobufAlt_FOUND ${protobuf_FOUND}) ++if(protobufAlt_FOUND) ++ set(protobufAlt_VERSION ${protobuf_VERSION}) ++ set(protobufAlt_VERSION_MAJOR ${protobuf_VERSION_MAJOR}) ++ set(protobufAlt_VERSION_MINOR ${protobuf_VERSION_MINOR}) ++ set(protobufAlt_VERSION_PATCH ${protobuf_VERSION_PATCH}) ++ set(protobufAlt_VERSION_TWEEK ${protobuf_VERSION_TWEEK}) + endif() +diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake +index 94f926039..8efc98323 100644 +--- a/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cmake_modules/ThirdpartyToolchain.cmake +@@ -63,7 +63,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES + opentelemetry-cpp + ORC + re2 +- Protobuf ++ protobuf + RapidJSON + Snappy + Substrait +@@ -189,7 +189,7 @@ macro(build_dependency DEPENDENCY_NAME) + build_opentelemetry() + elseif("${DEPENDENCY_NAME}" STREQUAL "ORC") + build_orc() +- elseif("${DEPENDENCY_NAME}" STREQUAL "Protobuf") ++ elseif("${DEPENDENCY_NAME}" STREQUAL "protobuf") + build_protobuf() + elseif("${DEPENDENCY_NAME}" STREQUAL "RapidJSON") + build_rapidjson() +@@ -1722,7 +1722,7 @@ if(ARROW_WITH_PROTOBUF) + # "3" for 3.12.0 are different. So 23.4 is rejected with 3.12.0. If + # we use FORCE_ANY_NEWER_VERSION here, we can bypass the check and + # use 23.4. +- resolve_dependency(Protobuf ++ resolve_dependency(protobuf + FORCE_ANY_NEWER_VERSION + TRUE + HAVE_ALT +@@ -1732,7 +1732,7 @@ if(ARROW_WITH_PROTOBUF) + PC_PACKAGE_NAMES + protobuf) + +- if(NOT Protobuf_USE_STATIC_LIBS AND MSVC_TOOLCHAIN) ++ if(NOT protobuf_USE_STATIC_LIBS AND MSVC_TOOLCHAIN) + add_definitions(-DPROTOBUF_USE_DLLS) + endif() + +@@ -1768,8 +1768,8 @@ if(ARROW_WITH_PROTOBUF) + message(STATUS "Found protoc: ${PROTOBUF_PROTOC_EXECUTABLE}") + get_target_property(PROTOBUF_TYPE ${ARROW_PROTOBUF_LIBPROTOBUF} TYPE) + if(NOT STREQUAL "INTERFACE_LIBRARY") +- # Protobuf_PROTOC_LIBRARY is set by all versions of FindProtobuf.cmake +- message(STATUS "Found libprotoc: ${Protobuf_PROTOC_LIBRARY}") ++ # protobuf_PROTOC_LIBRARY is set by all versions of FindProtobuf.cmake ++ message(STATUS "Found libprotoc: ${protobuf_PROTOC_LIBRARY}") + get_target_property(PROTOBUF_LIBRARY ${ARROW_PROTOBUF_LIBPROTOBUF} IMPORTED_LOCATION) + message(STATUS "Found libprotobuf: ${PROTOBUF_LIBRARY}") + message(STATUS "Found protobuf headers: ${PROTOBUF_INCLUDE_DIR}") +@@ -3986,7 +3986,7 @@ endmacro() + + if(ARROW_WITH_GRPC) + set(ARROW_GRPC_REQUIRED_VERSION "1.30.0") +- if(NOT Protobuf_SOURCE STREQUAL gRPC_SOURCE) ++ if(NOT protobuf_SOURCE STREQUAL gRPC_SOURCE) + # ARROW-15495: Protobuf/gRPC must come from the same source + message(STATUS "Forcing gRPC_SOURCE to Protobuf_SOURCE (${Protobuf_SOURCE})") + set(gRPC_SOURCE "${Protobuf_SOURCE}") diff --git a/pkgs/development/libraries/arrow-cpp/darwin.patch b/pkgs/development/libraries/arrow-cpp/darwin.patch deleted file mode 100644 index de9b1986ffed..000000000000 --- a/pkgs/development/libraries/arrow-cpp/darwin.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff --git a/cmake_modules/FindPythonLibsNew.cmake b/cmake_modules/FindPythonLibsNew.cmake ---- a/cmake_modules/FindPythonLibsNew.cmake -+++ b/cmake_modules/FindPythonLibsNew.cmake -@@ -117,6 +117,7 @@ list(GET _PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) - list(GET _PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) - list(GET _PYTHON_VALUES 8 PYTHON_LIBRARY_PATH) - list(GET _PYTHON_VALUES 9 PYTHON_OTHER_LIBS) -+string(REPLACE "-lncurses" "" PYTHON_OTHER_LIBS "${PYTHON_OTHER_LIBS}") - - # Make sure the Python has the same pointer-size as the chosen compiler - # Skip the check on OS X, it doesn't consistently have CMAKE_SIZEOF_VOID_P defined diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix index c1bbc3e303d1..ddf8db98b39b 100644 --- a/pkgs/development/libraries/arrow-cpp/default.nix +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -78,11 +78,11 @@ let in stdenv.mkDerivation rec { pname = "arrow-cpp"; - version = "12.0.1"; + version = "13.0.0"; src = fetchurl { url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz"; - hash = "sha256-NIHEETk6oVx16I2Tz4MV+vf0PhgP4HkBKNOEDUF96Fg="; + hash = "sha256-Nd/aGRJip1a+k07viv7o0JdiytJQIdqmJuskniUayeY="; }; sourceRoot = "apache-arrow-${version}/cpp"; @@ -120,8 +120,7 @@ stdenv.mkDerivation rec { }; patches = [ - # patch to fix python-test - ./darwin.patch + # Protobuf switched to lower case project name. ./cmake-find-protobuf.patch ]; From 219775241d845a2c6220420d649e1f362053dc33 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Sat, 2 Sep 2023 06:56:43 +0200 Subject: [PATCH 2/6] python3Packages.dask: disable failing test --- pkgs/development/python-modules/dask/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/development/python-modules/dask/default.nix b/pkgs/development/python-modules/dask/default.nix index 5166afde24de..355e16553be0 100644 --- a/pkgs/development/python-modules/dask/default.nix +++ b/pkgs/development/python-modules/dask/default.nix @@ -134,6 +134,8 @@ buildPythonPackage rec { # AttributeError: 'str' object has no attribute 'decode' "test_read_dir_nometa" ] ++ [ + # https://github.com/dask/dask/issues/10347#issuecomment-1589683941 + "test_concat_categorical" # AttributeError: 'ArrowStringArray' object has no attribute 'tobytes'. Did you mean: 'nbytes'? "test_dot" "test_dot_nan" From 52a3f592df60fe092964de85740499e35a7683c8 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Sat, 2 Sep 2023 14:30:19 +0200 Subject: [PATCH 3/6] python3Packages.intake: disable failing tests --- pkgs/development/python-modules/intake/default.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkgs/development/python-modules/intake/default.nix b/pkgs/development/python-modules/intake/default.nix index 98450dd31ea9..46c7ab88248a 100644 --- a/pkgs/development/python-modules/intake/default.nix +++ b/pkgs/development/python-modules/intake/default.nix @@ -107,6 +107,16 @@ buildPythonPackage rec { "test_python" # Timing-based, flaky on darwin and possibly others "TestServerV1Source.test_idle_timer" + # arrow-cpp-13 related + "test_read" + "test_pickle" + "test_read_dask" + "test_read_list" + "test_read_list_with_glob" + "test_to_dask" + "test_columns" + "test_df_transform" + "test_pipeline_apply" ] ++ lib.optionals (stdenv.isDarwin && lib.versionOlder stdenv.hostPlatform.darwinMinVersion "10.13") [ # Flaky with older low-res mtime on darwin < 10.13 (#143987) "test_second_load_timestamp" From 472525b1cf361b9a0aef3e3056d0a407708dcd37 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Sun, 3 Sep 2023 10:25:46 +0200 Subject: [PATCH 4/6] python3Packages.dask-awkward: adjust to pyarrow-13 --- pkgs/development/python-modules/dask-awkward/default.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkgs/development/python-modules/dask-awkward/default.nix b/pkgs/development/python-modules/dask-awkward/default.nix index d7ef1204bc3e..a0ae12d7c13e 100644 --- a/pkgs/development/python-modules/dask-awkward/default.nix +++ b/pkgs/development/python-modules/dask-awkward/default.nix @@ -3,6 +3,7 @@ , buildPythonPackage , dask , fetchFromGitHub +, fetchpatch , hatch-vcs , hatchling , pyarrow @@ -25,6 +26,14 @@ buildPythonPackage rec { hash = "sha256-sSsd35Psf3VEydkNxtd9mSBzV23S7fRM/jhbC9T62kY="; }; + patches = [ + (fetchpatch { + name = "dask-awkward-pyarrow13-test-fixes.patch"; + url = "https://github.com/dask-contrib/dask-awkward/commit/abe7f4504b4f926232e4d0dfa5c601d265773d85.patch"; + hash = "sha256-IYlKTV6YasuKIJutB4cCmHeglGWUwBcvFgx1MZw4hjU="; + }) + ]; + SETUPTOOLS_SCM_PRETEND_VERSION = version; pythonRelaxDeps = [ From 11c98fd1ec1d6fe769467754097fbc5e76f448d5 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Sun, 3 Sep 2023 11:11:37 +0200 Subject: [PATCH 5/6] python3Packages.{google-cloud-bigquery,insightface,worldengine}: mark as broken --- .../python-modules/google-cloud-bigquery/default.nix | 2 ++ pkgs/development/python-modules/insightface/default.nix | 3 +++ pkgs/development/python-modules/worldengine/default.nix | 1 + 3 files changed, 6 insertions(+) diff --git a/pkgs/development/python-modules/google-cloud-bigquery/default.nix b/pkgs/development/python-modules/google-cloud-bigquery/default.nix index 69a32d87a1f5..7edbedb80c6a 100644 --- a/pkgs/development/python-modules/google-cloud-bigquery/default.nix +++ b/pkgs/development/python-modules/google-cloud-bigquery/default.nix @@ -127,6 +127,8 @@ buildPythonPackage rec { ]; meta = with lib; { + # Not compatible with pyarrow13 yet. + broken = true; description = "Google BigQuery API client library"; homepage = "https://github.com/googleapis/python-bigquery"; changelog = "https://github.com/googleapis/python-bigquery/blob/v${version}/CHANGELOG.md"; diff --git a/pkgs/development/python-modules/insightface/default.nix b/pkgs/development/python-modules/insightface/default.nix index 6d5c4a66239c..e06b86f91802 100644 --- a/pkgs/development/python-modules/insightface/default.nix +++ b/pkgs/development/python-modules/insightface/default.nix @@ -69,6 +69,9 @@ buildPythonPackage rec { doCheck = false; # Upstream has no tests meta = with lib; { + # Both protobuf3 and protobuf4 in the build closure. + # related: https://github.com/onnx/onnx/issues/5563 + broken = true; description = "State-of-the-art 2D and 3D Face Analysis Project"; homepage = "https://github.com/deepinsight/insightface"; license = licenses.mit; diff --git a/pkgs/development/python-modules/worldengine/default.nix b/pkgs/development/python-modules/worldengine/default.nix index 32c69b001249..28d25317afa9 100644 --- a/pkgs/development/python-modules/worldengine/default.nix +++ b/pkgs/development/python-modules/worldengine/default.nix @@ -68,6 +68,7 @@ buildPythonPackage rec { ]; meta = with lib; { + broken = true; homepage = "https://github.com/mindwerks/worldengine"; description = "World generator using simulation of plates, rain shadow, erosion, etc"; license = licenses.mit; From 7964e8a38c95d4ad2c4ec9bbb28508c8442f1d4b Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Mon, 4 Sep 2023 11:18:49 +0200 Subject: [PATCH 6/6] ibis-framework: disable tests broken by pyarrow13 --- .../python-modules/ibis-framework/default.nix | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkgs/development/python-modules/ibis-framework/default.nix b/pkgs/development/python-modules/ibis-framework/default.nix index 1c5b134ce1ae..16274bc3f00d 100644 --- a/pkgs/development/python-modules/ibis-framework/default.nix +++ b/pkgs/development/python-modules/ibis-framework/default.nix @@ -130,6 +130,19 @@ buildPythonPackage rec { # tries to download duckdb extensions "--deselect=ibis/backends/duckdb/tests/test_register.py::test_register_sqlite" "--deselect=ibis/backends/duckdb/tests/test_register.py::test_read_sqlite" + # pyarrow13 is not supported yet. + "--deselect=ibis/backends/tests/test_temporal.py::test_date_truncate" + "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp" + "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp" + "--deselect=ibis/backends/tests/test_temporal.py::test_interval_add_cast_column" + "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp" + "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp" + "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp" + "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp" + "--deselect=ibis/backends/tests/test_timecontext.py::test_context_adjustment_filter_before_window" + "--deselect=ibis/backends/tests/test_timecontext.py::test_context_adjustment_window_udf" + "--deselect=ibis/backends/tests/test_timecontext.py::test_context_adjustment_window_udf" + "--deselect=ibis/backends/tests/test_aggregation.py::test_aggregate_grouped" ]; # patch out tests that check formatting with black