Merge pull request #232602 from cpcloud/bump-duckdb

duckdb: 0.7.1 -> 0.8.0

pkgs/development/libraries/duckdb/default.nix

@@ -1,7 +1,6 @@
 { lib
 , stdenv
 , fetchFromGitHub
-, fetchpatch
 , cmake
 , ninja
 , openssl
@@ -16,23 +15,19 @@ let
 in
 stdenv.mkDerivation rec {
   pname = "duckdb";
-  version = "0.7.1";
+  version = "0.8.0";
 
   src = fetchFromGitHub {
     owner = pname;
     repo = pname;
     rev = "v${version}";
-    sha256 = "sha256-dCPWrB/Jqm4/kS6J/3jcQG291tFKAZSEptEYLGOZsLo=";
+    hash = "sha256-w01QB3mufeKdo4d0BBm6xiidnnyo9CLpmntT6690leM=";
   };
 
   patches = [ ./version.patch ];
 
   postPatch = ''
     substituteInPlace CMakeLists.txt --subst-var-by DUCKDB_VERSION "v${version}"
-    substituteInPlace tools/shell/CMakeLists.txt \
-      --replace \
-      'install(TARGETS shell RUNTIME DESTINATION "''${PROJECT_BINARY_DIR}")' \
-      'install(TARGETS shell RUNTIME DESTINATION "''${INSTALL_BIN_DIR}")'
   '';
 
   nativeBuildInputs = [ cmake ninja ];
@@ -90,6 +85,16 @@ stdenv.mkDerivation rec {
         "test/sql/copy/parquet/delta_byte_array_length_mismatch.test"
         "test/sql/function/timestamp/test_icu_strptime.test"
         "test/sql/timezone/test_icu_timezone.test"
+        "test/sql/copy/parquet/snowflake_lineitem.test"
+        "test/sql/copy/parquet/test_parquet_force_download.test"
+        "test/sql/copy/parquet/delta_byte_array_multiple_pages.test"
+        "test/sql/copy/csv/test_csv_httpfs_prepared.test"
+        "test/sql/copy/csv/test_csv_httpfs.test"
+        "test/sql/copy/csv/parallel/test_parallel_csv.test"
+        "test/sql/copy/csv/parallel/csv_parallel_httpfs.test"
+        "test/common/test_cast_struct.test"
+        # test is order sensitive
+        "test/sql/copy/parquet/parquet_glob.test"
         # these are only hidden if no filters are passed in
         "[!hide]"
         # this test apparently never terminates

pkgs/development/python-modules/datafusion/default.nix

@@ -1,7 +1,6 @@
 { lib
 , stdenv
 , buildPythonPackage
-, fetchPypi
 , fetchFromGitHub
 , rustPlatform
 , pytestCheckHook
@@ -17,22 +16,22 @@ let
     name = "arrow-testing";
     owner = "apache";
     repo = "arrow-testing";
-    rev = "47f7b56b25683202c1fd957668e13f2abafc0f12";
-    hash = "sha256-ZDznR+yi0hm5O1s9as8zq5nh1QxJ8kXCRwbNQlzXpnI=";
+    rev = "5bab2f264a23f5af68f69ea93d24ef1e8e77fc88";
+    hash = "sha256-Pxx8ohUpXb5u1995IvXmxQMqWiDJ+7LAll/AjQP7ph8=";
   };
 
   parquet-testing = fetchFromGitHub {
     name = "parquet-testing";
     owner = "apache";
     repo = "parquet-testing";
-    rev = "b2e7cc755159196e3a068c8594f7acbaecfdaaac";
-    hash = "sha256-IFvGTOkaRSNgZOj8DziRj88yH5JRF+wgSDZ5N0GNvjk=";
+    rev = "e13af117de7c4f0a4d9908ae3827b3ab119868f3";
+    hash = "sha256-rVI9zyk9IRDlKv4u8BeMb0HRdWLfCpqOlYCeUdA7BB8=";
   };
 in
 
 buildPythonPackage rec {
   pname = "datafusion";
-  version = "23.0.0";
+  version = "25.0.0";
   format = "pyproject";
 
   src = fetchFromGitHub {
@@ -40,13 +39,13 @@ buildPythonPackage rec {
     owner = "apache";
     repo = "arrow-datafusion-python";
     rev = "refs/tags/${version}";
-    hash = "sha256-ndee7aNmoTtZyfl9UUXdNVHkp0GAuJWkyfZJyRrGwn8=";
+    hash = "sha256-oC+fp41a9rsdobpvShZ7sDdtYPJQQ7JLg6MFL+4Pksg=";
   };
 
   cargoDeps = rustPlatform.fetchCargoTarball {
     name = "datafusion-cargo-deps";
     inherit src pname version;
-    hash = "sha256-eDweEc+7dDbF0WBi6M5XAPIiHRjlYAdf2eNJdwj4D7c=";
+    hash = "sha256-0e0ZRgwcS/46mi4c2loAnBA2bsaD+/RiMh7oNg3EvHY=";
   };
 
   nativeBuildInputs = with rustPlatform; [

pkgs/development/python-modules/duckdb-engine/default.nix

@@ -7,13 +7,14 @@
 , hypothesis
 , ipython-sql
 , poetry-core
+, snapshottest
 , sqlalchemy
 , typing-extensions
 }:
 
 buildPythonPackage rec {
   pname = "duckdb-engine";
-  version = "0.7.0";
+  version = "0.7.3";
   format = "pyproject";
 
   disabled = pythonOlder "3.7";
@@ -22,7 +23,7 @@ buildPythonPackage rec {
     repo = "duckdb_engine";
     owner = "Mause";
     rev = "refs/tags/v${version}";
-    hash = "sha256-qLQjFkud9DivLQ9PignLrXlUVOAxsd28s7+2GdC5jKE=";
+    hash = "sha256-Z9m1+Bc/csWKdPDuwf82xX0qOiD1Y5LBgJjUlLntAO8=";
   };
 
   nativeBuildInputs = [
@@ -38,15 +39,19 @@ buildPythonPackage rec {
     export HOME="$(mktemp -d)"
   '';
 
-  # this test tries to download the httpfs extension
   disabledTests = [
+    # this test tries to download the httpfs extension
     "test_preload_extension"
+    # test should be skipped based on sqlalchemy version but isn't and fails
+    "test_commit"
   ];
 
   nativeCheckInputs = [
     pytestCheckHook
     hypothesis
     ipython-sql
+    # TODO(cpcloud): include pandas here when it supports sqlalchemy 2.0
+    snapshottest
     typing-extensions
   ];
 

pkgs/development/python-modules/duckdb/default.nix

@@ -1,6 +1,5 @@
 { lib
 , buildPythonPackage
-, fetchpatch
 , duckdb
 , google-cloud-storage
 , mypy
@@ -47,6 +46,15 @@ buildPythonPackage rec {
     pytestCheckHook
   ];
 
+  disabledTests = [
+    # tries to make http request
+    "test_install_non_existent_extension"
+  ];
+
+  preCheck = ''
+    export HOME="$(mktemp -d)"
+  '';
+
   pythonImportsCheck = [
     "duckdb"
   ];

pkgs/development/python-modules/ibis-framework/default.nix

@@ -139,6 +139,20 @@ buildPythonPackage rec {
     # tries to download duckdb extensions
     "--deselect=ibis/backends/duckdb/tests/test_register.py::test_register_sqlite"
     "--deselect=ibis/backends/duckdb/tests/test_register.py::test_read_sqlite"
+    # all the following tests are fixed in the next release (6.0.0)
+    "--deselect=ibis/backends/tests/test_export.py::test_column_to_pyarrow_table_schema"
+    "--deselect=ibis/backends/tests/test_export.py::test_table_to_pyarrow_batches"
+    "--deselect=ibis/backends/tests/test_export.py::test_column_to_pyarrow_batches"
+    "--deselect=ibis/backends/tests/test_export.py::test_to_pyarrow_batches_memtable"
+    "--deselect=ibis/backends/tests/test_export.py::test_table_to_pyarrow_table_schema"
+    "--deselect=ibis/backends/tests/test_export.py::test_to_pyarrow_batches_borked_types"
+    "--deselect=ibis/backends/tests/test_export.py::test_table_pyarrow_batch_chunk_size"
+    "--deselect=ibis/backends/tests/test_export.py::test_column_pyarrow_batch_chunk_size"
+    "--deselect=ibis/backends/tests/test_export.py::test_roundtrip_partitioned_parquet"
+    "--deselect=ibis/tests/sql/test_sqlalchemy.py::test_order_by"
+    "--deselect=ibis/tests/sql/test_sqlalchemy.py::test_no_cart_join"
+    "--deselect=ibis/backends/tests/test_temporal.py::test_temporal_binop"
+    "--deselect=ibis/backends/tests/test_temporal.py::test_interval_literal"
   ];
 
   # patch out tests that check formatting with black
@@ -146,6 +160,7 @@ buildPythonPackage rec {
     find ibis/tests -type f -name '*.py' -exec sed -i \
       -e '/^ *assert_decompile_roundtrip/d' \
       -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
+    substituteInPlace pyproject.toml --replace 'sqlglot = ">=10.4.3,<12"' 'sqlglot = "*"'
   '';
 
   preCheck = ''

pkgs/development/python-modules/sqlglot/default.nix

@@ -5,11 +5,11 @@
 , pytestCheckHook
 , python-dateutil
 , duckdb
-, pyspark
+, setuptools-scm
 }:
 buildPythonPackage rec {
   pname = "sqlglot";
-  version = "10.5.2";
+  version = "15.0.0";
 
   disabled = pythonOlder "3.8";
 
@@ -17,12 +17,17 @@ buildPythonPackage rec {
     repo = "sqlglot";
     owner = "tobymao";
     rev = "v${version}";
-    hash = "sha256-ZFc2aOhCTRFlrzgnYDSdIZxRqKZ8FvkYSZRU0OMHI34=";
+    hash = "sha256-F95JUYUgimWMyfPAV55e+bOcvWy5A95djKcJuSQ8RRs=";
   };
 
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
+
+  nativeBuildInputs = [ setuptools-scm ];
+
+  # optional dependency used in the sqlglot optimizer
   propagatedBuildInputs = [ python-dateutil ];
 
-  nativeCheckInputs = [ pytestCheckHook duckdb pyspark ];
+  nativeCheckInputs = [ pytestCheckHook duckdb ];
 
   # these integration tests assume a running Spark instance
   disabledTestPaths = [ "tests/dataframe/integration" ];