From b5500b878f855eb556bec62188ea958e167806e6 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sat, 30 Sep 2023 03:25:23 -0700
Subject: [PATCH 01/15] update duckdb and python duckdb to v0.9.0

---
 pkgs/development/libraries/duckdb/default.nix | 79 +++++++------------
 .../libraries/duckdb/version.patch            | 58 --------------
 .../python-modules/duckdb/default.nix         |  6 ++
 3 files changed, 33 insertions(+), 110 deletions(-)
 delete mode 100644 pkgs/development/libraries/duckdb/version.patch

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index f02dacb95034..b149c1bd3b65 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -2,6 +2,7 @@
 , stdenv
 , fetchFromGitHub
 , cmake
+, git
 , ninja
 , openssl
 , openjdk11
@@ -15,40 +16,26 @@ let
 in
 stdenv.mkDerivation rec {
   pname = "duckdb";
-  version = "0.8.1";
+  version = "0.9.0";
 
   src = fetchFromGitHub {
     owner = pname;
     repo = pname;
+    deepClone = true;
     rev = "v${version}";
-    hash = "sha256-LEv9yURkYvONObTbIA4CS+umwCRMH8gRQaDtzbCzID4=";
+    hash = "sha256-NFkeeTpsxazQOstKUUu0b27hXbnq3U5g/+24BIMqtJY=";
   };
 
-  patches = [ ./version.patch ];
-
-  postPatch = ''
-    substituteInPlace CMakeLists.txt --subst-var-by DUCKDB_VERSION "v${version}"
-  '';
-
-  nativeBuildInputs = [ cmake ninja ];
+  nativeBuildInputs = [ cmake git ninja ];
   buildInputs = [ openssl ]
     ++ lib.optionals withJdbc [ openjdk11 ]
     ++ lib.optionals withOdbc [ unixODBC ];
 
   cmakeFlags = [
-    "-DBUILD_AUTOCOMPLETE_EXTENSION=ON"
-    "-DBUILD_ICU_EXTENSION=ON"
-    "-DBUILD_PARQUET_EXTENSION=ON"
-    "-DBUILD_TPCH_EXTENSION=ON"
-    "-DBUILD_TPCDS_EXTENSION=ON"
-    "-DBUILD_FTS_EXTENSION=ON"
-    "-DBUILD_HTTPFS_EXTENSION=ON"
-    "-DBUILD_VISUALIZER_EXTENSION=ON"
-    "-DBUILD_JSON_EXTENSION=ON"
-    "-DBUILD_JEMALLOC_EXTENSION=ON"
-    "-DBUILD_EXCEL_EXTENSION=ON"
-    "-DBUILD_INET_EXTENSION=ON"
-    "-DBUILD_TPCE=ON"
+    "-DDEBUG_STACKTRACE=1"
+    "-DDUCKDB_EXTENSION_CONFIGS=${src}/.github/config/bundled_extensions.cmake"
+    "-DENABLE_EXTENSION_AUTOINSTALL=1"
+    "-DENABLE_EXTENSION_AUTOLOADING=1"
     "-DBUILD_ODBC_DRIVER=${enableFeature withOdbc}"
     "-DJDBC_DRIVER=${enableFeature withJdbc}"
   ] ++ lib.optionals doInstallCheck [
@@ -67,38 +54,26 @@ stdenv.mkDerivation rec {
   installCheckPhase =
     let
       excludes = map (pattern: "exclude:'${pattern}'") [
-        "[s3]"
-        "Test closing database during long running query"
-        "test/common/test_cast_hugeint.test"
-        "test/sql/copy/csv/test_csv_remote.test"
-        "test/sql/copy/parquet/test_parquet_remote.test"
-        "test/sql/copy/parquet/test_parquet_remote_foreign_files.test"
-        "test/sql/storage/compression/chimp/chimp_read.test"
-        "test/sql/storage/compression/chimp/chimp_read_float.test"
-        "test/sql/storage/compression/patas/patas_compression_ratio.test_coverage"
-        "test/sql/storage/compression/patas/patas_read.test"
-        "test/sql/json/read_json_objects.test"
-        "test/sql/json/read_json.test"
-        "test/sql/copy/parquet/parquet_5968.test"
-        "test/fuzzer/pedro/buffer_manager_out_of_memory.test"
-        "test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"
-        "test/sql/copy/parquet/delta_byte_array_length_mismatch.test"
-        "test/sql/function/timestamp/test_icu_strptime.test"
-        "test/sql/timezone/test_icu_timezone.test"
-        "test/sql/copy/parquet/snowflake_lineitem.test"
-        "test/sql/copy/parquet/test_parquet_force_download.test"
-        "test/sql/copy/parquet/delta_byte_array_multiple_pages.test"
-        "test/sql/copy/csv/test_csv_httpfs_prepared.test"
-        "test/sql/copy/csv/test_csv_httpfs.test"
-        "test/sql/copy/csv/parallel/test_parallel_csv.test"
-        "test/sql/copy/csv/parallel/csv_parallel_httpfs.test"
-        "test/common/test_cast_struct.test"
-        # test is order sensitive
-        "test/sql/copy/parquet/parquet_glob.test"
+	"test/fuzzer/pedro/buffer_manager_out_of_memory.test"
+	"test/sql/copy/csv/parallel/csv_parallel_httpfs.test"
+	"test/sql/copy/csv/parallel/test_parallel_csv.test"
+	"test/sql/copy/csv/test_csv_httpfs_prepared.test"
+	"test/sql/copy/csv/test_csv_httpfs.test"
+	"test/sql/copy/csv/test_csv_remote.test"
+	"test/sql/copy/parquet/delta_byte_array_length_mismatch.test"
+	"test/sql/copy/parquet/delta_byte_array_multiple_pages.test"
+	"test/sql/copy/parquet/parquet_5968.test"
+	"test/sql/copy/parquet/snowflake_lineitem.test"
+	"test/sql/copy/parquet/test_parquet_force_download.test"
+	"test/sql/copy/parquet/test_parquet_remote_foreign_files.test"
+	"test/sql/copy/parquet/test_parquet_remote.test"
+	"test/sql/json/table/read_json_objects.test"
+	"test/sql/json/table/read_json.test"
+	"test/sql/settings/test_disabled_file_system_httpfs.test"
+	"test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"
+
         # these are only hidden if no filters are passed in
         "[!hide]"
-        # this test apparently never terminates
-        "test/sql/copy/csv/auto/test_csv_auto.test"
       ] ++ lib.optionals stdenv.isAarch64 [
         "test/sql/aggregate/aggregates/test_kurtosis.test"
         "test/sql/aggregate/aggregates/test_skewness.test"
diff --git a/pkgs/development/libraries/duckdb/version.patch b/pkgs/development/libraries/duckdb/version.patch
deleted file mode 100644
index 1f52fdb3b935..000000000000
--- a/pkgs/development/libraries/duckdb/version.patch
+++ /dev/null
@@ -1,58 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 349af6acf7..7ffec0b4cb 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -196,52 +196,7 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
-   set(SUN TRUE)
- endif()
- 
--find_package(Git)
--if(Git_FOUND)
--  if (NOT DEFINED GIT_COMMIT_HASH)
--    execute_process(
--            COMMAND ${GIT_EXECUTABLE} log -1 --format=%h
--            WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
--            RESULT_VARIABLE GIT_RESULT
--            OUTPUT_VARIABLE GIT_COMMIT_HASH
--            OUTPUT_STRIP_TRAILING_WHITESPACE)
--  endif()
--  execute_process(
--          COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0
--          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
--          OUTPUT_VARIABLE GIT_LAST_TAG
--          OUTPUT_STRIP_TRAILING_WHITESPACE)
--  execute_process(
--          COMMAND ${GIT_EXECUTABLE} describe --tags --long
--          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
--          OUTPUT_VARIABLE GIT_ITERATION
--          OUTPUT_STRIP_TRAILING_WHITESPACE)
--else()
--  message("Git NOT FOUND")
--endif()
--
--if(GIT_RESULT EQUAL "0")
--  string(REGEX REPLACE "v([0-9]+).[0-9]+.[0-9]+" "\\1" DUCKDB_MAJOR_VERSION "${GIT_LAST_TAG}")
--  string(REGEX REPLACE "v[0-9]+.([0-9]+).[0-9]+" "\\1" DUCKDB_MINOR_VERSION "${GIT_LAST_TAG}")
--  string(REGEX REPLACE "v[0-9]+.[0-9]+.([0-9]+)" "\\1" DUCKDB_PATCH_VERSION "${GIT_LAST_TAG}")
--  string(REGEX REPLACE ".*-([0-9]+)-.*" "\\1" DUCKDB_DEV_ITERATION "${GIT_ITERATION}")
--
--  if(DUCKDB_DEV_ITERATION EQUAL 0)
--    # on a tag; directly use the version
--    set(DUCKDB_VERSION "${GIT_LAST_TAG}")
--  else()
--    # not on a tag, increment the patch version by one and add a -devX suffix
--    math(EXPR DUCKDB_PATCH_VERSION "${DUCKDB_PATCH_VERSION}+1")
--    set(DUCKDB_VERSION "v${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}.${DUCKDB_PATCH_VERSION}-dev${DUCKDB_DEV_ITERATION}")
--  endif()
--else()
--  # fallback for when building from tarball
--  set(DUCKDB_MAJOR_VERSION 0)
--  set(DUCKDB_MINOR_VERSION 0)
--  set(DUCKDB_PATCH_VERSION 1)
--  set(DUCKDB_DEV_ITERATION 0)
--  set(DUCKDB_VERSION "v${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}.${DUCKDB_PATCH_VERSION}-dev${DUCKDB_DEV_ITERATION}")
--endif()
-+set(DUCKDB_VERSION "@DUCKDB_VERSION@")
- 
- message(STATUS "git hash ${GIT_COMMIT_HASH}, version ${DUCKDB_VERSION}")
- 
diff --git a/pkgs/development/python-modules/duckdb/default.nix b/pkgs/development/python-modules/duckdb/default.nix
index f582ae9c3fdf..546646c3b35c 100644
--- a/pkgs/development/python-modules/duckdb/default.nix
+++ b/pkgs/development/python-modules/duckdb/default.nix
@@ -1,6 +1,8 @@
 { lib
 , buildPythonPackage
 , duckdb
+, fsspec
+, git
 , google-cloud-storage
 , numpy
 , pandas
@@ -31,6 +33,7 @@ buildPythonPackage rec {
   SETUPTOOLS_SCM_PRETEND_VERSION = version;
 
   nativeBuildInputs = [
+    git
     pybind11
     setuptools-scm
   ];
@@ -41,6 +44,7 @@ buildPythonPackage rec {
   ];
 
   nativeCheckInputs = [
+    fsspec
     google-cloud-storage
     psutil
     pytestCheckHook
@@ -53,6 +57,8 @@ buildPythonPackage rec {
 
   preCheck = ''
     export HOME="$(mktemp -d)"
+    # duckdb directory prevents loading the python module
+    mv duckdb duckdb.pk
   '';
 
   pythonImportsCheck = [

From b03d629461ca2216d4c31d8cdf70550fe7c731a3 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sat, 30 Sep 2023 06:41:41 -0700
Subject: [PATCH 02/15] make duckdb-engine build with duckdb 0.9.0. disable
 autoload and autoinstall as they seem buggy -- even if extension is built in
 they can download and load an extension automatically.

---
 pkgs/development/libraries/duckdb/default.nix |  3 +--
 .../python-modules/duckdb-engine/default.nix  |  9 +++++--
 .../duckdb-engine/remote_data.patch           | 24 +++++++++++++++++++
 .../python-modules/duckdb/default.nix         | 12 ++++++----
 .../python-modules/duckdb/setup.patch         | 22 +++++++++++++++++
 5 files changed, 62 insertions(+), 8 deletions(-)
 create mode 100644 pkgs/development/python-modules/duckdb-engine/remote_data.patch
 create mode 100644 pkgs/development/python-modules/duckdb/setup.patch

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index b149c1bd3b65..127f77b1a7b5 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -32,10 +32,9 @@ stdenv.mkDerivation rec {
     ++ lib.optionals withOdbc [ unixODBC ];
 
   cmakeFlags = [
+    # use similar flags to what is defined in ${src}/.github/workflow/{LinuxRelease,OSX}.yml
     "-DDEBUG_STACKTRACE=1"
     "-DDUCKDB_EXTENSION_CONFIGS=${src}/.github/config/bundled_extensions.cmake"
-    "-DENABLE_EXTENSION_AUTOINSTALL=1"
-    "-DENABLE_EXTENSION_AUTOLOADING=1"
     "-DBUILD_ODBC_DRIVER=${enableFeature withOdbc}"
     "-DJDBC_DRIVER=${enableFeature withJdbc}"
   ] ++ lib.optionals doInstallCheck [
diff --git a/pkgs/development/python-modules/duckdb-engine/default.nix b/pkgs/development/python-modules/duckdb-engine/default.nix
index f2b49cd0ea7b..afd31a54eee0 100644
--- a/pkgs/development/python-modules/duckdb-engine/default.nix
+++ b/pkgs/development/python-modules/duckdb-engine/default.nix
@@ -14,7 +14,7 @@
 
 buildPythonPackage rec {
   pname = "duckdb-engine";
-  version = "0.7.3";
+  version = "0.9.2";
   format = "pyproject";
 
   disabled = pythonOlder "3.7";
@@ -23,9 +23,11 @@ buildPythonPackage rec {
     repo = "duckdb_engine";
     owner = "Mause";
     rev = "refs/tags/v${version}";
-    hash = "sha256-Z9m1+Bc/csWKdPDuwf82xX0qOiD1Y5LBgJjUlLntAO8=";
+    hash = "sha256-T02nGF+YlughRQPinb0I3NC6xsarh4+qRhG8YfhTvhI=";
   };
 
+  patches = [ ./remote_data.patch ];
+
   nativeBuildInputs = [
     poetry-core
   ];
@@ -42,8 +44,11 @@ buildPythonPackage rec {
   disabledTests = [
     # this test tries to download the httpfs extension
     "test_preload_extension"
+    "test_motherduck"
     # test should be skipped based on sqlalchemy version but isn't and fails
     "test_commit"
+    # rowcount nolonger generates an attribute error.
+    "test_rowcount"
   ];
 
   nativeCheckInputs = [
diff --git a/pkgs/development/python-modules/duckdb-engine/remote_data.patch b/pkgs/development/python-modules/duckdb-engine/remote_data.patch
new file mode 100644
index 000000000000..1c887acf6cae
--- /dev/null
+++ b/pkgs/development/python-modules/duckdb-engine/remote_data.patch
@@ -0,0 +1,24 @@
+diff --git a/duckdb_engine/tests/test_basic.py b/duckdb_engine/tests/test_basic.py
+index 302636f..ed20f12 100644
+--- a/duckdb_engine/tests/test_basic.py
++++ b/duckdb_engine/tests/test_basic.py
+@@ -183,7 +183,6 @@ def test_get_views(engine: Engine) -> None:
+ 
+ 
+ @mark.skipif(os.uname().machine == "aarch64", reason="not supported on aarch64")
+-@mark.remote_data
+ def test_preload_extension() -> None:
+     duckdb.default_connection.execute("INSTALL httpfs")
+     engine = create_engine(
+diff --git a/duckdb_engine/tests/test_integration.py b/duckdb_engine/tests/test_integration.py
+index 349c976..bf3dbaa 100644
+--- a/duckdb_engine/tests/test_integration.py
++++ b/duckdb_engine/tests/test_integration.py
+@@ -24,7 +24,6 @@ def test_integration(engine: Engine) -> None:
+         conn.execute(text("select * from test_df"))
+ 
+ 
+-@mark.remote_data
+ @mark.skipif(
+     "dev" in duckdb.__version__, reason="md extension not available for dev builds"  # type: ignore[attr-defined]
+ )
diff --git a/pkgs/development/python-modules/duckdb/default.nix b/pkgs/development/python-modules/duckdb/default.nix
index 546646c3b35c..4736024d6810 100644
--- a/pkgs/development/python-modules/duckdb/default.nix
+++ b/pkgs/development/python-modules/duckdb/default.nix
@@ -5,6 +5,7 @@
 , git
 , google-cloud-storage
 , numpy
+, openssl
 , pandas
 , psutil
 , pybind11
@@ -13,9 +14,12 @@
 }:
 
 buildPythonPackage rec {
-  inherit (duckdb) pname version src patches;
+  inherit (duckdb) pname version src;
   format = "setuptools";
 
+  BUILD_HTTPFS = 1;
+  patches = [ ./setup.patch ];
+
   postPatch = ''
     # we can't use sourceRoot otherwise patches don't apply, because the patches apply to the C++ library
     cd tools/pythonpkg
@@ -23,14 +27,12 @@ buildPythonPackage rec {
     # 1. let nix control build cores
     # 2. unconstrain setuptools_scm version
     substituteInPlace setup.py \
-      --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES" \
-      --replace "setuptools_scm<7.0.0" "setuptools_scm"
+      --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES"
 
       # avoid dependency on mypy
       rm tests/stubs/test_stubs.py
   '';
 
-  SETUPTOOLS_SCM_PRETEND_VERSION = version;
 
   nativeBuildInputs = [
     git
@@ -38,6 +40,8 @@ buildPythonPackage rec {
     setuptools-scm
   ];
 
+  buildInputs = [ openssl ];
+
   propagatedBuildInputs = [
     numpy
     pandas
diff --git a/pkgs/development/python-modules/duckdb/setup.patch b/pkgs/development/python-modules/duckdb/setup.patch
new file mode 100644
index 000000000000..5532cc6c1cfc
--- /dev/null
+++ b/pkgs/development/python-modules/duckdb/setup.patch
@@ -0,0 +1,22 @@
+diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py
+index fdf2911019..c363cc518a 100644
+--- a/tools/pythonpkg/setup.py
++++ b/tools/pythonpkg/setup.py
+@@ -163,8 +163,6 @@ if 'BUILD_HTTPFS' in os.environ:
+ for ext in extensions:
+     toolchain_args.extend(['-DDUCKDB_EXTENSION_{}_LINKED'.format(ext.upper())])
+ 
+-toolchain_args.extend(['-DDUCKDB_EXTENSION_AUTOLOAD_DEFAULT=1', '-DDUCKDB_EXTENSION_AUTOINSTALL_DEFAULT=1'])
+-
+ 
+ class get_pybind_include(object):
+     def __init__(self, user=False):
+@@ -343,7 +341,7 @@ setup(
+     packages=packages,
+     include_package_data=True,
+     python_requires='>=3.7.0',
+-    setup_requires=setup_requires + ["setuptools_scm<7.0.0", 'pybind11>=2.6.0'],
++    setup_requires=setup_requires + ["setuptools_scm", 'pybind11>=2.6.0'],
+     use_scm_version=setuptools_scm_conf,
+     tests_require=['google-cloud-storage', 'mypy', 'pytest'],
+     classifiers=[

From 12678720a519272d5a15da7f97449cb7d3094d55 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sat, 30 Sep 2023 07:16:32 -0700
Subject: [PATCH 03/15] expand tabs

---
 pkgs/development/libraries/duckdb/default.nix | 34 +++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index 127f77b1a7b5..80a110562e48 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -53,23 +53,23 @@ stdenv.mkDerivation rec {
   installCheckPhase =
     let
       excludes = map (pattern: "exclude:'${pattern}'") [
-	"test/fuzzer/pedro/buffer_manager_out_of_memory.test"
-	"test/sql/copy/csv/parallel/csv_parallel_httpfs.test"
-	"test/sql/copy/csv/parallel/test_parallel_csv.test"
-	"test/sql/copy/csv/test_csv_httpfs_prepared.test"
-	"test/sql/copy/csv/test_csv_httpfs.test"
-	"test/sql/copy/csv/test_csv_remote.test"
-	"test/sql/copy/parquet/delta_byte_array_length_mismatch.test"
-	"test/sql/copy/parquet/delta_byte_array_multiple_pages.test"
-	"test/sql/copy/parquet/parquet_5968.test"
-	"test/sql/copy/parquet/snowflake_lineitem.test"
-	"test/sql/copy/parquet/test_parquet_force_download.test"
-	"test/sql/copy/parquet/test_parquet_remote_foreign_files.test"
-	"test/sql/copy/parquet/test_parquet_remote.test"
-	"test/sql/json/table/read_json_objects.test"
-	"test/sql/json/table/read_json.test"
-	"test/sql/settings/test_disabled_file_system_httpfs.test"
-	"test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"
+        "test/fuzzer/pedro/buffer_manager_out_of_memory.test"
+        "test/sql/copy/csv/parallel/csv_parallel_httpfs.test"
+        "test/sql/copy/csv/parallel/test_parallel_csv.test"
+        "test/sql/copy/csv/test_csv_httpfs_prepared.test"
+        "test/sql/copy/csv/test_csv_httpfs.test"
+        "test/sql/copy/csv/test_csv_remote.test"
+        "test/sql/copy/parquet/delta_byte_array_length_mismatch.test"
+        "test/sql/copy/parquet/delta_byte_array_multiple_pages.test"
+        "test/sql/copy/parquet/parquet_5968.test"
+        "test/sql/copy/parquet/snowflake_lineitem.test"
+        "test/sql/copy/parquet/test_parquet_force_download.test"
+        "test/sql/copy/parquet/test_parquet_remote_foreign_files.test"
+        "test/sql/copy/parquet/test_parquet_remote.test"
+        "test/sql/json/table/read_json_objects.test"
+        "test/sql/json/table/read_json.test"
+        "test/sql/settings/test_disabled_file_system_httpfs.test"
+        "test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"
 
         # these are only hidden if no filters are passed in
         "[!hide]"

From d81e04ede94a2336eb15286eb685594d17cbc94f Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 01:28:56 -0700
Subject: [PATCH 04/15] use the full hash and shallow git clone

---
 pkgs/development/libraries/duckdb/default.nix |  6 +-
 .../libraries/duckdb/version.patch            | 64 +++++++++++++++++++
 .../python-modules/duckdb/default.nix         |  7 +-
 .../python-modules/duckdb/setup.patch         | 22 -------
 4 files changed, 71 insertions(+), 28 deletions(-)
 create mode 100644 pkgs/development/libraries/duckdb/version.patch
 delete mode 100644 pkgs/development/python-modules/duckdb/setup.patch

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index 80a110562e48..96fd06754b97 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -21,11 +21,13 @@ stdenv.mkDerivation rec {
   src = fetchFromGitHub {
     owner = pname;
     repo = pname;
-    deepClone = true;
+    leaveDotGit = true;
     rev = "v${version}";
     hash = "sha256-NFkeeTpsxazQOstKUUu0b27hXbnq3U5g/+24BIMqtJY=";
   };
 
+  patches = [ ./version.patch ];
+
   nativeBuildInputs = [ cmake git ninja ];
   buildInputs = [ openssl ]
     ++ lib.optionals withJdbc [ openjdk11 ]
@@ -33,8 +35,8 @@ stdenv.mkDerivation rec {
 
   cmakeFlags = [
     # use similar flags to what is defined in ${src}/.github/workflow/{LinuxRelease,OSX}.yml
-    "-DDEBUG_STACKTRACE=1"
     "-DDUCKDB_EXTENSION_CONFIGS=${src}/.github/config/bundled_extensions.cmake"
+    "-DGIT_LAST_TAG=${src.rev}"
     "-DBUILD_ODBC_DRIVER=${enableFeature withOdbc}"
     "-DJDBC_DRIVER=${enableFeature withJdbc}"
   ] ++ lib.optionals doInstallCheck [
diff --git a/pkgs/development/libraries/duckdb/version.patch b/pkgs/development/libraries/duckdb/version.patch
new file mode 100644
index 000000000000..1ba73557b0b5
--- /dev/null
+++ b/pkgs/development/libraries/duckdb/version.patch
@@ -0,0 +1,64 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 2b49e11288..0a3eaa7f4e 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -248,22 +248,13 @@ find_package(Git)
+ if(Git_FOUND)
+   if (NOT DEFINED GIT_COMMIT_HASH)
+     execute_process(
+-            COMMAND ${GIT_EXECUTABLE} log -1 --format=%h
++            COMMAND ${GIT_EXECUTABLE} log -1 --format=%H
+             WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+             RESULT_VARIABLE GIT_RESULT
+             OUTPUT_VARIABLE GIT_COMMIT_HASH
+             OUTPUT_STRIP_TRAILING_WHITESPACE)
+   endif()
+-  execute_process(
+-          COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0
+-          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+-          OUTPUT_VARIABLE GIT_LAST_TAG
+-          OUTPUT_STRIP_TRAILING_WHITESPACE)
+-  execute_process(
+-          COMMAND ${GIT_EXECUTABLE} describe --tags --long
+-          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+-          OUTPUT_VARIABLE GIT_ITERATION
+-          OUTPUT_STRIP_TRAILING_WHITESPACE)
++  set(GIT_ITERATION "${GIT_LAST_TAG}-0-g${GIT_COMMIT_HASH}")
+ else()
+   message("Git NOT FOUND")
+ endif()
+diff --git a/scripts/package_build.py b/scripts/package_build.py
+index f0d7535ed0..350d74f877 100644
+--- a/scripts/package_build.py
++++ b/scripts/package_build.py
+@@ -130,7 +130,7 @@ def git_commit_hash():
+     if 'SETUPTOOLS_SCM_PRETEND_HASH' in os.environ:
+         return os.environ['SETUPTOOLS_SCM_PRETEND_HASH']
+     try:
+-        return subprocess.check_output(['git', 'log', '-1', '--format=%h']).strip().decode('utf8')
++        return subprocess.check_output(['git', 'log', '-1', '--format=%H']).strip().decode('utf8')
+     except:
+         return "deadbeeff"
+ 
+diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py
+index fdf2911019..c363cc518a 100644
+--- a/tools/pythonpkg/setup.py
++++ b/tools/pythonpkg/setup.py
+@@ -163,8 +163,6 @@ if 'BUILD_HTTPFS' in os.environ:
+ for ext in extensions:
+     toolchain_args.extend(['-DDUCKDB_EXTENSION_{}_LINKED'.format(ext.upper())])
+ 
+-toolchain_args.extend(['-DDUCKDB_EXTENSION_AUTOLOAD_DEFAULT=1', '-DDUCKDB_EXTENSION_AUTOINSTALL_DEFAULT=1'])
+-
+ 
+ class get_pybind_include(object):
+     def __init__(self, user=False):
+@@ -343,7 +341,7 @@ setup(
+     packages=packages,
+     include_package_data=True,
+     python_requires='>=3.7.0',
+-    setup_requires=setup_requires + ["setuptools_scm<7.0.0", 'pybind11>=2.6.0'],
++    setup_requires=setup_requires + ["setuptools_scm", 'pybind11>=2.6.0'],
+     use_scm_version=setuptools_scm_conf,
+     tests_require=['google-cloud-storage', 'mypy', 'pytest'],
+     classifiers=[
diff --git a/pkgs/development/python-modules/duckdb/default.nix b/pkgs/development/python-modules/duckdb/default.nix
index 4736024d6810..7bf204133f7f 100644
--- a/pkgs/development/python-modules/duckdb/default.nix
+++ b/pkgs/development/python-modules/duckdb/default.nix
@@ -14,12 +14,9 @@
 }:
 
 buildPythonPackage rec {
-  inherit (duckdb) pname version src;
+  inherit (duckdb) pname version src patches;
   format = "setuptools";
 
-  BUILD_HTTPFS = 1;
-  patches = [ ./setup.patch ];
-
   postPatch = ''
     # we can't use sourceRoot otherwise patches don't apply, because the patches apply to the C++ library
     cd tools/pythonpkg
@@ -33,6 +30,8 @@ buildPythonPackage rec {
       rm tests/stubs/test_stubs.py
   '';
 
+  BUILD_HTTPFS = 1;
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
 
   nativeBuildInputs = [
     git
diff --git a/pkgs/development/python-modules/duckdb/setup.patch b/pkgs/development/python-modules/duckdb/setup.patch
deleted file mode 100644
index 5532cc6c1cfc..000000000000
--- a/pkgs/development/python-modules/duckdb/setup.patch
+++ /dev/null
@@ -1,22 +0,0 @@
-diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py
-index fdf2911019..c363cc518a 100644
---- a/tools/pythonpkg/setup.py
-+++ b/tools/pythonpkg/setup.py
-@@ -163,8 +163,6 @@ if 'BUILD_HTTPFS' in os.environ:
- for ext in extensions:
-     toolchain_args.extend(['-DDUCKDB_EXTENSION_{}_LINKED'.format(ext.upper())])
- 
--toolchain_args.extend(['-DDUCKDB_EXTENSION_AUTOLOAD_DEFAULT=1', '-DDUCKDB_EXTENSION_AUTOINSTALL_DEFAULT=1'])
--
- 
- class get_pybind_include(object):
-     def __init__(self, user=False):
-@@ -343,7 +341,7 @@ setup(
-     packages=packages,
-     include_package_data=True,
-     python_requires='>=3.7.0',
--    setup_requires=setup_requires + ["setuptools_scm<7.0.0", 'pybind11>=2.6.0'],
-+    setup_requires=setup_requires + ["setuptools_scm", 'pybind11>=2.6.0'],
-     use_scm_version=setuptools_scm_conf,
-     tests_require=['google-cloud-storage', 'mypy', 'pytest'],
-     classifiers=[

From db9265e1975c88964e1aebc7df8f5bc890f99f98 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 04:09:39 -0700
Subject: [PATCH 05/15] skip failing duckdb tests.

---
 .../python-modules/ibis-framework/default.nix         | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/pkgs/development/python-modules/ibis-framework/default.nix b/pkgs/development/python-modules/ibis-framework/default.nix
index 16274bc3f00d..9b77a6be51d0 100644
--- a/pkgs/development/python-modules/ibis-framework/default.nix
+++ b/pkgs/development/python-modules/ibis-framework/default.nix
@@ -130,6 +130,17 @@ buildPythonPackage rec {
     # tries to download duckdb extensions
     "--deselect=ibis/backends/duckdb/tests/test_register.py::test_register_sqlite"
     "--deselect=ibis/backends/duckdb/tests/test_register.py::test_read_sqlite"
+
+    # duckdb does not respect sample_size=2 (reads 3 lines of csv).
+    "--deselect=ibis/backends/tests/test_register.py::test_csv_reregister_schema"
+
+    # duckdb fails with:
+    # "This function can not be called with an active transaction!, commit or abort the existing one first"
+    "--deselect=ibis/backends/tests/test_udf.py::test_vectorized_udf"
+    "--deselect=ibis/backends/tests/test_udf.py::test_map_merge_udf"
+    "--deselect=ibis/backends/tests/test_udf.py::test_udf"
+    "--deselect=ibis/backends/tests/test_udf.py::test_map_udf"
+
     # pyarrow13 is not supported yet.
     "--deselect=ibis/backends/tests/test_temporal.py::test_date_truncate"
     "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp"

From 7418da6f809b3fecb6b2f82dedfed1af96b42687 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 18:12:27 -0700
Subject: [PATCH 06/15] use --inplace for python build

---
 pkgs/development/python-modules/duckdb/default.nix | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/pkgs/development/python-modules/duckdb/default.nix b/pkgs/development/python-modules/duckdb/default.nix
index 7bf204133f7f..52d2da21b486 100644
--- a/pkgs/development/python-modules/duckdb/default.nix
+++ b/pkgs/development/python-modules/duckdb/default.nix
@@ -60,10 +60,12 @@ buildPythonPackage rec {
 
   preCheck = ''
     export HOME="$(mktemp -d)"
-    # duckdb directory prevents loading the python module
-    mv duckdb duckdb.pk
   '';
 
+  setupPyBuildFlags = [
+    "--inplace"
+  ];
+
   pythonImportsCheck = [
     "duckdb"
   ];

From a1a557bd4345cc2e490f7f56b5f9403248732ebd Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 18:17:34 -0700
Subject: [PATCH 07/15] restore the original set of unittest excludes and add
 "Test using a remote optimizer pass in case thats important to someone"

---
 pkgs/development/libraries/duckdb/default.nix | 41 ++++++++++++-------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index 96fd06754b97..d4e50143942d 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -55,26 +55,39 @@ stdenv.mkDerivation rec {
   installCheckPhase =
     let
       excludes = map (pattern: "exclude:'${pattern}'") [
-        "test/fuzzer/pedro/buffer_manager_out_of_memory.test"
-        "test/sql/copy/csv/parallel/csv_parallel_httpfs.test"
-        "test/sql/copy/csv/parallel/test_parallel_csv.test"
-        "test/sql/copy/csv/test_csv_httpfs_prepared.test"
-        "test/sql/copy/csv/test_csv_httpfs.test"
+        "[s3]"
+        "Test closing database during long running query"
+        "Test using a remote optimizer pass in case thats important to someone"
+        "test/common/test_cast_hugeint.test"
         "test/sql/copy/csv/test_csv_remote.test"
-        "test/sql/copy/parquet/delta_byte_array_length_mismatch.test"
-        "test/sql/copy/parquet/delta_byte_array_multiple_pages.test"
+        "test/sql/copy/parquet/test_parquet_remote.test"
+        "test/sql/copy/parquet/test_parquet_remote_foreign_files.test"
+        "test/sql/storage/compression/chimp/chimp_read.test"
+        "test/sql/storage/compression/chimp/chimp_read_float.test"
+        "test/sql/storage/compression/patas/patas_compression_ratio.test_coverage"
+        "test/sql/storage/compression/patas/patas_read.test"
+        "test/sql/json/read_json_objects.test"
+        "test/sql/json/read_json.test"
         "test/sql/copy/parquet/parquet_5968.test"
+        "test/fuzzer/pedro/buffer_manager_out_of_memory.test"
+        "test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"
+        "test/sql/copy/parquet/delta_byte_array_length_mismatch.test"
+        "test/sql/function/timestamp/test_icu_strptime.test"
+        "test/sql/timezone/test_icu_timezone.test"
         "test/sql/copy/parquet/snowflake_lineitem.test"
         "test/sql/copy/parquet/test_parquet_force_download.test"
-        "test/sql/copy/parquet/test_parquet_remote_foreign_files.test"
-        "test/sql/copy/parquet/test_parquet_remote.test"
-        "test/sql/json/table/read_json_objects.test"
-        "test/sql/json/table/read_json.test"
-        "test/sql/settings/test_disabled_file_system_httpfs.test"
-        "test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"
-
+        "test/sql/copy/parquet/delta_byte_array_multiple_pages.test"
+        "test/sql/copy/csv/test_csv_httpfs_prepared.test"
+        "test/sql/copy/csv/test_csv_httpfs.test"
+        "test/sql/copy/csv/parallel/test_parallel_csv.test"
+        "test/sql/copy/csv/parallel/csv_parallel_httpfs.test"
+        "test/common/test_cast_struct.test"
+        # test is order sensitive
+        "test/sql/copy/parquet/parquet_glob.test"
         # these are only hidden if no filters are passed in
         "[!hide]"
+        # this test apparently never terminates
+        "test/sql/copy/csv/auto/test_csv_auto.test"
       ] ++ lib.optionals stdenv.isAarch64 [
         "test/sql/aggregate/aggregates/test_kurtosis.test"
         "test/sql/aggregate/aggregates/test_skewness.test"

From fd2795d0bcd0d001378914728510a71ae20f7421 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 18:43:03 -0700
Subject: [PATCH 08/15] restore original version strategy with no git hash.

---
 pkgs/development/libraries/duckdb/default.nix | 11 +--
 .../libraries/duckdb/version.patch            | 70 ++++++++++++-------
 .../python-modules/duckdb/default.nix         |  2 -
 3 files changed, 49 insertions(+), 34 deletions(-)

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index d4e50143942d..bde43d4c8915 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -2,7 +2,6 @@
 , stdenv
 , fetchFromGitHub
 , cmake
-, git
 , ninja
 , openssl
 , openjdk11
@@ -21,14 +20,17 @@ stdenv.mkDerivation rec {
   src = fetchFromGitHub {
     owner = pname;
     repo = pname;
-    leaveDotGit = true;
     rev = "v${version}";
-    hash = "sha256-NFkeeTpsxazQOstKUUu0b27hXbnq3U5g/+24BIMqtJY=";
+    hash = "sha256-EKvDH7RwOC4Gu/lturrfnGpzXnJ9azIwAFeuVoa6L/Y=";
   };
 
   patches = [ ./version.patch ];
 
-  nativeBuildInputs = [ cmake git ninja ];
+  postPatch = ''
+    substituteInPlace CMakeLists.txt --subst-var-by DUCKDB_VERSION "v${version}"
+  '';
+
+  nativeBuildInputs = [ cmake ninja ];
   buildInputs = [ openssl ]
     ++ lib.optionals withJdbc [ openjdk11 ]
     ++ lib.optionals withOdbc [ unixODBC ];
@@ -36,7 +38,6 @@ stdenv.mkDerivation rec {
   cmakeFlags = [
     # use similar flags to what is defined in ${src}/.github/workflow/{LinuxRelease,OSX}.yml
     "-DDUCKDB_EXTENSION_CONFIGS=${src}/.github/config/bundled_extensions.cmake"
-    "-DGIT_LAST_TAG=${src.rev}"
     "-DBUILD_ODBC_DRIVER=${enableFeature withOdbc}"
     "-DJDBC_DRIVER=${enableFeature withJdbc}"
   ] ++ lib.optionals doInstallCheck [
diff --git a/pkgs/development/libraries/duckdb/version.patch b/pkgs/development/libraries/duckdb/version.patch
index 1ba73557b0b5..9b368eac5dbc 100644
--- a/pkgs/development/libraries/duckdb/version.patch
+++ b/pkgs/development/libraries/duckdb/version.patch
@@ -1,18 +1,21 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 2b49e11288..0a3eaa7f4e 100644
+index 2b49e11288..0a4a69b9a0 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -248,22 +248,13 @@ find_package(Git)
- if(Git_FOUND)
-   if (NOT DEFINED GIT_COMMIT_HASH)
-     execute_process(
+@@ -244,52 +244,7 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+   set(SUN TRUE)
+ endif()
+ 
+-find_package(Git)
+-if(Git_FOUND)
+-  if (NOT DEFINED GIT_COMMIT_HASH)
+-    execute_process(
 -            COMMAND ${GIT_EXECUTABLE} log -1 --format=%h
-+            COMMAND ${GIT_EXECUTABLE} log -1 --format=%H
-             WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-             RESULT_VARIABLE GIT_RESULT
-             OUTPUT_VARIABLE GIT_COMMIT_HASH
-             OUTPUT_STRIP_TRAILING_WHITESPACE)
-   endif()
+-            WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+-            RESULT_VARIABLE GIT_RESULT
+-            OUTPUT_VARIABLE GIT_COMMIT_HASH
+-            OUTPUT_STRIP_TRAILING_WHITESPACE)
+-  endif()
 -  execute_process(
 -          COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0
 -          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -23,22 +26,35 @@ index 2b49e11288..0a3eaa7f4e 100644
 -          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 -          OUTPUT_VARIABLE GIT_ITERATION
 -          OUTPUT_STRIP_TRAILING_WHITESPACE)
-+  set(GIT_ITERATION "${GIT_LAST_TAG}-0-g${GIT_COMMIT_HASH}")
- else()
-   message("Git NOT FOUND")
- endif()
-diff --git a/scripts/package_build.py b/scripts/package_build.py
-index f0d7535ed0..350d74f877 100644
---- a/scripts/package_build.py
-+++ b/scripts/package_build.py
-@@ -130,7 +130,7 @@ def git_commit_hash():
-     if 'SETUPTOOLS_SCM_PRETEND_HASH' in os.environ:
-         return os.environ['SETUPTOOLS_SCM_PRETEND_HASH']
-     try:
--        return subprocess.check_output(['git', 'log', '-1', '--format=%h']).strip().decode('utf8')
-+        return subprocess.check_output(['git', 'log', '-1', '--format=%H']).strip().decode('utf8')
-     except:
-         return "deadbeeff"
+-else()
+-  message("Git NOT FOUND")
+-endif()
+-
+-if(GIT_RESULT EQUAL "0")
+-  string(REGEX REPLACE "v([0-9]+).[0-9]+.[0-9]+" "\\1" DUCKDB_MAJOR_VERSION "${GIT_LAST_TAG}")
+-  string(REGEX REPLACE "v[0-9]+.([0-9]+).[0-9]+" "\\1" DUCKDB_MINOR_VERSION "${GIT_LAST_TAG}")
+-  string(REGEX REPLACE "v[0-9]+.[0-9]+.([0-9]+)" "\\1" DUCKDB_PATCH_VERSION "${GIT_LAST_TAG}")
+-  string(REGEX REPLACE ".*-([0-9]+)-.*" "\\1" DUCKDB_DEV_ITERATION "${GIT_ITERATION}")
+-
+-  if(DUCKDB_DEV_ITERATION EQUAL 0)
+-    # on a tag; directly use the version
+-    set(DUCKDB_VERSION "${GIT_LAST_TAG}")
+-  else()
+-    # not on a tag, increment the patch version by one and add a -devX suffix
+-    math(EXPR DUCKDB_PATCH_VERSION "${DUCKDB_PATCH_VERSION}+1")
+-    set(DUCKDB_VERSION "v${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}.${DUCKDB_PATCH_VERSION}-dev${DUCKDB_DEV_ITERATION}")
+-  endif()
+-else()
+-  # fallback for when building from tarball
+-  set(DUCKDB_MAJOR_VERSION 0)
+-  set(DUCKDB_MINOR_VERSION 0)
+-  set(DUCKDB_PATCH_VERSION 1)
+-  set(DUCKDB_DEV_ITERATION 0)
+-  set(DUCKDB_VERSION "v${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}.${DUCKDB_PATCH_VERSION}-dev${DUCKDB_DEV_ITERATION}")
+-endif()
++set(DUCKDB_VERSION "@DUCKDB_VERSION@")
+ 
+ message(STATUS "git hash ${GIT_COMMIT_HASH}, version ${DUCKDB_VERSION}")
  
 diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py
 index fdf2911019..c363cc518a 100644
diff --git a/pkgs/development/python-modules/duckdb/default.nix b/pkgs/development/python-modules/duckdb/default.nix
index 52d2da21b486..986ac205b5d9 100644
--- a/pkgs/development/python-modules/duckdb/default.nix
+++ b/pkgs/development/python-modules/duckdb/default.nix
@@ -2,7 +2,6 @@
 , buildPythonPackage
 , duckdb
 , fsspec
-, git
 , google-cloud-storage
 , numpy
 , openssl
@@ -34,7 +33,6 @@ buildPythonPackage rec {
   SETUPTOOLS_SCM_PRETEND_VERSION = version;
 
   nativeBuildInputs = [
-    git
     pybind11
     setuptools-scm
   ];

From 85318c391531d7c292727382c7c8661f75548152 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 18:53:21 -0700
Subject: [PATCH 09/15] httpfs extension was not bundled. just bundle all of
 the in tree extensions

---
 pkgs/development/libraries/duckdb/default.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index bde43d4c8915..1a549502f74d 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -36,8 +36,7 @@ stdenv.mkDerivation rec {
     ++ lib.optionals withOdbc [ unixODBC ];
 
   cmakeFlags = [
-    # use similar flags to what is defined in ${src}/.github/workflow/{LinuxRelease,OSX}.yml
-    "-DDUCKDB_EXTENSION_CONFIGS=${src}/.github/config/bundled_extensions.cmake"
+    "-DDUCKDB_EXTENSION_CONFIGS=${src}/.github/config/in_tree_extensions.cmake"
     "-DBUILD_ODBC_DRIVER=${enableFeature withOdbc}"
     "-DJDBC_DRIVER=${enableFeature withJdbc}"
   ] ++ lib.optionals doInstallCheck [

From bfbe80f1b4d0521aaa9e99234291979fe2f6f6d8 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 19:00:10 -0700
Subject: [PATCH 10/15] exclude additional httpfs tests which access network

---
 pkgs/development/libraries/duckdb/default.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index 1a549502f74d..70c5c5fc8949 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -68,6 +68,7 @@ stdenv.mkDerivation rec {
         "test/sql/storage/compression/patas/patas_read.test"
         "test/sql/json/read_json_objects.test"
         "test/sql/json/read_json.test"
+        "test/sql/json/table/read_json.test"
         "test/sql/copy/parquet/parquet_5968.test"
         "test/fuzzer/pedro/buffer_manager_out_of_memory.test"
         "test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"
@@ -79,6 +80,7 @@ stdenv.mkDerivation rec {
         "test/sql/copy/parquet/delta_byte_array_multiple_pages.test"
         "test/sql/copy/csv/test_csv_httpfs_prepared.test"
         "test/sql/copy/csv/test_csv_httpfs.test"
+        "test/sql/settings/test_disabled_file_system_httpfs.test"
         "test/sql/copy/csv/parallel/test_parallel_csv.test"
         "test/sql/copy/csv/parallel/csv_parallel_httpfs.test"
         "test/common/test_cast_struct.test"

From 430a6cb7902dd709d8d47504738aa91537d1fbf4 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 19:11:38 -0700
Subject: [PATCH 11/15] disable one more httpfs test
 test/sql/json/table/read_json_objects.test

---
 pkgs/development/libraries/duckdb/default.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index 70c5c5fc8949..b047b8ebffd0 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -68,6 +68,7 @@ stdenv.mkDerivation rec {
         "test/sql/storage/compression/patas/patas_read.test"
         "test/sql/json/read_json_objects.test"
         "test/sql/json/read_json.test"
+	"test/sql/json/table/read_json_objects.test"
         "test/sql/json/table/read_json.test"
         "test/sql/copy/parquet/parquet_5968.test"
         "test/fuzzer/pedro/buffer_manager_out_of_memory.test"

From a9d95ded63e499febe6d12c43e225407755f2cb4 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 19:21:24 -0700
Subject: [PATCH 12/15] remove tabs

---
 pkgs/development/libraries/duckdb/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index b047b8ebffd0..ea152c0cc099 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -68,7 +68,7 @@ stdenv.mkDerivation rec {
         "test/sql/storage/compression/patas/patas_read.test"
         "test/sql/json/read_json_objects.test"
         "test/sql/json/read_json.test"
-	"test/sql/json/table/read_json_objects.test"
+        "test/sql/json/table/read_json_objects.test"
         "test/sql/json/table/read_json.test"
         "test/sql/copy/parquet/parquet_5968.test"
         "test/fuzzer/pedro/buffer_manager_out_of_memory.test"

From e7943aa168894768e350f525f74775f38b48489d Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 22:16:31 -0700
Subject: [PATCH 13/15] fix bad merge

---
 pkgs/development/python-modules/duckdb/default.nix | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/pkgs/development/python-modules/duckdb/default.nix b/pkgs/development/python-modules/duckdb/default.nix
index 810ee8efe921..1332688f414b 100644
--- a/pkgs/development/python-modules/duckdb/default.nix
+++ b/pkgs/development/python-modules/duckdb/default.nix
@@ -16,8 +16,6 @@ buildPythonPackage rec {
   inherit (duckdb) pname version src;
   format = "setuptools";
 
-  BUILD_HTTPFS = 1;
-
   postPatch = ''
     # we can't use sourceRoot otherwise patches don't apply, because the patches apply to the C++ library
     cd tools/pythonpkg
@@ -35,7 +33,6 @@ buildPythonPackage rec {
   SETUPTOOLS_SCM_PRETEND_VERSION = version;
 
   nativeBuildInputs = [
-    git
     pybind11
     setuptools-scm
   ];

From fef9a2a11b64817ab9fbdf29b8a305e6e11f8304 Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 22:17:57 -0700
Subject: [PATCH 14/15] fix bad merge

---
 pkgs/development/python-modules/duckdb/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgs/development/python-modules/duckdb/default.nix b/pkgs/development/python-modules/duckdb/default.nix
index 1332688f414b..e9aac74d835e 100644
--- a/pkgs/development/python-modules/duckdb/default.nix
+++ b/pkgs/development/python-modules/duckdb/default.nix
@@ -13,7 +13,7 @@
 }:
 
 buildPythonPackage rec {
-  inherit (duckdb) pname version src;
+  inherit (duckdb) pname version src patches;
   format = "setuptools";
 
   postPatch = ''

From d5942e4513b6fb571b7df328ff7b060c56dbb88e Mon Sep 17 00:00:00 2001
From: Rob <86313040+robert-manchester@users.noreply.github.com>
Date: Sun, 1 Oct 2023 22:21:14 -0700
Subject: [PATCH 15/15] fix bad merge

---
 pkgs/development/libraries/duckdb/default.nix | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/pkgs/development/libraries/duckdb/default.nix b/pkgs/development/libraries/duckdb/default.nix
index b844e0defd2f..ea152c0cc099 100644
--- a/pkgs/development/libraries/duckdb/default.nix
+++ b/pkgs/development/libraries/duckdb/default.nix
@@ -2,7 +2,6 @@
 , stdenv
 , fetchFromGitHub
 , cmake
-, git
 , ninja
 , openssl
 , openjdk11
@@ -21,13 +20,17 @@ stdenv.mkDerivation rec {
   src = fetchFromGitHub {
     owner = pname;
     repo = pname;
-    deepClone = true;
     rev = "v${version}";
     hash = "sha256-EKvDH7RwOC4Gu/lturrfnGpzXnJ9azIwAFeuVoa6L/Y=";
   };
 
-  nativeBuildInputs = [ cmake git ninja ];
+  patches = [ ./version.patch ];
 
+  postPatch = ''
+    substituteInPlace CMakeLists.txt --subst-var-by DUCKDB_VERSION "v${version}"
+  '';
+
+  nativeBuildInputs = [ cmake ninja ];
   buildInputs = [ openssl ]
     ++ lib.optionals withJdbc [ openjdk11 ]
     ++ lib.optionals withOdbc [ unixODBC ];
@@ -51,7 +54,7 @@ stdenv.mkDerivation rec {
 
   installCheckPhase =
     let
-      excludes = map (pattern: "exclude:'${pattern}'") ([
+      excludes = map (pattern: "exclude:'${pattern}'") [
         "[s3]"
         "Test closing database during long running query"
         "Test using a remote optimizer pass in case thats important to someone"
@@ -92,7 +95,7 @@ stdenv.mkDerivation rec {
         "test/sql/aggregate/aggregates/test_kurtosis.test"
         "test/sql/aggregate/aggregates/test_skewness.test"
         "test/sql/function/list/aggregates/skewness.test"
-      ]);
+      ];
     in
     ''
       runHook preInstallCheck