From 7dd6767f44a8c6726620a1e75fdb7ae232fec1e6 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 21 Apr 2025 14:56:25 +0200 Subject: [PATCH 1/5] python312Packages.google-cloud-bigquery: cleanup --- .../google-cloud-bigquery/default.nix | 79 ++++++++++--------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/pkgs/development/python-modules/google-cloud-bigquery/default.nix b/pkgs/development/python-modules/google-cloud-bigquery/default.nix index 660b87e6c63c..e53652bb3c38 100644 --- a/pkgs/development/python-modules/google-cloud-bigquery/default.nix +++ b/pkgs/development/python-modules/google-cloud-bigquery/default.nix @@ -1,31 +1,38 @@ { lib, buildPythonPackage, - db-dtypes, fetchPypi, - freezegun, + + # build-system + setuptools, + + # dependencies google-api-core, google-cloud-bigquery-storage, google-cloud-core, + google-resumable-media, + grpcio, + proto-plus, + protobuf, + python-dateutil, + requests, + + # optional-dependencies + pyarrow, + db-dtypes, + pandas, + tqdm, + ipython, + + # tests + freezegun, google-cloud-datacatalog, google-cloud-storage, google-cloud-testutils, - google-resumable-media, - grpcio, - ipython, mock, - pandas, - proto-plus, - protobuf, psutil, - pyarrow, pytest-xdist, pytestCheckHook, - python-dateutil, - pythonOlder, - requests, - setuptools, - tqdm, }: buildPythonPackage rec { @@ -33,8 +40,6 @@ buildPythonPackage rec { version = "3.31.0"; pyproject = true; - disabled = pythonOlder "3.7"; - src = fetchPypi { pname = "google_cloud_bigquery"; inherit version; @@ -44,15 +49,15 @@ buildPythonPackage rec { build-system = [ setuptools ]; dependencies = [ - grpcio google-api-core - google-cloud-core google-cloud-bigquery-storage + google-cloud-core google-resumable-media + grpcio proto-plus protobuf - requests python-dateutil + requests ] ++ google-api-core.optional-dependencies.grpc; optional-dependencies = { @@ -73,13 +78,13 @@ buildPythonPackage rec { nativeCheckInputs = [ freezegun + google-cloud-datacatalog + google-cloud-storage google-cloud-testutils mock psutil - google-cloud-datacatalog - google-cloud-storage - pytestCheckHook pytest-xdist + pytestCheckHook ] ++ optional-dependencies.pandas ++ optional-dependencies.ipython; @@ -91,39 +96,41 @@ buildPythonPackage rec { disabledTests = [ # requires credentials - "test_bigquery_magic" "TestBigQuery" - "test_context_with_no_query_cache_from_context" "test_arrow_extension_types_same_for_storage_and_REST_APIs_894" + "test_bigquery_magic" + "test_context_with_no_query_cache_from_context" + "test_dry_run" "test_list_rows_empty_table" "test_list_rows_page_size" - "test_list_rows_range_csv" "test_list_rows_range" - "test_list_rows_scalars_extreme" + "test_list_rows_range_csv" "test_list_rows_scalars" - "test_dry_run" + "test_list_rows_scalars_extreme" "test_session" + # Mocking of _ensure_bqstorage_client fails "test_to_arrow_ensure_bqstorage_client_wo_bqstorage" + # requires network + "test__initiate_resumable_upload" + "test__initiate_resumable_upload_mtls" + "test__initiate_resumable_upload_with_retry" + "test_context_with_custom_connection" + "test_context_with_default_connection" "test_dbapi_create_view" "test_list_rows_nullable_scalars_dtypes" "test_parameterized_types_round_trip" "test_structs" - "test_table_snapshots" - "test__initiate_resumable_upload" - "test__initiate_resumable_upload_mtls" - "test__initiate_resumable_upload_with_retry" "test_table_clones" - "test_context_with_default_connection" - "test_context_with_custom_connection" + "test_table_snapshots" ]; disabledTestPaths = [ # Tests require credentials - "tests/system/test_query.py" "tests/system/test_job_retry.py" "tests/system/test_pandas.py" + "tests/system/test_query.py" ]; pythonImportsCheck = [ @@ -131,11 +138,11 @@ buildPythonPackage rec { "google.cloud.bigquery_v2" ]; - meta = with lib; { + meta = { description = "Google BigQuery API client library"; homepage = "https://github.com/googleapis/python-bigquery"; changelog = "https://github.com/googleapis/python-bigquery/blob/v${version}/CHANGELOG.md"; - license = licenses.asl20; + license = lib.licenses.asl20; maintainers = [ ]; }; } From 81ad0ff8866e5b47fcd5aefe055b7930bf8ed5b8 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 21 Apr 2025 15:00:34 +0200 Subject: [PATCH 2/5] python312Packages.google-cloud-bigquery: skip failing tests --- .../python-modules/google-cloud-bigquery/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/development/python-modules/google-cloud-bigquery/default.nix b/pkgs/development/python-modules/google-cloud-bigquery/default.nix index e53652bb3c38..7dc93f48590b 100644 --- a/pkgs/development/python-modules/google-cloud-bigquery/default.nix +++ b/pkgs/development/python-modules/google-cloud-bigquery/default.nix @@ -108,6 +108,7 @@ buildPythonPackage rec { "test_list_rows_scalars" "test_list_rows_scalars_extreme" "test_session" + "test_to_arrow_query_with_empty_result" # Mocking of _ensure_bqstorage_client fails "test_to_arrow_ensure_bqstorage_client_wo_bqstorage" @@ -131,6 +132,9 @@ buildPythonPackage rec { "tests/system/test_job_retry.py" "tests/system/test_pandas.py" "tests/system/test_query.py" + + # ModuleNotFoundError: No module named 'google.cloud.resourcemanager_v3' + "tests/system/test_client.py" ]; pythonImportsCheck = [ From 321262155769a62fa8dc6f6b3c357223ec163b65 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 21 Apr 2025 15:30:40 +0200 Subject: [PATCH 3/5] python312Packages.dbt-bigquery: cleanup & relax google-cloud-storage dependency --- .../python-modules/dbt-bigquery/default.nix | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pkgs/development/python-modules/dbt-bigquery/default.nix b/pkgs/development/python-modules/dbt-bigquery/default.nix index c5ae4e4ef2e8..311c22b7e478 100644 --- a/pkgs/development/python-modules/dbt-bigquery/default.nix +++ b/pkgs/development/python-modules/dbt-bigquery/default.nix @@ -1,17 +1,22 @@ { lib, - agate, buildPythonPackage, + fetchFromGitHub, + + # build-system + setuptools, + + # dependencies + agate, dbt-adapters, dbt-common, dbt-core, - fetchFromGitHub, google-cloud-bigquery, google-cloud-dataproc, google-cloud-storage, + + # tests pytestCheckHook, - pythonOlder, - setuptools, }: buildPythonPackage rec { @@ -19,8 +24,6 @@ buildPythonPackage rec { version = "1.9.1"; pyproject = true; - disabled = pythonOlder "3.8"; - src = fetchFromGitHub { owner = "dbt-labs"; repo = "dbt-bigquery"; @@ -28,7 +31,10 @@ buildPythonPackage rec { hash = "sha256-YZA8lcUGoq5jMNS1GlbBd036X2F3khsZWr5Pv65zpPI="; }; - pythonRelaxDeps = [ "agate" ]; + pythonRelaxDeps = [ + "agate" + "google-cloud-storage" + ]; build-system = [ setuptools @@ -36,12 +42,12 @@ buildPythonPackage rec { dependencies = [ agate - dbt-common dbt-adapters + dbt-common dbt-core google-cloud-bigquery - google-cloud-storage google-cloud-dataproc + google-cloud-storage ]; nativeCheckInputs = [ pytestCheckHook ]; @@ -50,11 +56,11 @@ buildPythonPackage rec { pythonImportsCheck = [ "dbt.adapters.bigquery" ]; - meta = with lib; { + meta = { description = "Plugin enabling dbt to operate on a BigQuery database"; homepage = "https://github.com/dbt-labs/dbt-bigquery"; changelog = "https://github.com/dbt-labs/dbt-bigquery/blob/${version}/CHANGELOG.md"; - license = licenses.asl20; - maintainers = with maintainers; [ tjni ]; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ tjni ]; }; } From 32694b6682166c87229b7bd66c49db45908d8957 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 21 Apr 2025 15:42:24 +0200 Subject: [PATCH 4/5] python312Packages.dbt-common: 1.14.0 -> 1.22.0 Diff: https://github.com/dbt-labs/dbt-common/compare/965ad815f0dd546678d2595a3010d81f344f8b73...965ad815f0dd546678d2595a3010d81f344f8b73 Changelog: https://github.com/dbt-labs/dbt-common/blob/1.22.0/CHANGELOG.md --- .../python-modules/dbt-common/default.nix | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/pkgs/development/python-modules/dbt-common/default.nix b/pkgs/development/python-modules/dbt-common/default.nix index 1ee4ca70b6c6..334c2bd37b55 100644 --- a/pkgs/development/python-modules/dbt-common/default.nix +++ b/pkgs/development/python-modules/dbt-common/default.nix @@ -1,8 +1,7 @@ { lib, - fetchFromGitHub, buildPythonPackage, - pythonOlder, + fetchPypi, # build-system hatchling, @@ -29,19 +28,14 @@ buildPythonPackage rec { pname = "dbt-common"; - version = "1.14.0"; + version = "1.22.0"; pyproject = true; - disabled = pythonOlder "3.9"; - - src = fetchFromGitHub { - owner = "dbt-labs"; - repo = "dbt-common"; - # Unfortunately, upstream doesn't tag commits on GitHub, and the pypi source - # doesn't include tests. TODO: Write an update script that will detect the - # version from `dbt_common/__about__.py`. - rev = "965ad815f0dd546678d2595a3010d81f344f8b73"; - hash = "sha256-63gWkETi52uOrO0FTPwM831UHECqcyCtb7wVHQuujnc="; + # No tags on GitHub + src = fetchPypi { + pname = "dbt_common"; + inherit version; + hash = "sha256-6cdTMVCCB6SNEUsQtzKUBnKuJgwfttl7o2+zBp8Fu5g="; }; build-system = [ hatchling ]; @@ -82,6 +76,9 @@ buildPythonPackage rec { "test_extra_dict_on_event" ]; + # No tests in the pypi archive + doCheck = false; + pythonImportsCheck = [ "dbt_common" ]; meta = { From ca33c51d51da959ec560b5cfc1e468749d930d88 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 21 Apr 2025 15:46:04 +0200 Subject: [PATCH 5/5] python312Packages.dbt-core: 1.9.3 -> 1.10.0b2 Diff: https://github.com/dbt-labs/dbt-core/compare/refs/tags/v1.9.3...refs/tags/v1.10.0b2 Changelog: https://github.com/dbt-labs/dbt-core/blob/v1.10.0b2/CHANGELOG.md --- .../python-modules/dbt-core/default.nix | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/pkgs/development/python-modules/dbt-core/default.nix b/pkgs/development/python-modules/dbt-core/default.nix index dce85a6395f1..fb57ff19af1c 100644 --- a/pkgs/development/python-modules/dbt-core/default.nix +++ b/pkgs/development/python-modules/dbt-core/default.nix @@ -1,14 +1,19 @@ { lib, - agate, buildPythonPackage, + fetchFromGitHub, + + # build-system + setuptools, + + # dependencies + agate, click, daff, dbt-adapters, dbt-common, dbt-extractor, dbt-semantic-interfaces, - fetchFromGitHub, jinja2, logbook, mashumaro, @@ -16,35 +21,41 @@ packaging, pathspec, protobuf, - callPackage, - pythonOlder, + pydantic, + pydantic-settings, pytz, pyyaml, requests, - setuptools, snowplow-tracker, sqlparse, typing-extensions, + + # passthru + callPackage, }: buildPythonPackage rec { pname = "dbt-core"; - version = "1.9.3"; + version = "1.10.0b2"; pyproject = true; - disabled = pythonOlder "3.9"; - src = fetchFromGitHub { owner = "dbt-labs"; repo = "dbt-core"; tag = "v${version}"; - hash = "sha256-kJyeXKETYYZIHyx1LeOVcC1ELQ9NkHFhkd8gb6cuzZc="; + hash = "sha256-MTrdpbPqdakFDmLKRFJ23u9hLgGhZ5T+r4om9HPBjkw="; }; + postPatch = '' + substituteInPlace dbt/utils/artifact_upload.py \ + --replace-fail \ + "from pydantic import BaseSettings" \ + "from pydantic_settings import BaseSettings" + ''; + sourceRoot = "${src.name}/core"; pythonRelaxDeps = [ - "protobuf" "agate" "click" "dbt-common" @@ -54,6 +65,7 @@ buildPythonPackage rec { "networkx" "pathspec" "protobuf" + "pydantic" "urllib3" ]; @@ -76,6 +88,8 @@ buildPythonPackage rec { packaging pathspec protobuf + pydantic + pydantic-settings pytz pyyaml requests @@ -91,7 +105,7 @@ buildPythonPackage rec { withAdapters = callPackage ./with-adapters.nix { }; }; - meta = with lib; { + meta = { description = "Enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications"; longDescription = '' The dbt tool needs adapters to data sources in order to work. The available @@ -111,8 +125,8 @@ buildPythonPackage rec { ''; homepage = "https://github.com/dbt-labs/dbt-core"; changelog = "https://github.com/dbt-labs/dbt-core/blob/${src.tag}/CHANGELOG.md"; - license = licenses.asl20; - maintainers = with maintainers; [ + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ mausch tjni ];