From 01ea739747b04264059adc839afb8d60bafaf51e Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 28 Apr 2025 14:31:55 +0200 Subject: [PATCH 1/5] python312Packages.findspark: init at 2.0.1 --- .../python-modules/findspark/default.nix | 36 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 ++ 2 files changed, 38 insertions(+) create mode 100644 pkgs/development/python-modules/findspark/default.nix diff --git a/pkgs/development/python-modules/findspark/default.nix b/pkgs/development/python-modules/findspark/default.nix new file mode 100644 index 000000000000..9a0e3f5ff82d --- /dev/null +++ b/pkgs/development/python-modules/findspark/default.nix @@ -0,0 +1,36 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + setuptools, +}: + +buildPythonPackage rec { + pname = "findspark"; + version = "2.0.1"; + pyproject = true; + + src = fetchFromGitHub { + owner = "minrk"; + repo = "findspark"; + tag = version; + hash = "sha256-/+b1Pf+ySwlv6XP1wtHx1tx4WfYdu6GuxJVQkcX3MY8="; + }; + + build-system = [ + setuptools + ]; + + pythonImportsCheck = [ "findspark" ]; + + # No tests + doCheck = false; + + meta = { + description = "Find pyspark to make it importable"; + homepage = "https://github.com/minrk/findspark"; + changelog = "https://github.com/minrk/findspark/blob/${version}/CHANGELOG.md"; + license = lib.licenses.bsd3; + maintainers = with lib.maintainers; [ GaetanLepage ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 725886628211..5613aef78020 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -4900,6 +4900,8 @@ self: super: with self; { findpython = callPackage ../development/python-modules/findpython { }; + findspark = callPackage ../development/python-modules/findspark { }; + finetuning-scheduler = callPackage ../development/python-modules/finetuning-scheduler { }; fingerprints = callPackage ../development/python-modules/fingerprints { }; From 7b9f1842b28f13dc91c9943c812dc82092bb8d34 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 28 Apr 2025 14:49:51 +0200 Subject: [PATCH 2/5] python312Packages.sqlglot: 25.20.1 -> 26.16.2 Diff: https://github.com/tobymao/sqlglot/compare/refs/tags/v25.20.1...refs/tags/v26.16.2 Changelog: https://github.com/tobymao/sqlglot/blob/v26.16.2/CHANGELOG.md --- .../python-modules/sqlglot/default.nix | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pkgs/development/python-modules/sqlglot/default.nix b/pkgs/development/python-modules/sqlglot/default.nix index 6945e853adc3..7b109e1f2590 100644 --- a/pkgs/development/python-modules/sqlglot/default.nix +++ b/pkgs/development/python-modules/sqlglot/default.nix @@ -1,35 +1,38 @@ { lib, buildPythonPackage, - duckdb, fetchFromGitHub, - pytestCheckHook, - python-dateutil, - pythonOlder, + + # build-system setuptools, setuptools-scm, + + # dependencies + python-dateutil, + + # tests + pytestCheckHook, + duckdb, }: buildPythonPackage rec { pname = "sqlglot"; - version = "25.20.1"; + version = "26.16.2"; pyproject = true; - disabled = pythonOlder "3.8"; - src = fetchFromGitHub { repo = "sqlglot"; owner = "tobymao"; tag = "v${version}"; - hash = "sha256-RE9Hbb3g6j4j5X2ksjcBZ610RcV7Zd3YaKaBIUyD2vU="; + hash = "sha256-uX72AHr4IC+u5AYkW/3myruVPs5NZ1V3THVg+9GWxpg="; }; - nativeBuildInputs = [ + build-system = [ setuptools setuptools-scm ]; - propagatedBuildInputs = [ + dependencies = [ # Optional dependency used in the sqlglot optimizer python-dateutil ]; @@ -41,11 +44,11 @@ buildPythonPackage rec { pythonImportsCheck = [ "sqlglot" ]; - meta = with lib; { + meta = { description = "No dependency Python SQL parser, transpiler, and optimizer"; homepage = "https://github.com/tobymao/sqlglot"; changelog = "https://github.com/tobymao/sqlglot/blob/v${version}/CHANGELOG.md"; - license = licenses.mit; - maintainers = with maintainers; [ cpcloud ]; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ cpcloud ]; }; } From b1a2691333f669354d4686f60161d838aa066340 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 28 Apr 2025 14:23:22 +0200 Subject: [PATCH 3/5] python312Packages.sqlframe: init at 3.31.2 --- .../python-modules/sqlframe/default.nix | 79 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 81 insertions(+) create mode 100644 pkgs/development/python-modules/sqlframe/default.nix diff --git a/pkgs/development/python-modules/sqlframe/default.nix b/pkgs/development/python-modules/sqlframe/default.nix new file mode 100644 index 000000000000..b09674bbe8ac --- /dev/null +++ b/pkgs/development/python-modules/sqlframe/default.nix @@ -0,0 +1,79 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + + # build-system + setuptools-scm, + + # dependencies + prettytable, + sqlglot, + typing-extensions, + + # tests + databricks-sql-connector, + duckdb, + findspark, + google-cloud-bigquery, + pyspark, + pytest-postgresql, + pytest-xdist, + pytestCheckHook, +}: + +buildPythonPackage rec { + pname = "sqlframe"; + version = "3.31.2"; + pyproject = true; + + src = fetchFromGitHub { + owner = "eakmanrq"; + repo = "sqlframe"; + tag = "v${version}"; + hash = "sha256-X7KZSbq9KKUE4lXZIHj++koJmjWYI1sMhA6LfClr9pU="; + }; + + build-system = [ + setuptools-scm + ]; + + dependencies = [ + prettytable + sqlglot + typing-extensions + ]; + + pythonImportsCheck = [ "sqlframe" ]; + + nativeCheckInputs = [ + databricks-sql-connector + duckdb + findspark + google-cloud-bigquery + pyspark + pytest-postgresql + pytest-xdist + pytestCheckHook + ]; + + disabledTests = [ + # Requires google-cloud credentials + # google.auth.exceptions.DefaultCredentialsErro + "test_activate_bigquery_default_dataset" + ]; + + disabledTestPaths = [ + # duckdb.duckdb.CatalogException: Catalog Error: Table Function with name "dsdgen" is not in the catalog, but it exists in the tpcds extension. + # "tests/integration/test_int_dataframe.py" + "tests/integration/" + ]; + + meta = { + description = "Turning PySpark Into a Universal DataFrame API"; + homepage = "https://github.com/eakmanrq/sqlframe"; + changelog = "https://github.com/eakmanrq/sqlframe/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ GaetanLepage ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 5613aef78020..7a32edf17fe0 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -16485,6 +16485,8 @@ self: super: with self; { sqlfmt = callPackage ../development/python-modules/sqlfmt { }; + sqlframe = callPackage ../development/python-modules/sqlframe { }; + sqlglot = callPackage ../development/python-modules/sqlglot { }; sqlite-anyio = callPackage ../development/python-modules/sqlite-anyio { }; From 423ef5cdb89790dc081d3918b1d85c8c51f6e4bf Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 28 Apr 2025 14:19:40 +0200 Subject: [PATCH 4/5] python312Packages.narwhals: 1.30.0 -> 1.37.0 Diff: https://github.com/narwhals-dev/narwhals/compare/refs/tags/v1.30.0...refs/tags/v1.37.0 Changelog: https://github.com/narwhals-dev/narwhals/releases/tag/v1.37.0 --- pkgs/development/python-modules/narwhals/default.nix | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/narwhals/default.nix b/pkgs/development/python-modules/narwhals/default.nix index 3633431ddc39..6c9e162220d0 100644 --- a/pkgs/development/python-modules/narwhals/default.nix +++ b/pkgs/development/python-modules/narwhals/default.nix @@ -13,6 +13,8 @@ pandas, polars, pyarrow, + pyspark, + sqlframe, # tests duckdb, @@ -23,14 +25,14 @@ buildPythonPackage rec { pname = "narwhals"; - version = "1.30.0"; + version = "1.37.0"; pyproject = true; src = fetchFromGitHub { owner = "narwhals-dev"; repo = "narwhals"; tag = "v${version}"; - hash = "sha256-jqrrQRviWllzZQEnlOTZ6oJM3WYQ3YlDvareTTBcNl4="; + hash = "sha256-AYgpHJwQVP+F2kr5YJtjnLNYedc81RvRcX1Cfh7c0xw="; }; build-system = [ @@ -46,6 +48,8 @@ buildPythonPackage rec { pandas = [ pandas ]; polars = [ polars ]; pyarrow = [ pyarrow ]; + pyspark = [ pyspark ]; + sqlframe = [ sqlframe ]; }; nativeCheckInputs = [ From d215254fbae64346cb051b66850341d56edd144e Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 28 Apr 2025 15:11:39 +0200 Subject: [PATCH 5/5] python312Packages.ibis-framework: 10.4.0 -> 10.5.0 Diff: https://github.com/ibis-project/ibis/compare/refs/tags/10.4.0...refs/tags/10.5.0 Changelog: https://github.com/ibis-project/ibis/blob/10.5.0/docs/release_notes.md --- .../python-modules/ibis-framework/default.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/ibis-framework/default.nix b/pkgs/development/python-modules/ibis-framework/default.nix index 28bd56d7968e..7d10c0d4c3d5 100644 --- a/pkgs/development/python-modules/ibis-framework/default.nix +++ b/pkgs/development/python-modules/ibis-framework/default.nix @@ -90,7 +90,7 @@ let ibisTestingData = fetchFromGitHub { owner = "ibis-project"; repo = "testing-data"; - # https://github.com/ibis-project/ibis/blob/10.4.0/nix/overlay.nix#L94-L100 + # https://github.com/ibis-project/ibis/blob/10.5.0/nix/overlay.nix#L94-L100 rev = "b26bd40cf29004372319df620c4bbe41420bb6f8"; hash = "sha256-1fenQNQB+Q0pbb0cbK2S/UIwZDE4PXXG15MH3aVbyLU="; }; @@ -98,14 +98,14 @@ in buildPythonPackage rec { pname = "ibis-framework"; - version = "10.4.0"; + version = "10.5.0"; pyproject = true; src = fetchFromGitHub { owner = "ibis-project"; repo = "ibis"; tag = version; - hash = "sha256-N6T3Hx4UIb08P+Xqg6RFb9q/pmAHvldW0yaFoRxmMv0="; + hash = "sha256-KJPl5bkD/tQlHY2k0b9zok5YCPekaXw7Y9z8P4AD3FQ="; }; build-system = [ @@ -157,13 +157,19 @@ buildPythonPackage rec { "test_read_sqlite" "test_register_sqlite" "test_roundtrip_xlsx" + # AssertionError: value does not match the expected value in snapshot "test_union_aliasing" + # requires network connection "test_s3_403_fallback" "test_hugging_face" + # requires pytest 8.2+ "test_roundtrip_delta" + + # AssertionError: value does not match the expected value in snapshot ibis/backends/tests/snapshots/test_sql/test_rewrite_context/sqlite/out.sql + "test_rewrite_context" ]; # patch out tests that check formatting with black