From 6efd19704d833400212300191ba70b9f205ce01b Mon Sep 17 00:00:00 2001 From: Sergey Volkov Date: Sat, 9 Aug 2025 12:57:30 +0200 Subject: [PATCH 01/16] python3Packages.svcs: init at 25.1.0 --- .../python-modules/svcs/default.nix | 73 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 75 insertions(+) create mode 100644 pkgs/development/python-modules/svcs/default.nix diff --git a/pkgs/development/python-modules/svcs/default.nix b/pkgs/development/python-modules/svcs/default.nix new file mode 100644 index 000000000000..08a1452b9bfb --- /dev/null +++ b/pkgs/development/python-modules/svcs/default.nix @@ -0,0 +1,73 @@ +{ + lib, + stdenv, + buildPythonPackage, + fetchFromGitHub, + + attrs, + hatch-fancy-pypi-readme, + hatch-vcs, + hatchling, + # test dependencies + aiohttp, + fastapi, + flask, + httpx, + pyramid, + pytest-asyncio, + pytestCheckHook, + starlette, + sybil, +}: + +buildPythonPackage rec { + pname = "svcs"; + version = "25.1.0"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "hynek"; + repo = "svcs"; + tag = version; + hash = "sha256-dDPmOKGifAGmAH3TD0NzJvR8lUB5qDWbxIwzHtNeF+4="; + }; + + build-system = [ + hatch-fancy-pypi-readme + hatch-vcs + hatchling + ]; + + dependencies = [ + attrs + ]; + + nativeCheckInputs = [ + aiohttp + fastapi + flask + httpx + pyramid + pytest-asyncio + pytestCheckHook + starlette + sybil + ]; + + disabledTests = lib.optionals stdenv.hostPlatform.isDarwin [ + "test_aclose_registry_ok" + "test_registrations" + "test_get_pings" + "test_client_pool_register_value" + ]; + + pythonImportsCheck = [ "svcs" ]; + + meta = { + description = "Flexible Service Locator for Python"; + homepage = "https://github.com/hynek/svcs"; + changelog = "https://github.com/hynek/svcs/releases/tag/${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 3240f13cb592..68bfaf6ca7af 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -18406,6 +18406,8 @@ self: super: with self; { sv-ttk = callPackage ../development/python-modules/sv-ttk { }; + svcs = callPackage ../development/python-modules/svcs { }; + svg-path = callPackage ../development/python-modules/svg-path { }; svg-py = callPackage ../development/python-modules/svg-py { }; From 5d883c1ecff2a55a74f3bc939c0ba4e51fa18bc6 Mon Sep 17 00:00:00 2001 From: Sergey Volkov Date: Sat, 9 Aug 2025 12:56:33 +0200 Subject: [PATCH 02/16] python3Packages.issubclass: init at 0.1.2 --- .../python-modules/issubclass/default.nix | 30 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 ++ 2 files changed, 32 insertions(+) create mode 100644 pkgs/development/python-modules/issubclass/default.nix diff --git a/pkgs/development/python-modules/issubclass/default.nix b/pkgs/development/python-modules/issubclass/default.nix new file mode 100644 index 000000000000..041ec408e62e --- /dev/null +++ b/pkgs/development/python-modules/issubclass/default.nix @@ -0,0 +1,30 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + poetry-core, +}: + +buildPythonPackage rec { + pname = "issubclass"; + version = "0.1.2"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "zmievsa"; + repo = "issubclass"; + tag = version; + hash = "sha256-a4EUbQY5duicnxuwtpBQ/7UBmtmeU+ZaySEnT4lmgLs="; + }; + + build-system = [ poetry-core ]; + + pythonImportsCheck = [ "issubclass" ]; + + meta = { + description = "issubclass() builtin that doesn't raise TypeError when arguments are not classes"; + homepage = "https://pypi.org/project/issubclass"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 68bfaf6ca7af..da4f85fe560a 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -7578,6 +7578,8 @@ self: super: with self; { israel-rail-api = callPackage ../development/python-modules/israel-rail-api { }; + issubclass = callPackage ../development/python-modules/issubclass { }; + itables = callPackage ../development/python-modules/itables { }; itanium-demangler = callPackage ../development/python-modules/itanium-demangler { }; From 19406b750c4fe71617174c73b284b83fe30486b3 Mon Sep 17 00:00:00 2001 From: Sergey Volkov Date: Thu, 27 Mar 2025 10:06:28 +0100 Subject: [PATCH 03/16] python3Packages.pytest-fixture-classes: init at 1.0.3 --- .../pytest-fixture-classes/default.nix | 40 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 42 insertions(+) create mode 100644 pkgs/development/python-modules/pytest-fixture-classes/default.nix diff --git a/pkgs/development/python-modules/pytest-fixture-classes/default.nix b/pkgs/development/python-modules/pytest-fixture-classes/default.nix new file mode 100644 index 000000000000..6765247d940f --- /dev/null +++ b/pkgs/development/python-modules/pytest-fixture-classes/default.nix @@ -0,0 +1,40 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + + poetry-core, + pytest, + pytestCheckHook, + typing-extensions, +}: + +buildPythonPackage rec { + pname = "pytest-fixture-classes"; + version = "1.0.3"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "zmievsa"; + repo = "pytest-fixture-classes"; + tag = version; + hash = "sha256-A3HsDhCGxoJnkl841tZ/7lIn8UyGy5NRjVWODZV7aOQ="; + }; + + build-system = [ poetry-core ]; + + dependencies = [ + typing-extensions + ]; + + pythonImportsCheck = [ "pytest_fixture_classes" ]; + + nativeCheckInputs = [ pytestCheckHook ]; + + meta = { + description = "Fixtures as classes that work well with dependency injection, autocompletetion, type checkers, and language servers"; + homepage = "https://github.com/zmievsa/pytest-fixture-classes"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index da4f85fe560a..596879868cb3 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -14965,6 +14965,8 @@ self: super: with self; { pytest-filter-subpackage = callPackage ../development/python-modules/pytest-filter-subpackage { }; + pytest-fixture-classes = callPackage ../development/python-modules/pytest-fixture-classes { }; + pytest-fixture-config = callPackage ../development/python-modules/pytest-fixture-config { }; pytest-flake8 = callPackage ../development/python-modules/pytest-flake8 { }; From ba61bcfe8c72c6f853c2f7c22e52bad17f66ec44 Mon Sep 17 00:00:00 2001 From: Sergey Volkov Date: Wed, 26 Mar 2025 10:33:42 +0100 Subject: [PATCH 04/16] python3Packages.cadwyn: init at 5.6.1 Newer versions currently don't build with fastapi 0.121 Co-authored-by: Ethan Carter Edwards --- .../python-modules/cadwyn/default.nix | 73 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 75 insertions(+) create mode 100644 pkgs/development/python-modules/cadwyn/default.nix diff --git a/pkgs/development/python-modules/cadwyn/default.nix b/pkgs/development/python-modules/cadwyn/default.nix new file mode 100644 index 000000000000..0049172411ce --- /dev/null +++ b/pkgs/development/python-modules/cadwyn/default.nix @@ -0,0 +1,73 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + hatchling, + pytestCheckHook, + # runtime dependencies + fastapi, + issubclass, + jinja2, + pydantic, + starlette, + typing-extensions, + typing-inspection, + # test dependencies + dirty-equals, + httpx, + inline-snapshot, + pydantic-settings, + pytest-fixture-classes, + python-multipart, + svcs, + typer, + uvicorn, +}: + +buildPythonPackage rec { + pname = "cadwyn"; + version = "5.6.1"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "zmievsa"; + repo = "cadwyn"; + tag = version; + hash = "sha256-VVi79c/Y1mu520H/7gy9lGdBIVuKsYedU49P501NQao="; + }; + + build-system = [ hatchling ]; + + dependencies = [ + fastapi + issubclass + jinja2 + pydantic + starlette + typing-extensions + typing-inspection + ]; + + pythonImportsCheck = [ "cadwyn" ]; + + nativeCheckInputs = [ + dirty-equals + httpx + inline-snapshot + pydantic-settings + pytest-fixture-classes + pytestCheckHook + python-multipart + svcs + typer + uvicorn + ]; + + meta = { + description = "Production-ready community-driven modern Stripe-like API versioning in FastAPI"; + homepage = "https://github.com/zmievsa/cadwyn"; + changelog = "https://github.com/zmievsa/cadwyn/releases/tag/${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 596879868cb3..5846bd415e0e 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2383,6 +2383,8 @@ self: super: with self; { cachy = callPackage ../development/python-modules/cachy { }; + cadwyn = callPackage ../development/python-modules/cadwyn { }; + caffe = toPythonModule ( pkgs.caffe.override { pythonSupport = true; From f96cdc8f978adfe69819e295d2de7fab15747d2a Mon Sep 17 00:00:00 2001 From: Sergey Volkov Date: Mon, 31 Mar 2025 15:47:32 +0200 Subject: [PATCH 05/16] python3Packages.retryhttp: init at 1.3.1 --- .../python-modules/retryhttp/default.nix | 50 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 52 insertions(+) create mode 100644 pkgs/development/python-modules/retryhttp/default.nix diff --git a/pkgs/development/python-modules/retryhttp/default.nix b/pkgs/development/python-modules/retryhttp/default.nix new file mode 100644 index 000000000000..8cc37882c709 --- /dev/null +++ b/pkgs/development/python-modules/retryhttp/default.nix @@ -0,0 +1,50 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + # build system + setuptools, + setuptools-scm, + # runtime + httpx, + pydantic, + requests, + tenacity, + types-requests, +}: + +buildPythonPackage rec { + pname = "retryhttp"; + version = "1.3.1"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "austind"; + repo = "retryhttp"; + tag = "release/v${version}"; + hash = "sha256-Jz073CTeIfPjcOQUGdb6/Q5OfEXZW5hTiEPKrfAK0Gg="; + }; + + build-system = [ + setuptools + setuptools-scm + ]; + + dependencies = [ + httpx + pydantic + requests + tenacity + types-requests + ]; + + pythonImportsCheck = [ "retryhttp" ]; + + meta = { + description = "Retry potentially transient HTTP errors in Python"; + homepage = "https://github.com/austind/retryhttp"; + changelog = "https://github.com/austind/retryhttp/releases/tag/release%2Fv${version}"; + license = lib.licenses.apsl20; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 5846bd415e0e..60009f33a1cf 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -16482,6 +16482,8 @@ self: super: with self; { retry2 = callPackage ../development/python-modules/retry2 { }; + retryhttp = callPackage ../development/python-modules/retryhttp { }; + retrying = callPackage ../development/python-modules/retrying { }; returns = callPackage ../development/python-modules/returns { }; From af47bd53b67539a7dd51e9a5da4de5be003ae103 Mon Sep 17 00:00:00 2001 From: Sergey Volkov Date: Wed, 8 Oct 2025 09:26:30 +0200 Subject: [PATCH 06/16] python3Packages.greenback: init at 1.2.1 --- .../python-modules/greenback/default.nix | 47 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 49 insertions(+) create mode 100644 pkgs/development/python-modules/greenback/default.nix diff --git a/pkgs/development/python-modules/greenback/default.nix b/pkgs/development/python-modules/greenback/default.nix new file mode 100644 index 000000000000..fcf90d4ce115 --- /dev/null +++ b/pkgs/development/python-modules/greenback/default.nix @@ -0,0 +1,47 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + + asgiref, + greenlet, + importlib-metadata, + outcome, + setuptools, + sniffio, +}: + +buildPythonPackage rec { + pname = "greenback"; + version = "1.2.1"; + pyproject = true; + + src = fetchFromGitHub { + owner = "oremanj"; + repo = "greenback"; + tag = "v${version}"; + hash = "sha256-u7kihdE5EH3Xbq4dQGY4ojiootjw3muYnoI7d2MrMkE="; + }; + + build-system = [ + importlib-metadata + setuptools + ]; + + dependencies = [ + asgiref + greenlet + outcome + sniffio + ]; + + pythonImportsCheck = [ "greenback" ]; + + meta = { + description = "Use anyio and asyncio from synchronous code"; + homepage = "https://github.com/oremanj/greenback"; + changelog = "https://github.com/oremanj/greenback/blob/${src.rev}/CHANGES.rst"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 60009f33a1cf..cd7ea6ef5339 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -6578,6 +6578,8 @@ self: super: with self; { green = callPackage ../development/python-modules/green { }; + greenback = callPackage ../development/python-modules/greenback { }; + greeneye-monitor = callPackage ../development/python-modules/greeneye-monitor { }; # built-in for pypi From 8626fc4fd8c6108ed3576febd43ad34567bc3674 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 7 Jan 2026 16:31:09 +0100 Subject: [PATCH 07/16] apache-airflow: refactor Use replace-fail, tag and sort some deps and use writableTmpDirAsHomeHook. --- pkgs/by-name/ap/apache-airflow/package.nix | 6 +++--- pkgs/by-name/ap/apache-airflow/python-package.nix | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pkgs/by-name/ap/apache-airflow/package.nix b/pkgs/by-name/ap/apache-airflow/package.nix index ba341a97bdf5..2565d93106c5 100644 --- a/pkgs/by-name/ap/apache-airflow/package.nix +++ b/pkgs/by-name/ap/apache-airflow/package.nix @@ -55,7 +55,7 @@ let ]; postPatch = '' substituteInPlace connexion/__init__.py \ - --replace "2020.0.dev1" "${version}" + --replace-fail "2020.0.dev1" "${version}" ''; }; werkzeug = pySuper.werkzeug.overridePythonAttrs rec { @@ -155,7 +155,7 @@ let # with pytest-httpbin 1.x preCheck = '' substituteInPlace pyproject.toml \ - --replace '[tool.pytest.ini_options]' '[tool.notpytest.ini_options]' + --replace-fail '[tool.pytest.ini_options]' '[tool.notpytest.ini_options]' ''; }; pytest-httpbin = pySuper.pytest-httpbin.overridePythonAttrs rec { @@ -221,7 +221,7 @@ let src = fetchFromGitHub { owner = "gitpython-developers"; repo = "smmap"; - rev = "refs/tags/v${version}"; + tag = "v${version}"; hash = "sha256-0Y175kjv/8UJpSxtLpWH4/VT7JrcVPAq79Nf3rtHZZM="; }; }; diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index bce488ed44ec..2f5dab9fb159 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -3,6 +3,7 @@ stdenv, fetchFromGitHub, writeScript, + writableTmpDirAsHomeHook, # javascript fetchYarnDeps, @@ -92,10 +93,10 @@ let }; nativeBuildInputs = [ - yarnConfigHook - yarnBuildHook nodejs webpack-cli + yarnBuildHook + yarnConfigHook ]; # The webpack license plugin tries to create /3rd-party-licenses when given the @@ -205,7 +206,9 @@ buildPythonPackage rec { src = airflow-src; pyproject = true; - nativeBuildInputs = [ hatchling ]; + nativeBuildInputs = [ writableTmpDirAsHomeHook ]; + + build-system = [ hatchling ]; dependencies = [ alembic @@ -283,8 +286,6 @@ buildPythonPackage rec { postInstall = '' cp -rv ${airflow-frontend}/static/dist $out/${python.sitePackages}/airflow/www/static - # Needed for pythonImportsCheck below - export HOME=$(mktemp -d) ''; pythonImportsCheck = [ @@ -293,9 +294,7 @@ buildPythonPackage rec { ++ providerImports; preCheck = '' - export AIRFLOW_HOME=$HOME export AIRFLOW__CORE__UNIT_TEST_MODE=True - export AIRFLOW_DB="$HOME/airflow.db" export PATH=$PATH:$out/bin ''; From cf4435aea53d7c52bb4327a337b4e7ea23c54dee Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 8 Jan 2026 12:30:01 +0100 Subject: [PATCH 08/16] apache-airflow: move build dependenies to build-system --- .../ap/apache-airflow/python-package.nix | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index 2f5dab9fb159..cfc670fa04aa 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -208,7 +208,17 @@ buildPythonPackage rec { nativeBuildInputs = [ writableTmpDirAsHomeHook ]; - build-system = [ hatchling ]; + build-system = [ + gitdb + gitpython + hatchling + packaging + pathspec + pluggy + smmap + tomli + trove-classifiers + ]; dependencies = [ alembic @@ -224,8 +234,6 @@ buildPythonPackage rec { flask-login flask-session fsspec - gitdb - gitpython gunicorn lazy-object-proxy linkify-it-py @@ -234,25 +242,19 @@ buildPythonPackage rec { methodtools opentelemetry-api opentelemetry-exporter-otlp - packaging pandas - pathspec pendulum - pluggy psutil python-daemon python-nvd3 python-slugify rich-argparse setproctitle - smmap sqlalchemy sqlalchemy-jsonfield tabulate tenacity termcolor - tomli - trove-classifiers universal-pathlib ] ++ providerPackages; From f4efd2c84eff1dd78da76a7c07cd2635b5645382 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 7 Jan 2026 16:31:34 +0100 Subject: [PATCH 09/16] apache-airflow: remove left-overs Once upon a time, these were important, but not anymore. --- pkgs/by-name/ap/apache-airflow/python-package.nix | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index cfc670fa04aa..58978b74e492 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -73,15 +73,9 @@ let owner = "apache"; repo = "airflow"; tag = version; - # Download using the git protocol rather than using tarballs, because the - # GitHub archive tarballs don't appear to include tests - forceFetchGit = true; - hash = "sha256-q5/CM+puXE31+15F3yZmcrR74LrqHppdCDUqjLQXPfk="; + hash = "sha256-APmZ/ylAcZuW5du2JJdMeAQ9g0VNAsXB+ZBPr0OEprg="; }; - # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree. - # This replicates this shell script, fixing bugs in yarn.lock and package.json - airflow-frontend = stdenv.mkDerivation rec { name = "airflow-frontend"; @@ -105,10 +99,6 @@ let sed -i 's!../../../../3rd-party-licenses/LICENSES-ui.txt!/3rd-party-licenses/LICENSES-ui.txt!' webpack.config.js ''; - postBuild = '' - find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5 - ''; - installPhase = '' mkdir -p $out/static/ cp -r static/dist $out/static From e53f050baa46e79b9ec58ac4c2e4d22d835c564f Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 8 Jan 2026 12:04:28 +0100 Subject: [PATCH 10/16] apache-airflow: move update script to separate file --- .../ap/apache-airflow/python-package.nix | 23 +------------------ pkgs/by-name/ap/apache-airflow/update.sh | 18 +++++++++++++++ 2 files changed, 19 insertions(+), 22 deletions(-) create mode 100755 pkgs/by-name/ap/apache-airflow/update.sh diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index 58978b74e492..e3056ce7c4d0 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -2,7 +2,6 @@ lib, stdenv, fetchFromGitHub, - writeScript, writableTmpDirAsHomeHook, # javascript @@ -298,27 +297,7 @@ buildPythonPackage rec { "bash_operator_kill" # psutil.AccessDenied ]; - # Updates yarn.lock and package.json - passthru.updateScript = writeScript "update.sh" '' - #!/usr/bin/env nix-shell - #!nix-shell -i bash -p common-updater-scripts curl "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix - - set -euo pipefail - - # Get new version - new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html | - grep -oE 'Airflow [0-9.]+' | head -1 | grep -oE '[0-9.]+')" - update-source-version ${pname} "$new_version" - - # Update frontend - cd ./pkgs/servers/apache-airflow - curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock - curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json - yarn2nix > yarn.nix - - # update provider dependencies - ./update-providers.py - ''; + passthru.updateScript = ./update.sh; # Note on testing the web UI: # You can (manually) test the web UI as follows: diff --git a/pkgs/by-name/ap/apache-airflow/update.sh b/pkgs/by-name/ap/apache-airflow/update.sh new file mode 100755 index 000000000000..319e9c70f3d6 --- /dev/null +++ b/pkgs/by-name/ap/apache-airflow/update.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix + +set -euo pipefail + +# Get new version +new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html | + pcregrep -o1 'Airflow ([0-9.]+).' | head -1)" +update-source-version apache-airflow "$new_version" + +# Update frontend +cd ./pkgs/servers/apache-airflow +curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock +curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json +yarn2nix > yarn.nix + +# update provider dependencies +./update-providers.py From d47876cf52e446e9ac4eadc20900ce9b7ce28bd4 Mon Sep 17 00:00:00 2001 From: Sergei Volkov Date: Wed, 21 Jan 2026 18:58:59 +0100 Subject: [PATCH 11/16] python3Packages.cadwyn: disable for Python 3.14 --- pkgs/development/python-modules/cadwyn/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/development/python-modules/cadwyn/default.nix b/pkgs/development/python-modules/cadwyn/default.nix index 0049172411ce..34fc2f74030e 100644 --- a/pkgs/development/python-modules/cadwyn/default.nix +++ b/pkgs/development/python-modules/cadwyn/default.nix @@ -22,6 +22,7 @@ svcs, typer, uvicorn, + pythonAtLeast, }: buildPythonPackage rec { @@ -36,6 +37,8 @@ buildPythonPackage rec { hash = "sha256-VVi79c/Y1mu520H/7gy9lGdBIVuKsYedU49P501NQao="; }; + disabled = pythonAtLeast "3.14"; + build-system = [ hatchling ]; dependencies = [ From c971beb7148226bc71c076e40e6428df618d4bf3 Mon Sep 17 00:00:00 2001 From: Sergey Volkov Date: Mon, 11 Aug 2025 12:06:47 +0200 Subject: [PATCH 12/16] apache-airflow: 2.10.5 -> 3.1.0 Co-authored-by: Wolfgang Walther --- pkgs/by-name/ap/apache-airflow/package.nix | 264 +- pkgs/by-name/ap/apache-airflow/providers.nix | 2261 +++-------------- .../ap/apache-airflow/python-package.nix | 440 ++-- .../ap/apache-airflow/update-providers.py | 234 +- pkgs/by-name/ap/apache-airflow/update.sh | 57 +- 5 files changed, 867 insertions(+), 2389 deletions(-) diff --git a/pkgs/by-name/ap/apache-airflow/package.nix b/pkgs/by-name/ap/apache-airflow/package.nix index 2565d93106c5..7e8e9065b3ff 100644 --- a/pkgs/by-name/ap/apache-airflow/package.nix +++ b/pkgs/by-name/ap/apache-airflow/package.nix @@ -9,213 +9,6 @@ let python = python3.override { self = python; packageOverrides = pySelf: pySuper: { - connexion = pySuper.connexion.overridePythonAttrs rec { - version = "2.14.2"; - src = fetchFromGitHub { - owner = "spec-first"; - repo = "connexion"; - tag = version; - hash = "sha256-1v1xCHY3ZnZG/Vu9wN/it7rLKC/StoDefoMNs+hMjIs="; - }; - nativeBuildInputs = with pySelf; [ - setuptools - ]; - pythonRelaxDeps = [ - "werkzeug" - ]; - propagatedBuildInputs = with pySelf; [ - aiohttp - aiohttp-jinja2 - aiohttp-swagger - clickclick - flask - inflection - jsonschema - openapi-spec-validator - packaging - pyyaml - requests - swagger-ui-bundle - ]; - nativeCheckInputs = with pySelf; [ - aiohttp-remotes - decorator - pytest-aiohttp - pytest-tornasync - pytestCheckHook - testfixtures - ]; - disabledTests = [ - "test_aiohttp_simple_api" - "test_app" - "test_invalid_type" # https://github.com/spec-first/connexion/issues/1969 - "test_openapi_yaml_behind_proxy" - "test_run_with_wsgi_containers" - "test_swagger_ui" - ]; - postPatch = '' - substituteInPlace connexion/__init__.py \ - --replace-fail "2020.0.dev1" "${version}" - ''; - }; - werkzeug = pySuper.werkzeug.overridePythonAttrs rec { - version = "2.3.8"; - src = fetchPypi { - pname = "werkzeug"; - inherit version; - hash = "sha256-VUslfHS763oNJUFgpPj/4YUkP1KlIDUGC3Ycpi2XfwM="; - }; - nativeCheckInputs = with pySelf; [ - pytest-xprocess - ]; - }; - # flask's test-suite needs click 8.1.8 - # TypeError: CliRunner.__init__() got an unexpected keyword argument 'mix_stderr' - click = pySuper.click.overridePythonAttrs rec { - version = "8.1.8"; - src = fetchPypi { - pname = "click"; - inherit version; - hash = "sha256-7VPJ2JkNg8Kifermjk7jN0c/YzDAQKMdQiXJV00WCWo="; - }; - }; - flask = pySuper.flask.overridePythonAttrs (o: rec { - version = "2.2.5"; - src = fetchPypi { - pname = "Flask"; - inherit version; - hash = "sha256-7e6bCn/yZiG9WowQ/0hK4oc3okENmbC7mmhQx/uXeqA="; - }; - nativeBuildInputs = (o.nativeBuildInputs or [ ]) ++ [ - pySelf.setuptools - ]; - pytestFlagsArray = [ - # tests that are marked with filterwarnings fail with - # DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for - # removal in Python 3.14; use importlib.util.find_spec() instead - "-W ignore::DeprecationWarning" - ]; - }); - flask-login = pySuper.flask-login.overridePythonAttrs rec { - version = "0.6.3"; - src = fetchFromGitHub { - owner = "maxcountryman"; - repo = "flask-login"; - tag = version; - hash = "sha256-Sn7Ond67P/3+OmKKFE/KfA6FE4IajhiRXVVrXKJtY3I="; - }; - nativeBuildInputs = with pySelf; [ setuptools ]; - pytestFlagsArray = [ - # DeprecationWarning: datetime.datetime.utcnow() is deprecated - # and scheduled for removal in a future version. - # Use timezone-aware objects to represent datetimes in UTC: - # datetime.datetime.now(datetime.UTC). - "-W ignore::DeprecationWarning" - ]; - }; - flask-session = pySuper.flask-session.overridePythonAttrs rec { - version = "0.5.0"; - src = fetchFromGitHub { - owner = "palletc-eco"; - repo = "flask-session"; - tag = version; - hash = "sha256-t8w6ZS4gBDpnnKvL3DLtn+rRLQNJbrT2Hxm4f3+a3Xc="; - }; - nativeCheckInputs = with pySelf; [ pytestCheckHook ]; - pytestFlagsArray = [ - "-k" - "'null_session or filesystem_session'" - ]; - dependencies = with pySelf; [ - flask-sqlalchemy - cachelib - ]; - disabledTests = [ ]; - disabledTestPaths = [ ]; - preCheck = ""; - postCheck = ""; - }; - # flask-appbuilder doesn't work with sqlalchemy 2.x, flask-appbuilder 3.x - # https://github.com/dpgaspar/Flask-AppBuilder/issues/2038 - flask-appbuilder = pySuper.flask-appbuilder.overridePythonAttrs { - meta.broken = false; - }; - # a knock-on effect from overriding the sqlalchemy version - flask-sqlalchemy = pySuper.flask-sqlalchemy.overridePythonAttrs { - src = fetchPypi { - pname = "Flask-SQLAlchemy"; - version = "2.5.1"; - hash = "sha256-K9pEtD58rLFdTgX/PMH4vJeTbMRkYjQkECv8LDXpWRI="; - }; - format = "setuptools"; - }; - httpcore = pySuper.httpcore.overridePythonAttrs { - # nullify upstream's pytest flags which cause - # "TLS/SSL connection has been closed (EOF)" - # with pytest-httpbin 1.x - preCheck = '' - substituteInPlace pyproject.toml \ - --replace-fail '[tool.pytest.ini_options]' '[tool.notpytest.ini_options]' - ''; - }; - pytest-httpbin = pySuper.pytest-httpbin.overridePythonAttrs rec { - version = "1.0.2"; - src = fetchFromGitHub { - owner = "kevin1024"; - repo = "pytest-httpbin"; - tag = "v${version}"; - hash = "sha256-S4ThQx4H3UlKhunJo35esPClZiEn7gX/Qwo4kE1QMTI="; - }; - # Use unmerged PR #65 to fix older version: - # https://github.com/kevin1024/pytest-httpbin/pull/65/ - # It was closed in favour of another which isn't compatible with the overriden version. - patches = [ - (fetchpatch { - url = "https://github.com/kevin1024/pytest-httpbin/commit/4e325f877ff8f77dec9f380bd8e53bb42976775c.patch"; - hash = "sha256-a33XcdMupD+7ZzvUibePdldGImmPLDNU2sxRbwpveDA="; - }) - (fetchpatch { - url = "https://github.com/kevin1024/pytest-httpbin/commit/463afb9b200563ac6fe7ae535f7a7a3c818b0418.patch"; - hash = "sha256-HFmuLtAtEjnB6heSG1YNnqxtz2phXNkHbQaZyB5bLJs="; - }) - ]; - disabledTests = [ - "test_httpbin_secure_accepts_get_requests" - "test_httpbin_secure_accepts_lots_of_get_requests" - "test_httpbin_both[https]" - "test_chunked_encoding[https]" - "TestClassBassedTests::test_http_secure" - "test_dont_crash_on_certificate_problems" - "test_redirect_location_is_https_for_secure_server" - "test_httpbin_secure_accepts_get_requests" - "test_http_secure" - ]; - }; - # apache-airflow doesn't work with sqlalchemy 2.x - # https://github.com/apache/airflow/issues/28723 - sqlalchemy = pySuper.sqlalchemy_1_4; - gitpython = pySuper.gitpython.overridePythonAttrs rec { - version = "3.1.44"; - src = fetchFromGitHub { - owner = "gitpython-developers"; - repo = "gitpython"; - rev = version; - hash = "sha256-KnKaBv/tKk4wiGWUWCEgd1vgrTouwUhqxJ1/nMjRaWk="; - }; - }; - # ValueError: Unknown classifier in field `project.classifiers`: Programming Language :: Python :: Free Threading :: 2 - Beta - urllib3 = pySuper.urllib3.overridePythonAttrs rec { - version = "2.5.0"; - src = fetchPypi { - pname = "urllib3"; - inherit version; - hash = "sha256-P8R3M8fkGdS8P2s9wrT4kLt0OQajDVa6Slv6S7/5J2A="; - }; - postPatch = '' - substituteInPlace pyproject.toml \ - --replace-fail ', "setuptools-scm>=8,<9"' "" - ''; - }; smmap = pySuper.smmap.overridePythonAttrs rec { version = "5.0.2"; src = fetchFromGitHub { @@ -225,53 +18,22 @@ let hash = "sha256-0Y175kjv/8UJpSxtLpWH4/VT7JrcVPAq79Nf3rtHZZM="; }; }; - trove-classifiers = pySuper.trove-classifiers.overridePythonAttrs rec { - version = "2024.10.21.16"; - src = fetchPypi { - pname = "trove_classifiers"; - inherit version; - hash = "sha256-F8vQVdZ9Xp2d5jKTqHMpQ/q8IVdOTHt07fEStJKM9fM="; - }; - postPatch = ""; - }; - packaging = pySuper.packaging.overridePythonAttrs rec { - version = "24.2"; - src = fetchPypi { - pname = "packaging"; - inherit version; - hash = "sha256-wiim3F6TLTRrxXOTeRCdSeiFPdgiNXHHxbVSYO3AuX8="; - }; - }; - pluggy = pySuper.pluggy.overridePythonAttrs rec { - version = "1.5.0"; + structlog = pySuper.structlog.overridePythonAttrs (o: rec { + version = "25.4.0"; src = fetchFromGitHub { - owner = "pytest-dev"; - repo = "pluggy"; + owner = "hynek"; + repo = "structlog"; tag = version; - hash = "sha256-f0DxyZZk6RoYtOEXLACcsOn2B+Hot4U4g5Ogr/hKmOE="; + hash = "sha256-iNnUogcICQJvHBZO2J8uk4NleQY/ra3ZzxQgnSRKr30="; }; - }; - pyproject-api = pySuper.pyproject-api.overridePythonAttrs rec { - version = "1.8.0"; - src = fetchPypi { - pname = "pyproject_api"; - inherit version; - hash = "sha256-d7gEny/rXTPu/MIbV/HieWNid6isita1hxA3skN3hJY="; - }; - disabledTests = [ - # AssertionError: assert ['magic>3', 'requests>2'] == ['magic >3', 'requests >2'] - "test_frontend_setuptools" - ]; - }; - tox = pySuper.tox.overridePythonAttrs rec { - version = "4.27.0"; - src = fetchFromGitHub { - owner = "tox-dev"; - repo = "tox"; - tag = version; - hash = "sha256-Z3qUK4w1ebPvdZD4ZuKgZXJPUu5lG0G41vn/pc9gC/0="; - }; - }; + nativeCheckInputs = + with pySelf; + o.nativeCheckInputs + ++ [ + freezegun + pretend + ]; + }); apache-airflow = pySelf.callPackage ./python-package.nix { }; }; }; diff --git a/pkgs/by-name/ap/apache-airflow/providers.nix b/pkgs/by-name/ap/apache-airflow/providers.nix index d2da8c6a4c65..a33d6fd234d1 100644 --- a/pkgs/by-name/ap/apache-airflow/providers.nix +++ b/pkgs/by-name/ap/apache-airflow/providers.nix @@ -1,851 +1,361 @@ # Warning: generated by update-providers.py, do not update manually { airbyte = { - deps = [ - "aiohttp" - "asgiref" - "requests" - "requests-toolbelt" - ]; - imports = [ - "airflow.providers.airbyte.hooks.airbyte" - "airflow.providers.airbyte.operators.airbyte" - ]; + deps = [ "requests" ]; + imports = [ "airflow.providers.airbyte.get_provider_info" ]; + version = "5.2.3"; }; + alibaba = { deps = [ "oss2" ]; - imports = [ - "airflow.providers.alibaba.cloud.hooks.analyticdb_spark" - "airflow.providers.alibaba.cloud.hooks.oss" - "airflow.providers.alibaba.cloud.operators.analyticdb_spark" - "airflow.providers.alibaba.cloud.operators.oss" - ]; + imports = [ "airflow.providers.alibaba.get_provider_info" ]; + version = "3.2.2"; }; + amazon = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" + "aiobotocore" "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" "boto3" "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" "inflection" "jmespath" - "json-merge-patch" "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" + "lxml" + "marshmallow" "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" + "python3-saml" "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.amazon.aws.hooks.appflow" - "airflow.providers.amazon.aws.hooks.athena" - "airflow.providers.amazon.aws.hooks.athena_sql" - "airflow.providers.amazon.aws.hooks.base_aws" - "airflow.providers.amazon.aws.hooks.batch_client" - "airflow.providers.amazon.aws.hooks.batch_waiters" - "airflow.providers.amazon.aws.hooks.bedrock" - "airflow.providers.amazon.aws.hooks.chime" - "airflow.providers.amazon.aws.hooks.cloud_formation" - "airflow.providers.amazon.aws.hooks.comprehend" - "airflow.providers.amazon.aws.hooks.datasync" - "airflow.providers.amazon.aws.hooks.dms" - "airflow.providers.amazon.aws.hooks.dynamodb" - "airflow.providers.amazon.aws.hooks.ec2" - "airflow.providers.amazon.aws.hooks.ecr" - "airflow.providers.amazon.aws.hooks.ecs" - "airflow.providers.amazon.aws.hooks.eks" - "airflow.providers.amazon.aws.hooks.elasticache_replication_group" - "airflow.providers.amazon.aws.hooks.emr" - "airflow.providers.amazon.aws.hooks.emr" - "airflow.providers.amazon.aws.hooks.eventbridge" - "airflow.providers.amazon.aws.hooks.glacier" - "airflow.providers.amazon.aws.hooks.glue" - "airflow.providers.amazon.aws.hooks.glue_catalog" - "airflow.providers.amazon.aws.hooks.glue_crawler" - "airflow.providers.amazon.aws.hooks.glue_databrew" - "airflow.providers.amazon.aws.hooks.kinesis" - "airflow.providers.amazon.aws.hooks.kinesis_analytics" - "airflow.providers.amazon.aws.hooks.lambda_function" - "airflow.providers.amazon.aws.hooks.logs" - "airflow.providers.amazon.aws.hooks.neptune" - "airflow.providers.amazon.aws.hooks.opensearch_serverless" - "airflow.providers.amazon.aws.hooks.quicksight" - "airflow.providers.amazon.aws.hooks.rds" - "airflow.providers.amazon.aws.hooks.redshift_cluster" - "airflow.providers.amazon.aws.hooks.redshift_data" - "airflow.providers.amazon.aws.hooks.redshift_sql" - "airflow.providers.amazon.aws.hooks.s3" - "airflow.providers.amazon.aws.hooks.sagemaker" - "airflow.providers.amazon.aws.hooks.secrets_manager" - "airflow.providers.amazon.aws.hooks.ses" - "airflow.providers.amazon.aws.hooks.sns" - "airflow.providers.amazon.aws.hooks.sqs" - "airflow.providers.amazon.aws.hooks.ssm" - "airflow.providers.amazon.aws.hooks.step_function" - "airflow.providers.amazon.aws.hooks.sts" - "airflow.providers.amazon.aws.hooks.verified_permissions" - "airflow.providers.amazon.aws.operators.appflow" - "airflow.providers.amazon.aws.operators.athena" - "airflow.providers.amazon.aws.operators.base_aws" - "airflow.providers.amazon.aws.operators.batch" - "airflow.providers.amazon.aws.operators.bedrock" - "airflow.providers.amazon.aws.operators.cloud_formation" - "airflow.providers.amazon.aws.operators.comprehend" - "airflow.providers.amazon.aws.operators.datasync" - "airflow.providers.amazon.aws.operators.dms" - "airflow.providers.amazon.aws.operators.ec2" - "airflow.providers.amazon.aws.operators.ecs" - "airflow.providers.amazon.aws.operators.eks" - "airflow.providers.amazon.aws.operators.emr" - "airflow.providers.amazon.aws.operators.emr" - "airflow.providers.amazon.aws.operators.eventbridge" - "airflow.providers.amazon.aws.operators.glacier" - "airflow.providers.amazon.aws.operators.glue" - "airflow.providers.amazon.aws.operators.glue_crawler" - "airflow.providers.amazon.aws.operators.glue_databrew" - "airflow.providers.amazon.aws.operators.kinesis_analytics" - "airflow.providers.amazon.aws.operators.lambda_function" - "airflow.providers.amazon.aws.operators.neptune" - "airflow.providers.amazon.aws.operators.quicksight" - "airflow.providers.amazon.aws.operators.rds" - "airflow.providers.amazon.aws.operators.redshift_cluster" - "airflow.providers.amazon.aws.operators.redshift_data" - "airflow.providers.amazon.aws.operators.s3" - "airflow.providers.amazon.aws.operators.sagemaker" - "airflow.providers.amazon.aws.operators.sns" - "airflow.providers.amazon.aws.operators.sqs" - "airflow.providers.amazon.aws.operators.step_function" + "s3fs" + "xmlsec" ]; + imports = [ "airflow.providers.amazon.get_provider_info" ]; + version = "9.13.0"; }; + apache_beam = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" + "numpy" "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.apache.beam.hooks.beam" - "airflow.providers.apache.beam.operators.beam" ]; + imports = [ "airflow.providers.apache.beam.get_provider_info" ]; + version = "6.1.4"; }; + apache_cassandra = { deps = [ "cassandra-driver" ]; - imports = [ "airflow.providers.apache.cassandra.hooks.cassandra" ]; + imports = [ "airflow.providers.apache.cassandra.get_provider_info" ]; + version = "3.8.2"; }; + apache_drill = { deps = [ ]; - imports = [ - "airflow.providers.apache.drill.hooks.drill" - "airflow.providers.apache.drill.operators.drill" - ]; + imports = [ "airflow.providers.apache.drill.get_provider_info" ]; + version = "3.1.2"; }; + apache_druid = { - deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pydruid" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.apache.druid.hooks.druid" - "airflow.providers.apache.druid.operators.druid" - "airflow.providers.apache.druid.operators.druid_check" - ]; + deps = [ "pydruid" ]; + imports = [ "airflow.providers.apache.druid.get_provider_info" ]; + version = "4.3.0"; }; + apache_flink = { - deps = [ - "aiofiles" - "asgiref" - "cryptography" - "google-re2" - "kubernetes" - "kubernetes-asyncio" - ]; - imports = [ "airflow.providers.apache.flink.operators.flink_kubernetes" ]; + deps = [ "cryptography" ]; + imports = [ "airflow.providers.apache.flink.get_provider_info" ]; + version = "1.7.2"; }; + apache_hdfs = { - deps = [ "pandas" ]; - imports = [ "airflow.providers.apache.hdfs.hooks.webhdfs" ]; + deps = [ + "hdfs" + "pandas" + ]; + imports = [ "airflow.providers.apache.hdfs.get_provider_info" ]; + version = "4.10.2"; }; + apache_hive = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.apache.hive.hooks.hive" - "airflow.providers.apache.hive.operators.hive" - "airflow.providers.apache.hive.operators.hive_stats" ]; + imports = [ "airflow.providers.apache.hive.get_provider_info" ]; + version = "9.1.2"; }; + apache_iceberg = { deps = [ ]; - imports = [ "airflow.providers.apache.iceberg.hooks.iceberg" ]; + imports = [ "airflow.providers.apache.iceberg.get_provider_info" ]; + version = "1.3.2"; }; + apache_impala = { - deps = [ ]; - imports = [ "airflow.providers.apache.impala.hooks.impala" ]; + deps = [ "kerberos" ]; + imports = [ "airflow.providers.apache.impala.get_provider_info" ]; + version = "1.7.2"; }; + apache_kafka = { deps = [ "asgiref" "confluent-kafka" ]; - imports = [ - "airflow.providers.apache.kafka.hooks.base" - "airflow.providers.apache.kafka.hooks.client" - "airflow.providers.apache.kafka.hooks.consume" - "airflow.providers.apache.kafka.hooks.produce" - "airflow.providers.apache.kafka.operators.consume" - "airflow.providers.apache.kafka.operators.produce" - ]; + imports = [ "airflow.providers.apache.kafka.get_provider_info" ]; + version = "1.10.3"; }; + apache_kylin = { deps = [ ]; - imports = [ - "airflow.providers.apache.kylin.hooks.kylin" - "airflow.providers.apache.kylin.operators.kylin_cube" - ]; + imports = [ "airflow.providers.apache.kylin.get_provider_info" ]; + version = "3.9.2"; }; + apache_livy = { deps = [ "aiohttp" "asgiref" - "requests" - "requests-toolbelt" - ]; - imports = [ - "airflow.providers.apache.livy.hooks.livy" - "airflow.providers.apache.livy.operators.livy" ]; + imports = [ "airflow.providers.apache.livy.get_provider_info" ]; + version = "4.4.2"; }; + apache_pig = { deps = [ ]; - imports = [ - "airflow.providers.apache.pig.hooks.pig" - "airflow.providers.apache.pig.operators.pig" - ]; + imports = [ "airflow.providers.apache.pig.get_provider_info" ]; + version = "4.7.2"; }; + apache_pinot = { deps = [ ]; - imports = [ "airflow.providers.apache.pinot.hooks.pinot" ]; + imports = [ "airflow.providers.apache.pinot.get_provider_info" ]; + version = "4.8.2"; }; + apache_spark = { deps = [ - "aiofiles" - "asgiref" - "cryptography" - "google-re2" "grpcio-status" - "kubernetes" - "kubernetes-asyncio" "pyspark" ]; - imports = [ - "airflow.providers.apache.spark.hooks.spark_connect" - "airflow.providers.apache.spark.hooks.spark_jdbc" - "airflow.providers.apache.spark.hooks.spark_jdbc_script" - "airflow.providers.apache.spark.hooks.spark_sql" - "airflow.providers.apache.spark.hooks.spark_submit" - "airflow.providers.apache.spark.operators.spark_jdbc" - "airflow.providers.apache.spark.operators.spark_sql" - "airflow.providers.apache.spark.operators.spark_submit" - ]; + imports = [ "airflow.providers.apache.spark.get_provider_info" ]; + version = "5.3.2"; }; + + apache_tinkerpop = { + deps = [ "gremlinpython" ]; + imports = [ "airflow.providers.apache.tinkerpop.get_provider_info" ]; + version = "1.0.3"; + }; + apprise = { deps = [ "apprise" ]; - imports = [ "airflow.providers.apprise.hooks.apprise" ]; + imports = [ "airflow.providers.apprise.get_provider_info" ]; + version = "2.1.2"; }; + arangodb = { deps = [ "python-arango" ]; - imports = [ - "airflow.providers.arangodb.hooks.arangodb" - "airflow.providers.arangodb.operators.arangodb" - ]; + imports = [ "airflow.providers.arangodb.get_provider_info" ]; + version = "2.8.2"; }; + asana = { deps = [ "asana" ]; - imports = [ - "airflow.providers.asana.hooks.asana" - "airflow.providers.asana.operators.asana_tasks" - ]; + imports = [ "airflow.providers.asana.get_provider_info" ]; + version = "2.10.2"; }; + atlassian_jira = { deps = [ "atlassian-python-api" ]; - imports = [ - "airflow.providers.atlassian.jira.hooks.jira" - "airflow.providers.atlassian.jira.operators.jira" - ]; + imports = [ "airflow.providers.atlassian.jira.get_provider_info" ]; + version = "3.1.2"; }; + celery = { deps = [ - "aiofiles" - "asgiref" - "cryptography" + "celery" "flower" - "google-re2" - "kubernetes" - "kubernetes-asyncio" ]; - imports = [ ]; + imports = [ "airflow.providers.celery.get_provider_info" ]; + version = "3.12.3"; }; + cloudant = { deps = [ ]; - imports = [ "airflow.providers.cloudant.hooks.cloudant" ]; + imports = [ "airflow.providers.cloudant.get_provider_info" ]; + version = "4.2.2"; }; + cncf_kubernetes = { deps = [ "aiofiles" "asgiref" "cryptography" - "google-re2" "kubernetes" "kubernetes-asyncio" ]; - imports = [ - "airflow.providers.cncf.kubernetes.hooks.kubernetes" - "airflow.providers.cncf.kubernetes.operators.custom_object_launcher" - "airflow.providers.cncf.kubernetes.operators.job" - "airflow.providers.cncf.kubernetes.operators.pod" - "airflow.providers.cncf.kubernetes.operators.resource" - "airflow.providers.cncf.kubernetes.operators.spark_kubernetes" - ]; + imports = [ "airflow.providers.cncf.kubernetes.get_provider_info" ]; + version = "10.8.0"; }; + cohere = { deps = [ "cohere" ]; - imports = [ - "airflow.providers.cohere.hooks.cohere" - "airflow.providers.cohere.operators.embedding" - ]; + imports = [ "airflow.providers.cohere.get_provider_info" ]; + version = "1.5.3"; }; + common_compat = { deps = [ ]; - imports = [ ]; + imports = [ "airflow.providers.common.compat.get_provider_info" ]; + version = "1.7.3"; }; + common_io = { - deps = [ "attrs" ]; - imports = [ "airflow.providers.common.io.operators.file_transfer" ]; + deps = [ ]; + imports = [ "airflow.providers.common.io.get_provider_info" ]; + version = "1.6.2"; }; + + common_messaging = { + deps = [ ]; + imports = [ "airflow.providers.common.messaging.get_provider_info" ]; + version = "2.0.0"; + }; + common_sql = { deps = [ - "attrs" + "methodtools" "more-itertools" + "pandas" + "polars" "sqlparse" ]; - imports = [ - "airflow.providers.common.sql.hooks.sql" - "airflow.providers.common.sql.operators.sql" - ]; + imports = [ "airflow.providers.common.sql.get_provider_info" ]; + version = "1.28.0"; }; + databricks = { deps = [ "aiohttp" + "azure-identity" + "databricks-sdk" "databricks-sql-connector" "mergedeep" "pandas" "pyarrow" "requests" ]; - imports = [ - "airflow.providers.databricks.hooks.databricks" - "airflow.providers.databricks.hooks.databricks_base" - "airflow.providers.databricks.hooks.databricks_sql" - "airflow.providers.databricks.operators.databricks" - "airflow.providers.databricks.operators.databricks_repos" - "airflow.providers.databricks.operators.databricks_sql" - "airflow.providers.databricks.operators.databricks_workflow" - ]; + imports = [ "airflow.providers.databricks.get_provider_info" ]; + version = "7.7.2"; }; + datadog = { deps = [ "datadog" ]; - imports = [ "airflow.providers.datadog.hooks.datadog" ]; + imports = [ "airflow.providers.datadog.get_provider_info" ]; + version = "3.9.2"; }; + dbt_cloud = { deps = [ "aiohttp" "asgiref" - "attrs" - "requests" - "requests-toolbelt" - ]; - imports = [ - "airflow.providers.dbt.cloud.hooks.dbt" - "airflow.providers.dbt.cloud.operators.dbt" ]; + imports = [ "airflow.providers.dbt.cloud.get_provider_info" ]; + version = "4.4.3"; }; + dingding = { - deps = [ - "aiohttp" - "asgiref" - "requests" - "requests-toolbelt" - ]; - imports = [ - "airflow.providers.dingding.hooks.dingding" - "airflow.providers.dingding.operators.dingding" - ]; + deps = [ ]; + imports = [ "airflow.providers.dingding.get_provider_info" ]; + version = "3.8.2"; }; + discord = { - deps = [ - "aiohttp" - "asgiref" - "requests" - "requests-toolbelt" - ]; - imports = [ - "airflow.providers.discord.hooks.discord_webhook" - "airflow.providers.discord.operators.discord_webhook" - ]; + deps = [ ]; + imports = [ "airflow.providers.discord.get_provider_info" ]; + version = "3.10.2"; }; + docker = { deps = [ "docker" "python-dotenv" ]; - imports = [ - "airflow.providers.docker.hooks.docker" - "airflow.providers.docker.operators.docker" - "airflow.providers.docker.operators.docker_swarm" - ]; + imports = [ "airflow.providers.docker.get_provider_info" ]; + version = "4.4.2"; }; + + edge3 = { + deps = [ + "pydantic" + "retryhttp" + ]; + imports = [ "airflow.providers.edge3.get_provider_info" ]; + version = "1.2.0"; + }; + elasticsearch = { deps = [ "elasticsearch" ]; - imports = [ "airflow.providers.elasticsearch.hooks.elasticsearch" ]; + imports = [ "airflow.providers.elasticsearch.get_provider_info" ]; + version = "6.3.2"; }; + exasol = { deps = [ "pandas" ]; - imports = [ - "airflow.providers.exasol.hooks.exasol" - "airflow.providers.exasol.operators.exasol" - ]; + imports = [ "airflow.providers.exasol.get_provider_info" ]; + version = "4.8.2"; }; + fab = { deps = [ + "blinker" + "connexion" "flask" "flask-appbuilder" + "flask-limiter" "flask-login" - "google-re2" + "flask-session" + "flask-wtf" "jmespath" + "kerberos" + "werkzeug" + "wtforms" ]; - imports = [ ]; + imports = [ "airflow.providers.fab.get_provider_info" ]; + version = "2.4.2"; }; + facebook = { deps = [ ]; - imports = [ "airflow.providers.facebook.ads.hooks.ads" ]; + imports = [ "airflow.providers.facebook.get_provider_info" ]; + version = "3.8.2"; }; + ftp = { - deps = [ "attrs" ]; - imports = [ - "airflow.providers.ftp.hooks.ftp" - "airflow.providers.ftp.operators.ftp" - ]; + deps = [ ]; + imports = [ "airflow.providers.ftp.get_provider_info" ]; + version = "3.13.2"; }; + + git = { + deps = [ "gitpython" ]; + imports = [ "airflow.providers.git.get_provider_info" ]; + version = "0.0.7"; + }; + github = { deps = [ "pygithub" ]; - imports = [ - "airflow.providers.github.hooks.github" - "airflow.providers.github.operators.github" - ]; + imports = [ "airflow.providers.github.get_provider_info" ]; + version = "2.9.2"; }; + google = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" "apache-beam" "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" "dill" - "dnspython" "gcsfs" "google-api-core" "google-api-python-client" @@ -854,6 +364,7 @@ "google-cloud-automl" "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" + "google-cloud-bigquery-storage" "google-cloud-bigtable" "google-cloud-compute" "google-cloud-container" @@ -876,285 +387,42 @@ "google-cloud-videointelligence" "google-cloud-vision" "google-cloud-workflows" - "google-re2" "grpcio-gcp" "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" + "immutabledict" "pandas" - "paramiko" + "plyvel" "proto-plus" - "psycopg2" "pyarrow" - "pyathena" - "pymongo" - "pymssql" "pyopenssl" - "pysftp" "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" + "ray" "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.google.ads.hooks.ads" - "airflow.providers.google.ads.operators.ads" - "airflow.providers.google.cloud.hooks.automl" - "airflow.providers.google.cloud.hooks.bigquery" - "airflow.providers.google.cloud.hooks.bigquery_dts" - "airflow.providers.google.cloud.hooks.bigtable" - "airflow.providers.google.cloud.hooks.cloud_batch" - "airflow.providers.google.cloud.hooks.cloud_build" - "airflow.providers.google.cloud.hooks.cloud_composer" - "airflow.providers.google.cloud.hooks.cloud_memorystore" - "airflow.providers.google.cloud.hooks.cloud_run" - "airflow.providers.google.cloud.hooks.cloud_sql" - "airflow.providers.google.cloud.hooks.cloud_storage_transfer_service" - "airflow.providers.google.cloud.hooks.compute" - "airflow.providers.google.cloud.hooks.compute_ssh" - "airflow.providers.google.cloud.hooks.datacatalog" - "airflow.providers.google.cloud.hooks.dataflow" - "airflow.providers.google.cloud.hooks.dataform" - "airflow.providers.google.cloud.hooks.datafusion" - "airflow.providers.google.cloud.hooks.datapipeline" - "airflow.providers.google.cloud.hooks.dataplex" - "airflow.providers.google.cloud.hooks.dataprep" - "airflow.providers.google.cloud.hooks.dataproc" - "airflow.providers.google.cloud.hooks.dataproc_metastore" - "airflow.providers.google.cloud.hooks.datastore" - "airflow.providers.google.cloud.hooks.dlp" - "airflow.providers.google.cloud.hooks.functions" - "airflow.providers.google.cloud.hooks.gcs" - "airflow.providers.google.cloud.hooks.gdm" - "airflow.providers.google.cloud.hooks.kms" - "airflow.providers.google.cloud.hooks.kubernetes_engine" - "airflow.providers.google.cloud.hooks.life_sciences" - "airflow.providers.google.cloud.hooks.looker" - "airflow.providers.google.cloud.hooks.mlengine" - "airflow.providers.google.cloud.hooks.natural_language" - "airflow.providers.google.cloud.hooks.os_login" - "airflow.providers.google.cloud.hooks.pubsub" - "airflow.providers.google.cloud.hooks.secret_manager" - "airflow.providers.google.cloud.hooks.spanner" - "airflow.providers.google.cloud.hooks.speech_to_text" - "airflow.providers.google.cloud.hooks.stackdriver" - "airflow.providers.google.cloud.hooks.tasks" - "airflow.providers.google.cloud.hooks.text_to_speech" - "airflow.providers.google.cloud.hooks.translate" - "airflow.providers.google.cloud.hooks.vertex_ai.auto_ml" - "airflow.providers.google.cloud.hooks.vertex_ai.batch_prediction_job" - "airflow.providers.google.cloud.hooks.vertex_ai.custom_job" - "airflow.providers.google.cloud.hooks.vertex_ai.dataset" - "airflow.providers.google.cloud.hooks.vertex_ai.endpoint_service" - "airflow.providers.google.cloud.hooks.vertex_ai.generative_model" - "airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job" - "airflow.providers.google.cloud.hooks.vertex_ai.model_service" - "airflow.providers.google.cloud.hooks.vertex_ai.pipeline_job" - "airflow.providers.google.cloud.hooks.vertex_ai.prediction_service" - "airflow.providers.google.cloud.hooks.video_intelligence" - "airflow.providers.google.cloud.hooks.vision" - "airflow.providers.google.cloud.hooks.workflows" - "airflow.providers.google.cloud.operators.automl" - "airflow.providers.google.cloud.operators.bigquery" - "airflow.providers.google.cloud.operators.bigquery_dts" - "airflow.providers.google.cloud.operators.bigtable" - "airflow.providers.google.cloud.operators.cloud_base" - "airflow.providers.google.cloud.operators.cloud_batch" - "airflow.providers.google.cloud.operators.cloud_build" - "airflow.providers.google.cloud.operators.cloud_composer" - "airflow.providers.google.cloud.operators.cloud_memorystore" - "airflow.providers.google.cloud.operators.cloud_run" - "airflow.providers.google.cloud.operators.cloud_sql" - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service" - "airflow.providers.google.cloud.operators.compute" - "airflow.providers.google.cloud.operators.datacatalog" - "airflow.providers.google.cloud.operators.dataflow" - "airflow.providers.google.cloud.operators.dataform" - "airflow.providers.google.cloud.operators.datafusion" - "airflow.providers.google.cloud.operators.datapipeline" - "airflow.providers.google.cloud.operators.dataplex" - "airflow.providers.google.cloud.operators.dataprep" - "airflow.providers.google.cloud.operators.dataproc" - "airflow.providers.google.cloud.operators.dataproc_metastore" - "airflow.providers.google.cloud.operators.datastore" - "airflow.providers.google.cloud.operators.dlp" - "airflow.providers.google.cloud.operators.functions" - "airflow.providers.google.cloud.operators.gcs" - "airflow.providers.google.cloud.operators.kubernetes_engine" - "airflow.providers.google.cloud.operators.life_sciences" - "airflow.providers.google.cloud.operators.looker" - "airflow.providers.google.cloud.operators.mlengine" - "airflow.providers.google.cloud.operators.natural_language" - "airflow.providers.google.cloud.operators.pubsub" - "airflow.providers.google.cloud.operators.spanner" - "airflow.providers.google.cloud.operators.speech_to_text" - "airflow.providers.google.cloud.operators.stackdriver" - "airflow.providers.google.cloud.operators.tasks" - "airflow.providers.google.cloud.operators.text_to_speech" - "airflow.providers.google.cloud.operators.translate" - "airflow.providers.google.cloud.operators.translate_speech" - "airflow.providers.google.cloud.operators.translate_speech" - "airflow.providers.google.cloud.operators.vertex_ai.auto_ml" - "airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job" - "airflow.providers.google.cloud.operators.vertex_ai.custom_job" - "airflow.providers.google.cloud.operators.vertex_ai.dataset" - "airflow.providers.google.cloud.operators.vertex_ai.endpoint_service" - "airflow.providers.google.cloud.operators.vertex_ai.generative_model" - "airflow.providers.google.cloud.operators.vertex_ai.hyperparameter_tuning_job" - "airflow.providers.google.cloud.operators.vertex_ai.model_service" - "airflow.providers.google.cloud.operators.vertex_ai.pipeline_job" - "airflow.providers.google.cloud.operators.video_intelligence" - "airflow.providers.google.cloud.operators.vision" - "airflow.providers.google.cloud.operators.workflows" - "airflow.providers.google.common.hooks.base_google" - "airflow.providers.google.common.hooks.discovery_api" - "airflow.providers.google.firebase.hooks.firestore" - "airflow.providers.google.firebase.operators.firestore" - "airflow.providers.google.leveldb.hooks.leveldb" - "airflow.providers.google.leveldb.operators.leveldb" - "airflow.providers.google.marketing_platform.hooks.analytics" - "airflow.providers.google.marketing_platform.hooks.analytics_admin" - "airflow.providers.google.marketing_platform.hooks.campaign_manager" - "airflow.providers.google.marketing_platform.hooks.display_video" - "airflow.providers.google.marketing_platform.hooks.search_ads" - "airflow.providers.google.marketing_platform.operators.analytics" - "airflow.providers.google.marketing_platform.operators.analytics_admin" - "airflow.providers.google.marketing_platform.operators.campaign_manager" - "airflow.providers.google.marketing_platform.operators.display_video" - "airflow.providers.google.marketing_platform.operators.search_ads" - "airflow.providers.google.suite.hooks.calendar" - "airflow.providers.google.suite.hooks.drive" - "airflow.providers.google.suite.hooks.sheets" - "airflow.providers.google.suite.operators.sheets" + "types-protobuf" ]; + imports = [ "airflow.providers.google.get_provider_info" ]; + version = "17.2.0"; }; + grpc = { deps = [ "google-auth" "google-auth-httplib2" "grpcio" ]; - imports = [ - "airflow.providers.grpc.hooks.grpc" - "airflow.providers.grpc.operators.grpc" - ]; + imports = [ "airflow.providers.grpc.get_provider_info" ]; + version = "3.8.2"; }; + hashicorp = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" "hvac" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" ]; - imports = [ "airflow.providers.hashicorp.hooks.vault" ]; + imports = [ "airflow.providers.hashicorp.get_provider_info" ]; + version = "4.3.2"; }; + http = { deps = [ "aiohttp" @@ -1162,49 +430,47 @@ "requests" "requests-toolbelt" ]; - imports = [ - "airflow.providers.http.hooks.http" - "airflow.providers.http.operators.http" - ]; + imports = [ "airflow.providers.http.get_provider_info" ]; + version = "5.3.4"; }; + imap = { deps = [ ]; - imports = [ "airflow.providers.imap.hooks.imap" ]; + imports = [ "airflow.providers.imap.get_provider_info" ]; + version = "3.9.2"; }; + influxdb = { deps = [ "influxdb-client" "requests" ]; - imports = [ - "airflow.providers.influxdb.hooks.influxdb" - "airflow.providers.influxdb.operators.influxdb" - ]; + imports = [ "airflow.providers.influxdb.get_provider_info" ]; + version = "2.9.3"; }; + jdbc = { deps = [ "jaydebeapi" ]; - imports = [ - "airflow.providers.jdbc.hooks.jdbc" - "airflow.providers.jdbc.operators.jdbc" - ]; + imports = [ "airflow.providers.jdbc.get_provider_info" ]; + version = "5.2.3"; }; + jenkins = { deps = [ "python-jenkins" ]; - imports = [ - "airflow.providers.jenkins.hooks.jenkins" - "airflow.providers.jenkins.operators.jenkins_job_trigger" - ]; + imports = [ "airflow.providers.jenkins.get_provider_info" ]; + version = "4.1.2"; }; + + keycloak = { + deps = [ "python-keycloak" ]; + imports = [ "airflow.providers.keycloak.get_provider_info" ]; + version = "0.1.0"; + }; + microsoft_azure = { deps = [ "adal" "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" @@ -1224,1085 +490,316 @@ "azure-storage-file-share" "azure-synapse-artifacts" "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" "microsoft-kiota-abstractions" + "microsoft-kiota-authentication-azure" + "microsoft-kiota-http" + "microsoft-kiota-serialization-json" + "microsoft-kiota-serialization-text" + "msal-extensions" "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.microsoft.azure.hooks.adx" - "airflow.providers.microsoft.azure.hooks.asb" - "airflow.providers.microsoft.azure.hooks.base_azure" - "airflow.providers.microsoft.azure.hooks.batch" - "airflow.providers.microsoft.azure.hooks.container_instance" - "airflow.providers.microsoft.azure.hooks.container_registry" - "airflow.providers.microsoft.azure.hooks.container_volume" - "airflow.providers.microsoft.azure.hooks.cosmos" - "airflow.providers.microsoft.azure.hooks.data_factory" - "airflow.providers.microsoft.azure.hooks.data_lake" - "airflow.providers.microsoft.azure.hooks.data_lake" - "airflow.providers.microsoft.azure.hooks.fileshare" - "airflow.providers.microsoft.azure.hooks.msgraph" - "airflow.providers.microsoft.azure.hooks.synapse" - "airflow.providers.microsoft.azure.hooks.wasb" - "airflow.providers.microsoft.azure.operators.adls" - "airflow.providers.microsoft.azure.operators.adx" - "airflow.providers.microsoft.azure.operators.asb" - "airflow.providers.microsoft.azure.operators.batch" - "airflow.providers.microsoft.azure.operators.container_instances" - "airflow.providers.microsoft.azure.operators.cosmos" - "airflow.providers.microsoft.azure.operators.data_factory" - "airflow.providers.microsoft.azure.operators.msgraph" - "airflow.providers.microsoft.azure.operators.synapse" - "airflow.providers.microsoft.azure.operators.wasb_delete_blob" ]; + imports = [ "airflow.providers.microsoft.azure.get_provider_info" ]; + version = "12.7.0"; }; + microsoft_mssql = { - deps = [ "pymssql" ]; - imports = [ - "airflow.providers.microsoft.mssql.hooks.mssql" - "airflow.providers.microsoft.mssql.operators.mssql" + deps = [ + "methodtools" + "pymssql" ]; + imports = [ "airflow.providers.microsoft.mssql.get_provider_info" ]; + version = "4.3.2"; }; + microsoft_psrp = { deps = [ "pypsrp" ]; - imports = [ - "airflow.providers.microsoft.psrp.hooks.psrp" - "airflow.providers.microsoft.psrp.operators.psrp" - ]; + imports = [ "airflow.providers.microsoft.psrp.get_provider_info" ]; + version = "3.1.2"; }; + microsoft_winrm = { deps = [ "pywinrm" ]; - imports = [ - "airflow.providers.microsoft.winrm.hooks.winrm" - "airflow.providers.microsoft.winrm.operators.winrm" - ]; + imports = [ "airflow.providers.microsoft.winrm.get_provider_info" ]; + version = "3.11.0"; }; + mongo = { deps = [ "dnspython" "pymongo" ]; - imports = [ "airflow.providers.mongo.hooks.mongo" ]; + imports = [ "airflow.providers.mongo.get_provider_info" ]; + version = "5.2.2"; }; + mysql = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" + "aiomysql" "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.mysql.hooks.mysql" - "airflow.providers.mysql.operators.mysql" ]; + imports = [ "airflow.providers.mysql.get_provider_info" ]; + version = "6.3.4"; }; + neo4j = { deps = [ "neo4j" ]; - imports = [ - "airflow.providers.neo4j.hooks.neo4j" - "airflow.providers.neo4j.operators.neo4j" - ]; + imports = [ "airflow.providers.neo4j.get_provider_info" ]; + version = "3.10.1"; }; + odbc = { deps = [ "pyodbc" ]; - imports = [ "airflow.providers.odbc.hooks.odbc" ]; + imports = [ "airflow.providers.odbc.get_provider_info" ]; + version = "4.10.2"; }; + openai = { - deps = [ ]; - imports = [ - "airflow.providers.openai.hooks.openai" - "airflow.providers.openai.operators.openai" - ]; + deps = [ "openai" ]; + imports = [ "airflow.providers.openai.get_provider_info" ]; + version = "1.6.2"; }; + openfaas = { deps = [ ]; - imports = [ "airflow.providers.openfaas.hooks.openfaas" ]; + imports = [ "airflow.providers.openfaas.get_provider_info" ]; + version = "3.8.2"; }; + openlineage = { deps = [ "attrs" ]; - imports = [ ]; + imports = [ "airflow.providers.openlineage.get_provider_info" ]; + version = "2.7.0"; }; + opensearch = { deps = [ "opensearch-py" ]; - imports = [ - "airflow.providers.opensearch.hooks.opensearch" - "airflow.providers.opensearch.operators.opensearch" - ]; + imports = [ "airflow.providers.opensearch.get_provider_info" ]; + version = "1.7.2"; }; + opsgenie = { deps = [ ]; - imports = [ - "airflow.providers.opsgenie.hooks.opsgenie" - "airflow.providers.opsgenie.operators.opsgenie" - ]; + imports = [ "airflow.providers.opsgenie.get_provider_info" ]; + version = "5.9.2"; }; + oracle = { - deps = [ "oracledb" ]; - imports = [ - "airflow.providers.oracle.hooks.oracle" - "airflow.providers.oracle.operators.oracle" + deps = [ + "numpy" + "oracledb" ]; + imports = [ "airflow.providers.oracle.get_provider_info" ]; + version = "4.2.0"; }; + pagerduty = { deps = [ ]; - imports = [ - "airflow.providers.pagerduty.hooks.pagerduty" - "airflow.providers.pagerduty.hooks.pagerduty_events" - ]; + imports = [ "airflow.providers.pagerduty.get_provider_info" ]; + version = "5.0.2"; }; + papermill = { deps = [ "ipykernel" + "nbconvert" "pandas" + "papermill" ]; - imports = [ - "airflow.providers.papermill.hooks.kernel" - "airflow.providers.papermill.operators.papermill" - ]; + imports = [ "airflow.providers.papermill.get_provider_info" ]; + version = "3.11.2"; }; + pgvector = { - deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.pgvector.hooks.pgvector" - "airflow.providers.pgvector.operators.pgvector" - ]; + deps = [ "pgvector" ]; + imports = [ "airflow.providers.pgvector.get_provider_info" ]; + version = "1.5.2"; }; + pinecone = { - deps = [ "pinecone-client" ]; - imports = [ - "airflow.providers.pinecone.hooks.pinecone" - "airflow.providers.pinecone.operators.pinecone" - ]; + deps = [ ]; + imports = [ "airflow.providers.pinecone.get_provider_info" ]; + version = "2.3.4"; }; + postgres = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" + "asyncpg" "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.postgres.hooks.postgres" - "airflow.providers.postgres.operators.postgres" + "polars" + "psycopg" + "psycopg2-binary" ]; + imports = [ "airflow.providers.postgres.get_provider_info" ]; + version = "6.3.0"; }; + presto = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" + "presto-python-client" ]; - imports = [ "airflow.providers.presto.hooks.presto" ]; + imports = [ "airflow.providers.presto.get_provider_info" ]; + version = "5.9.2"; }; + qdrant = { deps = [ "qdrant-client" ]; - imports = [ - "airflow.providers.qdrant.hooks.qdrant" - "airflow.providers.qdrant.operators.qdrant" - ]; + imports = [ "airflow.providers.qdrant.get_provider_info" ]; + version = "1.4.3"; }; + redis = { deps = [ "redis" ]; - imports = [ - "airflow.providers.redis.hooks.redis" - "airflow.providers.redis.operators.redis_publish" - ]; + imports = [ "airflow.providers.redis.get_provider_info" ]; + version = "4.3.0"; }; + salesforce = { deps = [ "pandas" "simple-salesforce" ]; - imports = [ - "airflow.providers.salesforce.hooks.salesforce" - "airflow.providers.salesforce.operators.bulk" - "airflow.providers.salesforce.operators.salesforce_apex_rest" - ]; + imports = [ "airflow.providers.salesforce.get_provider_info" ]; + version = "5.11.3"; }; + samba = { - deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ "airflow.providers.samba.hooks.samba" ]; + deps = [ "smbprotocol" ]; + imports = [ "airflow.providers.samba.get_provider_info" ]; + version = "4.11.0"; }; + segment = { deps = [ ]; - imports = [ - "airflow.providers.segment.hooks.segment" - "airflow.providers.segment.operators.segment_track_event" - ]; + imports = [ "airflow.providers.segment.get_provider_info" ]; + version = "3.8.2"; }; + sendgrid = { - deps = [ "sendgrid" ]; - imports = [ ]; + deps = [ + "sendgrid" + "werkzeug" + ]; + imports = [ "airflow.providers.sendgrid.get_provider_info" ]; + version = "4.1.3"; }; + sftp = { deps = [ "asyncssh" - "attrs" "paramiko" "pysftp" - "sshtunnel" - ]; - imports = [ - "airflow.providers.sftp.hooks.sftp" - "airflow.providers.sftp.operators.sftp" ]; + imports = [ "airflow.providers.sftp.get_provider_info" ]; + version = "5.4.0"; }; + singularity = { deps = [ "spython" ]; - imports = [ "airflow.providers.singularity.operators.singularity" ]; + imports = [ "airflow.providers.singularity.get_provider_info" ]; + version = "3.8.2"; }; + slack = { deps = [ "slack-sdk" ]; - imports = [ - "airflow.providers.slack.hooks.slack" - "airflow.providers.slack.hooks.slack_webhook" - "airflow.providers.slack.operators.slack" - "airflow.providers.slack.operators.slack_webhook" - ]; + imports = [ "airflow.providers.slack.get_provider_info" ]; + version = "9.2.0"; }; + smtp = { deps = [ ]; - imports = [ - "airflow.providers.smtp.hooks.smtp" - "airflow.providers.smtp.operators.smtp" - ]; + imports = [ "airflow.providers.smtp.get_provider_info" ]; + version = "2.2.1"; }; + snowflake = { deps = [ - "attrs" "pandas" "pyarrow" "snowflake-connector-python" "snowflake-sqlalchemy" ]; - imports = [ - "airflow.providers.snowflake.hooks.snowflake" - "airflow.providers.snowflake.hooks.snowflake_sql_api" - "airflow.providers.snowflake.operators.snowflake" - ]; + imports = [ "airflow.providers.snowflake.get_provider_info" ]; + version = "6.5.3"; }; + sqlite = { deps = [ ]; - imports = [ - "airflow.providers.sqlite.hooks.sqlite" - "airflow.providers.sqlite.operators.sqlite" - ]; + imports = [ "airflow.providers.sqlite.get_provider_info" ]; + version = "4.1.2"; }; + ssh = { deps = [ "paramiko" "sshtunnel" ]; - imports = [ - "airflow.providers.ssh.hooks.ssh" - "airflow.providers.ssh.operators.ssh" - ]; + imports = [ "airflow.providers.ssh.get_provider_info" ]; + version = "4.1.3"; }; + + standard = { + deps = [ ]; + imports = [ "airflow.providers.standard.get_provider_info" ]; + version = "1.7.0"; + }; + tableau = { deps = [ "tableauserverclient" ]; - imports = [ - "airflow.providers.tableau.hooks.tableau" - "airflow.providers.tableau.operators.tableau" - ]; - }; - tabular = { - deps = [ ]; - imports = [ ]; + imports = [ "airflow.providers.tableau.get_provider_info" ]; + version = "5.2.0"; }; + telegram = { deps = [ "python-telegram-bot" ]; - imports = [ - "airflow.providers.telegram.hooks.telegram" - "airflow.providers.telegram.operators.telegram" - ]; + imports = [ "airflow.providers.telegram.get_provider_info" ]; + version = "4.8.2"; }; + teradata = { - deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.teradata.hooks.teradata" - "airflow.providers.teradata.operators.teradata" - "airflow.providers.teradata.operators.teradata_compute_cluster" - ]; + deps = [ ]; + imports = [ "airflow.providers.teradata.get_provider_info" ]; + version = "3.2.1"; }; + trino = { - deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; - imports = [ - "airflow.providers.trino.hooks.trino" - "airflow.providers.trino.operators.trino" - ]; + deps = [ "pandas" ]; + imports = [ "airflow.providers.trino.get_provider_info" ]; + version = "6.3.3"; }; + vertica = { deps = [ "vertica-python" ]; - imports = [ - "airflow.providers.vertica.hooks.vertica" - "airflow.providers.vertica.operators.vertica" - ]; + imports = [ "airflow.providers.vertica.get_provider_info" ]; + version = "4.1.2"; }; + weaviate = { deps = [ "httpx" "pandas" "weaviate-client" ]; - imports = [ - "airflow.providers.weaviate.hooks.weaviate" - "airflow.providers.weaviate.operators.weaviate" - ]; + imports = [ "airflow.providers.weaviate.get_provider_info" ]; + version = "3.2.3"; }; + yandex = { deps = [ ]; - imports = [ - "airflow.providers.yandex.hooks.dataproc" - "airflow.providers.yandex.hooks.yandex" - "airflow.providers.yandex.hooks.yq" - "airflow.providers.yandex.operators.dataproc" - "airflow.providers.yandex.operators.yq" - ]; + imports = [ "airflow.providers.yandex.get_provider_info" ]; + version = "4.2.0"; }; + ydb = { deps = [ ]; - imports = [ - "airflow.providers.ydb.hooks._vendor.dbapi.connection" - "airflow.providers.ydb.hooks._vendor.dbapi.constants" - "airflow.providers.ydb.hooks._vendor.dbapi.cursor" - "airflow.providers.ydb.hooks._vendor.dbapi.errors" - "airflow.providers.ydb.hooks.ydb" - "airflow.providers.ydb.operators.ydb" - ]; + imports = [ "airflow.providers.ydb.get_provider_info" ]; + version = "2.2.2"; }; + zendesk = { deps = [ ]; - imports = [ "airflow.providers.zendesk.hooks.zendesk" ]; + imports = [ "airflow.providers.zendesk.get_provider_info" ]; + version = "4.10.2"; }; + } diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index e3056ce7c4d0..9d71376a95a3 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -5,102 +5,152 @@ writableTmpDirAsHomeHook, # javascript - fetchYarnDeps, + fetchPnpmDeps, nodejs, - webpack-cli, - yarnBuildHook, - yarnConfigHook, + pnpm, + pnpmConfigHook, # python + a2wsgi, + aiosqlite, alembic, argcomplete, + asgiref, + attrs, buildPythonPackage, + cadwyn, colorlog, - configupdater, - connexion, cron-descriptor, croniter, cryptography, + deprecated, dill, - flask-caching, - flask-login, - flask-session, + fastapi, + flit-core, fsspec, gitdb, gitpython, - gunicorn, + greenback, hatchling, + httpx, + importlib-metadata, + itsdangerous, + jinja2, + jsonschema, lazy-object-proxy, + libcst, linkify-it-py, lockfile, - marshmallow-oneofschema, - mdit-py-plugins, methodtools, + msgspec, + natsort, opentelemetry-api, opentelemetry-exporter-otlp, packaging, - pandas, pathspec, pendulum, pluggy, psutil, - pytest-asyncio, - pytestCheckHook, + pydantic, + pygments, + pygtrie, + pyjwt, python, python-daemon, - python-nvd3, + python-dateutil, python-slugify, + pyyaml, + requests, + retryhttp, + rich, rich-argparse, + rich-click, setproctitle, smmap, sqlalchemy, sqlalchemy-jsonfield, + sqlalchemy-utils, + starlette, + structlog, + svcs, tabulate, tenacity, termcolor, tomli, trove-classifiers, + types-requests, + typing-extensions, universal-pathlib, + uuid6, # Extra airflow providers to enable enabledProviders ? [ ], }: let - version = "2.10.5"; + version = "3.1.0"; - airflow-src = fetchFromGitHub { + src = fetchFromGitHub { owner = "apache"; repo = "airflow"; tag = version; - hash = "sha256-APmZ/ylAcZuW5du2JJdMeAQ9g0VNAsXB+ZBPr0OEprg="; + hash = "sha256-bg7PYW2xjDVWgBr8IduF+duIm5QnM23E23zY+fcKkns="; }; - airflow-frontend = stdenv.mkDerivation rec { - name = "airflow-frontend"; - - src = "${airflow-src}/airflow/www"; - - offlineCache = fetchYarnDeps { - yarnLock = "${src}/yarn.lock"; - hash = "sha256-hKgtMH4c8sPRDLPLVn+H8rmwc2Q6ei6U4er6fGuFn4I="; - }; + airflowUi = stdenv.mkDerivation rec { + pname = "airflow-ui-assets"; + inherit src version; + sourceRoot = "${src.name}/airflow-core/src/airflow/ui"; nativeBuildInputs = [ nodejs - webpack-cli - yarnBuildHook - yarnConfigHook + pnpm + pnpmConfigHook ]; - # The webpack license plugin tries to create /3rd-party-licenses when given the - # original relative path - postPatch = '' - sed -i 's!../../../../3rd-party-licenses/LICENSES-ui.txt!/3rd-party-licenses/LICENSES-ui.txt!' webpack.config.js + pnpmDeps = fetchPnpmDeps { + pname = "airflow-ui"; + inherit sourceRoot src version; + fetcherVersion = 1; + hash = "sha256-bGWt9KKvseuwDe7I+GbAUGUz3yGjDA2semKX5EmoNPM="; + }; + + buildPhase = '' + pnpm install + pnpm build ''; installPhase = '' - mkdir -p $out/static/ - cp -r static/dist $out/static + mkdir -p $out/share/airflow/ui + cp -r dist $out/share/airflow/ui/ + ''; + }; + + airflowSimpleAuthUi = stdenv.mkDerivation rec { + pname = "airflow-simple-ui-assets"; + inherit src version; + sourceRoot = "${src.name}/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui"; + + nativeBuildInputs = [ + nodejs + pnpm + pnpmConfigHook + ]; + + pnpmDeps = fetchPnpmDeps { + pname = "simple-auth-manager-ui"; + inherit sourceRoot src version; + fetcherVersion = 1; + hash = "sha256-XXA6Ynxvt/KnVT8EsV4z5Nxd5QejZO8iDDEcS04B57A="; + }; + + buildPhase = '' + pnpm install + pnpm build + ''; + + installPhase = '' + mkdir -p $out/share/airflow/simple-ui + cp -r dist $out/share/airflow/simple-ui/ ''; }; @@ -108,93 +158,179 @@ let "common_compat" "common_io" "common_sql" - "fab" - "ftp" - "http" - "imap" "smtp" - "sqlite" + "standard" ]; - # Import generated file with metadata for provider dependencies and imports. - # Enable additional providers using enabledProviders above. providers = import ./providers.nix; - getProviderPath = provider: lib.replaceStrings [ "_" ] [ "/" ] provider; - getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps; - getProviderImports = provider: providers.${provider}.imports; - providerImports = lib.concatMap getProviderImports enabledProviders; buildProvider = provider: - let - providerPath = getProviderPath provider; - in - python.pkgs.buildPythonPackage { + buildPythonPackage { pname = "apache-airflow-providers-${provider}"; - version = "unstable"; # will be extracted in the build phase - pyproject = false; # providers packages don't have pyproject.toml nor setup.py + version = providers.${provider}.version; + pyproject = true; - src = airflow-src; + inherit src; + sourceRoot = "${src.name}/providers/${lib.replaceStrings [ "_" ] [ "/" ] provider}"; - propagatedBuildInputs = getProviderDeps provider; - dependencies = [ packaging ]; + buildInputs = [ flit-core ]; - buildPhase = '' - # extract version from the provider's __init__.py file - if [ -f "airflow/providers/${providerPath}/__init__.py" ]; then - version=$(grep -oP "(?<=__version__ = ')[^']+" "airflow/providers/${providerPath}/__init__.py" || echo "0.0.0") - echo "Provider ${provider} version: $version" - else - echo "Error: __init__.py not found for provider ${provider} at path airflow/providers/${providerPath}" - exit 1 - fi - ''; + dependencies = map (dep: python.pkgs.${dep}) providers.${provider}.deps; - installPhase = '' - # create directory structure - mkdir -p $out/${python.sitePackages}/airflow/providers + pythonRemoveDeps = [ + "apache-airflow" + ]; - # copy the provider directory - if [ -d "airflow/providers/${providerPath}" ]; then - mkdir -p $out/${python.sitePackages}/airflow/providers/$(dirname "${providerPath}") - cp -r airflow/providers/${providerPath} $out/${python.sitePackages}/airflow/providers/$(dirname "${providerPath}") - - # create parent __init__.py files - touch $out/${python.sitePackages}/airflow/__init__.py - touch $out/${python.sitePackages}/airflow/providers/__init__.py - - # create any needed intermediate __init__.py files for nested providers - providerDir=$(dirname "${providerPath}") - while [ "$providerDir" != "." ] && [ -n "$providerDir" ]; do - mkdir -p $out/${python.sitePackages}/airflow/providers/$providerDir - touch $out/${python.sitePackages}/airflow/providers/$providerDir/__init__.py - providerDir=$(dirname "$providerDir") - done - - # create egg-info for package discovery - mkdir -p $out/${python.sitePackages}/apache_airflow_providers_${provider}.egg-info - cat > $out/${python.sitePackages}/apache_airflow_providers_${provider}.egg-info/PKG-INFO < Path: - return Path(os.path.dirname(sys.argv[0])) / "../../../.." + return Path(os.path.dirname(sys.argv[0])) / "../../../../" def dump_packages() -> Dict[str, Dict[str, str]]: @@ -70,27 +73,33 @@ def dump_packages() -> Dict[str, Dict[str, str]]: def remove_version_constraint(req: str) -> str: - return re.sub(r"[=><~].*$", "", req) + parsed_req = Requirement(req) + name = parsed_req.name + extras = ",".join(sorted(parsed_req.extras)) + if extras: + return f"{name}[{extras}]" + return name def name_to_attr_path(req: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]: - if req in PKG_PREFERENCES: - return f"{PKG_SET}.{PKG_PREFERENCES[req]}" + # Extract the base name, removing any extras (e.g., '[flask]') + base_req_name = req.split("[")[0] + logging.debug(f"Searching for base_req_name: {base_req_name}") + if base_req_name in PKG_PREFERENCES: + return f"{PKG_SET}.{PKG_PREFERENCES[base_req_name]}" attr_paths = [] - names = [req] + names = [base_req_name] # E.g. python-mpd2 is actually called python3.6-mpd2 # instead of python-3.6-python-mpd2 inside Nixpkgs - if req.startswith("python-") or req.startswith("python_"): - names.append(req[len("python-") :]) + if base_req_name.startswith("python-") or base_req_name.startswith("python_"): + names.append(base_req_name[len("python-") :]) for name in names: # treat "-" and "_" equally name = re.sub("[-_]", "[-_]", name) # python(minor).(major)-(pname)-(version or unstable-date) # we need the version qualifier, or we'll have multiple matches # (e.g. pyserial and pyserial-asyncio when looking for pyserial) - pattern = re.compile( - f"^python\\d+\\.\\d+-{name}-(?:\\d|unstable-.*)", re.I - ) + pattern = re.compile(rf"^python\d+\.\d+-{name}-(?:[\d\.]+|unstable-.*)", re.I) for attr_path, package in packages.items(): # logging.debug("Checking match for %s with %s", name, package["name"]) if pattern.match(package["name"]): @@ -130,30 +139,81 @@ def get_cross_provider_reqs( if len(cross_provider_deps[provider]) > 0: reqs.update( chain.from_iterable( - get_cross_provider_reqs( - d, provider_reqs, cross_provider_deps, seen + [provider] + ( + get_cross_provider_reqs( + d, provider_reqs, cross_provider_deps, seen + [provider] + ) + if d not in seen + else [] ) - if d not in seen - else [] for d in cross_provider_deps[provider] ) ) return reqs -def get_provider_reqs(version: str, packages: Dict) -> Dict: - provider_dependencies = get_file_from_github( - version, "generated/provider_dependencies.json" - ) +def parse_pyproject_toml(version: str, provider_name: str) -> Dict: + provider_dir = provider_name.replace(".", "/") + path = f"providers/{provider_dir}/pyproject.toml" + try: + content = get_file_from_github(version, path) + data = tomli.loads(content.decode("utf-8")) + + dependencies = data.get("project", {}).get("dependencies", []) + + # Extract optional dependencies + optional_dependencies = data.get("project", {}).get("optional-dependencies", {}) + for opt_deps_list in optional_dependencies.values(): + dependencies.extend(opt_deps_list) + + imports = [] + # Heuristic to generate imports based on provider name + # This might not be exhaustive but covers common cases + base_import_path = f"airflow.providers.{provider_name.replace('-', '.')}" + + # Try to get more specific imports from provider_info entry point if available + provider_info_entry_point = ( + data.get("project", {}) + .get("entry-points", {}) + .get("apache_airflow_provider", {}) + .get("provider_info") + ) + if provider_info_entry_point: + module_path = provider_info_entry_point.split(":")[0] + if module_path not in imports: + imports.append(module_path) + + return { + "deps": dependencies, + "imports": sorted(list(set(imports))), # Remove duplicates and sort + "cross-providers-deps": [], # pyproject.toml doesn't directly list cross-provider deps + "version": data.get("project", {}).get("version", "unknown"), + } + except HTTPError: + logging.warning("Couldn't get pyproject.toml for %s", provider_name) + return {"deps": [], "imports": [], "cross-providers-deps": []} + except Exception as e: + logging.error(f"Error parsing pyproject.toml for {provider_name}: {e}") + return {"deps": [], "imports": [], "cross-providers-deps": []} + + +def get_provider_reqs(version: str, packages: Dict, provider_names: List[str]) -> Dict: + provider_data = {} + for provider in provider_names: + data = parse_pyproject_toml(version, provider) + provider_data[provider] = { + "deps": data["deps"], + "cross-providers-deps": data["cross-providers-deps"], + } + provider_reqs = {} cross_provider_deps = {} - for provider, provider_data in provider_dependencies.items(): + for provider, data in provider_data.items(): provider_reqs[provider] = list( - provider_reqs_to_attr_paths(provider_data["deps"], packages) + provider_reqs_to_attr_paths(data["deps"], packages) ) + EXTRA_REQS.get(provider, []) - cross_provider_deps[provider] = [ - d for d in provider_data["cross-providers-deps"] if d != "common.sql" - ] + cross_provider_deps[provider] = data["cross-providers-deps"] + transitive_provider_reqs = {} # Add transitive cross-provider reqs for provider in provider_reqs: @@ -163,39 +223,25 @@ def get_provider_reqs(version: str, packages: Dict) -> Dict: return transitive_provider_reqs -def get_provider_yaml(version: str, provider: str) -> Dict: - provider_dir = provider.replace(".", "/") - path = f"airflow/providers/{provider_dir}/provider.yaml" - try: - return get_file_from_github(version, path) - except HTTPError: - logging.warning("Couldn't get provider yaml for %s", provider) - return {} - - -def get_provider_imports(version: str, providers) -> Dict: +def get_provider_imports(version: str, provider_names: List[str]) -> Dict: provider_imports = {} - for provider in providers: - provider_yaml = get_provider_yaml(version, provider) - imports: List[str] = [] - if "hooks" in provider_yaml: - imports.extend( - chain.from_iterable( - hook["python-modules"] for hook in provider_yaml["hooks"] - ) - ) - if "operators" in provider_yaml: - imports.extend( - chain.from_iterable( - operator["python-modules"] - for operator in provider_yaml["operators"] - ) - ) - provider_imports[provider] = imports + for provider in provider_names: + data = parse_pyproject_toml(version, provider) + provider_imports[provider] = data["imports"] return provider_imports -def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None: +def get_provider_versions(version: str, provider_names: List[str]) -> Dict: + provider_versions = {} + for provider in provider_names: + data = parse_pyproject_toml(version, provider) + provider_versions[provider] = data["version"] + return provider_versions + + +def to_nix_expr( + provider_reqs: Dict, provider_imports: Dict, provider_versions: Dict, fh: TextIO +) -> None: fh.write("# Warning: generated by update-providers.py, do not update manually\n") fh.write("{\n") for provider, reqs in provider_reqs.items(): @@ -209,19 +255,83 @@ def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None + " ".join(sorted(f'"{imp}"' for imp in provider_imports[provider])) + " ];\n" ) + fh.write(f' version = "{provider_versions[provider]}";\n') fh.write(" };\n") + fh.write("\n") fh.write("}\n") +def get_all_providers_from_github(version: str) -> List[str]: + """ + Fetches all provider paths that contain a pyproject.toml file from the airflow repository. + This is done by recursively fetching the git tree for the providers directory. + """ + providers = [] + try: + # Get the SHA of the 'providers' directory for the given version + with urlopen( + f"https://api.github.com/repos/apache/airflow/contents/?ref={version}" + ) as response: + root_contents = json.loads(response.read()) + + providers_dir_info = next( + ( + item + for item in root_contents + if item["name"] == "providers" and item["type"] == "dir" + ), + None, + ) + + if not providers_dir_info: + logging.error( + "Could not find 'providers' directory in the root of the repository." + ) + sys.exit(1) + + providers_dir_sha = providers_dir_info["sha"] + + # Now, get the tree for the 'providers' directory recursively + with urlopen( + f"https://api.github.com/repos/apache/airflow/git/trees/{providers_dir_sha}?recursive=1" + ) as response: + tree = json.loads(response.read()) + + pyproject_paths = [ + item["path"] + for item in tree["tree"] + if item["type"] == "blob" and item["path"].endswith("/pyproject.toml") + ] + + for path in pyproject_paths: + # remove /pyproject.toml suffix + provider_path = path[: -len("/pyproject.toml")] + # Exclude tests directory and empty paths (root pyproject.toml) + if provider_path and not provider_path.startswith("tests/"): + provider_name = provider_path.replace("/", ".") + providers.append(provider_name) + + except HTTPError as e: + logging.error(f"Error fetching provider list from GitHub: {e}") + sys.exit(1) + + return sorted(providers) + + def main() -> None: - logging.basicConfig(level=logging.INFO) + logging.basicConfig(level=logging.DEBUG) version = get_version() packages = dump_packages() logging.info("Generating providers.nix for version %s", version) - provider_reqs = get_provider_reqs(version, packages) - provider_imports = get_provider_imports(version, provider_reqs.keys()) - with open("providers.nix", "w") as fh: - to_nix_expr(provider_reqs, provider_imports, fh) + + # Fetch provider names from GitHub API + provider_names = get_all_providers_from_github(version) + + provider_reqs = get_provider_reqs(version, packages, provider_names) + provider_imports = get_provider_imports(version, provider_names) + provider_versions = get_provider_versions(version, provider_names) + with open(os.path.join(os.path.dirname(sys.argv[0]), "providers.nix"), "w") as fh: + to_nix_expr(provider_reqs, provider_imports, provider_versions, fh) if __name__ == "__main__": diff --git a/pkgs/by-name/ap/apache-airflow/update.sh b/pkgs/by-name/ap/apache-airflow/update.sh index 319e9c70f3d6..cf957d3ba778 100755 --- a/pkgs/by-name/ap/apache-airflow/update.sh +++ b/pkgs/by-name/ap/apache-airflow/update.sh @@ -1,18 +1,51 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix +#!nix-shell -i bash -p common-updater-scripts curl jq set -euo pipefail -# Get new version -new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html | - pcregrep -o1 'Airflow ([0-9.]+).' | head -1)" -update-source-version apache-airflow "$new_version" +# This script updates apache-airflow to the latest tag for a given major version. +# It defaults to major version 3. +# +# Usage: ./update.sh [MAJOR_VERSION] +# Example: ./update.sh 2 -# Update frontend -cd ./pkgs/servers/apache-airflow -curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock -curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json -yarn2nix > yarn.nix +MAJOR_VERSION="${1:-3}" +PACKAGE_DIR=$(dirname "$0") +PACKAGE_NAME="apache-airflow" + +# Fetches the latest git tag for the given major version from the GitHub API. +# It filters for tags that look like version numbers and sorts them to find the latest. +get_latest_tag() { + TEMP_FILE=$(mktemp) + curl -s "https://api.github.com/repos/apache/airflow/releases" > "$TEMP_FILE" + LATEST_TAG=$(jq -r --arg major_version "$MAJOR_VERSION" ' + .[] + | select(.tag_name | test("^" + $major_version + "\\.\\d+\\.\\d+$")) + | .tag_name + ' "$TEMP_FILE" | sort -V | tail -n 1) + rm "$TEMP_FILE" + echo "$LATEST_TAG" +} + +echo "Looking for latest tag for major version $MAJOR_VERSION..." +LATEST_TAG=$(get_latest_tag) + +if [[ -z "$LATEST_TAG" ]]; then + echo "No new tag found for major version $MAJOR_VERSION." + exit 0 +fi + +echo "Latest tag found: $LATEST_TAG" + +# Update the version and hash in the package definition +# This uses the standard nixpkgs script `update-source-version`. +echo "Updating source version for $PACKAGE_NAME to $LATEST_TAG..." +update-source-version "$PACKAGE_NAME" "$LATEST_TAG" + +# After updating the main package, run the providers update script. +# It will automatically pick up the new version from the nix file. +echo "Updating provider dependencies..." +"$PACKAGE_DIR/update-providers.py" + +echo "Update complete. Please check the git diff for changes." -# update provider dependencies -./update-providers.py From 69d641f2e161fb7ecafa4200a9d06c3fc5000044 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 8 Jan 2026 11:55:28 +0100 Subject: [PATCH 13/16] apache-airflow: expose enabledProviders Not sure how these were supposed to be enabled, without exposing them on the package interface itself. --- pkgs/by-name/ap/apache-airflow/package.nix | 5 ++++- pkgs/by-name/ap/apache-airflow/python-package.nix | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/ap/apache-airflow/package.nix b/pkgs/by-name/ap/apache-airflow/package.nix index 7e8e9065b3ff..f8d5ee6ab778 100644 --- a/pkgs/by-name/ap/apache-airflow/package.nix +++ b/pkgs/by-name/ap/apache-airflow/package.nix @@ -3,6 +3,9 @@ fetchpatch, fetchPypi, python3, + + # Extra airflow providers to enable + enabledProviders ? [ ], }: let @@ -34,7 +37,7 @@ let pretend ]; }); - apache-airflow = pySelf.callPackage ./python-package.nix { }; + apache-airflow = pySelf.callPackage ./python-package.nix { inherit enabledProviders; }; }; }; in diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index 9d71376a95a3..a83ccef41919 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -83,8 +83,7 @@ universal-pathlib, uuid6, - # Extra airflow providers to enable - enabledProviders ? [ ], + enabledProviders, }: let version = "3.1.0"; From acd825e7746b750be7e5ac12d78cf7d06268a41e Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 8 Jan 2026 13:25:33 +0100 Subject: [PATCH 14/16] apache-airflow: 3.1.0 -> 3.1.5 --- pkgs/by-name/ap/apache-airflow/providers.nix | 11 +++++----- .../ap/apache-airflow/python-package.nix | 20 ++++++++++++++----- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pkgs/by-name/ap/apache-airflow/providers.nix b/pkgs/by-name/ap/apache-airflow/providers.nix index a33d6fd234d1..24698f9536fb 100644 --- a/pkgs/by-name/ap/apache-airflow/providers.nix +++ b/pkgs/by-name/ap/apache-airflow/providers.nix @@ -209,13 +209,13 @@ common_compat = { deps = [ ]; imports = [ "airflow.providers.common.compat.get_provider_info" ]; - version = "1.7.3"; + version = "1.7.4"; }; common_io = { deps = [ ]; imports = [ "airflow.providers.common.io.get_provider_info" ]; - version = "1.6.2"; + version = "1.6.3"; }; common_messaging = { @@ -233,7 +233,7 @@ "sqlparse" ]; imports = [ "airflow.providers.common.sql.get_provider_info" ]; - version = "1.28.0"; + version = "1.28.1"; }; databricks = { @@ -377,6 +377,7 @@ "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" + "google-cloud-run" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" @@ -709,7 +710,7 @@ smtp = { deps = [ ]; imports = [ "airflow.providers.smtp.get_provider_info" ]; - version = "2.2.1"; + version = "2.3.1"; }; snowflake = { @@ -741,7 +742,7 @@ standard = { deps = [ ]; imports = [ "airflow.providers.standard.get_provider_info" ]; - version = "1.7.0"; + version = "1.9.0"; }; tableau = { diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index a83ccef41919..f5a26cd82234 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -17,6 +17,7 @@ argcomplete, asgiref, attrs, + babel, buildPythonPackage, cadwyn, colorlog, @@ -82,17 +83,18 @@ typing-extensions, universal-pathlib, uuid6, + uvicorn, enabledProviders, }: let - version = "3.1.0"; + version = "3.1.5"; src = fetchFromGitHub { owner = "apache"; repo = "airflow"; tag = version; - hash = "sha256-bg7PYW2xjDVWgBr8IduF+duIm5QnM23E23zY+fcKkns="; + hash = "sha256-C6QujoXUHN3xLHjLhjgcecdT1VQQMQ9v2f3x97i0NpM="; }; airflowUi = stdenv.mkDerivation rec { @@ -110,7 +112,7 @@ let pname = "airflow-ui"; inherit sourceRoot src version; fetcherVersion = 1; - hash = "sha256-bGWt9KKvseuwDe7I+GbAUGUz3yGjDA2semKX5EmoNPM="; + hash = "sha256-oB6fbQ9FwBJNkbXVN1ovMYvegbmyG1xDSYBYqHKKh+M="; }; buildPhase = '' @@ -139,7 +141,7 @@ let pname = "simple-auth-manager-ui"; inherit sourceRoot src version; fetcherVersion = 1; - hash = "sha256-XXA6Ynxvt/KnVT8EsV4z5Nxd5QejZO8iDDEcS04B57A="; + hash = "sha256-rUmrDHdC/Pqpe9njAM7cmoBLuPSgh36yLCT9j8ImN58="; }; buildPhase = '' @@ -242,6 +244,7 @@ let lockfile methodtools msgspec + natsort opentelemetry-api opentelemetry-exporter-otlp packaging @@ -275,8 +278,15 @@ let typing-extensions universal-pathlib uuid6 + uvicorn ] ++ (map buildProvider requiredProviders); + + pythonRelaxDeps = [ + # Temporary to fix CI only: + # https://github.com/apache/airflow/commit/c474be9ff06cf16bf96f93de9a09e30ffc476bee + "fastapi" + ]; }; taskSdk = buildPythonPackage { @@ -298,6 +308,7 @@ let dependencies = [ asgiref attrs + babel colorlog fsspec greenback @@ -346,7 +357,6 @@ buildPythonPackage rec { dependencies = [ airflowCore # subpackage from airflow src - natsort taskSdk # subpackage from airflow src ] ++ (map buildProvider enabledProviders); From 0b2284a690d6e9f525fdb7109c2a5dc33a81b470 Mon Sep 17 00:00:00 2001 From: Sergei Volkov Date: Wed, 21 Jan 2026 15:34:21 +0100 Subject: [PATCH 15/16] apache-airflow: 3.1.5 -> 3.1.6 --- pkgs/by-name/ap/apache-airflow/providers.nix | 486 ++++++++++++++---- .../ap/apache-airflow/python-package.nix | 10 +- 2 files changed, 395 insertions(+), 101 deletions(-) diff --git a/pkgs/by-name/ap/apache-airflow/providers.nix b/pkgs/by-name/ap/apache-airflow/providers.nix index 24698f9536fb..d20c06f64e48 100644 --- a/pkgs/by-name/ap/apache-airflow/providers.nix +++ b/pkgs/by-name/ap/apache-airflow/providers.nix @@ -2,13 +2,19 @@ { airbyte = { deps = [ "requests" ]; - imports = [ "airflow.providers.airbyte.get_provider_info" ]; + imports = [ + "airflow.providers.airbyte" + "airflow.providers.airbyte.get_provider_info" + ]; version = "5.2.3"; }; alibaba = { deps = [ "oss2" ]; - imports = [ "airflow.providers.alibaba.get_provider_info" ]; + imports = [ + "airflow.providers.alibaba" + "airflow.providers.alibaba.get_provider_info" + ]; version = "3.2.2"; }; @@ -29,7 +35,10 @@ "s3fs" "xmlsec" ]; - imports = [ "airflow.providers.amazon.get_provider_info" ]; + imports = [ + "airflow.providers.amazon" + "airflow.providers.amazon.get_provider_info" + ]; version = "9.13.0"; }; @@ -39,31 +48,46 @@ "numpy" "pyarrow" ]; - imports = [ "airflow.providers.apache.beam.get_provider_info" ]; + imports = [ + "airflow.providers.apache.beam" + "airflow.providers.apache.beam.get_provider_info" + ]; version = "6.1.4"; }; apache_cassandra = { deps = [ "cassandra-driver" ]; - imports = [ "airflow.providers.apache.cassandra.get_provider_info" ]; + imports = [ + "airflow.providers.apache.cassandra" + "airflow.providers.apache.cassandra.get_provider_info" + ]; version = "3.8.2"; }; apache_drill = { deps = [ ]; - imports = [ "airflow.providers.apache.drill.get_provider_info" ]; + imports = [ + "airflow.providers.apache.drill" + "airflow.providers.apache.drill.get_provider_info" + ]; version = "3.1.2"; }; apache_druid = { deps = [ "pydruid" ]; - imports = [ "airflow.providers.apache.druid.get_provider_info" ]; + imports = [ + "airflow.providers.apache.druid" + "airflow.providers.apache.druid.get_provider_info" + ]; version = "4.3.0"; }; apache_flink = { deps = [ "cryptography" ]; - imports = [ "airflow.providers.apache.flink.get_provider_info" ]; + imports = [ + "airflow.providers.apache.flink" + "airflow.providers.apache.flink.get_provider_info" + ]; version = "1.7.2"; }; @@ -72,7 +96,10 @@ "hdfs" "pandas" ]; - imports = [ "airflow.providers.apache.hdfs.get_provider_info" ]; + imports = [ + "airflow.providers.apache.hdfs" + "airflow.providers.apache.hdfs.get_provider_info" + ]; version = "4.10.2"; }; @@ -82,19 +109,28 @@ "pandas" "thrift" ]; - imports = [ "airflow.providers.apache.hive.get_provider_info" ]; + imports = [ + "airflow.providers.apache.hive" + "airflow.providers.apache.hive.get_provider_info" + ]; version = "9.1.2"; }; apache_iceberg = { deps = [ ]; - imports = [ "airflow.providers.apache.iceberg.get_provider_info" ]; + imports = [ + "airflow.providers.apache.iceberg" + "airflow.providers.apache.iceberg.get_provider_info" + ]; version = "1.3.2"; }; apache_impala = { deps = [ "kerberos" ]; - imports = [ "airflow.providers.apache.impala.get_provider_info" ]; + imports = [ + "airflow.providers.apache.impala" + "airflow.providers.apache.impala.get_provider_info" + ]; version = "1.7.2"; }; @@ -103,13 +139,19 @@ "asgiref" "confluent-kafka" ]; - imports = [ "airflow.providers.apache.kafka.get_provider_info" ]; + imports = [ + "airflow.providers.apache.kafka" + "airflow.providers.apache.kafka.get_provider_info" + ]; version = "1.10.3"; }; apache_kylin = { deps = [ ]; - imports = [ "airflow.providers.apache.kylin.get_provider_info" ]; + imports = [ + "airflow.providers.apache.kylin" + "airflow.providers.apache.kylin.get_provider_info" + ]; version = "3.9.2"; }; @@ -118,19 +160,28 @@ "aiohttp" "asgiref" ]; - imports = [ "airflow.providers.apache.livy.get_provider_info" ]; + imports = [ + "airflow.providers.apache.livy" + "airflow.providers.apache.livy.get_provider_info" + ]; version = "4.4.2"; }; apache_pig = { deps = [ ]; - imports = [ "airflow.providers.apache.pig.get_provider_info" ]; + imports = [ + "airflow.providers.apache.pig" + "airflow.providers.apache.pig.get_provider_info" + ]; version = "4.7.2"; }; apache_pinot = { deps = [ ]; - imports = [ "airflow.providers.apache.pinot.get_provider_info" ]; + imports = [ + "airflow.providers.apache.pinot" + "airflow.providers.apache.pinot.get_provider_info" + ]; version = "4.8.2"; }; @@ -139,37 +190,55 @@ "grpcio-status" "pyspark" ]; - imports = [ "airflow.providers.apache.spark.get_provider_info" ]; + imports = [ + "airflow.providers.apache.spark" + "airflow.providers.apache.spark.get_provider_info" + ]; version = "5.3.2"; }; apache_tinkerpop = { deps = [ "gremlinpython" ]; - imports = [ "airflow.providers.apache.tinkerpop.get_provider_info" ]; + imports = [ + "airflow.providers.apache.tinkerpop" + "airflow.providers.apache.tinkerpop.get_provider_info" + ]; version = "1.0.3"; }; apprise = { deps = [ "apprise" ]; - imports = [ "airflow.providers.apprise.get_provider_info" ]; + imports = [ + "airflow.providers.apprise" + "airflow.providers.apprise.get_provider_info" + ]; version = "2.1.2"; }; arangodb = { deps = [ "python-arango" ]; - imports = [ "airflow.providers.arangodb.get_provider_info" ]; + imports = [ + "airflow.providers.arangodb" + "airflow.providers.arangodb.get_provider_info" + ]; version = "2.8.2"; }; asana = { deps = [ "asana" ]; - imports = [ "airflow.providers.asana.get_provider_info" ]; + imports = [ + "airflow.providers.asana" + "airflow.providers.asana.get_provider_info" + ]; version = "2.10.2"; }; atlassian_jira = { deps = [ "atlassian-python-api" ]; - imports = [ "airflow.providers.atlassian.jira.get_provider_info" ]; + imports = [ + "airflow.providers.atlassian.jira" + "airflow.providers.atlassian.jira.get_provider_info" + ]; version = "3.1.2"; }; @@ -178,13 +247,19 @@ "celery" "flower" ]; - imports = [ "airflow.providers.celery.get_provider_info" ]; + imports = [ + "airflow.providers.celery" + "airflow.providers.celery.get_provider_info" + ]; version = "3.12.3"; }; cloudant = { deps = [ ]; - imports = [ "airflow.providers.cloudant.get_provider_info" ]; + imports = [ + "airflow.providers.cloudant" + "airflow.providers.cloudant.get_provider_info" + ]; version = "4.2.2"; }; @@ -195,32 +270,48 @@ "cryptography" "kubernetes" "kubernetes-asyncio" + "urllib3" + ]; + imports = [ + "airflow.providers.cncf.kubernetes" + "airflow.providers.cncf.kubernetes.get_provider_info" ]; - imports = [ "airflow.providers.cncf.kubernetes.get_provider_info" ]; version = "10.8.0"; }; cohere = { deps = [ "cohere" ]; - imports = [ "airflow.providers.cohere.get_provider_info" ]; + imports = [ + "airflow.providers.cohere" + "airflow.providers.cohere.get_provider_info" + ]; version = "1.5.3"; }; common_compat = { deps = [ ]; - imports = [ "airflow.providers.common.compat.get_provider_info" ]; + imports = [ + "airflow.providers.common.compat" + "airflow.providers.common.compat.get_provider_info" + ]; version = "1.7.4"; }; common_io = { deps = [ ]; - imports = [ "airflow.providers.common.io.get_provider_info" ]; + imports = [ + "airflow.providers.common.io" + "airflow.providers.common.io.get_provider_info" + ]; version = "1.6.3"; }; common_messaging = { deps = [ ]; - imports = [ "airflow.providers.common.messaging.get_provider_info" ]; + imports = [ + "airflow.providers.common.messaging" + "airflow.providers.common.messaging.get_provider_info" + ]; version = "2.0.0"; }; @@ -232,7 +323,10 @@ "polars" "sqlparse" ]; - imports = [ "airflow.providers.common.sql.get_provider_info" ]; + imports = [ + "airflow.providers.common.sql" + "airflow.providers.common.sql.get_provider_info" + ]; version = "1.28.1"; }; @@ -247,13 +341,19 @@ "pyarrow" "requests" ]; - imports = [ "airflow.providers.databricks.get_provider_info" ]; + imports = [ + "airflow.providers.databricks" + "airflow.providers.databricks.get_provider_info" + ]; version = "7.7.2"; }; datadog = { deps = [ "datadog" ]; - imports = [ "airflow.providers.datadog.get_provider_info" ]; + imports = [ + "airflow.providers.datadog" + "airflow.providers.datadog.get_provider_info" + ]; version = "3.9.2"; }; @@ -262,19 +362,28 @@ "aiohttp" "asgiref" ]; - imports = [ "airflow.providers.dbt.cloud.get_provider_info" ]; + imports = [ + "airflow.providers.dbt.cloud" + "airflow.providers.dbt.cloud.get_provider_info" + ]; version = "4.4.3"; }; dingding = { deps = [ ]; - imports = [ "airflow.providers.dingding.get_provider_info" ]; + imports = [ + "airflow.providers.dingding" + "airflow.providers.dingding.get_provider_info" + ]; version = "3.8.2"; }; discord = { deps = [ ]; - imports = [ "airflow.providers.discord.get_provider_info" ]; + imports = [ + "airflow.providers.discord" + "airflow.providers.discord.get_provider_info" + ]; version = "3.10.2"; }; @@ -283,7 +392,10 @@ "docker" "python-dotenv" ]; - imports = [ "airflow.providers.docker.get_provider_info" ]; + imports = [ + "airflow.providers.docker" + "airflow.providers.docker.get_provider_info" + ]; version = "4.4.2"; }; @@ -292,19 +404,28 @@ "pydantic" "retryhttp" ]; - imports = [ "airflow.providers.edge3.get_provider_info" ]; + imports = [ + "airflow.providers.edge3" + "airflow.providers.edge3.get_provider_info" + ]; version = "1.2.0"; }; elasticsearch = { deps = [ "elasticsearch" ]; - imports = [ "airflow.providers.elasticsearch.get_provider_info" ]; + imports = [ + "airflow.providers.elasticsearch" + "airflow.providers.elasticsearch.get_provider_info" + ]; version = "6.3.2"; }; exasol = { deps = [ "pandas" ]; - imports = [ "airflow.providers.exasol.get_provider_info" ]; + imports = [ + "airflow.providers.exasol" + "airflow.providers.exasol.get_provider_info" + ]; version = "4.8.2"; }; @@ -323,31 +444,46 @@ "werkzeug" "wtforms" ]; - imports = [ "airflow.providers.fab.get_provider_info" ]; + imports = [ + "airflow.providers.fab" + "airflow.providers.fab.get_provider_info" + ]; version = "2.4.2"; }; facebook = { deps = [ ]; - imports = [ "airflow.providers.facebook.get_provider_info" ]; + imports = [ + "airflow.providers.facebook" + "airflow.providers.facebook.get_provider_info" + ]; version = "3.8.2"; }; ftp = { deps = [ ]; - imports = [ "airflow.providers.ftp.get_provider_info" ]; + imports = [ + "airflow.providers.ftp" + "airflow.providers.ftp.get_provider_info" + ]; version = "3.13.2"; }; git = { deps = [ "gitpython" ]; - imports = [ "airflow.providers.git.get_provider_info" ]; + imports = [ + "airflow.providers.git" + "airflow.providers.git.get_provider_info" + ]; version = "0.0.7"; }; github = { deps = [ "pygithub" ]; - imports = [ "airflow.providers.github.get_provider_info" ]; + imports = [ + "airflow.providers.github" + "airflow.providers.github.get_provider_info" + ]; version = "2.9.2"; }; @@ -401,7 +537,10 @@ "tenacity" "types-protobuf" ]; - imports = [ "airflow.providers.google.get_provider_info" ]; + imports = [ + "airflow.providers.google" + "airflow.providers.google.get_provider_info" + ]; version = "17.2.0"; }; @@ -411,7 +550,10 @@ "google-auth-httplib2" "grpcio" ]; - imports = [ "airflow.providers.grpc.get_provider_info" ]; + imports = [ + "airflow.providers.grpc" + "airflow.providers.grpc.get_provider_info" + ]; version = "3.8.2"; }; @@ -420,7 +562,10 @@ "boto3" "hvac" ]; - imports = [ "airflow.providers.hashicorp.get_provider_info" ]; + imports = [ + "airflow.providers.hashicorp" + "airflow.providers.hashicorp.get_provider_info" + ]; version = "4.3.2"; }; @@ -431,13 +576,19 @@ "requests" "requests-toolbelt" ]; - imports = [ "airflow.providers.http.get_provider_info" ]; + imports = [ + "airflow.providers.http" + "airflow.providers.http.get_provider_info" + ]; version = "5.3.4"; }; imap = { deps = [ ]; - imports = [ "airflow.providers.imap.get_provider_info" ]; + imports = [ + "airflow.providers.imap" + "airflow.providers.imap.get_provider_info" + ]; version = "3.9.2"; }; @@ -446,25 +597,37 @@ "influxdb-client" "requests" ]; - imports = [ "airflow.providers.influxdb.get_provider_info" ]; + imports = [ + "airflow.providers.influxdb" + "airflow.providers.influxdb.get_provider_info" + ]; version = "2.9.3"; }; jdbc = { deps = [ "jaydebeapi" ]; - imports = [ "airflow.providers.jdbc.get_provider_info" ]; + imports = [ + "airflow.providers.jdbc" + "airflow.providers.jdbc.get_provider_info" + ]; version = "5.2.3"; }; jenkins = { deps = [ "python-jenkins" ]; - imports = [ "airflow.providers.jenkins.get_provider_info" ]; + imports = [ + "airflow.providers.jenkins" + "airflow.providers.jenkins.get_provider_info" + ]; version = "4.1.2"; }; keycloak = { deps = [ "python-keycloak" ]; - imports = [ "airflow.providers.keycloak.get_provider_info" ]; + imports = [ + "airflow.providers.keycloak" + "airflow.providers.keycloak.get_provider_info" + ]; version = "0.1.0"; }; @@ -499,7 +662,10 @@ "msal-extensions" "msgraph-core" ]; - imports = [ "airflow.providers.microsoft.azure.get_provider_info" ]; + imports = [ + "airflow.providers.microsoft.azure" + "airflow.providers.microsoft.azure.get_provider_info" + ]; version = "12.7.0"; }; @@ -508,19 +674,28 @@ "methodtools" "pymssql" ]; - imports = [ "airflow.providers.microsoft.mssql.get_provider_info" ]; + imports = [ + "airflow.providers.microsoft.mssql" + "airflow.providers.microsoft.mssql.get_provider_info" + ]; version = "4.3.2"; }; microsoft_psrp = { deps = [ "pypsrp" ]; - imports = [ "airflow.providers.microsoft.psrp.get_provider_info" ]; + imports = [ + "airflow.providers.microsoft.psrp" + "airflow.providers.microsoft.psrp.get_provider_info" + ]; version = "3.1.2"; }; microsoft_winrm = { deps = [ "pywinrm" ]; - imports = [ "airflow.providers.microsoft.winrm.get_provider_info" ]; + imports = [ + "airflow.providers.microsoft.winrm" + "airflow.providers.microsoft.winrm.get_provider_info" + ]; version = "3.11.0"; }; @@ -529,7 +704,10 @@ "dnspython" "pymongo" ]; - imports = [ "airflow.providers.mongo.get_provider_info" ]; + imports = [ + "airflow.providers.mongo" + "airflow.providers.mongo.get_provider_info" + ]; version = "5.2.2"; }; @@ -538,49 +716,73 @@ "aiomysql" "mysqlclient" ]; - imports = [ "airflow.providers.mysql.get_provider_info" ]; + imports = [ + "airflow.providers.mysql" + "airflow.providers.mysql.get_provider_info" + ]; version = "6.3.4"; }; neo4j = { deps = [ "neo4j" ]; - imports = [ "airflow.providers.neo4j.get_provider_info" ]; + imports = [ + "airflow.providers.neo4j" + "airflow.providers.neo4j.get_provider_info" + ]; version = "3.10.1"; }; odbc = { deps = [ "pyodbc" ]; - imports = [ "airflow.providers.odbc.get_provider_info" ]; + imports = [ + "airflow.providers.odbc" + "airflow.providers.odbc.get_provider_info" + ]; version = "4.10.2"; }; openai = { deps = [ "openai" ]; - imports = [ "airflow.providers.openai.get_provider_info" ]; + imports = [ + "airflow.providers.openai" + "airflow.providers.openai.get_provider_info" + ]; version = "1.6.2"; }; openfaas = { deps = [ ]; - imports = [ "airflow.providers.openfaas.get_provider_info" ]; + imports = [ + "airflow.providers.openfaas" + "airflow.providers.openfaas.get_provider_info" + ]; version = "3.8.2"; }; openlineage = { deps = [ "attrs" ]; - imports = [ "airflow.providers.openlineage.get_provider_info" ]; + imports = [ + "airflow.providers.openlineage" + "airflow.providers.openlineage.get_provider_info" + ]; version = "2.7.0"; }; opensearch = { deps = [ "opensearch-py" ]; - imports = [ "airflow.providers.opensearch.get_provider_info" ]; + imports = [ + "airflow.providers.opensearch" + "airflow.providers.opensearch.get_provider_info" + ]; version = "1.7.2"; }; opsgenie = { deps = [ ]; - imports = [ "airflow.providers.opsgenie.get_provider_info" ]; + imports = [ + "airflow.providers.opsgenie" + "airflow.providers.opsgenie.get_provider_info" + ]; version = "5.9.2"; }; @@ -589,13 +791,19 @@ "numpy" "oracledb" ]; - imports = [ "airflow.providers.oracle.get_provider_info" ]; + imports = [ + "airflow.providers.oracle" + "airflow.providers.oracle.get_provider_info" + ]; version = "4.2.0"; }; pagerduty = { deps = [ ]; - imports = [ "airflow.providers.pagerduty.get_provider_info" ]; + imports = [ + "airflow.providers.pagerduty" + "airflow.providers.pagerduty.get_provider_info" + ]; version = "5.0.2"; }; @@ -606,19 +814,28 @@ "pandas" "papermill" ]; - imports = [ "airflow.providers.papermill.get_provider_info" ]; + imports = [ + "airflow.providers.papermill" + "airflow.providers.papermill.get_provider_info" + ]; version = "3.11.2"; }; pgvector = { deps = [ "pgvector" ]; - imports = [ "airflow.providers.pgvector.get_provider_info" ]; + imports = [ + "airflow.providers.pgvector" + "airflow.providers.pgvector.get_provider_info" + ]; version = "1.5.2"; }; pinecone = { deps = [ ]; - imports = [ "airflow.providers.pinecone.get_provider_info" ]; + imports = [ + "airflow.providers.pinecone" + "airflow.providers.pinecone.get_provider_info" + ]; version = "2.3.4"; }; @@ -630,7 +847,10 @@ "psycopg" "psycopg2-binary" ]; - imports = [ "airflow.providers.postgres.get_provider_info" ]; + imports = [ + "airflow.providers.postgres" + "airflow.providers.postgres.get_provider_info" + ]; version = "6.3.0"; }; @@ -639,19 +859,28 @@ "pandas" "presto-python-client" ]; - imports = [ "airflow.providers.presto.get_provider_info" ]; + imports = [ + "airflow.providers.presto" + "airflow.providers.presto.get_provider_info" + ]; version = "5.9.2"; }; qdrant = { deps = [ "qdrant-client" ]; - imports = [ "airflow.providers.qdrant.get_provider_info" ]; + imports = [ + "airflow.providers.qdrant" + "airflow.providers.qdrant.get_provider_info" + ]; version = "1.4.3"; }; redis = { deps = [ "redis" ]; - imports = [ "airflow.providers.redis.get_provider_info" ]; + imports = [ + "airflow.providers.redis" + "airflow.providers.redis.get_provider_info" + ]; version = "4.3.0"; }; @@ -660,19 +889,28 @@ "pandas" "simple-salesforce" ]; - imports = [ "airflow.providers.salesforce.get_provider_info" ]; + imports = [ + "airflow.providers.salesforce" + "airflow.providers.salesforce.get_provider_info" + ]; version = "5.11.3"; }; samba = { deps = [ "smbprotocol" ]; - imports = [ "airflow.providers.samba.get_provider_info" ]; + imports = [ + "airflow.providers.samba" + "airflow.providers.samba.get_provider_info" + ]; version = "4.11.0"; }; segment = { deps = [ ]; - imports = [ "airflow.providers.segment.get_provider_info" ]; + imports = [ + "airflow.providers.segment" + "airflow.providers.segment.get_provider_info" + ]; version = "3.8.2"; }; @@ -681,7 +919,10 @@ "sendgrid" "werkzeug" ]; - imports = [ "airflow.providers.sendgrid.get_provider_info" ]; + imports = [ + "airflow.providers.sendgrid" + "airflow.providers.sendgrid.get_provider_info" + ]; version = "4.1.3"; }; @@ -691,25 +932,37 @@ "paramiko" "pysftp" ]; - imports = [ "airflow.providers.sftp.get_provider_info" ]; + imports = [ + "airflow.providers.sftp" + "airflow.providers.sftp.get_provider_info" + ]; version = "5.4.0"; }; singularity = { deps = [ "spython" ]; - imports = [ "airflow.providers.singularity.get_provider_info" ]; + imports = [ + "airflow.providers.singularity" + "airflow.providers.singularity.get_provider_info" + ]; version = "3.8.2"; }; slack = { deps = [ "slack-sdk" ]; - imports = [ "airflow.providers.slack.get_provider_info" ]; + imports = [ + "airflow.providers.slack" + "airflow.providers.slack.get_provider_info" + ]; version = "9.2.0"; }; smtp = { deps = [ ]; - imports = [ "airflow.providers.smtp.get_provider_info" ]; + imports = [ + "airflow.providers.smtp" + "airflow.providers.smtp.get_provider_info" + ]; version = "2.3.1"; }; @@ -720,13 +973,19 @@ "snowflake-connector-python" "snowflake-sqlalchemy" ]; - imports = [ "airflow.providers.snowflake.get_provider_info" ]; + imports = [ + "airflow.providers.snowflake" + "airflow.providers.snowflake.get_provider_info" + ]; version = "6.5.3"; }; sqlite = { deps = [ ]; - imports = [ "airflow.providers.sqlite.get_provider_info" ]; + imports = [ + "airflow.providers.sqlite" + "airflow.providers.sqlite.get_provider_info" + ]; version = "4.1.2"; }; @@ -735,43 +994,64 @@ "paramiko" "sshtunnel" ]; - imports = [ "airflow.providers.ssh.get_provider_info" ]; + imports = [ + "airflow.providers.ssh" + "airflow.providers.ssh.get_provider_info" + ]; version = "4.1.3"; }; standard = { deps = [ ]; - imports = [ "airflow.providers.standard.get_provider_info" ]; + imports = [ + "airflow.providers.standard" + "airflow.providers.standard.get_provider_info" + ]; version = "1.9.0"; }; tableau = { deps = [ "tableauserverclient" ]; - imports = [ "airflow.providers.tableau.get_provider_info" ]; + imports = [ + "airflow.providers.tableau" + "airflow.providers.tableau.get_provider_info" + ]; version = "5.2.0"; }; telegram = { deps = [ "python-telegram-bot" ]; - imports = [ "airflow.providers.telegram.get_provider_info" ]; + imports = [ + "airflow.providers.telegram" + "airflow.providers.telegram.get_provider_info" + ]; version = "4.8.2"; }; teradata = { deps = [ ]; - imports = [ "airflow.providers.teradata.get_provider_info" ]; + imports = [ + "airflow.providers.teradata" + "airflow.providers.teradata.get_provider_info" + ]; version = "3.2.1"; }; trino = { deps = [ "pandas" ]; - imports = [ "airflow.providers.trino.get_provider_info" ]; + imports = [ + "airflow.providers.trino" + "airflow.providers.trino.get_provider_info" + ]; version = "6.3.3"; }; vertica = { deps = [ "vertica-python" ]; - imports = [ "airflow.providers.vertica.get_provider_info" ]; + imports = [ + "airflow.providers.vertica" + "airflow.providers.vertica.get_provider_info" + ]; version = "4.1.2"; }; @@ -781,25 +1061,37 @@ "pandas" "weaviate-client" ]; - imports = [ "airflow.providers.weaviate.get_provider_info" ]; + imports = [ + "airflow.providers.weaviate" + "airflow.providers.weaviate.get_provider_info" + ]; version = "3.2.3"; }; yandex = { deps = [ ]; - imports = [ "airflow.providers.yandex.get_provider_info" ]; + imports = [ + "airflow.providers.yandex" + "airflow.providers.yandex.get_provider_info" + ]; version = "4.2.0"; }; ydb = { deps = [ ]; - imports = [ "airflow.providers.ydb.get_provider_info" ]; + imports = [ + "airflow.providers.ydb" + "airflow.providers.ydb.get_provider_info" + ]; version = "2.2.2"; }; zendesk = { deps = [ ]; - imports = [ "airflow.providers.zendesk.get_provider_info" ]; + imports = [ + "airflow.providers.zendesk" + "airflow.providers.zendesk.get_provider_info" + ]; version = "4.10.2"; }; diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index f5a26cd82234..d08ed53b8079 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -88,13 +88,13 @@ enabledProviders, }: let - version = "3.1.5"; + version = "3.1.6"; src = fetchFromGitHub { owner = "apache"; repo = "airflow"; tag = version; - hash = "sha256-C6QujoXUHN3xLHjLhjgcecdT1VQQMQ9v2f3x97i0NpM="; + hash = "sha256-wC6C0jhCA76/+KhBQbe3WeSGqR6FwaudCT5xPV39Z6c="; }; airflowUi = stdenv.mkDerivation rec { @@ -112,7 +112,7 @@ let pname = "airflow-ui"; inherit sourceRoot src version; fetcherVersion = 1; - hash = "sha256-oB6fbQ9FwBJNkbXVN1ovMYvegbmyG1xDSYBYqHKKh+M="; + hash = "sha256-UcEFQkDZ9Ye+VfyJ9rdZKe0wilTgO4dMsULABWfL2Co="; }; buildPhase = '' @@ -141,7 +141,7 @@ let pname = "simple-auth-manager-ui"; inherit sourceRoot src version; fetcherVersion = 1; - hash = "sha256-rUmrDHdC/Pqpe9njAM7cmoBLuPSgh36yLCT9j8ImN58="; + hash = "sha256-8nZdWnhERUkiaY8USyy/a/j+dMksjmEzCabSkysndSE="; }; buildPhase = '' @@ -382,6 +382,8 @@ buildPythonPackage rec { ++ lib.concatMap (provider: providers.${provider}.imports) (requiredProviders ++ enabledProviders); passthru.updateScript = ./update.sh; + passthru.airflowUi = airflowUi; + passthru.airflowSimpleAuthUi = airflowSimpleAuthUi; # Note on testing the web UI: # You can (manually) test the web UI as follows: From 988222f90720565a534f9a708947f28d63e075e3 Mon Sep 17 00:00:00 2001 From: Sergei Volkov Date: Thu, 22 Jan 2026 14:34:04 +0100 Subject: [PATCH 16/16] apache-airflow: improve update scripts - add nixfmt-tree call to format generated providers.nix - fix update-providers script to include correct imports --- pkgs/by-name/ap/apache-airflow/update-providers.py | 1 + pkgs/by-name/ap/apache-airflow/update.sh | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgs/by-name/ap/apache-airflow/update-providers.py b/pkgs/by-name/ap/apache-airflow/update-providers.py index 74fc728f1a60..01ee2bd49fe0 100755 --- a/pkgs/by-name/ap/apache-airflow/update-providers.py +++ b/pkgs/by-name/ap/apache-airflow/update-providers.py @@ -170,6 +170,7 @@ def parse_pyproject_toml(version: str, provider_name: str) -> Dict: # Heuristic to generate imports based on provider name # This might not be exhaustive but covers common cases base_import_path = f"airflow.providers.{provider_name.replace('-', '.')}" + imports.append(base_import_path) # Try to get more specific imports from provider_info entry point if available provider_info_entry_point = ( diff --git a/pkgs/by-name/ap/apache-airflow/update.sh b/pkgs/by-name/ap/apache-airflow/update.sh index cf957d3ba778..943f73dcc6bf 100755 --- a/pkgs/by-name/ap/apache-airflow/update.sh +++ b/pkgs/by-name/ap/apache-airflow/update.sh @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p common-updater-scripts curl jq +#!nix-shell -i bash -p common-updater-scripts curl jq nixfmt-tree set -euo pipefail @@ -46,6 +46,8 @@ update-source-version "$PACKAGE_NAME" "$LATEST_TAG" # It will automatically pick up the new version from the nix file. echo "Updating provider dependencies..." "$PACKAGE_DIR/update-providers.py" +echo "Formatting generated providers.nix" +treefmt "$PACKAGE_DIR" echo "Update complete. Please check the git diff for changes."