From d3fbf4a5097feb7b94eda1775aeb6a64cba4d9a2 Mon Sep 17 00:00:00 2001 From: Sergey Volkov Date: Sun, 6 Apr 2025 13:33:01 +0200 Subject: [PATCH 1/6] apache-airflow: add taranarmo to maintainers --- pkgs/by-name/ap/apache-airflow/python-package.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index a8a8fd9ecaac..4423b17bc54f 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -340,6 +340,7 @@ buildPythonPackage rec { maintainers = with lib.maintainers; [ gbpdt ingenieroariel + taranarmo ]; knownVulnerabilities = [ "CVE-2023-50943" From 1ab93d0cc5e22f3800cb8df27bf03efc7d02ca83 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 6 Jan 2026 16:30:36 +0100 Subject: [PATCH 2/6] apache-airflow: cleanup Remove some unused arguments and use tag instead of rev in fetchers. --- pkgs/by-name/ap/apache-airflow/package.nix | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkgs/by-name/ap/apache-airflow/package.nix b/pkgs/by-name/ap/apache-airflow/package.nix index 5ed986239f0e..c7eec6a7b68a 100644 --- a/pkgs/by-name/ap/apache-airflow/package.nix +++ b/pkgs/by-name/ap/apache-airflow/package.nix @@ -8,12 +8,12 @@ let python = python3.override { self = python; packageOverrides = pySelf: pySuper: { - connexion = pySuper.connexion.overridePythonAttrs (o: rec { + connexion = pySuper.connexion.overridePythonAttrs rec { version = "2.14.2"; src = fetchFromGitHub { owner = "spec-first"; repo = "connexion"; - rev = "refs/tags/${version}"; + tag = version; hash = "sha256-1v1xCHY3ZnZG/Vu9wN/it7rLKC/StoDefoMNs+hMjIs="; }; nativeBuildInputs = with pySelf; [ @@ -62,19 +62,19 @@ let }); # flask-appbuilder doesn't work with sqlalchemy 2.x, flask-appbuilder 3.x # https://github.com/dpgaspar/Flask-AppBuilder/issues/2038 - flask-appbuilder = pySuper.flask-appbuilder.overridePythonAttrs (o: { + flask-appbuilder = pySuper.flask-appbuilder.overridePythonAttrs { meta.broken = false; - }); + }; # a knock-on effect from overriding the sqlalchemy version - flask-sqlalchemy = pySuper.flask-sqlalchemy.overridePythonAttrs (o: { + flask-sqlalchemy = pySuper.flask-sqlalchemy.overridePythonAttrs { src = fetchPypi { pname = "Flask-SQLAlchemy"; version = "2.5.1"; hash = "sha256-K9pEtD58rLFdTgX/PMH4vJeTbMRkYjQkECv8LDXpWRI="; }; format = "setuptools"; - }); - httpcore = pySuper.httpcore.overridePythonAttrs (o: { + }; + httpcore = pySuper.httpcore.overridePythonAttrs { # nullify upstream's pytest flags which cause # "TLS/SSL connection has been closed (EOF)" # with pytest-httpbin 1.x @@ -82,16 +82,16 @@ let substituteInPlace pyproject.toml \ --replace '[tool.pytest.ini_options]' '[tool.notpytest.ini_options]' ''; - }); - pytest-httpbin = pySuper.pytest-httpbin.overridePythonAttrs (o: rec { + }; + pytest-httpbin = pySuper.pytest-httpbin.overridePythonAttrs rec { version = "1.0.2"; src = fetchFromGitHub { owner = "kevin1024"; repo = "pytest-httpbin"; - rev = "refs/tags/v${version}"; + tag = "v${version}"; hash = "sha256-S4ThQx4H3UlKhunJo35esPClZiEn7gX/Qwo4kE1QMTI="; }; - }); + }; # apache-airflow doesn't work with sqlalchemy 2.x # https://github.com/apache/airflow/issues/28723 sqlalchemy = pySuper.sqlalchemy_1_4; From 424f6d30551ede3fe748a5c309ac8e4efd203b17 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 6 Jan 2026 17:16:31 +0100 Subject: [PATCH 3/6] apache-airflow: sort arguments and dependencies --- .../ap/apache-airflow/python-package.nix | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index 4423b17bc54f..a5a003f05d05 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -1,14 +1,23 @@ { lib, stdenv, - python, - buildPythonPackage, fetchFromGitHub, + writeScript, + + # javascript + fetchYarnDeps, + nodejs, + webpack-cli, + yarnBuildHook, + yarnConfigHook, + + # python alembic, argcomplete, asgiref, attrs, blinker, + buildPythonPackage, cached-property, cattrs, clickclick, @@ -21,20 +30,21 @@ deprecated, dill, flask, - flask-login, flask-appbuilder, flask-caching, + flask-login, flask-session, flask-wtf, + freezegun, gitpython, google-re2, graphviz, gunicorn, httpx, - iso8601, - importlib-resources, importlib-metadata, + importlib-resources, inflection, + iso8601, itsdangerous, jinja2, jsonschema, @@ -56,6 +66,9 @@ pydantic, pygments, pyjwt, + pytest-asyncio, + pytestCheckHook, + python, python-daemon, python-dateutil, python-nvd3, @@ -72,19 +85,10 @@ tabulate, tenacity, termcolor, + time-machine, typing-extensions, unicodecsv, werkzeug, - freezegun, - pytest-asyncio, - pytestCheckHook, - time-machine, - yarnConfigHook, - yarnBuildHook, - nodejs, - webpack-cli, - fetchYarnDeps, - writeScript, # Extra airflow providers to enable enabledProviders ? [ ], @@ -174,17 +178,17 @@ buildPythonPackage rec { flask flask-appbuilder flask-caching + flask-login flask-session flask-wtf - flask-login gitpython google-re2 graphviz gunicorn httpx - iso8601 importlib-resources inflection + iso8601 itsdangerous jinja2 jsonschema From 17abb21855e6767956fb641284903933c198d8a6 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 6 Jan 2026 16:31:01 +0100 Subject: [PATCH 4/6] apache-airflow: fix build of dependencies Some dependencies have advanced and now need to be pinned. apache-airflow itself does not build, yet, because it needs pendulum 2.x, which requires python < 3.12, I believe. Co-authored-by: Sergey Volkov --- pkgs/by-name/ap/apache-airflow/package.nix | 75 +++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/pkgs/by-name/ap/apache-airflow/package.nix b/pkgs/by-name/ap/apache-airflow/package.nix index c7eec6a7b68a..712235851693 100644 --- a/pkgs/by-name/ap/apache-airflow/package.nix +++ b/pkgs/by-name/ap/apache-airflow/package.nix @@ -1,5 +1,6 @@ { fetchFromGitHub, + fetchpatch, fetchPypi, python3, }: @@ -40,15 +41,40 @@ let aiohttp-remotes decorator pytest-aiohttp + pytest-tornasync pytestCheckHook testfixtures ]; disabledTests = [ + "test_aiohttp_simple_api" "test_app" + "test_invalid_type" # https://github.com/spec-first/connexion/issues/1969 "test_openapi_yaml_behind_proxy" + "test_run_with_wsgi_containers" "test_swagger_ui" ]; - }); + }; + werkzeug = pySuper.werkzeug.overridePythonAttrs rec { + version = "2.3.8"; + src = fetchPypi { + pname = "werkzeug"; + inherit version; + hash = "sha256-VUslfHS763oNJUFgpPj/4YUkP1KlIDUGC3Ycpi2XfwM="; + }; + nativeCheckInputs = with pySelf; [ + pytest-xprocess + ]; + }; + # flask's test-suite needs click 8.1.8 + # TypeError: CliRunner.__init__() got an unexpected keyword argument 'mix_stderr' + click = pySuper.click.overridePythonAttrs rec { + version = "8.1.8"; + src = fetchPypi { + pname = "click"; + inherit version; + hash = "sha256-7VPJ2JkNg8Kifermjk7jN0c/YzDAQKMdQiXJV00WCWo="; + }; + }; flask = pySuper.flask.overridePythonAttrs (o: rec { version = "2.2.5"; src = fetchPypi { @@ -59,7 +85,30 @@ let nativeBuildInputs = (o.nativeBuildInputs or [ ]) ++ [ pySelf.setuptools ]; + pytestFlagsArray = [ + # tests that are marked with filterwarnings fail with + # DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for + # removal in Python 3.14; use importlib.util.find_spec() instead + "-W ignore::DeprecationWarning" + ]; }); + flask-login = pySuper.flask-login.overridePythonAttrs rec { + version = "0.6.3"; + src = fetchFromGitHub { + owner = "maxcountryman"; + repo = "flask-login"; + tag = version; + hash = "sha256-Sn7Ond67P/3+OmKKFE/KfA6FE4IajhiRXVVrXKJtY3I="; + }; + nativeBuildInputs = with pySelf; [ setuptools ]; + pytestFlagsArray = [ + # DeprecationWarning: datetime.datetime.utcnow() is deprecated + # and scheduled for removal in a future version. + # Use timezone-aware objects to represent datetimes in UTC: + # datetime.datetime.now(datetime.UTC). + "-W ignore::DeprecationWarning" + ]; + }; # flask-appbuilder doesn't work with sqlalchemy 2.x, flask-appbuilder 3.x # https://github.com/dpgaspar/Flask-AppBuilder/issues/2038 flask-appbuilder = pySuper.flask-appbuilder.overridePythonAttrs { @@ -91,6 +140,30 @@ let tag = "v${version}"; hash = "sha256-S4ThQx4H3UlKhunJo35esPClZiEn7gX/Qwo4kE1QMTI="; }; + # Use unmerged PR #65 to fix older version: + # https://github.com/kevin1024/pytest-httpbin/pull/65/ + # It was closed in favour of another which isn't compatible with the overriden version. + patches = [ + (fetchpatch { + url = "https://github.com/kevin1024/pytest-httpbin/commit/4e325f877ff8f77dec9f380bd8e53bb42976775c.patch"; + hash = "sha256-a33XcdMupD+7ZzvUibePdldGImmPLDNU2sxRbwpveDA="; + }) + (fetchpatch { + url = "https://github.com/kevin1024/pytest-httpbin/commit/463afb9b200563ac6fe7ae535f7a7a3c818b0418.patch"; + hash = "sha256-HFmuLtAtEjnB6heSG1YNnqxtz2phXNkHbQaZyB5bLJs="; + }) + ]; + disabledTests = [ + "test_httpbin_secure_accepts_get_requests" + "test_httpbin_secure_accepts_lots_of_get_requests" + "test_httpbin_both[https]" + "test_chunked_encoding[https]" + "TestClassBassedTests::test_http_secure" + "test_dont_crash_on_certificate_problems" + "test_redirect_location_is_https_for_secure_server" + "test_httpbin_secure_accepts_get_requests" + "test_http_secure" + ]; }; # apache-airflow doesn't work with sqlalchemy 2.x # https://github.com/apache/airflow/issues/28723 From 1a277d101587269be76ad32772574a3081d1f45e Mon Sep 17 00:00:00 2001 From: Sergey Volkov Date: Sun, 6 Apr 2025 13:31:51 +0200 Subject: [PATCH 5/6] apache-airflow: 2.7.3 -> 2.10.5 --- pkgs/by-name/ap/apache-airflow/package.nix | 105 +- pkgs/by-name/ap/apache-airflow/providers.nix | 983 ++++++++++++++++-- .../ap/apache-airflow/python-package.nix | 231 ++-- .../ap/apache-airflow/update-providers.py | 4 +- 4 files changed, 1092 insertions(+), 231 deletions(-) diff --git a/pkgs/by-name/ap/apache-airflow/package.nix b/pkgs/by-name/ap/apache-airflow/package.nix index 712235851693..ba341a97bdf5 100644 --- a/pkgs/by-name/ap/apache-airflow/package.nix +++ b/pkgs/by-name/ap/apache-airflow/package.nix @@ -53,6 +53,10 @@ let "test_run_with_wsgi_containers" "test_swagger_ui" ]; + postPatch = '' + substituteInPlace connexion/__init__.py \ + --replace "2020.0.dev1" "${version}" + ''; }; werkzeug = pySuper.werkzeug.overridePythonAttrs rec { version = "2.3.8"; @@ -109,6 +113,28 @@ let "-W ignore::DeprecationWarning" ]; }; + flask-session = pySuper.flask-session.overridePythonAttrs rec { + version = "0.5.0"; + src = fetchFromGitHub { + owner = "palletc-eco"; + repo = "flask-session"; + tag = version; + hash = "sha256-t8w6ZS4gBDpnnKvL3DLtn+rRLQNJbrT2Hxm4f3+a3Xc="; + }; + nativeCheckInputs = with pySelf; [ pytestCheckHook ]; + pytestFlagsArray = [ + "-k" + "'null_session or filesystem_session'" + ]; + dependencies = with pySelf; [ + flask-sqlalchemy + cachelib + ]; + disabledTests = [ ]; + disabledTestPaths = [ ]; + preCheck = ""; + postCheck = ""; + }; # flask-appbuilder doesn't work with sqlalchemy 2.x, flask-appbuilder 3.x # https://github.com/dpgaspar/Flask-AppBuilder/issues/2038 flask-appbuilder = pySuper.flask-appbuilder.overridePythonAttrs { @@ -168,7 +194,84 @@ let # apache-airflow doesn't work with sqlalchemy 2.x # https://github.com/apache/airflow/issues/28723 sqlalchemy = pySuper.sqlalchemy_1_4; - + gitpython = pySuper.gitpython.overridePythonAttrs rec { + version = "3.1.44"; + src = fetchFromGitHub { + owner = "gitpython-developers"; + repo = "gitpython"; + rev = version; + hash = "sha256-KnKaBv/tKk4wiGWUWCEgd1vgrTouwUhqxJ1/nMjRaWk="; + }; + }; + # ValueError: Unknown classifier in field `project.classifiers`: Programming Language :: Python :: Free Threading :: 2 - Beta + urllib3 = pySuper.urllib3.overridePythonAttrs rec { + version = "2.5.0"; + src = fetchPypi { + pname = "urllib3"; + inherit version; + hash = "sha256-P8R3M8fkGdS8P2s9wrT4kLt0OQajDVa6Slv6S7/5J2A="; + }; + postPatch = '' + substituteInPlace pyproject.toml \ + --replace-fail ', "setuptools-scm>=8,<9"' "" + ''; + }; + smmap = pySuper.smmap.overridePythonAttrs rec { + version = "5.0.2"; + src = fetchFromGitHub { + owner = "gitpython-developers"; + repo = "smmap"; + rev = "refs/tags/v${version}"; + hash = "sha256-0Y175kjv/8UJpSxtLpWH4/VT7JrcVPAq79Nf3rtHZZM="; + }; + }; + trove-classifiers = pySuper.trove-classifiers.overridePythonAttrs rec { + version = "2024.10.21.16"; + src = fetchPypi { + pname = "trove_classifiers"; + inherit version; + hash = "sha256-F8vQVdZ9Xp2d5jKTqHMpQ/q8IVdOTHt07fEStJKM9fM="; + }; + postPatch = ""; + }; + packaging = pySuper.packaging.overridePythonAttrs rec { + version = "24.2"; + src = fetchPypi { + pname = "packaging"; + inherit version; + hash = "sha256-wiim3F6TLTRrxXOTeRCdSeiFPdgiNXHHxbVSYO3AuX8="; + }; + }; + pluggy = pySuper.pluggy.overridePythonAttrs rec { + version = "1.5.0"; + src = fetchFromGitHub { + owner = "pytest-dev"; + repo = "pluggy"; + tag = version; + hash = "sha256-f0DxyZZk6RoYtOEXLACcsOn2B+Hot4U4g5Ogr/hKmOE="; + }; + }; + pyproject-api = pySuper.pyproject-api.overridePythonAttrs rec { + version = "1.8.0"; + src = fetchPypi { + pname = "pyproject_api"; + inherit version; + hash = "sha256-d7gEny/rXTPu/MIbV/HieWNid6isita1hxA3skN3hJY="; + }; + disabledTests = [ + # AssertionError: assert ['magic>3', 'requests>2'] == ['magic >3', 'requests >2'] + "test_frontend_setuptools" + ]; + }; + tox = pySuper.tox.overridePythonAttrs rec { + version = "4.27.0"; + src = fetchFromGitHub { + owner = "tox-dev"; + repo = "tox"; + tag = version; + hash = "sha256-Z3qUK4w1ebPvdZD4ZuKgZXJPUu5lG0G41vn/pc9gC/0="; + }; + }; apache-airflow = pySelf.callPackage ./python-package.nix { }; }; }; diff --git a/pkgs/by-name/ap/apache-airflow/providers.nix b/pkgs/by-name/ap/apache-airflow/providers.nix index 4edb68ac43f4..d2da8c6a4c65 100644 --- a/pkgs/by-name/ap/apache-airflow/providers.nix +++ b/pkgs/by-name/ap/apache-airflow/providers.nix @@ -2,6 +2,8 @@ { airbyte = { deps = [ + "aiohttp" + "asgiref" "requests" "requests-toolbelt" ]; @@ -13,38 +15,54 @@ alibaba = { deps = [ "oss2" ]; imports = [ + "airflow.providers.alibaba.cloud.hooks.analyticdb_spark" "airflow.providers.alibaba.cloud.hooks.oss" + "airflow.providers.alibaba.cloud.operators.analyticdb_spark" "airflow.providers.alibaba.cloud.operators.oss" ]; }; amazon = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -67,54 +85,74 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; imports = [ "airflow.providers.amazon.aws.hooks.appflow" "airflow.providers.amazon.aws.hooks.athena" + "airflow.providers.amazon.aws.hooks.athena_sql" "airflow.providers.amazon.aws.hooks.base_aws" "airflow.providers.amazon.aws.hooks.batch_client" "airflow.providers.amazon.aws.hooks.batch_waiters" + "airflow.providers.amazon.aws.hooks.bedrock" + "airflow.providers.amazon.aws.hooks.chime" "airflow.providers.amazon.aws.hooks.cloud_formation" + "airflow.providers.amazon.aws.hooks.comprehend" "airflow.providers.amazon.aws.hooks.datasync" "airflow.providers.amazon.aws.hooks.dms" "airflow.providers.amazon.aws.hooks.dynamodb" "airflow.providers.amazon.aws.hooks.ec2" + "airflow.providers.amazon.aws.hooks.ecr" "airflow.providers.amazon.aws.hooks.ecs" "airflow.providers.amazon.aws.hooks.eks" "airflow.providers.amazon.aws.hooks.elasticache_replication_group" "airflow.providers.amazon.aws.hooks.emr" "airflow.providers.amazon.aws.hooks.emr" + "airflow.providers.amazon.aws.hooks.eventbridge" "airflow.providers.amazon.aws.hooks.glacier" "airflow.providers.amazon.aws.hooks.glue" "airflow.providers.amazon.aws.hooks.glue_catalog" "airflow.providers.amazon.aws.hooks.glue_crawler" + "airflow.providers.amazon.aws.hooks.glue_databrew" "airflow.providers.amazon.aws.hooks.kinesis" + "airflow.providers.amazon.aws.hooks.kinesis_analytics" "airflow.providers.amazon.aws.hooks.lambda_function" "airflow.providers.amazon.aws.hooks.logs" + "airflow.providers.amazon.aws.hooks.neptune" + "airflow.providers.amazon.aws.hooks.opensearch_serverless" "airflow.providers.amazon.aws.hooks.quicksight" "airflow.providers.amazon.aws.hooks.rds" "airflow.providers.amazon.aws.hooks.redshift_cluster" @@ -126,13 +164,17 @@ "airflow.providers.amazon.aws.hooks.ses" "airflow.providers.amazon.aws.hooks.sns" "airflow.providers.amazon.aws.hooks.sqs" + "airflow.providers.amazon.aws.hooks.ssm" "airflow.providers.amazon.aws.hooks.step_function" "airflow.providers.amazon.aws.hooks.sts" + "airflow.providers.amazon.aws.hooks.verified_permissions" "airflow.providers.amazon.aws.operators.appflow" "airflow.providers.amazon.aws.operators.athena" - "airflow.providers.amazon.aws.operators.aws_lambda" + "airflow.providers.amazon.aws.operators.base_aws" "airflow.providers.amazon.aws.operators.batch" + "airflow.providers.amazon.aws.operators.bedrock" "airflow.providers.amazon.aws.operators.cloud_formation" + "airflow.providers.amazon.aws.operators.comprehend" "airflow.providers.amazon.aws.operators.datasync" "airflow.providers.amazon.aws.operators.dms" "airflow.providers.amazon.aws.operators.ec2" @@ -140,15 +182,18 @@ "airflow.providers.amazon.aws.operators.eks" "airflow.providers.amazon.aws.operators.emr" "airflow.providers.amazon.aws.operators.emr" + "airflow.providers.amazon.aws.operators.eventbridge" "airflow.providers.amazon.aws.operators.glacier" "airflow.providers.amazon.aws.operators.glue" "airflow.providers.amazon.aws.operators.glue_crawler" + "airflow.providers.amazon.aws.operators.glue_databrew" + "airflow.providers.amazon.aws.operators.kinesis_analytics" "airflow.providers.amazon.aws.operators.lambda_function" + "airflow.providers.amazon.aws.operators.neptune" "airflow.providers.amazon.aws.operators.quicksight" "airflow.providers.amazon.aws.operators.rds" "airflow.providers.amazon.aws.operators.redshift_cluster" "airflow.providers.amazon.aws.operators.redshift_data" - "airflow.providers.amazon.aws.operators.redshift_sql" "airflow.providers.amazon.aws.operators.s3" "airflow.providers.amazon.aws.operators.sagemaker" "airflow.providers.amazon.aws.operators.sns" @@ -159,31 +204,45 @@ apache_beam = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -206,28 +265,38 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; @@ -250,31 +319,45 @@ apache_druid = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -297,29 +380,39 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pydruid" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; @@ -329,41 +422,63 @@ "airflow.providers.apache.druid.operators.druid_check" ]; }; - apache_hdfs = { - deps = [ ]; - imports = [ - "airflow.providers.apache.hdfs.hooks.hdfs" - "airflow.providers.apache.hdfs.hooks.webhdfs" + apache_flink = { + deps = [ + "aiofiles" + "asgiref" + "cryptography" + "google-re2" + "kubernetes" + "kubernetes-asyncio" ]; + imports = [ "airflow.providers.apache.flink.operators.flink_kubernetes" ]; + }; + apache_hdfs = { + deps = [ "pandas" ]; + imports = [ "airflow.providers.apache.hdfs.hooks.webhdfs" ]; }; apache_hive = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -386,28 +501,38 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; @@ -417,6 +542,28 @@ "airflow.providers.apache.hive.operators.hive_stats" ]; }; + apache_iceberg = { + deps = [ ]; + imports = [ "airflow.providers.apache.iceberg.hooks.iceberg" ]; + }; + apache_impala = { + deps = [ ]; + imports = [ "airflow.providers.apache.impala.hooks.impala" ]; + }; + apache_kafka = { + deps = [ + "asgiref" + "confluent-kafka" + ]; + imports = [ + "airflow.providers.apache.kafka.hooks.base" + "airflow.providers.apache.kafka.hooks.client" + "airflow.providers.apache.kafka.hooks.consume" + "airflow.providers.apache.kafka.hooks.produce" + "airflow.providers.apache.kafka.operators.consume" + "airflow.providers.apache.kafka.operators.produce" + ]; + }; apache_kylin = { deps = [ ]; imports = [ @@ -426,6 +573,8 @@ }; apache_livy = { deps = [ + "aiohttp" + "asgiref" "requests" "requests-toolbelt" ]; @@ -446,8 +595,18 @@ imports = [ "airflow.providers.apache.pinot.hooks.pinot" ]; }; apache_spark = { - deps = [ "pyspark" ]; + deps = [ + "aiofiles" + "asgiref" + "cryptography" + "google-re2" + "grpcio-status" + "kubernetes" + "kubernetes-asyncio" + "pyspark" + ]; imports = [ + "airflow.providers.apache.spark.hooks.spark_connect" "airflow.providers.apache.spark.hooks.spark_jdbc" "airflow.providers.apache.spark.hooks.spark_jdbc_script" "airflow.providers.apache.spark.hooks.spark_sql" @@ -457,12 +616,9 @@ "airflow.providers.apache.spark.operators.spark_submit" ]; }; - apache_sqoop = { - deps = [ ]; - imports = [ - "airflow.providers.apache.sqoop.hooks.sqoop" - "airflow.providers.apache.sqoop.operators.sqoop" - ]; + apprise = { + deps = [ "apprise" ]; + imports = [ "airflow.providers.apprise.hooks.apprise" ]; }; arangodb = { deps = [ "python-arango" ]; @@ -479,7 +635,7 @@ ]; }; atlassian_jira = { - deps = [ "jira" ]; + deps = [ "atlassian-python-api" ]; imports = [ "airflow.providers.atlassian.jira.hooks.jira" "airflow.providers.atlassian.jira.operators.jira" @@ -487,8 +643,13 @@ }; celery = { deps = [ - "celery" + "aiofiles" + "asgiref" + "cryptography" "flower" + "google-re2" + "kubernetes" + "kubernetes-asyncio" ]; imports = [ ]; }; @@ -498,17 +659,43 @@ }; cncf_kubernetes = { deps = [ + "aiofiles" + "asgiref" "cryptography" + "google-re2" "kubernetes" + "kubernetes-asyncio" ]; imports = [ "airflow.providers.cncf.kubernetes.hooks.kubernetes" - "airflow.providers.cncf.kubernetes.operators.kubernetes_pod" + "airflow.providers.cncf.kubernetes.operators.custom_object_launcher" + "airflow.providers.cncf.kubernetes.operators.job" + "airflow.providers.cncf.kubernetes.operators.pod" + "airflow.providers.cncf.kubernetes.operators.resource" "airflow.providers.cncf.kubernetes.operators.spark_kubernetes" ]; }; + cohere = { + deps = [ "cohere" ]; + imports = [ + "airflow.providers.cohere.hooks.cohere" + "airflow.providers.cohere.operators.embedding" + ]; + }; + common_compat = { + deps = [ ]; + imports = [ ]; + }; + common_io = { + deps = [ "attrs" ]; + imports = [ "airflow.providers.common.io.operators.file_transfer" ]; + }; common_sql = { - deps = [ "sqlparse" ]; + deps = [ + "attrs" + "more-itertools" + "sqlparse" + ]; imports = [ "airflow.providers.common.sql.hooks.sql" "airflow.providers.common.sql.operators.sql" @@ -518,6 +705,9 @@ deps = [ "aiohttp" "databricks-sql-connector" + "mergedeep" + "pandas" + "pyarrow" "requests" ]; imports = [ @@ -527,6 +717,7 @@ "airflow.providers.databricks.operators.databricks" "airflow.providers.databricks.operators.databricks_repos" "airflow.providers.databricks.operators.databricks_sql" + "airflow.providers.databricks.operators.databricks_workflow" ]; }; datadog = { @@ -535,6 +726,9 @@ }; dbt_cloud = { deps = [ + "aiohttp" + "asgiref" + "attrs" "requests" "requests-toolbelt" ]; @@ -545,6 +739,8 @@ }; dingding = { deps = [ + "aiohttp" + "asgiref" "requests" "requests-toolbelt" ]; @@ -555,6 +751,8 @@ }; discord = { deps = [ + "aiohttp" + "asgiref" "requests" "requests-toolbelt" ]; @@ -575,10 +773,7 @@ ]; }; elasticsearch = { - deps = [ - "elasticsearch" - "elasticsearch-dsl" - ]; + deps = [ "elasticsearch" ]; imports = [ "airflow.providers.elasticsearch.hooks.elasticsearch" ]; }; exasol = { @@ -588,13 +783,26 @@ "airflow.providers.exasol.operators.exasol" ]; }; + fab = { + deps = [ + "flask" + "flask-appbuilder" + "flask-login" + "google-re2" + "jmespath" + ]; + imports = [ ]; + }; facebook = { deps = [ ]; imports = [ "airflow.providers.facebook.ads.hooks.ads" ]; }; ftp = { - deps = [ ]; - imports = [ "airflow.providers.ftp.hooks.ftp" ]; + deps = [ "attrs" ]; + imports = [ + "airflow.providers.ftp.hooks.ftp" + "airflow.providers.ftp.operators.ftp" + ]; }; github = { deps = [ "pygithub" ]; @@ -606,31 +814,45 @@ google = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -653,28 +875,38 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; @@ -685,9 +917,11 @@ "airflow.providers.google.cloud.hooks.bigquery" "airflow.providers.google.cloud.hooks.bigquery_dts" "airflow.providers.google.cloud.hooks.bigtable" + "airflow.providers.google.cloud.hooks.cloud_batch" "airflow.providers.google.cloud.hooks.cloud_build" "airflow.providers.google.cloud.hooks.cloud_composer" "airflow.providers.google.cloud.hooks.cloud_memorystore" + "airflow.providers.google.cloud.hooks.cloud_run" "airflow.providers.google.cloud.hooks.cloud_sql" "airflow.providers.google.cloud.hooks.cloud_storage_transfer_service" "airflow.providers.google.cloud.hooks.compute" @@ -696,6 +930,7 @@ "airflow.providers.google.cloud.hooks.dataflow" "airflow.providers.google.cloud.hooks.dataform" "airflow.providers.google.cloud.hooks.datafusion" + "airflow.providers.google.cloud.hooks.datapipeline" "airflow.providers.google.cloud.hooks.dataplex" "airflow.providers.google.cloud.hooks.dataprep" "airflow.providers.google.cloud.hooks.dataproc" @@ -725,8 +960,11 @@ "airflow.providers.google.cloud.hooks.vertex_ai.custom_job" "airflow.providers.google.cloud.hooks.vertex_ai.dataset" "airflow.providers.google.cloud.hooks.vertex_ai.endpoint_service" + "airflow.providers.google.cloud.hooks.vertex_ai.generative_model" "airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job" "airflow.providers.google.cloud.hooks.vertex_ai.model_service" + "airflow.providers.google.cloud.hooks.vertex_ai.pipeline_job" + "airflow.providers.google.cloud.hooks.vertex_ai.prediction_service" "airflow.providers.google.cloud.hooks.video_intelligence" "airflow.providers.google.cloud.hooks.vision" "airflow.providers.google.cloud.hooks.workflows" @@ -734,9 +972,12 @@ "airflow.providers.google.cloud.operators.bigquery" "airflow.providers.google.cloud.operators.bigquery_dts" "airflow.providers.google.cloud.operators.bigtable" + "airflow.providers.google.cloud.operators.cloud_base" + "airflow.providers.google.cloud.operators.cloud_batch" "airflow.providers.google.cloud.operators.cloud_build" "airflow.providers.google.cloud.operators.cloud_composer" "airflow.providers.google.cloud.operators.cloud_memorystore" + "airflow.providers.google.cloud.operators.cloud_run" "airflow.providers.google.cloud.operators.cloud_sql" "airflow.providers.google.cloud.operators.cloud_storage_transfer_service" "airflow.providers.google.cloud.operators.compute" @@ -744,6 +985,7 @@ "airflow.providers.google.cloud.operators.dataflow" "airflow.providers.google.cloud.operators.dataform" "airflow.providers.google.cloud.operators.datafusion" + "airflow.providers.google.cloud.operators.datapipeline" "airflow.providers.google.cloud.operators.dataplex" "airflow.providers.google.cloud.operators.dataprep" "airflow.providers.google.cloud.operators.dataproc" @@ -771,8 +1013,10 @@ "airflow.providers.google.cloud.operators.vertex_ai.custom_job" "airflow.providers.google.cloud.operators.vertex_ai.dataset" "airflow.providers.google.cloud.operators.vertex_ai.endpoint_service" + "airflow.providers.google.cloud.operators.vertex_ai.generative_model" "airflow.providers.google.cloud.operators.vertex_ai.hyperparameter_tuning_job" "airflow.providers.google.cloud.operators.vertex_ai.model_service" + "airflow.providers.google.cloud.operators.vertex_ai.pipeline_job" "airflow.providers.google.cloud.operators.video_intelligence" "airflow.providers.google.cloud.operators.vision" "airflow.providers.google.cloud.operators.workflows" @@ -783,10 +1027,12 @@ "airflow.providers.google.leveldb.hooks.leveldb" "airflow.providers.google.leveldb.operators.leveldb" "airflow.providers.google.marketing_platform.hooks.analytics" + "airflow.providers.google.marketing_platform.hooks.analytics_admin" "airflow.providers.google.marketing_platform.hooks.campaign_manager" "airflow.providers.google.marketing_platform.hooks.display_video" "airflow.providers.google.marketing_platform.hooks.search_ads" "airflow.providers.google.marketing_platform.operators.analytics" + "airflow.providers.google.marketing_platform.operators.analytics_admin" "airflow.providers.google.marketing_platform.operators.campaign_manager" "airflow.providers.google.marketing_platform.operators.display_video" "airflow.providers.google.marketing_platform.operators.search_ads" @@ -810,31 +1056,45 @@ hashicorp = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -857,29 +1117,39 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" "hvac" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; @@ -887,6 +1157,8 @@ }; http = { deps = [ + "aiohttp" + "asgiref" "requests" "requests-toolbelt" ]; @@ -926,31 +1198,45 @@ microsoft_azure = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -973,28 +1259,38 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; @@ -1009,7 +1305,9 @@ "airflow.providers.microsoft.azure.hooks.cosmos" "airflow.providers.microsoft.azure.hooks.data_factory" "airflow.providers.microsoft.azure.hooks.data_lake" + "airflow.providers.microsoft.azure.hooks.data_lake" "airflow.providers.microsoft.azure.hooks.fileshare" + "airflow.providers.microsoft.azure.hooks.msgraph" "airflow.providers.microsoft.azure.hooks.synapse" "airflow.providers.microsoft.azure.hooks.wasb" "airflow.providers.microsoft.azure.operators.adls" @@ -1019,12 +1317,13 @@ "airflow.providers.microsoft.azure.operators.container_instances" "airflow.providers.microsoft.azure.operators.cosmos" "airflow.providers.microsoft.azure.operators.data_factory" + "airflow.providers.microsoft.azure.operators.msgraph" "airflow.providers.microsoft.azure.operators.synapse" "airflow.providers.microsoft.azure.operators.wasb_delete_blob" ]; }; microsoft_mssql = { - deps = [ ]; + deps = [ "pymssql" ]; imports = [ "airflow.providers.microsoft.mssql.hooks.mssql" "airflow.providers.microsoft.mssql.operators.mssql" @@ -1054,31 +1353,45 @@ mysql = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -1101,28 +1414,38 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; @@ -1142,10 +1465,28 @@ deps = [ "pyodbc" ]; imports = [ "airflow.providers.odbc.hooks.odbc" ]; }; + openai = { + deps = [ ]; + imports = [ + "airflow.providers.openai.hooks.openai" + "airflow.providers.openai.operators.openai" + ]; + }; openfaas = { deps = [ ]; imports = [ "airflow.providers.openfaas.hooks.openfaas" ]; }; + openlineage = { + deps = [ "attrs" ]; + imports = [ ]; + }; + opensearch = { + deps = [ "opensearch-py" ]; + imports = [ + "airflow.providers.opensearch.hooks.opensearch" + "airflow.providers.opensearch.operators.opensearch" + ]; + }; opsgenie = { deps = [ ]; imports = [ @@ -1168,44 +1509,57 @@ ]; }; papermill = { - deps = [ ]; - imports = [ "airflow.providers.papermill.operators.papermill" ]; - }; - plexus = { - deps = [ "arrow" ]; + deps = [ + "ipykernel" + "pandas" + ]; imports = [ - "airflow.providers.plexus.hooks.plexus" - "airflow.providers.plexus.operators.job" + "airflow.providers.papermill.hooks.kernel" + "airflow.providers.papermill.operators.papermill" ]; }; - postgres = { + pgvector = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -1228,28 +1582,149 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" + "thrift" + "vertica-python" + ]; + imports = [ + "airflow.providers.pgvector.hooks.pgvector" + "airflow.providers.pgvector.operators.pgvector" + ]; + }; + pinecone = { + deps = [ "pinecone-client" ]; + imports = [ + "airflow.providers.pinecone.hooks.pinecone" + "airflow.providers.pinecone.operators.pinecone" + ]; + }; + postgres = { + deps = [ + "adal" + "adlfs" + "aiofiles" + "aiohttp" + "apache-beam" + "asgiref" + "asyncssh" + "attrs" + "azure-batch" + "azure-cosmos" + "azure-datalake-store" + "azure-identity" + "azure-keyvault-secrets" + "azure-kusto-data" + "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" + "azure-mgmt-datafactory" + "azure-mgmt-datalake-store" + "azure-mgmt-resource" + "azure-mgmt-storage" + "azure-servicebus" + "azure-storage-blob" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" + "azure-synapse-spark" + "boto3" + "botocore" + "cassandra-driver" + "cryptography" + "dill" + "dnspython" + "gcsfs" + "google-api-core" + "google-api-python-client" + "google-auth" + "google-auth-httplib2" + "google-cloud-automl" + "google-cloud-bigquery" + "google-cloud-bigquery-datatransfer" + "google-cloud-bigtable" + "google-cloud-compute" + "google-cloud-container" + "google-cloud-datacatalog" + "google-cloud-dataproc" + "google-cloud-dlp" + "google-cloud-kms" + "google-cloud-language" + "google-cloud-logging" + "google-cloud-monitoring" + "google-cloud-pubsub" + "google-cloud-redis" + "google-cloud-secret-manager" + "google-cloud-spanner" + "google-cloud-speech" + "google-cloud-storage" + "google-cloud-tasks" + "google-cloud-texttospeech" + "google-cloud-translate" + "google-cloud-videointelligence" + "google-cloud-vision" + "google-cloud-workflows" + "google-re2" + "grpcio-gcp" + "httpx" + "inflection" + "jmespath" + "json-merge-patch" + "jsonpath-ng" + "kubernetes" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" + "mysqlclient" + "oracledb" + "pandas" + "paramiko" + "proto-plus" + "psycopg2" + "pyarrow" + "pyathena" + "pymongo" + "pymssql" + "pyopenssl" + "pysftp" + "python-slugify" + "redshift-connector" + "requests" + "requests-toolbelt" + "simple-salesforce" + "smbprotocol" + "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; @@ -1261,31 +1736,45 @@ presto = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -1308,40 +1797,48 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; imports = [ "airflow.providers.presto.hooks.presto" ]; }; - qubole = { - deps = [ "qds-sdk" ]; + qdrant = { + deps = [ "qdrant-client" ]; imports = [ - "airflow.providers.qubole.hooks.qubole" - "airflow.providers.qubole.hooks.qubole_check" - "airflow.providers.qubole.operators.qubole" - "airflow.providers.qubole.operators.qubole_check" + "airflow.providers.qdrant.hooks.qdrant" + "airflow.providers.qdrant.operators.qdrant" ]; }; redis = { @@ -1363,7 +1860,104 @@ ]; }; samba = { - deps = [ "smbprotocol" ]; + deps = [ + "adal" + "adlfs" + "aiofiles" + "aiohttp" + "apache-beam" + "asgiref" + "asyncssh" + "attrs" + "azure-batch" + "azure-cosmos" + "azure-datalake-store" + "azure-identity" + "azure-keyvault-secrets" + "azure-kusto-data" + "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" + "azure-mgmt-datafactory" + "azure-mgmt-datalake-store" + "azure-mgmt-resource" + "azure-mgmt-storage" + "azure-servicebus" + "azure-storage-blob" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" + "azure-synapse-spark" + "boto3" + "botocore" + "cassandra-driver" + "cryptography" + "dill" + "dnspython" + "gcsfs" + "google-api-core" + "google-api-python-client" + "google-auth" + "google-auth-httplib2" + "google-cloud-automl" + "google-cloud-bigquery" + "google-cloud-bigquery-datatransfer" + "google-cloud-bigtable" + "google-cloud-compute" + "google-cloud-container" + "google-cloud-datacatalog" + "google-cloud-dataproc" + "google-cloud-dlp" + "google-cloud-kms" + "google-cloud-language" + "google-cloud-logging" + "google-cloud-monitoring" + "google-cloud-pubsub" + "google-cloud-redis" + "google-cloud-secret-manager" + "google-cloud-spanner" + "google-cloud-speech" + "google-cloud-storage" + "google-cloud-tasks" + "google-cloud-texttospeech" + "google-cloud-translate" + "google-cloud-videointelligence" + "google-cloud-vision" + "google-cloud-workflows" + "google-re2" + "grpcio-gcp" + "httpx" + "inflection" + "jmespath" + "json-merge-patch" + "jsonpath-ng" + "kubernetes" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" + "mysqlclient" + "oracledb" + "pandas" + "paramiko" + "proto-plus" + "psycopg2" + "pyarrow" + "pyathena" + "pymongo" + "pymssql" + "pyopenssl" + "pysftp" + "python-slugify" + "redshift-connector" + "requests" + "requests-toolbelt" + "simple-salesforce" + "smbprotocol" + "sshtunnel" + "tenacity" + "thrift" + "vertica-python" + ]; imports = [ "airflow.providers.samba.hooks.samba" ]; }; segment = { @@ -1379,6 +1973,8 @@ }; sftp = { deps = [ + "asyncssh" + "attrs" "paramiko" "pysftp" "sshtunnel" @@ -1389,7 +1985,7 @@ ]; }; singularity = { - deps = [ ]; + deps = [ "spython" ]; imports = [ "airflow.providers.singularity.operators.singularity" ]; }; slack = { @@ -1401,14 +1997,24 @@ "airflow.providers.slack.operators.slack_webhook" ]; }; + smtp = { + deps = [ ]; + imports = [ + "airflow.providers.smtp.hooks.smtp" + "airflow.providers.smtp.operators.smtp" + ]; + }; snowflake = { deps = [ - "slack-sdk" + "attrs" + "pandas" + "pyarrow" "snowflake-connector-python" "snowflake-sqlalchemy" ]; imports = [ "airflow.providers.snowflake.hooks.snowflake" + "airflow.providers.snowflake.hooks.snowflake_sql_api" "airflow.providers.snowflake.operators.snowflake" ]; }; @@ -1438,7 +2044,7 @@ }; tabular = { deps = [ ]; - imports = [ "airflow.providers.tabular.hooks.tabular" ]; + imports = [ ]; }; telegram = { deps = [ "python-telegram-bot" ]; @@ -1447,34 +2053,48 @@ "airflow.providers.telegram.operators.telegram" ]; }; - trino = { + teradata = { deps = [ "adal" + "adlfs" + "aiofiles" + "aiohttp" "apache-beam" "asgiref" + "asyncssh" + "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" + "azure-kusto-data" "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" + "azure-mgmt-storage" "azure-servicebus" "azure-storage-blob" - "azure-storage-common" - "azure-storage-file" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" "azure-synapse-spark" "boto3" + "botocore" "cassandra-driver" "cryptography" + "dill" "dnspython" + "gcsfs" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" + "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-compute" @@ -1497,28 +2117,143 @@ "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" + "google-cloud-workflows" + "google-re2" "grpcio-gcp" "httpx" + "inflection" + "jmespath" "json-merge-patch" "jsonpath-ng" "kubernetes" - "mypy-boto3-appflow" - "mypy-boto3-rds" - "mypy-boto3-redshift-data" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" "mysqlclient" "oracledb" "pandas" "paramiko" "proto-plus" - "protobuf" "psycopg2" + "pyarrow" + "pyathena" "pymongo" + "pymssql" "pyopenssl" "pysftp" + "python-slugify" "redshift-connector" + "requests" + "requests-toolbelt" "simple-salesforce" "smbprotocol" "sshtunnel" + "tenacity" + "thrift" + "vertica-python" + ]; + imports = [ + "airflow.providers.teradata.hooks.teradata" + "airflow.providers.teradata.operators.teradata" + "airflow.providers.teradata.operators.teradata_compute_cluster" + ]; + }; + trino = { + deps = [ + "adal" + "adlfs" + "aiofiles" + "aiohttp" + "apache-beam" + "asgiref" + "asyncssh" + "attrs" + "azure-batch" + "azure-cosmos" + "azure-datalake-store" + "azure-identity" + "azure-keyvault-secrets" + "azure-kusto-data" + "azure-mgmt-containerinstance" + "azure-mgmt-containerregistry" + "azure-mgmt-cosmosdb" + "azure-mgmt-datafactory" + "azure-mgmt-datalake-store" + "azure-mgmt-resource" + "azure-mgmt-storage" + "azure-servicebus" + "azure-storage-blob" + "azure-storage-file-datalake" + "azure-storage-file-share" + "azure-synapse-artifacts" + "azure-synapse-spark" + "boto3" + "botocore" + "cassandra-driver" + "cryptography" + "dill" + "dnspython" + "gcsfs" + "google-api-core" + "google-api-python-client" + "google-auth" + "google-auth-httplib2" + "google-cloud-automl" + "google-cloud-bigquery" + "google-cloud-bigquery-datatransfer" + "google-cloud-bigtable" + "google-cloud-compute" + "google-cloud-container" + "google-cloud-datacatalog" + "google-cloud-dataproc" + "google-cloud-dlp" + "google-cloud-kms" + "google-cloud-language" + "google-cloud-logging" + "google-cloud-monitoring" + "google-cloud-pubsub" + "google-cloud-redis" + "google-cloud-secret-manager" + "google-cloud-spanner" + "google-cloud-speech" + "google-cloud-storage" + "google-cloud-tasks" + "google-cloud-texttospeech" + "google-cloud-translate" + "google-cloud-videointelligence" + "google-cloud-vision" + "google-cloud-workflows" + "google-re2" + "grpcio-gcp" + "httpx" + "inflection" + "jmespath" + "json-merge-patch" + "jsonpath-ng" + "kubernetes" + "kubernetes-asyncio" + "microsoft-kiota-abstractions" + "msgraph-core" + "mysqlclient" + "oracledb" + "pandas" + "paramiko" + "proto-plus" + "psycopg2" + "pyarrow" + "pyathena" + "pymongo" + "pymssql" + "pyopenssl" + "pysftp" + "python-slugify" + "redshift-connector" + "requests" + "requests-toolbelt" + "simple-salesforce" + "smbprotocol" + "sshtunnel" + "tenacity" "thrift" "vertica-python" ]; @@ -1534,12 +2269,36 @@ "airflow.providers.vertica.operators.vertica" ]; }; + weaviate = { + deps = [ + "httpx" + "pandas" + "weaviate-client" + ]; + imports = [ + "airflow.providers.weaviate.hooks.weaviate" + "airflow.providers.weaviate.operators.weaviate" + ]; + }; yandex = { deps = [ ]; imports = [ + "airflow.providers.yandex.hooks.dataproc" "airflow.providers.yandex.hooks.yandex" - "airflow.providers.yandex.hooks.yandexcloud_dataproc" - "airflow.providers.yandex.operators.yandexcloud_dataproc" + "airflow.providers.yandex.hooks.yq" + "airflow.providers.yandex.operators.dataproc" + "airflow.providers.yandex.operators.yq" + ]; + }; + ydb = { + deps = [ ]; + imports = [ + "airflow.providers.ydb.hooks._vendor.dbapi.connection" + "airflow.providers.ydb.hooks._vendor.dbapi.constants" + "airflow.providers.ydb.hooks._vendor.dbapi.cursor" + "airflow.providers.ydb.hooks._vendor.dbapi.errors" + "airflow.providers.ydb.hooks.ydb" + "airflow.providers.ydb.operators.ydb" ]; }; zendesk = { diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index a5a003f05d05..5d156ca96f60 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -14,87 +14,59 @@ # python alembic, argcomplete, - asgiref, - attrs, - blinker, buildPythonPackage, - cached-property, - cattrs, - clickclick, colorlog, configupdater, connexion, cron-descriptor, croniter, cryptography, - deprecated, dill, - flask, - flask-appbuilder, flask-caching, flask-login, flask-session, - flask-wtf, - freezegun, + fsspec, + gitdb, gitpython, - google-re2, - graphviz, gunicorn, - httpx, - importlib-metadata, - importlib-resources, - inflection, - iso8601, - itsdangerous, - jinja2, - jsonschema, + hatchling, lazy-object-proxy, linkify-it-py, lockfile, - markdown, - markupsafe, marshmallow-oneofschema, mdit-py-plugins, - numpy, - openapi-spec-validator, + methodtools, opentelemetry-api, opentelemetry-exporter-otlp, + packaging, pandas, pathspec, pendulum, + pluggy, psutil, - pydantic, - pygments, - pyjwt, pytest-asyncio, pytestCheckHook, python, python-daemon, - python-dateutil, python-nvd3, python-slugify, - python3-openid, - pythonOlder, - pyyaml, - rich, rich-argparse, setproctitle, + smmap, sqlalchemy, sqlalchemy-jsonfield, - swagger-ui-bundle, tabulate, tenacity, termcolor, - time-machine, - typing-extensions, - unicodecsv, - werkzeug, + tomli, + trove-classifiers, + universal-pathlib, # Extra airflow providers to enable enabledProviders ? [ ], }: let - version = "2.7.3"; + version = "2.10.5"; airflow-src = fetchFromGitHub { owner = "apache"; @@ -103,7 +75,7 @@ let # Download using the git protocol rather than using tarballs, because the # GitHub archive tarballs don't appear to include tests forceFetchGit = true; - hash = "sha256-+YbiKFZLigSDbHPaUKIl97kpezW1rIt/j09MMa6lwhQ="; + hash = "sha256-q5/CM+puXE31+15F3yZmcrR74LrqHppdCDUqjLQXPfk="; }; # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree. @@ -116,7 +88,7 @@ let offlineCache = fetchYarnDeps { yarnLock = "${src}/yarn.lock"; - hash = "sha256-WQKuQgNp35fU6z7owequXOSwoUGJDJYcUgkjPDMOops="; + hash = "sha256-hKgtMH4c8sPRDLPLVn+H8rmwc2Q6ei6U4er6fGuFn4I="; }; nativeBuildInputs = [ @@ -126,10 +98,10 @@ let webpack-cli ]; - # The webpack license plugin tries to create /licenses when given the + # The webpack license plugin tries to create /3rd-party-licenses when given the # original relative path postPatch = '' - sed -i 's!../../../../licenses/LICENSES-ui.txt!licenses/LICENSES-ui.txt!' webpack.config.js + sed -i 's!../../../../3rd-party-licenses/LICENSES-ui.txt!/3rd-party-licenses/LICENSES-ui.txt!' webpack.config.js ''; postBuild = '' @@ -142,129 +114,165 @@ let ''; }; + requiredProviders = [ + "common_compat" + "common_io" + "common_sql" + "fab" + "ftp" + "http" + "imap" + "smtp" + "sqlite" + ]; + # Import generated file with metadata for provider dependencies and imports. # Enable additional providers using enabledProviders above. providers = import ./providers.nix; + getProviderPath = provider: lib.replaceStrings [ "_" ] [ "/" ] provider; getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps; getProviderImports = provider: providers.${provider}.imports; - providerDependencies = lib.concatMap getProviderDeps enabledProviders; providerImports = lib.concatMap getProviderImports enabledProviders; + + buildProvider = + provider: + let + providerPath = getProviderPath provider; + in + python.pkgs.buildPythonPackage { + pname = "apache-airflow-providers-${provider}"; + version = "unstable"; # will be extracted in the build phase + pyproject = false; # providers packages don't have pyproject.toml nor setup.py + + src = airflow-src; + + propagatedBuildInputs = getProviderDeps provider; + dependencies = [ packaging ]; + + buildPhase = '' + # extract version from the provider's __init__.py file + if [ -f "airflow/providers/${providerPath}/__init__.py" ]; then + version=$(grep -oP "(?<=__version__ = ')[^']+" "airflow/providers/${providerPath}/__init__.py" || echo "0.0.0") + echo "Provider ${provider} version: $version" + else + echo "Error: __init__.py not found for provider ${provider} at path airflow/providers/${providerPath}" + exit 1 + fi + ''; + + installPhase = '' + # create directory structure + mkdir -p $out/${python.sitePackages}/airflow/providers + + # copy the provider directory + if [ -d "airflow/providers/${providerPath}" ]; then + mkdir -p $out/${python.sitePackages}/airflow/providers/$(dirname "${providerPath}") + cp -r airflow/providers/${providerPath} $out/${python.sitePackages}/airflow/providers/$(dirname "${providerPath}") + + # create parent __init__.py files + touch $out/${python.sitePackages}/airflow/__init__.py + touch $out/${python.sitePackages}/airflow/providers/__init__.py + + # create any needed intermediate __init__.py files for nested providers + providerDir=$(dirname "${providerPath}") + while [ "$providerDir" != "." ] && [ -n "$providerDir" ]; do + mkdir -p $out/${python.sitePackages}/airflow/providers/$providerDir + touch $out/${python.sitePackages}/airflow/providers/$providerDir/__init__.py + providerDir=$(dirname "$providerDir") + done + + # create egg-info for package discovery + mkdir -p $out/${python.sitePackages}/apache_airflow_providers_${provider}.egg-info + cat > $out/${python.sitePackages}/apache_airflow_providers_${provider}.egg-info/PKG-INFO < Path: - return Path(os.path.dirname(sys.argv[0])) / "../../.." + return Path(os.path.dirname(sys.argv[0])) / "../../../.." def dump_packages() -> Dict[str, Dict[str, str]]: From 87d364eccfdbfd549126b87d67236d3444997e85 Mon Sep 17 00:00:00 2001 From: Sergey Volkov Date: Wed, 7 Jan 2026 14:51:24 +0100 Subject: [PATCH 6/6] apache-airflow: add meta.changelog --- pkgs/by-name/ap/apache-airflow/python-package.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index 5d156ca96f60..f9800b0700ec 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -343,6 +343,7 @@ buildPythonPackage rec { meta = { description = "Programmatically author, schedule and monitor data pipelines"; homepage = "https://airflow.apache.org/"; + changelog = "https://airflow.apache.org/docs/apache-airflow/${version}/release_notes.html"; license = lib.licenses.asl20; maintainers = with lib.maintainers; [ gbpdt