diff --git a/pkgs/by-name/ap/apache-airflow/package.nix b/pkgs/by-name/ap/apache-airflow/package.nix index ba341a97bdf5..f8d5ee6ab778 100644 --- a/pkgs/by-name/ap/apache-airflow/package.nix +++ b/pkgs/by-name/ap/apache-airflow/package.nix @@ -3,276 +3,41 @@ fetchpatch, fetchPypi, python3, + + # Extra airflow providers to enable + enabledProviders ? [ ], }: let python = python3.override { self = python; packageOverrides = pySelf: pySuper: { - connexion = pySuper.connexion.overridePythonAttrs rec { - version = "2.14.2"; - src = fetchFromGitHub { - owner = "spec-first"; - repo = "connexion"; - tag = version; - hash = "sha256-1v1xCHY3ZnZG/Vu9wN/it7rLKC/StoDefoMNs+hMjIs="; - }; - nativeBuildInputs = with pySelf; [ - setuptools - ]; - pythonRelaxDeps = [ - "werkzeug" - ]; - propagatedBuildInputs = with pySelf; [ - aiohttp - aiohttp-jinja2 - aiohttp-swagger - clickclick - flask - inflection - jsonschema - openapi-spec-validator - packaging - pyyaml - requests - swagger-ui-bundle - ]; - nativeCheckInputs = with pySelf; [ - aiohttp-remotes - decorator - pytest-aiohttp - pytest-tornasync - pytestCheckHook - testfixtures - ]; - disabledTests = [ - "test_aiohttp_simple_api" - "test_app" - "test_invalid_type" # https://github.com/spec-first/connexion/issues/1969 - "test_openapi_yaml_behind_proxy" - "test_run_with_wsgi_containers" - "test_swagger_ui" - ]; - postPatch = '' - substituteInPlace connexion/__init__.py \ - --replace "2020.0.dev1" "${version}" - ''; - }; - werkzeug = pySuper.werkzeug.overridePythonAttrs rec { - version = "2.3.8"; - src = fetchPypi { - pname = "werkzeug"; - inherit version; - hash = "sha256-VUslfHS763oNJUFgpPj/4YUkP1KlIDUGC3Ycpi2XfwM="; - }; - nativeCheckInputs = with pySelf; [ - pytest-xprocess - ]; - }; - # flask's test-suite needs click 8.1.8 - # TypeError: CliRunner.__init__() got an unexpected keyword argument 'mix_stderr' - click = pySuper.click.overridePythonAttrs rec { - version = "8.1.8"; - src = fetchPypi { - pname = "click"; - inherit version; - hash = "sha256-7VPJ2JkNg8Kifermjk7jN0c/YzDAQKMdQiXJV00WCWo="; - }; - }; - flask = pySuper.flask.overridePythonAttrs (o: rec { - version = "2.2.5"; - src = fetchPypi { - pname = "Flask"; - inherit version; - hash = "sha256-7e6bCn/yZiG9WowQ/0hK4oc3okENmbC7mmhQx/uXeqA="; - }; - nativeBuildInputs = (o.nativeBuildInputs or [ ]) ++ [ - pySelf.setuptools - ]; - pytestFlagsArray = [ - # tests that are marked with filterwarnings fail with - # DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for - # removal in Python 3.14; use importlib.util.find_spec() instead - "-W ignore::DeprecationWarning" - ]; - }); - flask-login = pySuper.flask-login.overridePythonAttrs rec { - version = "0.6.3"; - src = fetchFromGitHub { - owner = "maxcountryman"; - repo = "flask-login"; - tag = version; - hash = "sha256-Sn7Ond67P/3+OmKKFE/KfA6FE4IajhiRXVVrXKJtY3I="; - }; - nativeBuildInputs = with pySelf; [ setuptools ]; - pytestFlagsArray = [ - # DeprecationWarning: datetime.datetime.utcnow() is deprecated - # and scheduled for removal in a future version. - # Use timezone-aware objects to represent datetimes in UTC: - # datetime.datetime.now(datetime.UTC). - "-W ignore::DeprecationWarning" - ]; - }; - flask-session = pySuper.flask-session.overridePythonAttrs rec { - version = "0.5.0"; - src = fetchFromGitHub { - owner = "palletc-eco"; - repo = "flask-session"; - tag = version; - hash = "sha256-t8w6ZS4gBDpnnKvL3DLtn+rRLQNJbrT2Hxm4f3+a3Xc="; - }; - nativeCheckInputs = with pySelf; [ pytestCheckHook ]; - pytestFlagsArray = [ - "-k" - "'null_session or filesystem_session'" - ]; - dependencies = with pySelf; [ - flask-sqlalchemy - cachelib - ]; - disabledTests = [ ]; - disabledTestPaths = [ ]; - preCheck = ""; - postCheck = ""; - }; - # flask-appbuilder doesn't work with sqlalchemy 2.x, flask-appbuilder 3.x - # https://github.com/dpgaspar/Flask-AppBuilder/issues/2038 - flask-appbuilder = pySuper.flask-appbuilder.overridePythonAttrs { - meta.broken = false; - }; - # a knock-on effect from overriding the sqlalchemy version - flask-sqlalchemy = pySuper.flask-sqlalchemy.overridePythonAttrs { - src = fetchPypi { - pname = "Flask-SQLAlchemy"; - version = "2.5.1"; - hash = "sha256-K9pEtD58rLFdTgX/PMH4vJeTbMRkYjQkECv8LDXpWRI="; - }; - format = "setuptools"; - }; - httpcore = pySuper.httpcore.overridePythonAttrs { - # nullify upstream's pytest flags which cause - # "TLS/SSL connection has been closed (EOF)" - # with pytest-httpbin 1.x - preCheck = '' - substituteInPlace pyproject.toml \ - --replace '[tool.pytest.ini_options]' '[tool.notpytest.ini_options]' - ''; - }; - pytest-httpbin = pySuper.pytest-httpbin.overridePythonAttrs rec { - version = "1.0.2"; - src = fetchFromGitHub { - owner = "kevin1024"; - repo = "pytest-httpbin"; - tag = "v${version}"; - hash = "sha256-S4ThQx4H3UlKhunJo35esPClZiEn7gX/Qwo4kE1QMTI="; - }; - # Use unmerged PR #65 to fix older version: - # https://github.com/kevin1024/pytest-httpbin/pull/65/ - # It was closed in favour of another which isn't compatible with the overriden version. - patches = [ - (fetchpatch { - url = "https://github.com/kevin1024/pytest-httpbin/commit/4e325f877ff8f77dec9f380bd8e53bb42976775c.patch"; - hash = "sha256-a33XcdMupD+7ZzvUibePdldGImmPLDNU2sxRbwpveDA="; - }) - (fetchpatch { - url = "https://github.com/kevin1024/pytest-httpbin/commit/463afb9b200563ac6fe7ae535f7a7a3c818b0418.patch"; - hash = "sha256-HFmuLtAtEjnB6heSG1YNnqxtz2phXNkHbQaZyB5bLJs="; - }) - ]; - disabledTests = [ - "test_httpbin_secure_accepts_get_requests" - "test_httpbin_secure_accepts_lots_of_get_requests" - "test_httpbin_both[https]" - "test_chunked_encoding[https]" - "TestClassBassedTests::test_http_secure" - "test_dont_crash_on_certificate_problems" - "test_redirect_location_is_https_for_secure_server" - "test_httpbin_secure_accepts_get_requests" - "test_http_secure" - ]; - }; - # apache-airflow doesn't work with sqlalchemy 2.x - # https://github.com/apache/airflow/issues/28723 - sqlalchemy = pySuper.sqlalchemy_1_4; - gitpython = pySuper.gitpython.overridePythonAttrs rec { - version = "3.1.44"; - src = fetchFromGitHub { - owner = "gitpython-developers"; - repo = "gitpython"; - rev = version; - hash = "sha256-KnKaBv/tKk4wiGWUWCEgd1vgrTouwUhqxJ1/nMjRaWk="; - }; - }; - # ValueError: Unknown classifier in field `project.classifiers`: Programming Language :: Python :: Free Threading :: 2 - Beta - urllib3 = pySuper.urllib3.overridePythonAttrs rec { - version = "2.5.0"; - src = fetchPypi { - pname = "urllib3"; - inherit version; - hash = "sha256-P8R3M8fkGdS8P2s9wrT4kLt0OQajDVa6Slv6S7/5J2A="; - }; - postPatch = '' - substituteInPlace pyproject.toml \ - --replace-fail ', "setuptools-scm>=8,<9"' "" - ''; - }; smmap = pySuper.smmap.overridePythonAttrs rec { version = "5.0.2"; src = fetchFromGitHub { owner = "gitpython-developers"; repo = "smmap"; - rev = "refs/tags/v${version}"; + tag = "v${version}"; hash = "sha256-0Y175kjv/8UJpSxtLpWH4/VT7JrcVPAq79Nf3rtHZZM="; }; }; - trove-classifiers = pySuper.trove-classifiers.overridePythonAttrs rec { - version = "2024.10.21.16"; - src = fetchPypi { - pname = "trove_classifiers"; - inherit version; - hash = "sha256-F8vQVdZ9Xp2d5jKTqHMpQ/q8IVdOTHt07fEStJKM9fM="; - }; - postPatch = ""; - }; - packaging = pySuper.packaging.overridePythonAttrs rec { - version = "24.2"; - src = fetchPypi { - pname = "packaging"; - inherit version; - hash = "sha256-wiim3F6TLTRrxXOTeRCdSeiFPdgiNXHHxbVSYO3AuX8="; - }; - }; - pluggy = pySuper.pluggy.overridePythonAttrs rec { - version = "1.5.0"; + structlog = pySuper.structlog.overridePythonAttrs (o: rec { + version = "25.4.0"; src = fetchFromGitHub { - owner = "pytest-dev"; - repo = "pluggy"; + owner = "hynek"; + repo = "structlog"; tag = version; - hash = "sha256-f0DxyZZk6RoYtOEXLACcsOn2B+Hot4U4g5Ogr/hKmOE="; + hash = "sha256-iNnUogcICQJvHBZO2J8uk4NleQY/ra3ZzxQgnSRKr30="; }; - }; - pyproject-api = pySuper.pyproject-api.overridePythonAttrs rec { - version = "1.8.0"; - src = fetchPypi { - pname = "pyproject_api"; - inherit version; - hash = "sha256-d7gEny/rXTPu/MIbV/HieWNid6isita1hxA3skN3hJY="; - }; - disabledTests = [ - # AssertionError: assert ['magic>3', 'requests>2'] == ['magic >3', 'requests >2'] - "test_frontend_setuptools" - ]; - }; - tox = pySuper.tox.overridePythonAttrs rec { - version = "4.27.0"; - src = fetchFromGitHub { - owner = "tox-dev"; - repo = "tox"; - tag = version; - hash = "sha256-Z3qUK4w1ebPvdZD4ZuKgZXJPUu5lG0G41vn/pc9gC/0="; - }; - }; - apache-airflow = pySelf.callPackage ./python-package.nix { }; + nativeCheckInputs = + with pySelf; + o.nativeCheckInputs + ++ [ + freezegun + pretend + ]; + }); + apache-airflow = pySelf.callPackage ./python-package.nix { inherit enabledProviders; }; }; }; in diff --git a/pkgs/by-name/ap/apache-airflow/providers.nix b/pkgs/by-name/ap/apache-airflow/providers.nix index d2da8c6a4c65..d20c06f64e48 100644 --- a/pkgs/by-name/ap/apache-airflow/providers.nix +++ b/pkgs/by-name/ap/apache-airflow/providers.nix @@ -1,709 +1,340 @@ # Warning: generated by update-providers.py, do not update manually { airbyte = { - deps = [ - "aiohttp" - "asgiref" - "requests" - "requests-toolbelt" - ]; + deps = [ "requests" ]; imports = [ - "airflow.providers.airbyte.hooks.airbyte" - "airflow.providers.airbyte.operators.airbyte" + "airflow.providers.airbyte" + "airflow.providers.airbyte.get_provider_info" ]; + version = "5.2.3"; }; + alibaba = { deps = [ "oss2" ]; imports = [ - "airflow.providers.alibaba.cloud.hooks.analyticdb_spark" - "airflow.providers.alibaba.cloud.hooks.oss" - "airflow.providers.alibaba.cloud.operators.analyticdb_spark" - "airflow.providers.alibaba.cloud.operators.oss" + "airflow.providers.alibaba" + "airflow.providers.alibaba.get_provider_info" ]; + version = "3.2.2"; }; + amazon = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" + "aiobotocore" "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" "boto3" "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" "inflection" "jmespath" - "json-merge-patch" "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" + "lxml" + "marshmallow" "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" + "python3-saml" "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" + "s3fs" + "xmlsec" ]; imports = [ - "airflow.providers.amazon.aws.hooks.appflow" - "airflow.providers.amazon.aws.hooks.athena" - "airflow.providers.amazon.aws.hooks.athena_sql" - "airflow.providers.amazon.aws.hooks.base_aws" - "airflow.providers.amazon.aws.hooks.batch_client" - "airflow.providers.amazon.aws.hooks.batch_waiters" - "airflow.providers.amazon.aws.hooks.bedrock" - "airflow.providers.amazon.aws.hooks.chime" - "airflow.providers.amazon.aws.hooks.cloud_formation" - "airflow.providers.amazon.aws.hooks.comprehend" - "airflow.providers.amazon.aws.hooks.datasync" - "airflow.providers.amazon.aws.hooks.dms" - "airflow.providers.amazon.aws.hooks.dynamodb" - "airflow.providers.amazon.aws.hooks.ec2" - "airflow.providers.amazon.aws.hooks.ecr" - "airflow.providers.amazon.aws.hooks.ecs" - "airflow.providers.amazon.aws.hooks.eks" - "airflow.providers.amazon.aws.hooks.elasticache_replication_group" - "airflow.providers.amazon.aws.hooks.emr" - "airflow.providers.amazon.aws.hooks.emr" - "airflow.providers.amazon.aws.hooks.eventbridge" - "airflow.providers.amazon.aws.hooks.glacier" - "airflow.providers.amazon.aws.hooks.glue" - "airflow.providers.amazon.aws.hooks.glue_catalog" - "airflow.providers.amazon.aws.hooks.glue_crawler" - "airflow.providers.amazon.aws.hooks.glue_databrew" - "airflow.providers.amazon.aws.hooks.kinesis" - "airflow.providers.amazon.aws.hooks.kinesis_analytics" - "airflow.providers.amazon.aws.hooks.lambda_function" - "airflow.providers.amazon.aws.hooks.logs" - "airflow.providers.amazon.aws.hooks.neptune" - "airflow.providers.amazon.aws.hooks.opensearch_serverless" - "airflow.providers.amazon.aws.hooks.quicksight" - "airflow.providers.amazon.aws.hooks.rds" - "airflow.providers.amazon.aws.hooks.redshift_cluster" - "airflow.providers.amazon.aws.hooks.redshift_data" - "airflow.providers.amazon.aws.hooks.redshift_sql" - "airflow.providers.amazon.aws.hooks.s3" - "airflow.providers.amazon.aws.hooks.sagemaker" - "airflow.providers.amazon.aws.hooks.secrets_manager" - "airflow.providers.amazon.aws.hooks.ses" - "airflow.providers.amazon.aws.hooks.sns" - "airflow.providers.amazon.aws.hooks.sqs" - "airflow.providers.amazon.aws.hooks.ssm" - "airflow.providers.amazon.aws.hooks.step_function" - "airflow.providers.amazon.aws.hooks.sts" - "airflow.providers.amazon.aws.hooks.verified_permissions" - "airflow.providers.amazon.aws.operators.appflow" - "airflow.providers.amazon.aws.operators.athena" - "airflow.providers.amazon.aws.operators.base_aws" - "airflow.providers.amazon.aws.operators.batch" - "airflow.providers.amazon.aws.operators.bedrock" - "airflow.providers.amazon.aws.operators.cloud_formation" - "airflow.providers.amazon.aws.operators.comprehend" - "airflow.providers.amazon.aws.operators.datasync" - "airflow.providers.amazon.aws.operators.dms" - "airflow.providers.amazon.aws.operators.ec2" - "airflow.providers.amazon.aws.operators.ecs" - "airflow.providers.amazon.aws.operators.eks" - "airflow.providers.amazon.aws.operators.emr" - "airflow.providers.amazon.aws.operators.emr" - "airflow.providers.amazon.aws.operators.eventbridge" - "airflow.providers.amazon.aws.operators.glacier" - "airflow.providers.amazon.aws.operators.glue" - "airflow.providers.amazon.aws.operators.glue_crawler" - "airflow.providers.amazon.aws.operators.glue_databrew" - "airflow.providers.amazon.aws.operators.kinesis_analytics" - "airflow.providers.amazon.aws.operators.lambda_function" - "airflow.providers.amazon.aws.operators.neptune" - "airflow.providers.amazon.aws.operators.quicksight" - "airflow.providers.amazon.aws.operators.rds" - "airflow.providers.amazon.aws.operators.redshift_cluster" - "airflow.providers.amazon.aws.operators.redshift_data" - "airflow.providers.amazon.aws.operators.s3" - "airflow.providers.amazon.aws.operators.sagemaker" - "airflow.providers.amazon.aws.operators.sns" - "airflow.providers.amazon.aws.operators.sqs" - "airflow.providers.amazon.aws.operators.step_function" + "airflow.providers.amazon" + "airflow.providers.amazon.get_provider_info" ]; + version = "9.13.0"; }; + apache_beam = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" + "numpy" "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" ]; imports = [ - "airflow.providers.apache.beam.hooks.beam" - "airflow.providers.apache.beam.operators.beam" + "airflow.providers.apache.beam" + "airflow.providers.apache.beam.get_provider_info" ]; + version = "6.1.4"; }; + apache_cassandra = { deps = [ "cassandra-driver" ]; - imports = [ "airflow.providers.apache.cassandra.hooks.cassandra" ]; + imports = [ + "airflow.providers.apache.cassandra" + "airflow.providers.apache.cassandra.get_provider_info" + ]; + version = "3.8.2"; }; + apache_drill = { deps = [ ]; imports = [ - "airflow.providers.apache.drill.hooks.drill" - "airflow.providers.apache.drill.operators.drill" + "airflow.providers.apache.drill" + "airflow.providers.apache.drill.get_provider_info" ]; + version = "3.1.2"; }; + apache_druid = { + deps = [ "pydruid" ]; + imports = [ + "airflow.providers.apache.druid" + "airflow.providers.apache.druid.get_provider_info" + ]; + version = "4.3.0"; + }; + + apache_flink = { + deps = [ "cryptography" ]; + imports = [ + "airflow.providers.apache.flink" + "airflow.providers.apache.flink.get_provider_info" + ]; + version = "1.7.2"; + }; + + apache_hdfs = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" + "hdfs" "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pydruid" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" ]; imports = [ - "airflow.providers.apache.druid.hooks.druid" - "airflow.providers.apache.druid.operators.druid" - "airflow.providers.apache.druid.operators.druid_check" + "airflow.providers.apache.hdfs" + "airflow.providers.apache.hdfs.get_provider_info" ]; + version = "4.10.2"; }; - apache_flink = { - deps = [ - "aiofiles" - "asgiref" - "cryptography" - "google-re2" - "kubernetes" - "kubernetes-asyncio" - ]; - imports = [ "airflow.providers.apache.flink.operators.flink_kubernetes" ]; - }; - apache_hdfs = { - deps = [ "pandas" ]; - imports = [ "airflow.providers.apache.hdfs.hooks.webhdfs" ]; - }; + apache_hive = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" "thrift" - "vertica-python" ]; imports = [ - "airflow.providers.apache.hive.hooks.hive" - "airflow.providers.apache.hive.operators.hive" - "airflow.providers.apache.hive.operators.hive_stats" + "airflow.providers.apache.hive" + "airflow.providers.apache.hive.get_provider_info" ]; + version = "9.1.2"; }; + apache_iceberg = { deps = [ ]; - imports = [ "airflow.providers.apache.iceberg.hooks.iceberg" ]; + imports = [ + "airflow.providers.apache.iceberg" + "airflow.providers.apache.iceberg.get_provider_info" + ]; + version = "1.3.2"; }; + apache_impala = { - deps = [ ]; - imports = [ "airflow.providers.apache.impala.hooks.impala" ]; + deps = [ "kerberos" ]; + imports = [ + "airflow.providers.apache.impala" + "airflow.providers.apache.impala.get_provider_info" + ]; + version = "1.7.2"; }; + apache_kafka = { deps = [ "asgiref" "confluent-kafka" ]; imports = [ - "airflow.providers.apache.kafka.hooks.base" - "airflow.providers.apache.kafka.hooks.client" - "airflow.providers.apache.kafka.hooks.consume" - "airflow.providers.apache.kafka.hooks.produce" - "airflow.providers.apache.kafka.operators.consume" - "airflow.providers.apache.kafka.operators.produce" + "airflow.providers.apache.kafka" + "airflow.providers.apache.kafka.get_provider_info" ]; + version = "1.10.3"; }; + apache_kylin = { deps = [ ]; imports = [ - "airflow.providers.apache.kylin.hooks.kylin" - "airflow.providers.apache.kylin.operators.kylin_cube" + "airflow.providers.apache.kylin" + "airflow.providers.apache.kylin.get_provider_info" ]; + version = "3.9.2"; }; + apache_livy = { deps = [ "aiohttp" "asgiref" - "requests" - "requests-toolbelt" ]; imports = [ - "airflow.providers.apache.livy.hooks.livy" - "airflow.providers.apache.livy.operators.livy" + "airflow.providers.apache.livy" + "airflow.providers.apache.livy.get_provider_info" ]; + version = "4.4.2"; }; + apache_pig = { deps = [ ]; imports = [ - "airflow.providers.apache.pig.hooks.pig" - "airflow.providers.apache.pig.operators.pig" + "airflow.providers.apache.pig" + "airflow.providers.apache.pig.get_provider_info" ]; + version = "4.7.2"; }; + apache_pinot = { deps = [ ]; - imports = [ "airflow.providers.apache.pinot.hooks.pinot" ]; + imports = [ + "airflow.providers.apache.pinot" + "airflow.providers.apache.pinot.get_provider_info" + ]; + version = "4.8.2"; }; + apache_spark = { deps = [ - "aiofiles" - "asgiref" - "cryptography" - "google-re2" "grpcio-status" - "kubernetes" - "kubernetes-asyncio" "pyspark" ]; imports = [ - "airflow.providers.apache.spark.hooks.spark_connect" - "airflow.providers.apache.spark.hooks.spark_jdbc" - "airflow.providers.apache.spark.hooks.spark_jdbc_script" - "airflow.providers.apache.spark.hooks.spark_sql" - "airflow.providers.apache.spark.hooks.spark_submit" - "airflow.providers.apache.spark.operators.spark_jdbc" - "airflow.providers.apache.spark.operators.spark_sql" - "airflow.providers.apache.spark.operators.spark_submit" + "airflow.providers.apache.spark" + "airflow.providers.apache.spark.get_provider_info" ]; + version = "5.3.2"; }; + + apache_tinkerpop = { + deps = [ "gremlinpython" ]; + imports = [ + "airflow.providers.apache.tinkerpop" + "airflow.providers.apache.tinkerpop.get_provider_info" + ]; + version = "1.0.3"; + }; + apprise = { deps = [ "apprise" ]; - imports = [ "airflow.providers.apprise.hooks.apprise" ]; + imports = [ + "airflow.providers.apprise" + "airflow.providers.apprise.get_provider_info" + ]; + version = "2.1.2"; }; + arangodb = { deps = [ "python-arango" ]; imports = [ - "airflow.providers.arangodb.hooks.arangodb" - "airflow.providers.arangodb.operators.arangodb" + "airflow.providers.arangodb" + "airflow.providers.arangodb.get_provider_info" ]; + version = "2.8.2"; }; + asana = { deps = [ "asana" ]; imports = [ - "airflow.providers.asana.hooks.asana" - "airflow.providers.asana.operators.asana_tasks" + "airflow.providers.asana" + "airflow.providers.asana.get_provider_info" ]; + version = "2.10.2"; }; + atlassian_jira = { deps = [ "atlassian-python-api" ]; imports = [ - "airflow.providers.atlassian.jira.hooks.jira" - "airflow.providers.atlassian.jira.operators.jira" + "airflow.providers.atlassian.jira" + "airflow.providers.atlassian.jira.get_provider_info" ]; + version = "3.1.2"; }; + celery = { deps = [ - "aiofiles" - "asgiref" - "cryptography" + "celery" "flower" - "google-re2" - "kubernetes" - "kubernetes-asyncio" ]; - imports = [ ]; + imports = [ + "airflow.providers.celery" + "airflow.providers.celery.get_provider_info" + ]; + version = "3.12.3"; }; + cloudant = { deps = [ ]; - imports = [ "airflow.providers.cloudant.hooks.cloudant" ]; + imports = [ + "airflow.providers.cloudant" + "airflow.providers.cloudant.get_provider_info" + ]; + version = "4.2.2"; }; + cncf_kubernetes = { deps = [ "aiofiles" "asgiref" "cryptography" - "google-re2" "kubernetes" "kubernetes-asyncio" + "urllib3" ]; imports = [ - "airflow.providers.cncf.kubernetes.hooks.kubernetes" - "airflow.providers.cncf.kubernetes.operators.custom_object_launcher" - "airflow.providers.cncf.kubernetes.operators.job" - "airflow.providers.cncf.kubernetes.operators.pod" - "airflow.providers.cncf.kubernetes.operators.resource" - "airflow.providers.cncf.kubernetes.operators.spark_kubernetes" + "airflow.providers.cncf.kubernetes" + "airflow.providers.cncf.kubernetes.get_provider_info" ]; + version = "10.8.0"; }; + cohere = { deps = [ "cohere" ]; imports = [ - "airflow.providers.cohere.hooks.cohere" - "airflow.providers.cohere.operators.embedding" + "airflow.providers.cohere" + "airflow.providers.cohere.get_provider_info" ]; + version = "1.5.3"; }; + common_compat = { deps = [ ]; - imports = [ ]; + imports = [ + "airflow.providers.common.compat" + "airflow.providers.common.compat.get_provider_info" + ]; + version = "1.7.4"; }; + common_io = { - deps = [ "attrs" ]; - imports = [ "airflow.providers.common.io.operators.file_transfer" ]; + deps = [ ]; + imports = [ + "airflow.providers.common.io" + "airflow.providers.common.io.get_provider_info" + ]; + version = "1.6.3"; }; + + common_messaging = { + deps = [ ]; + imports = [ + "airflow.providers.common.messaging" + "airflow.providers.common.messaging.get_provider_info" + ]; + version = "2.0.0"; + }; + common_sql = { deps = [ - "attrs" + "methodtools" "more-itertools" + "pandas" + "polars" "sqlparse" ]; imports = [ - "airflow.providers.common.sql.hooks.sql" - "airflow.providers.common.sql.operators.sql" + "airflow.providers.common.sql" + "airflow.providers.common.sql.get_provider_info" ]; + version = "1.28.1"; }; + databricks = { deps = [ "aiohttp" + "azure-identity" + "databricks-sdk" "databricks-sql-connector" "mergedeep" "pandas" @@ -711,141 +342,156 @@ "requests" ]; imports = [ - "airflow.providers.databricks.hooks.databricks" - "airflow.providers.databricks.hooks.databricks_base" - "airflow.providers.databricks.hooks.databricks_sql" - "airflow.providers.databricks.operators.databricks" - "airflow.providers.databricks.operators.databricks_repos" - "airflow.providers.databricks.operators.databricks_sql" - "airflow.providers.databricks.operators.databricks_workflow" + "airflow.providers.databricks" + "airflow.providers.databricks.get_provider_info" ]; + version = "7.7.2"; }; + datadog = { deps = [ "datadog" ]; - imports = [ "airflow.providers.datadog.hooks.datadog" ]; + imports = [ + "airflow.providers.datadog" + "airflow.providers.datadog.get_provider_info" + ]; + version = "3.9.2"; }; + dbt_cloud = { deps = [ "aiohttp" "asgiref" - "attrs" - "requests" - "requests-toolbelt" ]; imports = [ - "airflow.providers.dbt.cloud.hooks.dbt" - "airflow.providers.dbt.cloud.operators.dbt" + "airflow.providers.dbt.cloud" + "airflow.providers.dbt.cloud.get_provider_info" ]; + version = "4.4.3"; }; + dingding = { - deps = [ - "aiohttp" - "asgiref" - "requests" - "requests-toolbelt" - ]; + deps = [ ]; imports = [ - "airflow.providers.dingding.hooks.dingding" - "airflow.providers.dingding.operators.dingding" + "airflow.providers.dingding" + "airflow.providers.dingding.get_provider_info" ]; + version = "3.8.2"; }; + discord = { - deps = [ - "aiohttp" - "asgiref" - "requests" - "requests-toolbelt" - ]; + deps = [ ]; imports = [ - "airflow.providers.discord.hooks.discord_webhook" - "airflow.providers.discord.operators.discord_webhook" + "airflow.providers.discord" + "airflow.providers.discord.get_provider_info" ]; + version = "3.10.2"; }; + docker = { deps = [ "docker" "python-dotenv" ]; imports = [ - "airflow.providers.docker.hooks.docker" - "airflow.providers.docker.operators.docker" - "airflow.providers.docker.operators.docker_swarm" + "airflow.providers.docker" + "airflow.providers.docker.get_provider_info" ]; + version = "4.4.2"; }; + + edge3 = { + deps = [ + "pydantic" + "retryhttp" + ]; + imports = [ + "airflow.providers.edge3" + "airflow.providers.edge3.get_provider_info" + ]; + version = "1.2.0"; + }; + elasticsearch = { deps = [ "elasticsearch" ]; - imports = [ "airflow.providers.elasticsearch.hooks.elasticsearch" ]; + imports = [ + "airflow.providers.elasticsearch" + "airflow.providers.elasticsearch.get_provider_info" + ]; + version = "6.3.2"; }; + exasol = { deps = [ "pandas" ]; imports = [ - "airflow.providers.exasol.hooks.exasol" - "airflow.providers.exasol.operators.exasol" + "airflow.providers.exasol" + "airflow.providers.exasol.get_provider_info" ]; + version = "4.8.2"; }; + fab = { deps = [ + "blinker" + "connexion" "flask" "flask-appbuilder" + "flask-limiter" "flask-login" - "google-re2" + "flask-session" + "flask-wtf" "jmespath" + "kerberos" + "werkzeug" + "wtforms" ]; - imports = [ ]; + imports = [ + "airflow.providers.fab" + "airflow.providers.fab.get_provider_info" + ]; + version = "2.4.2"; }; + facebook = { deps = [ ]; - imports = [ "airflow.providers.facebook.ads.hooks.ads" ]; - }; - ftp = { - deps = [ "attrs" ]; imports = [ - "airflow.providers.ftp.hooks.ftp" - "airflow.providers.ftp.operators.ftp" + "airflow.providers.facebook" + "airflow.providers.facebook.get_provider_info" ]; + version = "3.8.2"; }; + + ftp = { + deps = [ ]; + imports = [ + "airflow.providers.ftp" + "airflow.providers.ftp.get_provider_info" + ]; + version = "3.13.2"; + }; + + git = { + deps = [ "gitpython" ]; + imports = [ + "airflow.providers.git" + "airflow.providers.git.get_provider_info" + ]; + version = "0.0.7"; + }; + github = { deps = [ "pygithub" ]; imports = [ - "airflow.providers.github.hooks.github" - "airflow.providers.github.operators.github" + "airflow.providers.github" + "airflow.providers.github.get_provider_info" ]; + version = "2.9.2"; }; + google = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" "apache-beam" "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" "dill" - "dnspython" "gcsfs" "google-api-core" "google-api-python-client" @@ -854,6 +500,7 @@ "google-cloud-automl" "google-cloud-bigquery" "google-cloud-bigquery-datatransfer" + "google-cloud-bigquery-storage" "google-cloud-bigtable" "google-cloud-compute" "google-cloud-container" @@ -866,6 +513,7 @@ "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" + "google-cloud-run" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" @@ -876,172 +524,26 @@ "google-cloud-videointelligence" "google-cloud-vision" "google-cloud-workflows" - "google-re2" "grpcio-gcp" "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" + "immutabledict" "pandas" - "paramiko" + "plyvel" "proto-plus" - "psycopg2" "pyarrow" - "pyathena" - "pymongo" - "pymssql" "pyopenssl" - "pysftp" "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" + "ray" "tenacity" - "thrift" - "vertica-python" + "types-protobuf" ]; imports = [ - "airflow.providers.google.ads.hooks.ads" - "airflow.providers.google.ads.operators.ads" - "airflow.providers.google.cloud.hooks.automl" - "airflow.providers.google.cloud.hooks.bigquery" - "airflow.providers.google.cloud.hooks.bigquery_dts" - "airflow.providers.google.cloud.hooks.bigtable" - "airflow.providers.google.cloud.hooks.cloud_batch" - "airflow.providers.google.cloud.hooks.cloud_build" - "airflow.providers.google.cloud.hooks.cloud_composer" - "airflow.providers.google.cloud.hooks.cloud_memorystore" - "airflow.providers.google.cloud.hooks.cloud_run" - "airflow.providers.google.cloud.hooks.cloud_sql" - "airflow.providers.google.cloud.hooks.cloud_storage_transfer_service" - "airflow.providers.google.cloud.hooks.compute" - "airflow.providers.google.cloud.hooks.compute_ssh" - "airflow.providers.google.cloud.hooks.datacatalog" - "airflow.providers.google.cloud.hooks.dataflow" - "airflow.providers.google.cloud.hooks.dataform" - "airflow.providers.google.cloud.hooks.datafusion" - "airflow.providers.google.cloud.hooks.datapipeline" - "airflow.providers.google.cloud.hooks.dataplex" - "airflow.providers.google.cloud.hooks.dataprep" - "airflow.providers.google.cloud.hooks.dataproc" - "airflow.providers.google.cloud.hooks.dataproc_metastore" - "airflow.providers.google.cloud.hooks.datastore" - "airflow.providers.google.cloud.hooks.dlp" - "airflow.providers.google.cloud.hooks.functions" - "airflow.providers.google.cloud.hooks.gcs" - "airflow.providers.google.cloud.hooks.gdm" - "airflow.providers.google.cloud.hooks.kms" - "airflow.providers.google.cloud.hooks.kubernetes_engine" - "airflow.providers.google.cloud.hooks.life_sciences" - "airflow.providers.google.cloud.hooks.looker" - "airflow.providers.google.cloud.hooks.mlengine" - "airflow.providers.google.cloud.hooks.natural_language" - "airflow.providers.google.cloud.hooks.os_login" - "airflow.providers.google.cloud.hooks.pubsub" - "airflow.providers.google.cloud.hooks.secret_manager" - "airflow.providers.google.cloud.hooks.spanner" - "airflow.providers.google.cloud.hooks.speech_to_text" - "airflow.providers.google.cloud.hooks.stackdriver" - "airflow.providers.google.cloud.hooks.tasks" - "airflow.providers.google.cloud.hooks.text_to_speech" - "airflow.providers.google.cloud.hooks.translate" - "airflow.providers.google.cloud.hooks.vertex_ai.auto_ml" - "airflow.providers.google.cloud.hooks.vertex_ai.batch_prediction_job" - "airflow.providers.google.cloud.hooks.vertex_ai.custom_job" - "airflow.providers.google.cloud.hooks.vertex_ai.dataset" - "airflow.providers.google.cloud.hooks.vertex_ai.endpoint_service" - "airflow.providers.google.cloud.hooks.vertex_ai.generative_model" - "airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job" - "airflow.providers.google.cloud.hooks.vertex_ai.model_service" - "airflow.providers.google.cloud.hooks.vertex_ai.pipeline_job" - "airflow.providers.google.cloud.hooks.vertex_ai.prediction_service" - "airflow.providers.google.cloud.hooks.video_intelligence" - "airflow.providers.google.cloud.hooks.vision" - "airflow.providers.google.cloud.hooks.workflows" - "airflow.providers.google.cloud.operators.automl" - "airflow.providers.google.cloud.operators.bigquery" - "airflow.providers.google.cloud.operators.bigquery_dts" - "airflow.providers.google.cloud.operators.bigtable" - "airflow.providers.google.cloud.operators.cloud_base" - "airflow.providers.google.cloud.operators.cloud_batch" - "airflow.providers.google.cloud.operators.cloud_build" - "airflow.providers.google.cloud.operators.cloud_composer" - "airflow.providers.google.cloud.operators.cloud_memorystore" - "airflow.providers.google.cloud.operators.cloud_run" - "airflow.providers.google.cloud.operators.cloud_sql" - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service" - "airflow.providers.google.cloud.operators.compute" - "airflow.providers.google.cloud.operators.datacatalog" - "airflow.providers.google.cloud.operators.dataflow" - "airflow.providers.google.cloud.operators.dataform" - "airflow.providers.google.cloud.operators.datafusion" - "airflow.providers.google.cloud.operators.datapipeline" - "airflow.providers.google.cloud.operators.dataplex" - "airflow.providers.google.cloud.operators.dataprep" - "airflow.providers.google.cloud.operators.dataproc" - "airflow.providers.google.cloud.operators.dataproc_metastore" - "airflow.providers.google.cloud.operators.datastore" - "airflow.providers.google.cloud.operators.dlp" - "airflow.providers.google.cloud.operators.functions" - "airflow.providers.google.cloud.operators.gcs" - "airflow.providers.google.cloud.operators.kubernetes_engine" - "airflow.providers.google.cloud.operators.life_sciences" - "airflow.providers.google.cloud.operators.looker" - "airflow.providers.google.cloud.operators.mlengine" - "airflow.providers.google.cloud.operators.natural_language" - "airflow.providers.google.cloud.operators.pubsub" - "airflow.providers.google.cloud.operators.spanner" - "airflow.providers.google.cloud.operators.speech_to_text" - "airflow.providers.google.cloud.operators.stackdriver" - "airflow.providers.google.cloud.operators.tasks" - "airflow.providers.google.cloud.operators.text_to_speech" - "airflow.providers.google.cloud.operators.translate" - "airflow.providers.google.cloud.operators.translate_speech" - "airflow.providers.google.cloud.operators.translate_speech" - "airflow.providers.google.cloud.operators.vertex_ai.auto_ml" - "airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job" - "airflow.providers.google.cloud.operators.vertex_ai.custom_job" - "airflow.providers.google.cloud.operators.vertex_ai.dataset" - "airflow.providers.google.cloud.operators.vertex_ai.endpoint_service" - "airflow.providers.google.cloud.operators.vertex_ai.generative_model" - "airflow.providers.google.cloud.operators.vertex_ai.hyperparameter_tuning_job" - "airflow.providers.google.cloud.operators.vertex_ai.model_service" - "airflow.providers.google.cloud.operators.vertex_ai.pipeline_job" - "airflow.providers.google.cloud.operators.video_intelligence" - "airflow.providers.google.cloud.operators.vision" - "airflow.providers.google.cloud.operators.workflows" - "airflow.providers.google.common.hooks.base_google" - "airflow.providers.google.common.hooks.discovery_api" - "airflow.providers.google.firebase.hooks.firestore" - "airflow.providers.google.firebase.operators.firestore" - "airflow.providers.google.leveldb.hooks.leveldb" - "airflow.providers.google.leveldb.operators.leveldb" - "airflow.providers.google.marketing_platform.hooks.analytics" - "airflow.providers.google.marketing_platform.hooks.analytics_admin" - "airflow.providers.google.marketing_platform.hooks.campaign_manager" - "airflow.providers.google.marketing_platform.hooks.display_video" - "airflow.providers.google.marketing_platform.hooks.search_ads" - "airflow.providers.google.marketing_platform.operators.analytics" - "airflow.providers.google.marketing_platform.operators.analytics_admin" - "airflow.providers.google.marketing_platform.operators.campaign_manager" - "airflow.providers.google.marketing_platform.operators.display_video" - "airflow.providers.google.marketing_platform.operators.search_ads" - "airflow.providers.google.suite.hooks.calendar" - "airflow.providers.google.suite.hooks.drive" - "airflow.providers.google.suite.hooks.sheets" - "airflow.providers.google.suite.operators.sheets" + "airflow.providers.google" + "airflow.providers.google.get_provider_info" ]; + version = "17.2.0"; }; + grpc = { deps = [ "google-auth" @@ -1049,112 +551,24 @@ "grpcio" ]; imports = [ - "airflow.providers.grpc.hooks.grpc" - "airflow.providers.grpc.operators.grpc" + "airflow.providers.grpc" + "airflow.providers.grpc.get_provider_info" ]; + version = "3.8.2"; }; + hashicorp = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" "hvac" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" ]; - imports = [ "airflow.providers.hashicorp.hooks.vault" ]; + imports = [ + "airflow.providers.hashicorp" + "airflow.providers.hashicorp.get_provider_info" + ]; + version = "4.3.2"; }; + http = { deps = [ "aiohttp" @@ -1163,48 +577,64 @@ "requests-toolbelt" ]; imports = [ - "airflow.providers.http.hooks.http" - "airflow.providers.http.operators.http" + "airflow.providers.http" + "airflow.providers.http.get_provider_info" ]; + version = "5.3.4"; }; + imap = { deps = [ ]; - imports = [ "airflow.providers.imap.hooks.imap" ]; + imports = [ + "airflow.providers.imap" + "airflow.providers.imap.get_provider_info" + ]; + version = "3.9.2"; }; + influxdb = { deps = [ "influxdb-client" "requests" ]; imports = [ - "airflow.providers.influxdb.hooks.influxdb" - "airflow.providers.influxdb.operators.influxdb" + "airflow.providers.influxdb" + "airflow.providers.influxdb.get_provider_info" ]; + version = "2.9.3"; }; + jdbc = { deps = [ "jaydebeapi" ]; imports = [ - "airflow.providers.jdbc.hooks.jdbc" - "airflow.providers.jdbc.operators.jdbc" + "airflow.providers.jdbc" + "airflow.providers.jdbc.get_provider_info" ]; + version = "5.2.3"; }; + jenkins = { deps = [ "python-jenkins" ]; imports = [ - "airflow.providers.jenkins.hooks.jenkins" - "airflow.providers.jenkins.operators.jenkins_job_trigger" + "airflow.providers.jenkins" + "airflow.providers.jenkins.get_provider_info" ]; + version = "4.1.2"; }; + + keycloak = { + deps = [ "python-keycloak" ]; + imports = [ + "airflow.providers.keycloak" + "airflow.providers.keycloak.get_provider_info" + ]; + version = "0.1.0"; + }; + microsoft_azure = { deps = [ "adal" "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" "azure-batch" "azure-cosmos" "azure-datalake-store" @@ -1224,1051 +654,407 @@ "azure-storage-file-share" "azure-synapse-artifacts" "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" "microsoft-kiota-abstractions" + "microsoft-kiota-authentication-azure" + "microsoft-kiota-http" + "microsoft-kiota-serialization-json" + "microsoft-kiota-serialization-text" + "msal-extensions" "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" ]; imports = [ - "airflow.providers.microsoft.azure.hooks.adx" - "airflow.providers.microsoft.azure.hooks.asb" - "airflow.providers.microsoft.azure.hooks.base_azure" - "airflow.providers.microsoft.azure.hooks.batch" - "airflow.providers.microsoft.azure.hooks.container_instance" - "airflow.providers.microsoft.azure.hooks.container_registry" - "airflow.providers.microsoft.azure.hooks.container_volume" - "airflow.providers.microsoft.azure.hooks.cosmos" - "airflow.providers.microsoft.azure.hooks.data_factory" - "airflow.providers.microsoft.azure.hooks.data_lake" - "airflow.providers.microsoft.azure.hooks.data_lake" - "airflow.providers.microsoft.azure.hooks.fileshare" - "airflow.providers.microsoft.azure.hooks.msgraph" - "airflow.providers.microsoft.azure.hooks.synapse" - "airflow.providers.microsoft.azure.hooks.wasb" - "airflow.providers.microsoft.azure.operators.adls" - "airflow.providers.microsoft.azure.operators.adx" - "airflow.providers.microsoft.azure.operators.asb" - "airflow.providers.microsoft.azure.operators.batch" - "airflow.providers.microsoft.azure.operators.container_instances" - "airflow.providers.microsoft.azure.operators.cosmos" - "airflow.providers.microsoft.azure.operators.data_factory" - "airflow.providers.microsoft.azure.operators.msgraph" - "airflow.providers.microsoft.azure.operators.synapse" - "airflow.providers.microsoft.azure.operators.wasb_delete_blob" + "airflow.providers.microsoft.azure" + "airflow.providers.microsoft.azure.get_provider_info" ]; + version = "12.7.0"; }; + microsoft_mssql = { - deps = [ "pymssql" ]; - imports = [ - "airflow.providers.microsoft.mssql.hooks.mssql" - "airflow.providers.microsoft.mssql.operators.mssql" + deps = [ + "methodtools" + "pymssql" ]; + imports = [ + "airflow.providers.microsoft.mssql" + "airflow.providers.microsoft.mssql.get_provider_info" + ]; + version = "4.3.2"; }; + microsoft_psrp = { deps = [ "pypsrp" ]; imports = [ - "airflow.providers.microsoft.psrp.hooks.psrp" - "airflow.providers.microsoft.psrp.operators.psrp" + "airflow.providers.microsoft.psrp" + "airflow.providers.microsoft.psrp.get_provider_info" ]; + version = "3.1.2"; }; + microsoft_winrm = { deps = [ "pywinrm" ]; imports = [ - "airflow.providers.microsoft.winrm.hooks.winrm" - "airflow.providers.microsoft.winrm.operators.winrm" + "airflow.providers.microsoft.winrm" + "airflow.providers.microsoft.winrm.get_provider_info" ]; + version = "3.11.0"; }; + mongo = { deps = [ "dnspython" "pymongo" ]; - imports = [ "airflow.providers.mongo.hooks.mongo" ]; + imports = [ + "airflow.providers.mongo" + "airflow.providers.mongo.get_provider_info" + ]; + version = "5.2.2"; }; + mysql = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" + "aiomysql" "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" ]; imports = [ - "airflow.providers.mysql.hooks.mysql" - "airflow.providers.mysql.operators.mysql" + "airflow.providers.mysql" + "airflow.providers.mysql.get_provider_info" ]; + version = "6.3.4"; }; + neo4j = { deps = [ "neo4j" ]; imports = [ - "airflow.providers.neo4j.hooks.neo4j" - "airflow.providers.neo4j.operators.neo4j" + "airflow.providers.neo4j" + "airflow.providers.neo4j.get_provider_info" ]; + version = "3.10.1"; }; + odbc = { deps = [ "pyodbc" ]; - imports = [ "airflow.providers.odbc.hooks.odbc" ]; - }; - openai = { - deps = [ ]; imports = [ - "airflow.providers.openai.hooks.openai" - "airflow.providers.openai.operators.openai" + "airflow.providers.odbc" + "airflow.providers.odbc.get_provider_info" ]; + version = "4.10.2"; }; + + openai = { + deps = [ "openai" ]; + imports = [ + "airflow.providers.openai" + "airflow.providers.openai.get_provider_info" + ]; + version = "1.6.2"; + }; + openfaas = { deps = [ ]; - imports = [ "airflow.providers.openfaas.hooks.openfaas" ]; + imports = [ + "airflow.providers.openfaas" + "airflow.providers.openfaas.get_provider_info" + ]; + version = "3.8.2"; }; + openlineage = { deps = [ "attrs" ]; - imports = [ ]; + imports = [ + "airflow.providers.openlineage" + "airflow.providers.openlineage.get_provider_info" + ]; + version = "2.7.0"; }; + opensearch = { deps = [ "opensearch-py" ]; imports = [ - "airflow.providers.opensearch.hooks.opensearch" - "airflow.providers.opensearch.operators.opensearch" + "airflow.providers.opensearch" + "airflow.providers.opensearch.get_provider_info" ]; + version = "1.7.2"; }; + opsgenie = { deps = [ ]; imports = [ - "airflow.providers.opsgenie.hooks.opsgenie" - "airflow.providers.opsgenie.operators.opsgenie" + "airflow.providers.opsgenie" + "airflow.providers.opsgenie.get_provider_info" ]; + version = "5.9.2"; }; + oracle = { - deps = [ "oracledb" ]; - imports = [ - "airflow.providers.oracle.hooks.oracle" - "airflow.providers.oracle.operators.oracle" + deps = [ + "numpy" + "oracledb" ]; + imports = [ + "airflow.providers.oracle" + "airflow.providers.oracle.get_provider_info" + ]; + version = "4.2.0"; }; + pagerduty = { deps = [ ]; imports = [ - "airflow.providers.pagerduty.hooks.pagerduty" - "airflow.providers.pagerduty.hooks.pagerduty_events" + "airflow.providers.pagerduty" + "airflow.providers.pagerduty.get_provider_info" ]; + version = "5.0.2"; }; + papermill = { deps = [ "ipykernel" + "nbconvert" "pandas" + "papermill" ]; imports = [ - "airflow.providers.papermill.hooks.kernel" - "airflow.providers.papermill.operators.papermill" + "airflow.providers.papermill" + "airflow.providers.papermill.get_provider_info" ]; + version = "3.11.2"; }; + pgvector = { - deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; + deps = [ "pgvector" ]; imports = [ - "airflow.providers.pgvector.hooks.pgvector" - "airflow.providers.pgvector.operators.pgvector" + "airflow.providers.pgvector" + "airflow.providers.pgvector.get_provider_info" ]; + version = "1.5.2"; }; + pinecone = { - deps = [ "pinecone-client" ]; + deps = [ ]; imports = [ - "airflow.providers.pinecone.hooks.pinecone" - "airflow.providers.pinecone.operators.pinecone" + "airflow.providers.pinecone" + "airflow.providers.pinecone.get_provider_info" ]; + version = "2.3.4"; }; + postgres = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" + "asyncpg" "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" + "polars" + "psycopg" + "psycopg2-binary" ]; imports = [ - "airflow.providers.postgres.hooks.postgres" - "airflow.providers.postgres.operators.postgres" + "airflow.providers.postgres" + "airflow.providers.postgres.get_provider_info" ]; + version = "6.3.0"; }; + presto = { deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" + "presto-python-client" ]; - imports = [ "airflow.providers.presto.hooks.presto" ]; + imports = [ + "airflow.providers.presto" + "airflow.providers.presto.get_provider_info" + ]; + version = "5.9.2"; }; + qdrant = { deps = [ "qdrant-client" ]; imports = [ - "airflow.providers.qdrant.hooks.qdrant" - "airflow.providers.qdrant.operators.qdrant" + "airflow.providers.qdrant" + "airflow.providers.qdrant.get_provider_info" ]; + version = "1.4.3"; }; + redis = { deps = [ "redis" ]; imports = [ - "airflow.providers.redis.hooks.redis" - "airflow.providers.redis.operators.redis_publish" + "airflow.providers.redis" + "airflow.providers.redis.get_provider_info" ]; + version = "4.3.0"; }; + salesforce = { deps = [ "pandas" "simple-salesforce" ]; imports = [ - "airflow.providers.salesforce.hooks.salesforce" - "airflow.providers.salesforce.operators.bulk" - "airflow.providers.salesforce.operators.salesforce_apex_rest" + "airflow.providers.salesforce" + "airflow.providers.salesforce.get_provider_info" ]; + version = "5.11.3"; }; + samba = { - deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" + deps = [ "smbprotocol" ]; + imports = [ + "airflow.providers.samba" + "airflow.providers.samba.get_provider_info" ]; - imports = [ "airflow.providers.samba.hooks.samba" ]; + version = "4.11.0"; }; + segment = { deps = [ ]; imports = [ - "airflow.providers.segment.hooks.segment" - "airflow.providers.segment.operators.segment_track_event" + "airflow.providers.segment" + "airflow.providers.segment.get_provider_info" ]; + version = "3.8.2"; }; + sendgrid = { - deps = [ "sendgrid" ]; - imports = [ ]; + deps = [ + "sendgrid" + "werkzeug" + ]; + imports = [ + "airflow.providers.sendgrid" + "airflow.providers.sendgrid.get_provider_info" + ]; + version = "4.1.3"; }; + sftp = { deps = [ "asyncssh" - "attrs" "paramiko" "pysftp" - "sshtunnel" ]; imports = [ - "airflow.providers.sftp.hooks.sftp" - "airflow.providers.sftp.operators.sftp" + "airflow.providers.sftp" + "airflow.providers.sftp.get_provider_info" ]; + version = "5.4.0"; }; + singularity = { deps = [ "spython" ]; - imports = [ "airflow.providers.singularity.operators.singularity" ]; + imports = [ + "airflow.providers.singularity" + "airflow.providers.singularity.get_provider_info" + ]; + version = "3.8.2"; }; + slack = { deps = [ "slack-sdk" ]; imports = [ - "airflow.providers.slack.hooks.slack" - "airflow.providers.slack.hooks.slack_webhook" - "airflow.providers.slack.operators.slack" - "airflow.providers.slack.operators.slack_webhook" + "airflow.providers.slack" + "airflow.providers.slack.get_provider_info" ]; + version = "9.2.0"; }; + smtp = { deps = [ ]; imports = [ - "airflow.providers.smtp.hooks.smtp" - "airflow.providers.smtp.operators.smtp" + "airflow.providers.smtp" + "airflow.providers.smtp.get_provider_info" ]; + version = "2.3.1"; }; + snowflake = { deps = [ - "attrs" "pandas" "pyarrow" "snowflake-connector-python" "snowflake-sqlalchemy" ]; imports = [ - "airflow.providers.snowflake.hooks.snowflake" - "airflow.providers.snowflake.hooks.snowflake_sql_api" - "airflow.providers.snowflake.operators.snowflake" + "airflow.providers.snowflake" + "airflow.providers.snowflake.get_provider_info" ]; + version = "6.5.3"; }; + sqlite = { deps = [ ]; imports = [ - "airflow.providers.sqlite.hooks.sqlite" - "airflow.providers.sqlite.operators.sqlite" + "airflow.providers.sqlite" + "airflow.providers.sqlite.get_provider_info" ]; + version = "4.1.2"; }; + ssh = { deps = [ "paramiko" "sshtunnel" ]; imports = [ - "airflow.providers.ssh.hooks.ssh" - "airflow.providers.ssh.operators.ssh" + "airflow.providers.ssh" + "airflow.providers.ssh.get_provider_info" ]; + version = "4.1.3"; }; + + standard = { + deps = [ ]; + imports = [ + "airflow.providers.standard" + "airflow.providers.standard.get_provider_info" + ]; + version = "1.9.0"; + }; + tableau = { deps = [ "tableauserverclient" ]; imports = [ - "airflow.providers.tableau.hooks.tableau" - "airflow.providers.tableau.operators.tableau" + "airflow.providers.tableau" + "airflow.providers.tableau.get_provider_info" ]; + version = "5.2.0"; }; - tabular = { - deps = [ ]; - imports = [ ]; - }; + telegram = { deps = [ "python-telegram-bot" ]; imports = [ - "airflow.providers.telegram.hooks.telegram" - "airflow.providers.telegram.operators.telegram" + "airflow.providers.telegram" + "airflow.providers.telegram.get_provider_info" ]; + version = "4.8.2"; }; + teradata = { - deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; + deps = [ ]; imports = [ - "airflow.providers.teradata.hooks.teradata" - "airflow.providers.teradata.operators.teradata" - "airflow.providers.teradata.operators.teradata_compute_cluster" + "airflow.providers.teradata" + "airflow.providers.teradata.get_provider_info" ]; + version = "3.2.1"; }; + trino = { - deps = [ - "adal" - "adlfs" - "aiofiles" - "aiohttp" - "apache-beam" - "asgiref" - "asyncssh" - "attrs" - "azure-batch" - "azure-cosmos" - "azure-datalake-store" - "azure-identity" - "azure-keyvault-secrets" - "azure-kusto-data" - "azure-mgmt-containerinstance" - "azure-mgmt-containerregistry" - "azure-mgmt-cosmosdb" - "azure-mgmt-datafactory" - "azure-mgmt-datalake-store" - "azure-mgmt-resource" - "azure-mgmt-storage" - "azure-servicebus" - "azure-storage-blob" - "azure-storage-file-datalake" - "azure-storage-file-share" - "azure-synapse-artifacts" - "azure-synapse-spark" - "boto3" - "botocore" - "cassandra-driver" - "cryptography" - "dill" - "dnspython" - "gcsfs" - "google-api-core" - "google-api-python-client" - "google-auth" - "google-auth-httplib2" - "google-cloud-automl" - "google-cloud-bigquery" - "google-cloud-bigquery-datatransfer" - "google-cloud-bigtable" - "google-cloud-compute" - "google-cloud-container" - "google-cloud-datacatalog" - "google-cloud-dataproc" - "google-cloud-dlp" - "google-cloud-kms" - "google-cloud-language" - "google-cloud-logging" - "google-cloud-monitoring" - "google-cloud-pubsub" - "google-cloud-redis" - "google-cloud-secret-manager" - "google-cloud-spanner" - "google-cloud-speech" - "google-cloud-storage" - "google-cloud-tasks" - "google-cloud-texttospeech" - "google-cloud-translate" - "google-cloud-videointelligence" - "google-cloud-vision" - "google-cloud-workflows" - "google-re2" - "grpcio-gcp" - "httpx" - "inflection" - "jmespath" - "json-merge-patch" - "jsonpath-ng" - "kubernetes" - "kubernetes-asyncio" - "microsoft-kiota-abstractions" - "msgraph-core" - "mysqlclient" - "oracledb" - "pandas" - "paramiko" - "proto-plus" - "psycopg2" - "pyarrow" - "pyathena" - "pymongo" - "pymssql" - "pyopenssl" - "pysftp" - "python-slugify" - "redshift-connector" - "requests" - "requests-toolbelt" - "simple-salesforce" - "smbprotocol" - "sshtunnel" - "tenacity" - "thrift" - "vertica-python" - ]; + deps = [ "pandas" ]; imports = [ - "airflow.providers.trino.hooks.trino" - "airflow.providers.trino.operators.trino" + "airflow.providers.trino" + "airflow.providers.trino.get_provider_info" ]; + version = "6.3.3"; }; + vertica = { deps = [ "vertica-python" ]; imports = [ - "airflow.providers.vertica.hooks.vertica" - "airflow.providers.vertica.operators.vertica" + "airflow.providers.vertica" + "airflow.providers.vertica.get_provider_info" ]; + version = "4.1.2"; }; + weaviate = { deps = [ "httpx" @@ -2276,33 +1062,37 @@ "weaviate-client" ]; imports = [ - "airflow.providers.weaviate.hooks.weaviate" - "airflow.providers.weaviate.operators.weaviate" + "airflow.providers.weaviate" + "airflow.providers.weaviate.get_provider_info" ]; + version = "3.2.3"; }; + yandex = { deps = [ ]; imports = [ - "airflow.providers.yandex.hooks.dataproc" - "airflow.providers.yandex.hooks.yandex" - "airflow.providers.yandex.hooks.yq" - "airflow.providers.yandex.operators.dataproc" - "airflow.providers.yandex.operators.yq" + "airflow.providers.yandex" + "airflow.providers.yandex.get_provider_info" ]; + version = "4.2.0"; }; + ydb = { deps = [ ]; imports = [ - "airflow.providers.ydb.hooks._vendor.dbapi.connection" - "airflow.providers.ydb.hooks._vendor.dbapi.constants" - "airflow.providers.ydb.hooks._vendor.dbapi.cursor" - "airflow.providers.ydb.hooks._vendor.dbapi.errors" - "airflow.providers.ydb.hooks.ydb" - "airflow.providers.ydb.operators.ydb" + "airflow.providers.ydb" + "airflow.providers.ydb.get_provider_info" ]; + version = "2.2.2"; }; + zendesk = { deps = [ ]; - imports = [ "airflow.providers.zendesk.hooks.zendesk" ]; + imports = [ + "airflow.providers.zendesk" + "airflow.providers.zendesk.get_provider_info" + ]; + version = "4.10.2"; }; + } diff --git a/pkgs/by-name/ap/apache-airflow/python-package.nix b/pkgs/by-name/ap/apache-airflow/python-package.nix index bce488ed44ec..d08ed53b8079 100644 --- a/pkgs/by-name/ap/apache-airflow/python-package.nix +++ b/pkgs/by-name/ap/apache-airflow/python-package.nix @@ -2,115 +2,156 @@ lib, stdenv, fetchFromGitHub, - writeScript, + writableTmpDirAsHomeHook, # javascript - fetchYarnDeps, + fetchPnpmDeps, nodejs, - webpack-cli, - yarnBuildHook, - yarnConfigHook, + pnpm, + pnpmConfigHook, # python + a2wsgi, + aiosqlite, alembic, argcomplete, + asgiref, + attrs, + babel, buildPythonPackage, + cadwyn, colorlog, - configupdater, - connexion, cron-descriptor, croniter, cryptography, + deprecated, dill, - flask-caching, - flask-login, - flask-session, + fastapi, + flit-core, fsspec, gitdb, gitpython, - gunicorn, + greenback, hatchling, + httpx, + importlib-metadata, + itsdangerous, + jinja2, + jsonschema, lazy-object-proxy, + libcst, linkify-it-py, lockfile, - marshmallow-oneofschema, - mdit-py-plugins, methodtools, + msgspec, + natsort, opentelemetry-api, opentelemetry-exporter-otlp, packaging, - pandas, pathspec, pendulum, pluggy, psutil, - pytest-asyncio, - pytestCheckHook, + pydantic, + pygments, + pygtrie, + pyjwt, python, python-daemon, - python-nvd3, + python-dateutil, python-slugify, + pyyaml, + requests, + retryhttp, + rich, rich-argparse, + rich-click, setproctitle, smmap, sqlalchemy, sqlalchemy-jsonfield, + sqlalchemy-utils, + starlette, + structlog, + svcs, tabulate, tenacity, termcolor, tomli, trove-classifiers, + types-requests, + typing-extensions, universal-pathlib, + uuid6, + uvicorn, - # Extra airflow providers to enable - enabledProviders ? [ ], + enabledProviders, }: let - version = "2.10.5"; + version = "3.1.6"; - airflow-src = fetchFromGitHub { + src = fetchFromGitHub { owner = "apache"; repo = "airflow"; tag = version; - # Download using the git protocol rather than using tarballs, because the - # GitHub archive tarballs don't appear to include tests - forceFetchGit = true; - hash = "sha256-q5/CM+puXE31+15F3yZmcrR74LrqHppdCDUqjLQXPfk="; + hash = "sha256-wC6C0jhCA76/+KhBQbe3WeSGqR6FwaudCT5xPV39Z6c="; }; - # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree. - # This replicates this shell script, fixing bugs in yarn.lock and package.json - - airflow-frontend = stdenv.mkDerivation rec { - name = "airflow-frontend"; - - src = "${airflow-src}/airflow/www"; - - offlineCache = fetchYarnDeps { - yarnLock = "${src}/yarn.lock"; - hash = "sha256-hKgtMH4c8sPRDLPLVn+H8rmwc2Q6ei6U4er6fGuFn4I="; - }; + airflowUi = stdenv.mkDerivation rec { + pname = "airflow-ui-assets"; + inherit src version; + sourceRoot = "${src.name}/airflow-core/src/airflow/ui"; nativeBuildInputs = [ - yarnConfigHook - yarnBuildHook nodejs - webpack-cli + pnpm + pnpmConfigHook ]; - # The webpack license plugin tries to create /3rd-party-licenses when given the - # original relative path - postPatch = '' - sed -i 's!../../../../3rd-party-licenses/LICENSES-ui.txt!/3rd-party-licenses/LICENSES-ui.txt!' webpack.config.js - ''; + pnpmDeps = fetchPnpmDeps { + pname = "airflow-ui"; + inherit sourceRoot src version; + fetcherVersion = 1; + hash = "sha256-UcEFQkDZ9Ye+VfyJ9rdZKe0wilTgO4dMsULABWfL2Co="; + }; - postBuild = '' - find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5 + buildPhase = '' + pnpm install + pnpm build ''; installPhase = '' - mkdir -p $out/static/ - cp -r static/dist $out/static + mkdir -p $out/share/airflow/ui + cp -r dist $out/share/airflow/ui/ + ''; + }; + + airflowSimpleAuthUi = stdenv.mkDerivation rec { + pname = "airflow-simple-ui-assets"; + inherit src version; + sourceRoot = "${src.name}/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui"; + + nativeBuildInputs = [ + nodejs + pnpm + pnpmConfigHook + ]; + + pnpmDeps = fetchPnpmDeps { + pname = "simple-auth-manager-ui"; + inherit sourceRoot src version; + fetcherVersion = 1; + hash = "sha256-8nZdWnhERUkiaY8USyy/a/j+dMksjmEzCabSkysndSE="; + }; + + buildPhase = '' + pnpm install + pnpm build + ''; + + installPhase = '' + mkdir -p $out/share/airflow/simple-ui + cp -r dist $out/share/airflow/simple-ui/ ''; }; @@ -118,216 +159,231 @@ let "common_compat" "common_io" "common_sql" - "fab" - "ftp" - "http" - "imap" "smtp" - "sqlite" + "standard" ]; - # Import generated file with metadata for provider dependencies and imports. - # Enable additional providers using enabledProviders above. providers = import ./providers.nix; - getProviderPath = provider: lib.replaceStrings [ "_" ] [ "/" ] provider; - getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps; - getProviderImports = provider: providers.${provider}.imports; - providerImports = lib.concatMap getProviderImports enabledProviders; buildProvider = provider: - let - providerPath = getProviderPath provider; - in - python.pkgs.buildPythonPackage { + buildPythonPackage { pname = "apache-airflow-providers-${provider}"; - version = "unstable"; # will be extracted in the build phase - pyproject = false; # providers packages don't have pyproject.toml nor setup.py + version = providers.${provider}.version; + pyproject = true; - src = airflow-src; + inherit src; + sourceRoot = "${src.name}/providers/${lib.replaceStrings [ "_" ] [ "/" ] provider}"; - propagatedBuildInputs = getProviderDeps provider; - dependencies = [ packaging ]; + buildInputs = [ flit-core ]; - buildPhase = '' - # extract version from the provider's __init__.py file - if [ -f "airflow/providers/${providerPath}/__init__.py" ]; then - version=$(grep -oP "(?<=__version__ = ')[^']+" "airflow/providers/${providerPath}/__init__.py" || echo "0.0.0") - echo "Provider ${provider} version: $version" - else - echo "Error: __init__.py not found for provider ${provider} at path airflow/providers/${providerPath}" - exit 1 - fi - ''; + dependencies = map (dep: python.pkgs.${dep}) providers.${provider}.deps; - installPhase = '' - # create directory structure - mkdir -p $out/${python.sitePackages}/airflow/providers + pythonRemoveDeps = [ + "apache-airflow" + ]; - # copy the provider directory - if [ -d "airflow/providers/${providerPath}" ]; then - mkdir -p $out/${python.sitePackages}/airflow/providers/$(dirname "${providerPath}") - cp -r airflow/providers/${providerPath} $out/${python.sitePackages}/airflow/providers/$(dirname "${providerPath}") - - # create parent __init__.py files - touch $out/${python.sitePackages}/airflow/__init__.py - touch $out/${python.sitePackages}/airflow/providers/__init__.py - - # create any needed intermediate __init__.py files for nested providers - providerDir=$(dirname "${providerPath}") - while [ "$providerDir" != "." ] && [ -n "$providerDir" ]; do - mkdir -p $out/${python.sitePackages}/airflow/providers/$providerDir - touch $out/${python.sitePackages}/airflow/providers/$providerDir/__init__.py - providerDir=$(dirname "$providerDir") - done - - # create egg-info for package discovery - mkdir -p $out/${python.sitePackages}/apache_airflow_providers_${provider}.egg-info - cat > $out/${python.sitePackages}/apache_airflow_providers_${provider}.egg-info/PKG-INFO < yarn.nix - - # update provider dependencies - ./update-providers.py - ''; + passthru.updateScript = ./update.sh; + passthru.airflowUi = airflowUi; + passthru.airflowSimpleAuthUi = airflowSimpleAuthUi; # Note on testing the web UI: # You can (manually) test the web UI as follows: @@ -341,12 +397,13 @@ buildPythonPackage rec { # triggering the 'example_bash_operator' DAG and see if it reports success. meta = { - description = "Programmatically author, schedule and monitor data pipelines"; + description = "Platform to programmatically author, schedule and monitor workflows"; homepage = "https://airflow.apache.org/"; changelog = "https://airflow.apache.org/docs/apache-airflow/${version}/release_notes.html"; license = lib.licenses.asl20; maintainers = with lib.maintainers; [ taranarmo ]; + mainProgram = "airflow"; }; } diff --git a/pkgs/by-name/ap/apache-airflow/update-providers.py b/pkgs/by-name/ap/apache-airflow/update-providers.py index 32e24018d653..01ee2bd49fe0 100755 --- a/pkgs/by-name/ap/apache-airflow/update-providers.py +++ b/pkgs/by-name/ap/apache-airflow/update-providers.py @@ -1,4 +1,5 @@ -#! /usr/bin/env python3 +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p "python3.withPackages(ps: [ps.requests ps.tomli ps.packaging])" from itertools import chain import json @@ -11,7 +12,8 @@ import sys from typing import Dict, List, Optional, Set, TextIO from urllib.request import urlopen from urllib.error import HTTPError -import yaml +from packaging.requirements import Requirement +import tomli PKG_SET = "apache-airflow.pythonPackages" @@ -21,8 +23,9 @@ PKG_PREFERENCES = { "dnspython": "dnspython", "elasticsearch-dsl": "elasticsearch-dsl", "google-api-python-client": "google-api-python-client", + "numpy": "numpy", "protobuf": "protobuf", - "psycopg2-binary": "psycopg2", + "pydantic": "pydantic", "requests_toolbelt": "requests-toolbelt", } @@ -35,7 +38,7 @@ EXTRA_REQS = { def get_version(): with open(os.path.dirname(sys.argv[0]) + "/python-package.nix") as fh: # A version consists of digits, dots, and possibly a "b" (for beta) - m = re.search('version = "([\\d\\.b]+)";', fh.read()) + m = re.search(r'version = "([\d.b]+)";', fh.read()) return m.group(1) @@ -43,11 +46,11 @@ def get_file_from_github(version: str, path: str): with urlopen( f"https://raw.githubusercontent.com/apache/airflow/{version}/{path}" ) as response: - return yaml.safe_load(response) + return response.read() def repository_root() -> Path: - return Path(os.path.dirname(sys.argv[0])) / "../../../.." + return Path(os.path.dirname(sys.argv[0])) / "../../../../" def dump_packages() -> Dict[str, Dict[str, str]]: @@ -70,27 +73,33 @@ def dump_packages() -> Dict[str, Dict[str, str]]: def remove_version_constraint(req: str) -> str: - return re.sub(r"[=><~].*$", "", req) + parsed_req = Requirement(req) + name = parsed_req.name + extras = ",".join(sorted(parsed_req.extras)) + if extras: + return f"{name}[{extras}]" + return name def name_to_attr_path(req: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]: - if req in PKG_PREFERENCES: - return f"{PKG_SET}.{PKG_PREFERENCES[req]}" + # Extract the base name, removing any extras (e.g., '[flask]') + base_req_name = req.split("[")[0] + logging.debug(f"Searching for base_req_name: {base_req_name}") + if base_req_name in PKG_PREFERENCES: + return f"{PKG_SET}.{PKG_PREFERENCES[base_req_name]}" attr_paths = [] - names = [req] + names = [base_req_name] # E.g. python-mpd2 is actually called python3.6-mpd2 # instead of python-3.6-python-mpd2 inside Nixpkgs - if req.startswith("python-") or req.startswith("python_"): - names.append(req[len("python-") :]) + if base_req_name.startswith("python-") or base_req_name.startswith("python_"): + names.append(base_req_name[len("python-") :]) for name in names: # treat "-" and "_" equally name = re.sub("[-_]", "[-_]", name) # python(minor).(major)-(pname)-(version or unstable-date) # we need the version qualifier, or we'll have multiple matches # (e.g. pyserial and pyserial-asyncio when looking for pyserial) - pattern = re.compile( - f"^python\\d+\\.\\d+-{name}-(?:\\d|unstable-.*)", re.I - ) + pattern = re.compile(rf"^python\d+\.\d+-{name}-(?:[\d\.]+|unstable-.*)", re.I) for attr_path, package in packages.items(): # logging.debug("Checking match for %s with %s", name, package["name"]) if pattern.match(package["name"]): @@ -130,30 +139,82 @@ def get_cross_provider_reqs( if len(cross_provider_deps[provider]) > 0: reqs.update( chain.from_iterable( - get_cross_provider_reqs( - d, provider_reqs, cross_provider_deps, seen + [provider] + ( + get_cross_provider_reqs( + d, provider_reqs, cross_provider_deps, seen + [provider] + ) + if d not in seen + else [] ) - if d not in seen - else [] for d in cross_provider_deps[provider] ) ) return reqs -def get_provider_reqs(version: str, packages: Dict) -> Dict: - provider_dependencies = get_file_from_github( - version, "generated/provider_dependencies.json" - ) +def parse_pyproject_toml(version: str, provider_name: str) -> Dict: + provider_dir = provider_name.replace(".", "/") + path = f"providers/{provider_dir}/pyproject.toml" + try: + content = get_file_from_github(version, path) + data = tomli.loads(content.decode("utf-8")) + + dependencies = data.get("project", {}).get("dependencies", []) + + # Extract optional dependencies + optional_dependencies = data.get("project", {}).get("optional-dependencies", {}) + for opt_deps_list in optional_dependencies.values(): + dependencies.extend(opt_deps_list) + + imports = [] + # Heuristic to generate imports based on provider name + # This might not be exhaustive but covers common cases + base_import_path = f"airflow.providers.{provider_name.replace('-', '.')}" + imports.append(base_import_path) + + # Try to get more specific imports from provider_info entry point if available + provider_info_entry_point = ( + data.get("project", {}) + .get("entry-points", {}) + .get("apache_airflow_provider", {}) + .get("provider_info") + ) + if provider_info_entry_point: + module_path = provider_info_entry_point.split(":")[0] + if module_path not in imports: + imports.append(module_path) + + return { + "deps": dependencies, + "imports": sorted(list(set(imports))), # Remove duplicates and sort + "cross-providers-deps": [], # pyproject.toml doesn't directly list cross-provider deps + "version": data.get("project", {}).get("version", "unknown"), + } + except HTTPError: + logging.warning("Couldn't get pyproject.toml for %s", provider_name) + return {"deps": [], "imports": [], "cross-providers-deps": []} + except Exception as e: + logging.error(f"Error parsing pyproject.toml for {provider_name}: {e}") + return {"deps": [], "imports": [], "cross-providers-deps": []} + + +def get_provider_reqs(version: str, packages: Dict, provider_names: List[str]) -> Dict: + provider_data = {} + for provider in provider_names: + data = parse_pyproject_toml(version, provider) + provider_data[provider] = { + "deps": data["deps"], + "cross-providers-deps": data["cross-providers-deps"], + } + provider_reqs = {} cross_provider_deps = {} - for provider, provider_data in provider_dependencies.items(): + for provider, data in provider_data.items(): provider_reqs[provider] = list( - provider_reqs_to_attr_paths(provider_data["deps"], packages) + provider_reqs_to_attr_paths(data["deps"], packages) ) + EXTRA_REQS.get(provider, []) - cross_provider_deps[provider] = [ - d for d in provider_data["cross-providers-deps"] if d != "common.sql" - ] + cross_provider_deps[provider] = data["cross-providers-deps"] + transitive_provider_reqs = {} # Add transitive cross-provider reqs for provider in provider_reqs: @@ -163,39 +224,25 @@ def get_provider_reqs(version: str, packages: Dict) -> Dict: return transitive_provider_reqs -def get_provider_yaml(version: str, provider: str) -> Dict: - provider_dir = provider.replace(".", "/") - path = f"airflow/providers/{provider_dir}/provider.yaml" - try: - return get_file_from_github(version, path) - except HTTPError: - logging.warning("Couldn't get provider yaml for %s", provider) - return {} - - -def get_provider_imports(version: str, providers) -> Dict: +def get_provider_imports(version: str, provider_names: List[str]) -> Dict: provider_imports = {} - for provider in providers: - provider_yaml = get_provider_yaml(version, provider) - imports: List[str] = [] - if "hooks" in provider_yaml: - imports.extend( - chain.from_iterable( - hook["python-modules"] for hook in provider_yaml["hooks"] - ) - ) - if "operators" in provider_yaml: - imports.extend( - chain.from_iterable( - operator["python-modules"] - for operator in provider_yaml["operators"] - ) - ) - provider_imports[provider] = imports + for provider in provider_names: + data = parse_pyproject_toml(version, provider) + provider_imports[provider] = data["imports"] return provider_imports -def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None: +def get_provider_versions(version: str, provider_names: List[str]) -> Dict: + provider_versions = {} + for provider in provider_names: + data = parse_pyproject_toml(version, provider) + provider_versions[provider] = data["version"] + return provider_versions + + +def to_nix_expr( + provider_reqs: Dict, provider_imports: Dict, provider_versions: Dict, fh: TextIO +) -> None: fh.write("# Warning: generated by update-providers.py, do not update manually\n") fh.write("{\n") for provider, reqs in provider_reqs.items(): @@ -209,19 +256,83 @@ def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None + " ".join(sorted(f'"{imp}"' for imp in provider_imports[provider])) + " ];\n" ) + fh.write(f' version = "{provider_versions[provider]}";\n') fh.write(" };\n") + fh.write("\n") fh.write("}\n") +def get_all_providers_from_github(version: str) -> List[str]: + """ + Fetches all provider paths that contain a pyproject.toml file from the airflow repository. + This is done by recursively fetching the git tree for the providers directory. + """ + providers = [] + try: + # Get the SHA of the 'providers' directory for the given version + with urlopen( + f"https://api.github.com/repos/apache/airflow/contents/?ref={version}" + ) as response: + root_contents = json.loads(response.read()) + + providers_dir_info = next( + ( + item + for item in root_contents + if item["name"] == "providers" and item["type"] == "dir" + ), + None, + ) + + if not providers_dir_info: + logging.error( + "Could not find 'providers' directory in the root of the repository." + ) + sys.exit(1) + + providers_dir_sha = providers_dir_info["sha"] + + # Now, get the tree for the 'providers' directory recursively + with urlopen( + f"https://api.github.com/repos/apache/airflow/git/trees/{providers_dir_sha}?recursive=1" + ) as response: + tree = json.loads(response.read()) + + pyproject_paths = [ + item["path"] + for item in tree["tree"] + if item["type"] == "blob" and item["path"].endswith("/pyproject.toml") + ] + + for path in pyproject_paths: + # remove /pyproject.toml suffix + provider_path = path[: -len("/pyproject.toml")] + # Exclude tests directory and empty paths (root pyproject.toml) + if provider_path and not provider_path.startswith("tests/"): + provider_name = provider_path.replace("/", ".") + providers.append(provider_name) + + except HTTPError as e: + logging.error(f"Error fetching provider list from GitHub: {e}") + sys.exit(1) + + return sorted(providers) + + def main() -> None: - logging.basicConfig(level=logging.INFO) + logging.basicConfig(level=logging.DEBUG) version = get_version() packages = dump_packages() logging.info("Generating providers.nix for version %s", version) - provider_reqs = get_provider_reqs(version, packages) - provider_imports = get_provider_imports(version, provider_reqs.keys()) - with open("providers.nix", "w") as fh: - to_nix_expr(provider_reqs, provider_imports, fh) + + # Fetch provider names from GitHub API + provider_names = get_all_providers_from_github(version) + + provider_reqs = get_provider_reqs(version, packages, provider_names) + provider_imports = get_provider_imports(version, provider_names) + provider_versions = get_provider_versions(version, provider_names) + with open(os.path.join(os.path.dirname(sys.argv[0]), "providers.nix"), "w") as fh: + to_nix_expr(provider_reqs, provider_imports, provider_versions, fh) if __name__ == "__main__": diff --git a/pkgs/by-name/ap/apache-airflow/update.sh b/pkgs/by-name/ap/apache-airflow/update.sh new file mode 100755 index 000000000000..943f73dcc6bf --- /dev/null +++ b/pkgs/by-name/ap/apache-airflow/update.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p common-updater-scripts curl jq nixfmt-tree + +set -euo pipefail + +# This script updates apache-airflow to the latest tag for a given major version. +# It defaults to major version 3. +# +# Usage: ./update.sh [MAJOR_VERSION] +# Example: ./update.sh 2 + +MAJOR_VERSION="${1:-3}" +PACKAGE_DIR=$(dirname "$0") +PACKAGE_NAME="apache-airflow" + +# Fetches the latest git tag for the given major version from the GitHub API. +# It filters for tags that look like version numbers and sorts them to find the latest. +get_latest_tag() { + TEMP_FILE=$(mktemp) + curl -s "https://api.github.com/repos/apache/airflow/releases" > "$TEMP_FILE" + LATEST_TAG=$(jq -r --arg major_version "$MAJOR_VERSION" ' + .[] + | select(.tag_name | test("^" + $major_version + "\\.\\d+\\.\\d+$")) + | .tag_name + ' "$TEMP_FILE" | sort -V | tail -n 1) + rm "$TEMP_FILE" + echo "$LATEST_TAG" +} + +echo "Looking for latest tag for major version $MAJOR_VERSION..." +LATEST_TAG=$(get_latest_tag) + +if [[ -z "$LATEST_TAG" ]]; then + echo "No new tag found for major version $MAJOR_VERSION." + exit 0 +fi + +echo "Latest tag found: $LATEST_TAG" + +# Update the version and hash in the package definition +# This uses the standard nixpkgs script `update-source-version`. +echo "Updating source version for $PACKAGE_NAME to $LATEST_TAG..." +update-source-version "$PACKAGE_NAME" "$LATEST_TAG" + +# After updating the main package, run the providers update script. +# It will automatically pick up the new version from the nix file. +echo "Updating provider dependencies..." +"$PACKAGE_DIR/update-providers.py" +echo "Formatting generated providers.nix" +treefmt "$PACKAGE_DIR" + +echo "Update complete. Please check the git diff for changes." + diff --git a/pkgs/development/python-modules/cadwyn/default.nix b/pkgs/development/python-modules/cadwyn/default.nix new file mode 100644 index 000000000000..34fc2f74030e --- /dev/null +++ b/pkgs/development/python-modules/cadwyn/default.nix @@ -0,0 +1,76 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + hatchling, + pytestCheckHook, + # runtime dependencies + fastapi, + issubclass, + jinja2, + pydantic, + starlette, + typing-extensions, + typing-inspection, + # test dependencies + dirty-equals, + httpx, + inline-snapshot, + pydantic-settings, + pytest-fixture-classes, + python-multipart, + svcs, + typer, + uvicorn, + pythonAtLeast, +}: + +buildPythonPackage rec { + pname = "cadwyn"; + version = "5.6.1"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "zmievsa"; + repo = "cadwyn"; + tag = version; + hash = "sha256-VVi79c/Y1mu520H/7gy9lGdBIVuKsYedU49P501NQao="; + }; + + disabled = pythonAtLeast "3.14"; + + build-system = [ hatchling ]; + + dependencies = [ + fastapi + issubclass + jinja2 + pydantic + starlette + typing-extensions + typing-inspection + ]; + + pythonImportsCheck = [ "cadwyn" ]; + + nativeCheckInputs = [ + dirty-equals + httpx + inline-snapshot + pydantic-settings + pytest-fixture-classes + pytestCheckHook + python-multipart + svcs + typer + uvicorn + ]; + + meta = { + description = "Production-ready community-driven modern Stripe-like API versioning in FastAPI"; + homepage = "https://github.com/zmievsa/cadwyn"; + changelog = "https://github.com/zmievsa/cadwyn/releases/tag/${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/development/python-modules/greenback/default.nix b/pkgs/development/python-modules/greenback/default.nix new file mode 100644 index 000000000000..fcf90d4ce115 --- /dev/null +++ b/pkgs/development/python-modules/greenback/default.nix @@ -0,0 +1,47 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + + asgiref, + greenlet, + importlib-metadata, + outcome, + setuptools, + sniffio, +}: + +buildPythonPackage rec { + pname = "greenback"; + version = "1.2.1"; + pyproject = true; + + src = fetchFromGitHub { + owner = "oremanj"; + repo = "greenback"; + tag = "v${version}"; + hash = "sha256-u7kihdE5EH3Xbq4dQGY4ojiootjw3muYnoI7d2MrMkE="; + }; + + build-system = [ + importlib-metadata + setuptools + ]; + + dependencies = [ + asgiref + greenlet + outcome + sniffio + ]; + + pythonImportsCheck = [ "greenback" ]; + + meta = { + description = "Use anyio and asyncio from synchronous code"; + homepage = "https://github.com/oremanj/greenback"; + changelog = "https://github.com/oremanj/greenback/blob/${src.rev}/CHANGES.rst"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/development/python-modules/issubclass/default.nix b/pkgs/development/python-modules/issubclass/default.nix new file mode 100644 index 000000000000..041ec408e62e --- /dev/null +++ b/pkgs/development/python-modules/issubclass/default.nix @@ -0,0 +1,30 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + poetry-core, +}: + +buildPythonPackage rec { + pname = "issubclass"; + version = "0.1.2"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "zmievsa"; + repo = "issubclass"; + tag = version; + hash = "sha256-a4EUbQY5duicnxuwtpBQ/7UBmtmeU+ZaySEnT4lmgLs="; + }; + + build-system = [ poetry-core ]; + + pythonImportsCheck = [ "issubclass" ]; + + meta = { + description = "issubclass() builtin that doesn't raise TypeError when arguments are not classes"; + homepage = "https://pypi.org/project/issubclass"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/development/python-modules/pytest-fixture-classes/default.nix b/pkgs/development/python-modules/pytest-fixture-classes/default.nix new file mode 100644 index 000000000000..6765247d940f --- /dev/null +++ b/pkgs/development/python-modules/pytest-fixture-classes/default.nix @@ -0,0 +1,40 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + + poetry-core, + pytest, + pytestCheckHook, + typing-extensions, +}: + +buildPythonPackage rec { + pname = "pytest-fixture-classes"; + version = "1.0.3"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "zmievsa"; + repo = "pytest-fixture-classes"; + tag = version; + hash = "sha256-A3HsDhCGxoJnkl841tZ/7lIn8UyGy5NRjVWODZV7aOQ="; + }; + + build-system = [ poetry-core ]; + + dependencies = [ + typing-extensions + ]; + + pythonImportsCheck = [ "pytest_fixture_classes" ]; + + nativeCheckInputs = [ pytestCheckHook ]; + + meta = { + description = "Fixtures as classes that work well with dependency injection, autocompletetion, type checkers, and language servers"; + homepage = "https://github.com/zmievsa/pytest-fixture-classes"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/development/python-modules/retryhttp/default.nix b/pkgs/development/python-modules/retryhttp/default.nix new file mode 100644 index 000000000000..8cc37882c709 --- /dev/null +++ b/pkgs/development/python-modules/retryhttp/default.nix @@ -0,0 +1,50 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + # build system + setuptools, + setuptools-scm, + # runtime + httpx, + pydantic, + requests, + tenacity, + types-requests, +}: + +buildPythonPackage rec { + pname = "retryhttp"; + version = "1.3.1"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "austind"; + repo = "retryhttp"; + tag = "release/v${version}"; + hash = "sha256-Jz073CTeIfPjcOQUGdb6/Q5OfEXZW5hTiEPKrfAK0Gg="; + }; + + build-system = [ + setuptools + setuptools-scm + ]; + + dependencies = [ + httpx + pydantic + requests + tenacity + types-requests + ]; + + pythonImportsCheck = [ "retryhttp" ]; + + meta = { + description = "Retry potentially transient HTTP errors in Python"; + homepage = "https://github.com/austind/retryhttp"; + changelog = "https://github.com/austind/retryhttp/releases/tag/release%2Fv${version}"; + license = lib.licenses.apsl20; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/development/python-modules/svcs/default.nix b/pkgs/development/python-modules/svcs/default.nix new file mode 100644 index 000000000000..08a1452b9bfb --- /dev/null +++ b/pkgs/development/python-modules/svcs/default.nix @@ -0,0 +1,73 @@ +{ + lib, + stdenv, + buildPythonPackage, + fetchFromGitHub, + + attrs, + hatch-fancy-pypi-readme, + hatch-vcs, + hatchling, + # test dependencies + aiohttp, + fastapi, + flask, + httpx, + pyramid, + pytest-asyncio, + pytestCheckHook, + starlette, + sybil, +}: + +buildPythonPackage rec { + pname = "svcs"; + version = "25.1.0"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "hynek"; + repo = "svcs"; + tag = version; + hash = "sha256-dDPmOKGifAGmAH3TD0NzJvR8lUB5qDWbxIwzHtNeF+4="; + }; + + build-system = [ + hatch-fancy-pypi-readme + hatch-vcs + hatchling + ]; + + dependencies = [ + attrs + ]; + + nativeCheckInputs = [ + aiohttp + fastapi + flask + httpx + pyramid + pytest-asyncio + pytestCheckHook + starlette + sybil + ]; + + disabledTests = lib.optionals stdenv.hostPlatform.isDarwin [ + "test_aclose_registry_ok" + "test_registrations" + "test_get_pings" + "test_client_pool_register_value" + ]; + + pythonImportsCheck = [ "svcs" ]; + + meta = { + description = "Flexible Service Locator for Python"; + homepage = "https://github.com/hynek/svcs"; + changelog = "https://github.com/hynek/svcs/releases/tag/${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ taranarmo ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index e64a6ef38fc4..c0f172b404a1 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2383,6 +2383,8 @@ self: super: with self; { cachy = callPackage ../development/python-modules/cachy { }; + cadwyn = callPackage ../development/python-modules/cadwyn { }; + caffe = toPythonModule ( pkgs.caffe.override { pythonSupport = true; @@ -6576,6 +6578,8 @@ self: super: with self; { green = callPackage ../development/python-modules/green { }; + greenback = callPackage ../development/python-modules/greenback { }; + greeneye-monitor = callPackage ../development/python-modules/greeneye-monitor { }; # built-in for pypi @@ -7578,6 +7582,8 @@ self: super: with self; { israel-rail-api = callPackage ../development/python-modules/israel-rail-api { }; + issubclass = callPackage ../development/python-modules/issubclass { }; + itables = callPackage ../development/python-modules/itables { }; itanium-demangler = callPackage ../development/python-modules/itanium-demangler { }; @@ -14963,6 +14969,8 @@ self: super: with self; { pytest-filter-subpackage = callPackage ../development/python-modules/pytest-filter-subpackage { }; + pytest-fixture-classes = callPackage ../development/python-modules/pytest-fixture-classes { }; + pytest-fixture-config = callPackage ../development/python-modules/pytest-fixture-config { }; pytest-flake8 = callPackage ../development/python-modules/pytest-flake8 { }; @@ -16478,6 +16486,8 @@ self: super: with self; { retry2 = callPackage ../development/python-modules/retry2 { }; + retryhttp = callPackage ../development/python-modules/retryhttp { }; + retrying = callPackage ../development/python-modules/retrying { }; returns = callPackage ../development/python-modules/returns { }; @@ -18408,6 +18418,8 @@ self: super: with self; { sv-ttk = callPackage ../development/python-modules/sv-ttk { }; + svcs = callPackage ../development/python-modules/svcs { }; + svg-path = callPackage ../development/python-modules/svg-path { }; svg-py = callPackage ../development/python-modules/svg-py { };