diff --git a/pkgs/development/python-modules/apache-airflow/default.nix b/pkgs/development/python-modules/apache-airflow/default.nix index 62e2c2f02ac9..e33b7f96513f 100644 --- a/pkgs/development/python-modules/apache-airflow/default.nix +++ b/pkgs/development/python-modules/apache-airflow/default.nix @@ -71,6 +71,9 @@ , pytestCheckHook , freezegun , mkYarnPackage + +# Extra airflow providers to enable +, enabledProviders ? [] }: let version = "2.3.4"; @@ -123,6 +126,13 @@ let ''; }; + # Import generated file with metadata for provider dependencies and imports. + # Enable additional providers using enabledProviders above. + providers = import ./providers.nix; + getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps; + getProviderImports = provider: providers.${provider}.imports; + providerDependencies = lib.concatMap getProviderDeps enabledProviders; + providerImports = lib.concatMap getProviderImports enabledProviders; in buildPythonPackage rec { pname = "apache-airflow"; @@ -198,7 +208,7 @@ buildPythonPackage rec { dataclasses ] ++ lib.optionals (pythonOlder "3.9") [ importlib-metadata - ]; + ] ++ providerDependencies; buildInputs = [ airflow-frontend @@ -209,6 +219,9 @@ buildPythonPackage rec { pytestCheckHook ]; + # By default, source code of providers is included but unusable due to missing + # transitive dependencies. To enable a provider, add it to extraProviders + # above INSTALL_PROVIDERS_FROM_SOURCES = "true"; postPatch = '' @@ -228,7 +241,7 @@ buildPythonPackage rec { pythonImportsCheck = [ "airflow" - ]; + ] ++ providerImports; checkPhase = '' export HOME=$(mktemp -d) diff --git a/pkgs/development/python-modules/apache-airflow/providers.nix b/pkgs/development/python-modules/apache-airflow/providers.nix new file mode 100644 index 000000000000..a4ca33a45aa5 --- /dev/null +++ b/pkgs/development/python-modules/apache-airflow/providers.nix @@ -0,0 +1,303 @@ +# Warning: generated by update-providers.py, do not update manually +{ + airbyte = { + deps = [ "requests" "requests-toolbelt" ]; + imports = [ "airflow.providers.airbyte.hooks.airbyte" "airflow.providers.airbyte.operators.airbyte" ]; + }; + alibaba = { + deps = [ ]; + imports = [ "airflow.providers.alibaba.cloud.hooks.oss" "airflow.providers.alibaba.cloud.operators.oss" ]; + }; + amazon = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.amazon.aws.hooks.athena" "airflow.providers.amazon.aws.hooks.aws_dynamodb" "airflow.providers.amazon.aws.hooks.base_aws" "airflow.providers.amazon.aws.hooks.batch_client" "airflow.providers.amazon.aws.hooks.batch_waiters" "airflow.providers.amazon.aws.hooks.cloud_formation" "airflow.providers.amazon.aws.hooks.datasync" "airflow.providers.amazon.aws.hooks.dms" "airflow.providers.amazon.aws.hooks.dynamodb" "airflow.providers.amazon.aws.hooks.ec2" "airflow.providers.amazon.aws.hooks.eks" "airflow.providers.amazon.aws.hooks.elasticache_replication_group" "airflow.providers.amazon.aws.hooks.emr" "airflow.providers.amazon.aws.hooks.emr_containers" "airflow.providers.amazon.aws.hooks.glacier" "airflow.providers.amazon.aws.hooks.glue" "airflow.providers.amazon.aws.hooks.glue_catalog" "airflow.providers.amazon.aws.hooks.glue_crawler" "airflow.providers.amazon.aws.hooks.kinesis" "airflow.providers.amazon.aws.hooks.lambda_function" "airflow.providers.amazon.aws.hooks.logs" "airflow.providers.amazon.aws.hooks.rds" "airflow.providers.amazon.aws.hooks.redshift" "airflow.providers.amazon.aws.hooks.redshift_cluster" "airflow.providers.amazon.aws.hooks.redshift_data" "airflow.providers.amazon.aws.hooks.redshift_sql" "airflow.providers.amazon.aws.hooks.s3" "airflow.providers.amazon.aws.hooks.sagemaker" "airflow.providers.amazon.aws.hooks.secrets_manager" "airflow.providers.amazon.aws.hooks.ses" "airflow.providers.amazon.aws.hooks.sns" "airflow.providers.amazon.aws.hooks.sqs" "airflow.providers.amazon.aws.hooks.step_function" "airflow.providers.amazon.aws.operators.athena" "airflow.providers.amazon.aws.operators.aws_lambda" "airflow.providers.amazon.aws.operators.batch" "airflow.providers.amazon.aws.operators.cloud_formation" "airflow.providers.amazon.aws.operators.datasync" "airflow.providers.amazon.aws.operators.dms" "airflow.providers.amazon.aws.operators.dms_create_task" "airflow.providers.amazon.aws.operators.dms_delete_task" "airflow.providers.amazon.aws.operators.dms_describe_tasks" "airflow.providers.amazon.aws.operators.dms_start_task" "airflow.providers.amazon.aws.operators.dms_stop_task" "airflow.providers.amazon.aws.operators.ec2" "airflow.providers.amazon.aws.operators.ec2_start_instance" "airflow.providers.amazon.aws.operators.ec2_stop_instance" "airflow.providers.amazon.aws.operators.ecs" "airflow.providers.amazon.aws.operators.eks" "airflow.providers.amazon.aws.operators.emr" "airflow.providers.amazon.aws.operators.emr" "airflow.providers.amazon.aws.operators.emr_add_steps" "airflow.providers.amazon.aws.operators.emr_containers" "airflow.providers.amazon.aws.operators.emr_create_job_flow" "airflow.providers.amazon.aws.operators.emr_modify_cluster" "airflow.providers.amazon.aws.operators.emr_terminate_job_flow" "airflow.providers.amazon.aws.operators.glacier" "airflow.providers.amazon.aws.operators.glue" "airflow.providers.amazon.aws.operators.glue_crawler" "airflow.providers.amazon.aws.operators.rds" "airflow.providers.amazon.aws.operators.redshift" "airflow.providers.amazon.aws.operators.redshift_cluster" "airflow.providers.amazon.aws.operators.redshift_data" "airflow.providers.amazon.aws.operators.redshift_sql" "airflow.providers.amazon.aws.operators.s3" "airflow.providers.amazon.aws.operators.s3_bucket" "airflow.providers.amazon.aws.operators.s3_bucket_tagging" "airflow.providers.amazon.aws.operators.s3_copy_object" "airflow.providers.amazon.aws.operators.s3_delete_objects" "airflow.providers.amazon.aws.operators.s3_file_transform" "airflow.providers.amazon.aws.operators.s3_list" "airflow.providers.amazon.aws.operators.s3_list_prefixes" "airflow.providers.amazon.aws.operators.sagemaker" "airflow.providers.amazon.aws.operators.sagemaker_base" "airflow.providers.amazon.aws.operators.sagemaker_endpoint" "airflow.providers.amazon.aws.operators.sagemaker_endpoint_config" "airflow.providers.amazon.aws.operators.sagemaker_model" "airflow.providers.amazon.aws.operators.sagemaker_processing" "airflow.providers.amazon.aws.operators.sagemaker_training" "airflow.providers.amazon.aws.operators.sagemaker_transform" "airflow.providers.amazon.aws.operators.sagemaker_tuning" "airflow.providers.amazon.aws.operators.sns" "airflow.providers.amazon.aws.operators.sqs" "airflow.providers.amazon.aws.operators.step_function" "airflow.providers.amazon.aws.operators.step_function_get_execution_output" "airflow.providers.amazon.aws.operators.step_function_start_execution" ]; + }; + apache_beam = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.apache.beam.hooks.beam" "airflow.providers.apache.beam.operators.beam" ]; + }; + apache_cassandra = { + deps = [ "cassandra-driver" ]; + imports = [ "airflow.providers.apache.cassandra.hooks.cassandra" ]; + }; + apache_drill = { + deps = [ "sqlparse" ]; + imports = [ "airflow.providers.apache.drill.hooks.drill" "airflow.providers.apache.drill.operators.drill" ]; + }; + apache_druid = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.apache.druid.hooks.druid" "airflow.providers.apache.druid.operators.druid" "airflow.providers.apache.druid.operators.druid_check" ]; + }; + apache_hdfs = { + deps = [ ]; + imports = [ "airflow.providers.apache.hdfs.hooks.hdfs" "airflow.providers.apache.hdfs.hooks.webhdfs" ]; + }; + apache_hive = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.apache.hive.hooks.hive" "airflow.providers.apache.hive.operators.hive" "airflow.providers.apache.hive.operators.hive_stats" ]; + }; + apache_kylin = { + deps = [ ]; + imports = [ "airflow.providers.apache.kylin.hooks.kylin" "airflow.providers.apache.kylin.operators.kylin_cube" ]; + }; + apache_livy = { + deps = [ "requests" "requests-toolbelt" ]; + imports = [ "airflow.providers.apache.livy.hooks.livy" "airflow.providers.apache.livy.operators.livy" ]; + }; + apache_pig = { + deps = [ ]; + imports = [ "airflow.providers.apache.pig.hooks.pig" "airflow.providers.apache.pig.operators.pig" ]; + }; + apache_pinot = { + deps = [ ]; + imports = [ "airflow.providers.apache.pinot.hooks.pinot" ]; + }; + apache_spark = { + deps = [ "pyspark" ]; + imports = [ "airflow.providers.apache.spark.hooks.spark_jdbc" "airflow.providers.apache.spark.hooks.spark_jdbc_script" "airflow.providers.apache.spark.hooks.spark_sql" "airflow.providers.apache.spark.hooks.spark_submit" "airflow.providers.apache.spark.operators.spark_jdbc" "airflow.providers.apache.spark.operators.spark_sql" "airflow.providers.apache.spark.operators.spark_submit" ]; + }; + apache_sqoop = { + deps = [ ]; + imports = [ "airflow.providers.apache.sqoop.hooks.sqoop" "airflow.providers.apache.sqoop.operators.sqoop" ]; + }; + arangodb = { + deps = [ ]; + imports = [ "airflow.providers.arangodb.hooks.arangodb" "airflow.providers.arangodb.operators.arangodb" ]; + }; + asana = { + deps = [ "asana" ]; + imports = [ "airflow.providers.asana.hooks.asana" "airflow.providers.asana.operators.asana_tasks" ]; + }; + celery = { + deps = [ "celery" "flower" ]; + imports = [ ]; + }; + cloudant = { + deps = [ ]; + imports = [ "airflow.providers.cloudant.hooks.cloudant" ]; + }; + cncf_kubernetes = { + deps = [ "cryptography" "kubernetes" ]; + imports = [ "airflow.providers.cncf.kubernetes.hooks.kubernetes" "airflow.providers.cncf.kubernetes.operators.kubernetes_pod" "airflow.providers.cncf.kubernetes.operators.spark_kubernetes" ]; + }; + databricks = { + deps = [ "aiohttp" "requests" ]; + imports = [ "airflow.providers.databricks.hooks.databricks" "airflow.providers.databricks.hooks.databricks_base" "airflow.providers.databricks.hooks.databricks_sql" "airflow.providers.databricks.operators.databricks" "airflow.providers.databricks.operators.databricks_repos" "airflow.providers.databricks.operators.databricks_sql" ]; + }; + datadog = { + deps = [ "datadog" ]; + imports = [ "airflow.providers.datadog.hooks.datadog" ]; + }; + dbt_cloud = { + deps = [ "requests" "requests-toolbelt" ]; + imports = [ "airflow.providers.dbt.cloud.hooks.dbt" "airflow.providers.dbt.cloud.operators.dbt" ]; + }; + dingding = { + deps = [ "requests" "requests-toolbelt" ]; + imports = [ "airflow.providers.dingding.hooks.dingding" "airflow.providers.dingding.operators.dingding" ]; + }; + discord = { + deps = [ "requests" "requests-toolbelt" ]; + imports = [ "airflow.providers.discord.hooks.discord_webhook" "airflow.providers.discord.operators.discord_webhook" ]; + }; + docker = { + deps = [ "docker" ]; + imports = [ "airflow.providers.docker.hooks.docker" "airflow.providers.docker.operators.docker" "airflow.providers.docker.operators.docker_swarm" ]; + }; + elasticsearch = { + deps = [ "elasticsearch" "elasticsearch-dsl" ]; + imports = [ "airflow.providers.elasticsearch.hooks.elasticsearch" ]; + }; + exasol = { + deps = [ "pandas" ]; + imports = [ "airflow.providers.exasol.hooks.exasol" "airflow.providers.exasol.operators.exasol" ]; + }; + facebook = { + deps = [ ]; + imports = [ "airflow.providers.facebook.ads.hooks.ads" ]; + }; + ftp = { + deps = [ ]; + imports = [ "airflow.providers.ftp.hooks.ftp" ]; + }; + github = { + deps = [ "PyGithub" ]; + imports = [ "airflow.providers.github.hooks.github" "airflow.providers.github.operators.github" ]; + }; + google = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.google.ads.hooks.ads" "airflow.providers.google.ads.operators.ads" "airflow.providers.google.cloud.hooks.automl" "airflow.providers.google.cloud.hooks.bigquery" "airflow.providers.google.cloud.hooks.bigquery_dts" "airflow.providers.google.cloud.hooks.bigtable" "airflow.providers.google.cloud.hooks.cloud_build" "airflow.providers.google.cloud.hooks.cloud_composer" "airflow.providers.google.cloud.hooks.cloud_memorystore" "airflow.providers.google.cloud.hooks.cloud_sql" "airflow.providers.google.cloud.hooks.cloud_storage_transfer_service" "airflow.providers.google.cloud.hooks.compute" "airflow.providers.google.cloud.hooks.compute_ssh" "airflow.providers.google.cloud.hooks.datacatalog" "airflow.providers.google.cloud.hooks.dataflow" "airflow.providers.google.cloud.hooks.datafusion" "airflow.providers.google.cloud.hooks.dataplex" "airflow.providers.google.cloud.hooks.dataprep" "airflow.providers.google.cloud.hooks.dataproc" "airflow.providers.google.cloud.hooks.dataproc_metastore" "airflow.providers.google.cloud.hooks.datastore" "airflow.providers.google.cloud.hooks.dlp" "airflow.providers.google.cloud.hooks.functions" "airflow.providers.google.cloud.hooks.gcs" "airflow.providers.google.cloud.hooks.gdm" "airflow.providers.google.cloud.hooks.kms" "airflow.providers.google.cloud.hooks.kubernetes_engine" "airflow.providers.google.cloud.hooks.life_sciences" "airflow.providers.google.cloud.hooks.looker" "airflow.providers.google.cloud.hooks.mlengine" "airflow.providers.google.cloud.hooks.natural_language" "airflow.providers.google.cloud.hooks.os_login" "airflow.providers.google.cloud.hooks.pubsub" "airflow.providers.google.cloud.hooks.secret_manager" "airflow.providers.google.cloud.hooks.spanner" "airflow.providers.google.cloud.hooks.speech_to_text" "airflow.providers.google.cloud.hooks.stackdriver" "airflow.providers.google.cloud.hooks.tasks" "airflow.providers.google.cloud.hooks.text_to_speech" "airflow.providers.google.cloud.hooks.translate" "airflow.providers.google.cloud.hooks.vertex_ai.auto_ml" "airflow.providers.google.cloud.hooks.vertex_ai.batch_prediction_job" "airflow.providers.google.cloud.hooks.vertex_ai.custom_job" "airflow.providers.google.cloud.hooks.vertex_ai.dataset" "airflow.providers.google.cloud.hooks.vertex_ai.endpoint_service" "airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job" "airflow.providers.google.cloud.hooks.vertex_ai.model_service" "airflow.providers.google.cloud.hooks.video_intelligence" "airflow.providers.google.cloud.hooks.vision" "airflow.providers.google.cloud.hooks.workflows" "airflow.providers.google.cloud.operators.automl" "airflow.providers.google.cloud.operators.bigquery" "airflow.providers.google.cloud.operators.bigquery_dts" "airflow.providers.google.cloud.operators.bigtable" "airflow.providers.google.cloud.operators.cloud_build" "airflow.providers.google.cloud.operators.cloud_composer" "airflow.providers.google.cloud.operators.cloud_memorystore" "airflow.providers.google.cloud.operators.cloud_sql" "airflow.providers.google.cloud.operators.cloud_storage_transfer_service" "airflow.providers.google.cloud.operators.compute" "airflow.providers.google.cloud.operators.datacatalog" "airflow.providers.google.cloud.operators.dataflow" "airflow.providers.google.cloud.operators.datafusion" "airflow.providers.google.cloud.operators.dataplex" "airflow.providers.google.cloud.operators.dataprep" "airflow.providers.google.cloud.operators.dataproc" "airflow.providers.google.cloud.operators.dataproc_metastore" "airflow.providers.google.cloud.operators.datastore" "airflow.providers.google.cloud.operators.dlp" "airflow.providers.google.cloud.operators.functions" "airflow.providers.google.cloud.operators.gcs" "airflow.providers.google.cloud.operators.kubernetes_engine" "airflow.providers.google.cloud.operators.life_sciences" "airflow.providers.google.cloud.operators.looker" "airflow.providers.google.cloud.operators.mlengine" "airflow.providers.google.cloud.operators.natural_language" "airflow.providers.google.cloud.operators.pubsub" "airflow.providers.google.cloud.operators.spanner" "airflow.providers.google.cloud.operators.speech_to_text" "airflow.providers.google.cloud.operators.stackdriver" "airflow.providers.google.cloud.operators.tasks" "airflow.providers.google.cloud.operators.text_to_speech" "airflow.providers.google.cloud.operators.translate" "airflow.providers.google.cloud.operators.translate_speech" "airflow.providers.google.cloud.operators.translate_speech" "airflow.providers.google.cloud.operators.vertex_ai.auto_ml" "airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job" "airflow.providers.google.cloud.operators.vertex_ai.custom_job" "airflow.providers.google.cloud.operators.vertex_ai.dataset" "airflow.providers.google.cloud.operators.vertex_ai.endpoint_service" "airflow.providers.google.cloud.operators.vertex_ai.hyperparameter_tuning_job" "airflow.providers.google.cloud.operators.vertex_ai.model_service" "airflow.providers.google.cloud.operators.video_intelligence" "airflow.providers.google.cloud.operators.vision" "airflow.providers.google.cloud.operators.workflows" "airflow.providers.google.common.hooks.base_google" "airflow.providers.google.common.hooks.discovery_api" "airflow.providers.google.firebase.hooks.firestore" "airflow.providers.google.firebase.operators.firestore" "airflow.providers.google.leveldb.hooks.leveldb" "airflow.providers.google.leveldb.operators.leveldb" "airflow.providers.google.marketing_platform.hooks.analytics" "airflow.providers.google.marketing_platform.hooks.campaign_manager" "airflow.providers.google.marketing_platform.hooks.display_video" "airflow.providers.google.marketing_platform.hooks.search_ads" "airflow.providers.google.marketing_platform.operators.analytics" "airflow.providers.google.marketing_platform.operators.campaign_manager" "airflow.providers.google.marketing_platform.operators.display_video" "airflow.providers.google.marketing_platform.operators.search_ads" "airflow.providers.google.suite.hooks.calendar" "airflow.providers.google.suite.hooks.drive" "airflow.providers.google.suite.hooks.sheets" "airflow.providers.google.suite.operators.sheets" ]; + }; + grpc = { + deps = [ "google-auth" "google-auth-httplib2" "grpcio" ]; + imports = [ "airflow.providers.grpc.hooks.grpc" "airflow.providers.grpc.operators.grpc" ]; + }; + hashicorp = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "hvac" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.hashicorp.hooks.vault" ]; + }; + http = { + deps = [ "requests" "requests-toolbelt" ]; + imports = [ "airflow.providers.http.hooks.http" "airflow.providers.http.operators.http" ]; + }; + imap = { + deps = [ ]; + imports = [ "airflow.providers.imap.hooks.imap" ]; + }; + influxdb = { + deps = [ "influxdb-client" "requests" ]; + imports = [ "airflow.providers.influxdb.hooks.influxdb" "airflow.providers.influxdb.operators.influxdb" ]; + }; + jdbc = { + deps = [ "JayDeBeApi" ]; + imports = [ "airflow.providers.jdbc.hooks.jdbc" "airflow.providers.jdbc.operators.jdbc" ]; + }; + jenkins = { + deps = [ "python-jenkins" ]; + imports = [ "airflow.providers.jenkins.hooks.jenkins" "airflow.providers.jenkins.operators.jenkins_job_trigger" ]; + }; + jira = { + deps = [ "jira" ]; + imports = [ "airflow.providers.jira.hooks.jira" "airflow.providers.jira.operators.jira" ]; + }; + microsoft_azure = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.microsoft.azure.hooks.adx" "airflow.providers.microsoft.azure.hooks.azure_batch" "airflow.providers.microsoft.azure.hooks.azure_container_instance" "airflow.providers.microsoft.azure.hooks.azure_container_registry" "airflow.providers.microsoft.azure.hooks.azure_container_volume" "airflow.providers.microsoft.azure.hooks.azure_cosmos" "airflow.providers.microsoft.azure.hooks.azure_data_factory" "airflow.providers.microsoft.azure.hooks.azure_data_lake" "airflow.providers.microsoft.azure.hooks.azure_fileshare" "airflow.providers.microsoft.azure.hooks.base_azure" "airflow.providers.microsoft.azure.hooks.batch" "airflow.providers.microsoft.azure.hooks.container_instance" "airflow.providers.microsoft.azure.hooks.container_registry" "airflow.providers.microsoft.azure.hooks.container_volume" "airflow.providers.microsoft.azure.hooks.cosmos" "airflow.providers.microsoft.azure.hooks.data_factory" "airflow.providers.microsoft.azure.hooks.data_lake" "airflow.providers.microsoft.azure.hooks.fileshare" "airflow.providers.microsoft.azure.hooks.wasb" "airflow.providers.microsoft.azure.operators.adls" "airflow.providers.microsoft.azure.operators.adls_delete" "airflow.providers.microsoft.azure.operators.adls_list" "airflow.providers.microsoft.azure.operators.adx" "airflow.providers.microsoft.azure.operators.azure_batch" "airflow.providers.microsoft.azure.operators.azure_container_instances" "airflow.providers.microsoft.azure.operators.azure_cosmos" "airflow.providers.microsoft.azure.operators.batch" "airflow.providers.microsoft.azure.operators.container_instances" "airflow.providers.microsoft.azure.operators.cosmos" "airflow.providers.microsoft.azure.operators.data_factory" "airflow.providers.microsoft.azure.operators.wasb_delete_blob" ]; + }; + microsoft_mssql = { + deps = [ "pymssql" ]; + imports = [ "airflow.providers.microsoft.mssql.hooks.mssql" "airflow.providers.microsoft.mssql.operators.mssql" ]; + }; + microsoft_psrp = { + deps = [ ]; + imports = [ "airflow.providers.microsoft.psrp.hooks.psrp" "airflow.providers.microsoft.psrp.operators.psrp" ]; + }; + microsoft_winrm = { + deps = [ "pywinrm" ]; + imports = [ "airflow.providers.microsoft.winrm.hooks.winrm" "airflow.providers.microsoft.winrm.operators.winrm" ]; + }; + mongo = { + deps = [ "dnspython" "pymongo" ]; + imports = [ "airflow.providers.mongo.hooks.mongo" ]; + }; + mysql = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.mysql.hooks.mysql" "airflow.providers.mysql.operators.mysql" ]; + }; + neo4j = { + deps = [ ]; + imports = [ "airflow.providers.neo4j.hooks.neo4j" "airflow.providers.neo4j.operators.neo4j" ]; + }; + odbc = { + deps = [ "pyodbc" ]; + imports = [ "airflow.providers.odbc.hooks.odbc" ]; + }; + openfaas = { + deps = [ ]; + imports = [ "airflow.providers.openfaas.hooks.openfaas" ]; + }; + opsgenie = { + deps = [ ]; + imports = [ "airflow.providers.opsgenie.hooks.opsgenie" "airflow.providers.opsgenie.hooks.opsgenie_alert" "airflow.providers.opsgenie.operators.opsgenie" "airflow.providers.opsgenie.operators.opsgenie_alert" ]; + }; + oracle = { + deps = [ ]; + imports = [ "airflow.providers.oracle.hooks.oracle" "airflow.providers.oracle.operators.oracle" ]; + }; + pagerduty = { + deps = [ ]; + imports = [ "airflow.providers.pagerduty.hooks.pagerduty" "airflow.providers.pagerduty.hooks.pagerduty_events" ]; + }; + papermill = { + deps = [ ]; + imports = [ "airflow.providers.papermill.operators.papermill" ]; + }; + plexus = { + deps = [ "arrow" ]; + imports = [ "airflow.providers.plexus.hooks.plexus" "airflow.providers.plexus.operators.job" ]; + }; + postgres = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.postgres.hooks.postgres" "airflow.providers.postgres.operators.postgres" ]; + }; + presto = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.presto.hooks.presto" ]; + }; + qubole = { + deps = [ "qds_sdk" ]; + imports = [ "airflow.providers.qubole.hooks.qubole" "airflow.providers.qubole.hooks.qubole_check" "airflow.providers.qubole.operators.qubole" "airflow.providers.qubole.operators.qubole_check" ]; + }; + redis = { + deps = [ "redis" ]; + imports = [ "airflow.providers.redis.hooks.redis" "airflow.providers.redis.operators.redis_publish" ]; + }; + salesforce = { + deps = [ "pandas" "simple-salesforce" ]; + imports = [ "airflow.providers.salesforce.hooks.salesforce" "airflow.providers.salesforce.hooks.tableau" "airflow.providers.salesforce.operators.salesforce_apex_rest" "airflow.providers.salesforce.operators.tableau_refresh_workbook" ]; + }; + samba = { + deps = [ "smbprotocol" ]; + imports = [ "airflow.providers.samba.hooks.samba" ]; + }; + segment = { + deps = [ ]; + imports = [ "airflow.providers.segment.hooks.segment" "airflow.providers.segment.operators.segment_track_event" ]; + }; + sendgrid = { + deps = [ "sendgrid" ]; + imports = [ ]; + }; + sftp = { + deps = [ "paramiko" "pysftp" "sshtunnel" ]; + imports = [ "airflow.providers.sftp.hooks.sftp" "airflow.providers.sftp.operators.sftp" ]; + }; + singularity = { + deps = [ ]; + imports = [ "airflow.providers.singularity.operators.singularity" ]; + }; + slack = { + deps = [ "requests" "requests-toolbelt" "slack-sdk" ]; + imports = [ "airflow.providers.slack.hooks.slack" "airflow.providers.slack.hooks.slack_webhook" "airflow.providers.slack.operators.slack" "airflow.providers.slack.operators.slack_webhook" ]; + }; + snowflake = { + deps = [ "requests" "requests-toolbelt" "slack-sdk" "snowflake-connector-python" "snowflake-sqlalchemy" ]; + imports = [ "airflow.providers.snowflake.hooks.snowflake" "airflow.providers.snowflake.operators.snowflake" ]; + }; + sqlite = { + deps = [ ]; + imports = [ "airflow.providers.sqlite.hooks.sqlite" "airflow.providers.sqlite.operators.sqlite" ]; + }; + ssh = { + deps = [ "paramiko" "sshtunnel" ]; + imports = [ "airflow.providers.ssh.hooks.ssh" "airflow.providers.ssh.operators.ssh" ]; + }; + tableau = { + deps = [ "tableauserverclient" ]; + imports = [ "airflow.providers.tableau.hooks.tableau" "airflow.providers.tableau.operators.tableau" "airflow.providers.tableau.operators.tableau_refresh_workbook" ]; + }; + tabular = { + deps = [ ]; + imports = [ ]; + }; + telegram = { + deps = [ "python-telegram-bot" ]; + imports = [ "airflow.providers.telegram.hooks.telegram" "airflow.providers.telegram.operators.telegram" ]; + }; + trino = { + deps = [ "azure-batch" "azure-cosmos" "azure-datalake-store" "azure-identity" "azure-keyvault-secrets" "azure-mgmt-containerinstance" "azure-mgmt-datafactory" "azure-mgmt-datalake-store" "azure-mgmt-resource" "azure-servicebus" "azure-storage-blob" "azure-storage-common" "azure-storage-file" "boto3" "cassandra-driver" "cryptography" "dnspython" "google-api-core" "google-api-python-client" "google-auth" "google-auth-httplib2" "google-cloud-automl" "google-cloud-bigquery-datatransfer" "google-cloud-bigtable" "google-cloud-container" "google-cloud-datacatalog" "google-cloud-dataproc" "google-cloud-dlp" "google-cloud-kms" "google-cloud-language" "google-cloud-logging" "google-cloud-monitoring" "google-cloud-pubsub" "google-cloud-redis" "google-cloud-secret-manager" "google-cloud-spanner" "google-cloud-speech" "google-cloud-storage" "google-cloud-tasks" "google-cloud-texttospeech" "google-cloud-translate" "google-cloud-videointelligence" "google-cloud-vision" "grpcio-gcp" "httpx" "json-merge-patch" "jsonpath-ng" "kubernetes" "mysqlclient" "pandas" "paramiko" "presto-python-client" "proto-plus" "psycopg2" "pymongo" "pymssql" "pyopenssl" "pysftp" "requests" "requests-toolbelt" "simple-salesforce" "slack-sdk" "smbprotocol" "sshtunnel" "thrift" "vertica-python" ]; + imports = [ "airflow.providers.trino.hooks.trino" ]; + }; + vertica = { + deps = [ "vertica-python" ]; + imports = [ "airflow.providers.vertica.hooks.vertica" "airflow.providers.vertica.operators.vertica" ]; + }; + yandex = { + deps = [ ]; + imports = [ "airflow.providers.yandex.hooks.yandex" "airflow.providers.yandex.hooks.yandexcloud_dataproc" "airflow.providers.yandex.operators.yandexcloud_dataproc" ]; + }; + zendesk = { + deps = [ ]; + imports = [ "airflow.providers.zendesk.hooks.zendesk" ]; + }; +} diff --git a/pkgs/development/python-modules/apache-airflow/update-providers.py b/pkgs/development/python-modules/apache-airflow/update-providers.py new file mode 100755 index 000000000000..ed24bdaf01d7 --- /dev/null +++ b/pkgs/development/python-modules/apache-airflow/update-providers.py @@ -0,0 +1,226 @@ +#! /usr/bin/env python3 + +from itertools import chain +import json +import logging +from pathlib import Path +import os +import re +import subprocess +import sys +from typing import Dict, List, Optional, Set, TextIO +from urllib.request import urlopen +from urllib.error import HTTPError +import yaml + +PKG_SET = "pkgs.python3Packages" + +# If some requirements are matched by multiple or no Python packages, the +# following can be used to choose the correct one +PKG_PREFERENCES = { + "dnspython": "dnspython", + "google-api-python-client": "google-api-python-client", + "psycopg2-binary": "psycopg2", + "requests_toolbelt": "requests-toolbelt", +} + +# Requirements missing from the airflow provider metadata +EXTRA_REQS = { + "sftp": ["pysftp"], +} + + +def get_version(): + with open(os.path.dirname(sys.argv[0]) + "/default.nix") as fh: + # A version consists of digits, dots, and possibly a "b" (for beta) + m = re.search('version = "([\\d\\.b]+)";', fh.read()) + return m.group(1) + + +def get_file_from_github(version: str, path: str): + with urlopen( + f"https://raw.githubusercontent.com/apache/airflow/{version}/{path}" + ) as response: + return yaml.safe_load(response) + + +def repository_root() -> Path: + return Path(os.path.dirname(sys.argv[0])) / "../../../.." + + +def dump_packages() -> Dict[str, Dict[str, str]]: + # Store a JSON dump of Nixpkgs' python3Packages + output = subprocess.check_output( + [ + "nix-env", + "-f", + repository_root(), + "-qa", + "-A", + PKG_SET, + "--arg", + "config", + "{ allowAliases = false; }", + "--json", + ] + ) + return json.loads(output) + + +def remove_version_constraint(req: str) -> str: + return re.sub(r"[=><~].*$", "", req) + + +def name_to_attr_path(req: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]: + if req in PKG_PREFERENCES: + return f"{PKG_SET}.{PKG_PREFERENCES[req]}" + attr_paths = [] + names = [req] + # E.g. python-mpd2 is actually called python3.6-mpd2 + # instead of python-3.6-python-mpd2 inside Nixpkgs + if req.startswith("python-") or req.startswith("python_"): + names.append(req[len("python-") :]) + for name in names: + # treat "-" and "_" equally + name = re.sub("[-_]", "[-_]", name) + # python(minor).(major)-(pname)-(version or unstable-date) + # we need the version qualifier, or we'll have multiple matches + # (e.g. pyserial and pyserial-asyncio when looking for pyserial) + pattern = re.compile( + f"^python\\d+\\.\\d+-{name}-(?:\\d|unstable-.*)", re.I + ) + for attr_path, package in packages.items(): + # logging.debug("Checking match for %s with %s", name, package["name"]) + if pattern.match(package["name"]): + attr_paths.append(attr_path) + # Let's hope there's only one derivation with a matching name + assert len(attr_paths) <= 1, f"{req} matches more than one derivation: {attr_paths}" + if attr_paths: + return attr_paths[0] + return None + + +def provider_reqs_to_attr_paths(reqs: List, packages: Dict) -> List: + no_version_reqs = map(remove_version_constraint, reqs) + filtered_reqs = [ + req for req in no_version_reqs if not re.match(r"^apache-airflow", req) + ] + attr_paths = [] + for req in filtered_reqs: + attr_path = name_to_attr_path(req, packages) + if attr_path is not None: + # Add attribute path without "python3Packages." prefix + pname = attr_path[len(PKG_SET + ".") :] + attr_paths.append(pname) + else: + # If we can't find it, we just skip and warn the user + logging.warning("Could not find package attr for %s", req) + return attr_paths + + +def get_cross_provider_reqs( + provider: str, provider_reqs: Dict, cross_provider_deps: Dict, seen: List = None +) -> Set: + # Unfortunately there are circular cross-provider dependencies, so keep a + # list of ones we've seen already + seen = seen or [] + reqs = set(provider_reqs[provider]) + if len(cross_provider_deps[provider]) > 0: + reqs.update( + chain.from_iterable( + get_cross_provider_reqs( + d, provider_reqs, cross_provider_deps, seen + [provider] + ) + if d not in seen + else [] + for d in cross_provider_deps[provider] + ) + ) + return reqs + + +def get_provider_reqs(version: str, packages: Dict) -> Dict: + provider_dependencies = get_file_from_github( + version, "generated/provider_dependencies.json" + ) + provider_reqs = {} + cross_provider_deps = {} + for provider, provider_data in provider_dependencies.items(): + provider_reqs[provider] = list( + provider_reqs_to_attr_paths(provider_data["deps"], packages) + ) + EXTRA_REQS.get(provider, []) + cross_provider_deps[provider] = [ + d for d in provider_data["cross-providers-deps"] if d != "common.sql" + ] + transitive_provider_reqs = {} + # Add transitive cross-provider reqs + for provider in provider_reqs: + transitive_provider_reqs[provider] = get_cross_provider_reqs( + provider, provider_reqs, cross_provider_deps + ) + return transitive_provider_reqs + + +def get_provider_yaml(version: str, provider: str) -> Dict: + provider_dir = provider.replace(".", "/") + path = f"airflow/providers/{provider_dir}/provider.yaml" + try: + return get_file_from_github(version, path) + except HTTPError: + logging.warning("Couldn't get provider yaml for %s", provider) + return {} + + +def get_provider_imports(version: str, providers) -> Dict: + provider_imports = {} + for provider in providers: + provider_yaml = get_provider_yaml(version, provider) + imports: List[str] = [] + if "hooks" in provider_yaml: + imports.extend( + chain.from_iterable( + hook["python-modules"] for hook in provider_yaml["hooks"] + ) + ) + if "operators" in provider_yaml: + imports.extend( + chain.from_iterable( + operator["python-modules"] + for operator in provider_yaml["operators"] + ) + ) + provider_imports[provider] = imports + return provider_imports + + +def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None: + fh.write("# Warning: generated by update-providers.py, do not update manually\n") + fh.write("{\n") + for provider, reqs in provider_reqs.items(): + provider_name = provider.replace(".", "_") + fh.write(f" {provider_name} = {{\n") + fh.write( + " deps = [ " + " ".join(sorted(f'"{req}"' for req in reqs)) + " ];\n" + ) + fh.write( + " imports = [ " + + " ".join(sorted(f'"{imp}"' for imp in provider_imports[provider])) + + " ];\n" + ) + fh.write(" };\n") + fh.write("}\n") + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + version = get_version() + packages = dump_packages() + logging.info("Generating providers.nix for version %s", version) + provider_reqs = get_provider_reqs(version, packages) + provider_imports = get_provider_imports(version, provider_reqs.keys()) + with open("providers.nix", "w") as fh: + to_nix_expr(provider_reqs, provider_imports, fh) + + +if __name__ == "__main__": + main()