From cded24ddc2507f299ad19a91c320d8eb74c0c020 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Tue, 18 Feb 2025 16:33:09 +0100 Subject: [PATCH] python312Packages.apache-beam: 2.62.0 -> 2.63.0 Diff: https://github.com/apache/beam/compare/refs/tags/v2.62.0...v2.63.0 Changelog: https://github.com/apache/beam/blob/release-2.63.0/CHANGES.md --- .../python-modules/apache-beam/default.nix | 339 ++++++++++-------- 1 file changed, 182 insertions(+), 157 deletions(-) diff --git a/pkgs/development/python-modules/apache-beam/default.nix b/pkgs/development/python-modules/apache-beam/default.nix index 19df48ce1bd8..b38ad2e8ee05 100644 --- a/pkgs/development/python-modules/apache-beam/default.nix +++ b/pkgs/development/python-modules/apache-beam/default.nix @@ -42,6 +42,7 @@ # tests python, + docstring-parser, freezegun, hypothesis, mock, @@ -57,18 +58,19 @@ sqlalchemy, tenacity, testcontainers, + pythonAtLeast, }: buildPythonPackage rec { pname = "apache-beam"; - version = "2.62.0"; + version = "2.63.0"; pyproject = true; src = fetchFromGitHub { owner = "apache"; repo = "beam"; tag = "v${version}"; - hash = "sha256-vLvnRZAQg9nhUOI0SIUn+9Y8O7edK3445PkdhPbhO3Y="; + hash = "sha256-ixJstawgU3UGtNKVzkwMCLkdY7QKTbxNe6JJ7vG+vmA="; }; pythonRelaxDeps = [ @@ -148,6 +150,7 @@ buildPythonPackage rec { pythonImportsCheck = [ "apache_beam" ]; nativeCheckInputs = [ + docstring-parser freezegun hypothesis mock @@ -171,170 +174,192 @@ buildPythonPackage rec { cd $out/${python.sitePackages} ''; - disabledTestPaths = [ - # Fails with - # _______ ERROR collecting apache_beam/io/external/xlang_jdbcio_it_test.py _______ - # apache_beam/io/external/xlang_jdbcio_it_test.py:80: in - # class CrossLanguageJdbcIOTest(unittest.TestCase): - # apache_beam/io/external/xlang_jdbcio_it_test.py:99: in CrossLanguageJdbcIOTest - # container_init: Callable[[], Union[PostgresContainer, MySqlContainer]], - # E NameError: name 'MySqlContainer' is not defined - # - "apache_beam/io/external/xlang_jdbcio_it_test.py" + disabledTestPaths = + [ + # Fails with + # _______ ERROR collecting apache_beam/io/external/xlang_jdbcio_it_test.py _______ + # apache_beam/io/external/xlang_jdbcio_it_test.py:80: in + # class CrossLanguageJdbcIOTest(unittest.TestCase): + # apache_beam/io/external/xlang_jdbcio_it_test.py:99: in CrossLanguageJdbcIOTest + # container_init: Callable[[], Union[PostgresContainer, MySqlContainer]], + # E NameError: name 'MySqlContainer' is not defined + # + "apache_beam/io/external/xlang_jdbcio_it_test.py" - # These tests depend on the availability of specific servers backends. - "apache_beam/runners/portability/flink_runner_test.py" - "apache_beam/runners/portability/samza_runner_test.py" - "apache_beam/runners/portability/spark_runner_test.py" + # These tests depend on the availability of specific servers backends. + "apache_beam/runners/portability/flink_runner_test.py" + "apache_beam/runners/portability/samza_runner_test.py" + "apache_beam/runners/portability/spark_runner_test.py" - # Fails starting from dill 0.3.6 because it tries to pickle pytest globals: - # https://github.com/uqfoundation/dill/issues/482#issuecomment-1139017499. - "apache_beam/transforms/window_test.py" + # Fails starting from dill 0.3.6 because it tries to pickle pytest globals: + # https://github.com/uqfoundation/dill/issues/482#issuecomment-1139017499. + "apache_beam/transforms/window_test.py" - # See https://github.com/apache/beam/issues/25390. - "apache_beam/coders/slow_coders_test.py" - "apache_beam/dataframe/pandas_doctests_test.py" - "apache_beam/typehints/typed_pipeline_test.py" - "apache_beam/coders/fast_coders_test.py" - "apache_beam/dataframe/schemas_test.py" + # See https://github.com/apache/beam/issues/25390. + "apache_beam/coders/slow_coders_test.py" + "apache_beam/dataframe/pandas_doctests_test.py" + "apache_beam/typehints/typed_pipeline_test.py" + "apache_beam/coders/fast_coders_test.py" + "apache_beam/dataframe/schemas_test.py" - # Fails with TypeError: cannot pickle 'EncodedFile' instances - # Upstream issue https://github.com/apache/beam/issues/33889 - "apache_beam/options/pipeline_options_validator_test.py" - "apache_beam/yaml/main_test.py" - "apache_beam/yaml/programming_guide_test.py" - "apache_beam/yaml/readme_test.py" - "apache_beam/yaml/yaml_combine_test.py" - "apache_beam/yaml/yaml_enrichment_test.py" - "apache_beam/yaml/yaml_io_test.py" - "apache_beam/yaml/yaml_join_test.py" - "apache_beam/yaml/yaml_mapping_test.py" - "apache_beam/yaml/yaml_ml_test.py" - "apache_beam/yaml/yaml_provider_unit_test.py" + # Fails with TypeError: cannot pickle 'EncodedFile' instances + # Upstream issue https://github.com/apache/beam/issues/33889 + "apache_beam/options/pipeline_options_validator_test.py" + "apache_beam/yaml/main_test.py" + "apache_beam/yaml/programming_guide_test.py" + "apache_beam/yaml/readme_test.py" + "apache_beam/yaml/yaml_combine_test.py" + "apache_beam/yaml/yaml_enrichment_test.py" + "apache_beam/yaml/yaml_io_test.py" + "apache_beam/yaml/yaml_join_test.py" + "apache_beam/yaml/yaml_mapping_test.py" + "apache_beam/yaml/yaml_ml_test.py" + "apache_beam/yaml/yaml_provider_unit_test.py" - # FIXME All those fails due to a single- AttributeError: 'MaybeReshuffle' object has no attribute 'side_inputs' - # Upstream issue https://github.com/apache/beam/issues/33854 - "apache_beam/coders/row_coder_test.py" - "apache_beam/examples/avro_nyc_trips_test.py" - "apache_beam/examples/complete/autocomplete_test.py" - "apache_beam/examples/complete/estimate_pi_test.py" - "apache_beam/examples/complete/game/game_stats_test.py" - "apache_beam/examples/complete/game/hourly_team_score_test.py" - "apache_beam/examples/complete/game/leader_board_test.py" - "apache_beam/examples/complete/game/user_score_test.py" - "apache_beam/examples/complete/tfidf_test.py" - "apache_beam/examples/complete/top_wikipedia_sessions_test.py" - "apache_beam/examples/cookbook/bigquery_side_input_test.py" - "apache_beam/examples/cookbook/bigquery_tornadoes_test.py" - "apache_beam/examples/cookbook/coders_test.py" - "apache_beam/examples/cookbook/combiners_test.py" - "apache_beam/examples/cookbook/custom_ptransform_test.py" - "apache_beam/examples/cookbook/filters_test.py" - "apache_beam/examples/matrix_power_test.py" - "apache_beam/examples/snippets/snippets_test.py" - "apache_beam/examples/snippets/transforms/aggregation/approximatequantiles_test.py" - "apache_beam/examples/snippets/transforms/aggregation/approximateunique_test.py" - "apache_beam/examples/snippets/transforms/aggregation/batchelements_test.py" - "apache_beam/examples/snippets/transforms/aggregation/cogroupbykey_test.py" - "apache_beam/examples/snippets/transforms/aggregation/combineglobally_test.py" - "apache_beam/examples/snippets/transforms/aggregation/combineperkey_test.py" - "apache_beam/examples/snippets/transforms/aggregation/combinevalues_test.py" - "apache_beam/examples/snippets/transforms/aggregation/count_test.py" - "apache_beam/examples/snippets/transforms/aggregation/distinct_test.py" - "apache_beam/examples/snippets/transforms/aggregation/groupbykey_test.py" - "apache_beam/examples/snippets/transforms/aggregation/groupintobatches_test.py" - "apache_beam/examples/snippets/transforms/aggregation/latest_test.py" - "apache_beam/examples/snippets/transforms/aggregation/max_test.py" - "apache_beam/examples/snippets/transforms/aggregation/mean_test.py" - "apache_beam/examples/snippets/transforms/aggregation/min_test.py" - "apache_beam/examples/snippets/transforms/aggregation/sample_test.py" - "apache_beam/examples/snippets/transforms/aggregation/sum_test.py" - "apache_beam/examples/snippets/transforms/aggregation/tolist_test.py" - "apache_beam/examples/snippets/transforms/aggregation/top_test.py" - "apache_beam/examples/snippets/transforms/elementwise/filter_test.py" - "apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" - "apache_beam/examples/snippets/transforms/elementwise/keys_test.py" - "apache_beam/examples/snippets/transforms/elementwise/kvswap_test.py" - "apache_beam/examples/snippets/transforms/elementwise/map_test.py" - "apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" - "apache_beam/examples/snippets/transforms/elementwise/partition_test.py" - "apache_beam/examples/snippets/transforms/elementwise/regex_test.py" - "apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" - "apache_beam/examples/snippets/transforms/elementwise/values_test.py" - "apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" - "apache_beam/examples/snippets/transforms/other/create_test.py" - "apache_beam/examples/snippets/transforms/other/flatten_test.py" - "apache_beam/examples/snippets/transforms/other/window_test.py" - "apache_beam/examples/snippets/util_test.py" - "apache_beam/io/avroio_test.py" - "apache_beam/io/concat_source_test.py" - "apache_beam/io/filebasedsink_test.py" - "apache_beam/io/filebasedsource_test.py" - "apache_beam/io/fileio_test.py" - "apache_beam/io/mongodbio_test.py" - "apache_beam/io/parquetio_test.py" - "apache_beam/io/sources_test.py" - "apache_beam/io/textio_test.py" - "apache_beam/io/tfrecordio_test.py" - "apache_beam/metrics/metric_test.py" - "apache_beam/ml/inference/base_test.py" - "apache_beam/ml/inference/sklearn_inference_test.py" - "apache_beam/ml/inference/utils_test.py" - "apache_beam/ml/rag/chunking/base_test.py" - "apache_beam/pipeline_test.py" - "apache_beam/runners/direct/direct_runner_test.py" - "apache_beam/runners/direct/sdf_direct_runner_test.py" - "apache_beam/runners/interactive/interactive_beam_test.py" - "apache_beam/runners/interactive/interactive_runner_test.py" - "apache_beam/runners/interactive/non_interactive_runner_test.py" - "apache_beam/runners/interactive/recording_manager_test.py" - "apache_beam/runners/portability/fn_api_runner/translations_test.py" - "apache_beam/runners/portability/fn_api_runner/trigger_manager_test.py" - "apache_beam/runners/portability/stager_test.py" - "apache_beam/testing/synthetic_pipeline_test.py" - "apache_beam/testing/test_stream_test.py" - "apache_beam/testing/util_test.py" - "apache_beam/transforms/combiners_test.py" - "apache_beam/transforms/core_test.py" - "apache_beam/transforms/create_test.py" - "apache_beam/transforms/deduplicate_test.py" - "apache_beam/transforms/periodicsequence_test.py" - "apache_beam/transforms/ptransform_test.py" - "apache_beam/transforms/sideinputs_test.py" - "apache_beam/transforms/stats_test.py" - "apache_beam/transforms/transforms_keyword_only_args_test.py" - "apache_beam/transforms/trigger_test.py" - "apache_beam/transforms/userstate_test.py" - "apache_beam/transforms/util_test.py" - "apache_beam/transforms/write_ptransform_test.py" + # FIXME All those fails due to a single- AttributeError: 'MaybeReshuffle' object has no attribute 'side_inputs' + # Upstream issue https://github.com/apache/beam/issues/33854 + "apache_beam/coders/row_coder_test.py" + "apache_beam/examples/avro_nyc_trips_test.py" + "apache_beam/examples/complete/autocomplete_test.py" + "apache_beam/examples/complete/estimate_pi_test.py" + "apache_beam/examples/complete/game/game_stats_test.py" + "apache_beam/examples/complete/game/hourly_team_score_test.py" + "apache_beam/examples/complete/game/leader_board_test.py" + "apache_beam/examples/complete/game/user_score_test.py" + "apache_beam/examples/complete/tfidf_test.py" + "apache_beam/examples/complete/top_wikipedia_sessions_test.py" + "apache_beam/examples/cookbook/bigquery_side_input_test.py" + "apache_beam/examples/cookbook/bigquery_tornadoes_test.py" + "apache_beam/examples/cookbook/coders_test.py" + "apache_beam/examples/cookbook/combiners_test.py" + "apache_beam/examples/cookbook/custom_ptransform_test.py" + "apache_beam/examples/cookbook/filters_test.py" + "apache_beam/examples/matrix_power_test.py" + "apache_beam/examples/snippets/snippets_test.py" + "apache_beam/examples/snippets/transforms/aggregation/approximatequantiles_test.py" + "apache_beam/examples/snippets/transforms/aggregation/approximateunique_test.py" + "apache_beam/examples/snippets/transforms/aggregation/batchelements_test.py" + "apache_beam/examples/snippets/transforms/aggregation/cogroupbykey_test.py" + "apache_beam/examples/snippets/transforms/aggregation/combineglobally_test.py" + "apache_beam/examples/snippets/transforms/aggregation/combineperkey_test.py" + "apache_beam/examples/snippets/transforms/aggregation/combinevalues_test.py" + "apache_beam/examples/snippets/transforms/aggregation/count_test.py" + "apache_beam/examples/snippets/transforms/aggregation/distinct_test.py" + "apache_beam/examples/snippets/transforms/aggregation/groupbykey_test.py" + "apache_beam/examples/snippets/transforms/aggregation/groupintobatches_test.py" + "apache_beam/examples/snippets/transforms/aggregation/latest_test.py" + "apache_beam/examples/snippets/transforms/aggregation/max_test.py" + "apache_beam/examples/snippets/transforms/aggregation/mean_test.py" + "apache_beam/examples/snippets/transforms/aggregation/min_test.py" + "apache_beam/examples/snippets/transforms/aggregation/sample_test.py" + "apache_beam/examples/snippets/transforms/aggregation/sum_test.py" + "apache_beam/examples/snippets/transforms/aggregation/tolist_test.py" + "apache_beam/examples/snippets/transforms/aggregation/top_test.py" + "apache_beam/examples/snippets/transforms/elementwise/filter_test.py" + "apache_beam/examples/snippets/transforms/elementwise/flatmap_test.py" + "apache_beam/examples/snippets/transforms/elementwise/keys_test.py" + "apache_beam/examples/snippets/transforms/elementwise/kvswap_test.py" + "apache_beam/examples/snippets/transforms/elementwise/map_test.py" + "apache_beam/examples/snippets/transforms/elementwise/pardo_test.py" + "apache_beam/examples/snippets/transforms/elementwise/partition_test.py" + "apache_beam/examples/snippets/transforms/elementwise/regex_test.py" + "apache_beam/examples/snippets/transforms/elementwise/tostring_test.py" + "apache_beam/examples/snippets/transforms/elementwise/values_test.py" + "apache_beam/examples/snippets/transforms/elementwise/withtimestamps_test.py" + "apache_beam/examples/snippets/transforms/other/create_test.py" + "apache_beam/examples/snippets/transforms/other/flatten_test.py" + "apache_beam/examples/snippets/transforms/other/window_test.py" + "apache_beam/examples/snippets/util_test.py" + "apache_beam/io/avroio_test.py" + "apache_beam/io/concat_source_test.py" + "apache_beam/io/filebasedsink_test.py" + "apache_beam/io/filebasedsource_test.py" + "apache_beam/io/fileio_test.py" + "apache_beam/io/mongodbio_test.py" + "apache_beam/io/parquetio_test.py" + "apache_beam/io/sources_test.py" + "apache_beam/io/textio_test.py" + "apache_beam/io/tfrecordio_test.py" + "apache_beam/metrics/metric_test.py" + "apache_beam/ml/inference/base_test.py" + "apache_beam/ml/inference/sklearn_inference_test.py" + "apache_beam/ml/inference/utils_test.py" + "apache_beam/ml/rag/chunking/base_test.py" + "apache_beam/ml/rag/ingestion/base_test.py" + "apache_beam/pipeline_test.py" + "apache_beam/runners/direct/direct_runner_test.py" + "apache_beam/runners/direct/sdf_direct_runner_test.py" + "apache_beam/runners/interactive/interactive_beam_test.py" + "apache_beam/runners/interactive/interactive_runner_test.py" + "apache_beam/runners/interactive/non_interactive_runner_test.py" + "apache_beam/runners/interactive/recording_manager_test.py" + "apache_beam/runners/portability/fn_api_runner/translations_test.py" + "apache_beam/runners/portability/fn_api_runner/trigger_manager_test.py" + "apache_beam/runners/portability/stager_test.py" + "apache_beam/testing/synthetic_pipeline_test.py" + "apache_beam/testing/test_stream_test.py" + "apache_beam/testing/util_test.py" + "apache_beam/transforms/combiners_test.py" + "apache_beam/transforms/core_test.py" + "apache_beam/transforms/create_test.py" + "apache_beam/transforms/deduplicate_test.py" + "apache_beam/transforms/periodicsequence_test.py" + "apache_beam/transforms/ptransform_test.py" + "apache_beam/transforms/sideinputs_test.py" + "apache_beam/transforms/stats_test.py" + "apache_beam/transforms/transforms_keyword_only_args_test.py" + "apache_beam/transforms/trigger_test.py" + "apache_beam/transforms/userstate_test.py" + "apache_beam/transforms/util_test.py" + "apache_beam/transforms/write_ptransform_test.py" - # FIXME AttributeError: 'Namespace' object has no attribute 'test_pipeline_options' - # Upstream issue https://github.com/apache/beam/issues/33853 - "apache_beam/runners/portability/prism_runner_test.py" + # FIXME AttributeError: 'Namespace' object has no attribute 'test_pipeline_options' + # Upstream issue https://github.com/apache/beam/issues/33853 + "apache_beam/runners/portability/prism_runner_test.py" - # FIXME ValueError: Unable to run pipeline with requirement: unsupported_requirement - # Upstream issuehttps://github.com/apache/beam/issues/33853 - "apache_beam/yaml/yaml_transform_scope_test.py" - "apache_beam/yaml/yaml_transform_test.py" - "apache_beam/yaml/yaml_transform_unit_test.py" - "apache_beam/yaml/yaml_udf_test.py" - "apache_beam/dataframe/frames_test.py" + # FIXME ValueError: Unable to run pipeline with requirement: unsupported_requirement + # Upstream issuehttps://github.com/apache/beam/issues/33853 + "apache_beam/yaml/yaml_transform_scope_test.py" + "apache_beam/yaml/yaml_transform_test.py" + "apache_beam/yaml/yaml_transform_unit_test.py" + "apache_beam/yaml/yaml_udf_test.py" + "apache_beam/dataframe/frames_test.py" - # FIXME Those tests do not terminate due to a grpc error (threading issue) - # grpc_status:14, grpc_message:"Cancelling all calls"}" - # Upstream issue https://github.com/apache/beam/issues/33851 - "apache_beam/runners/portability/portable_runner_test.py" - ]; + # FIXME Those tests do not terminate due to a grpc error (threading issue) + # grpc_status:14, grpc_message:"Cancelling all calls"}" + # Upstream issue https://github.com/apache/beam/issues/33851 + "apache_beam/runners/portability/portable_runner_test.py" + ] + ++ lib.optionals (pythonAtLeast "3.13") [ + # > instruction = ofs_table[pc] + # E KeyError: 18 + "apache_beam/typehints/trivial_inference_test.py" + ]; - disabledTests = lib.optionals stdenv.hostPlatform.isDarwin [ - # PermissionError: [Errno 13] Permission denied: '/tmp/...' - "test_cache_manager_uses_local_ib_cache_root" - "test_describe_all_recordings" - "test_find_out_correct_user_pipeline" - "test_get_cache_manager_creates_cache_manager_if_absent" - "test_streaming_cache_uses_local_ib_cache_root" - "test_track_user_pipeline_cleanup_non_inspectable_pipeline" - ]; + disabledTests = + lib.optionals stdenv.hostPlatform.isDarwin [ + # PermissionError: [Errno 13] Permission denied: '/tmp/...' + "test_cache_manager_uses_local_ib_cache_root" + "test_describe_all_recordings" + "test_find_out_correct_user_pipeline" + "test_get_cache_manager_creates_cache_manager_if_absent" + "test_streaming_cache_uses_local_ib_cache_root" + "test_track_user_pipeline_cleanup_non_inspectable_pipeline" + ] + ++ lib.optionals (pythonAtLeast "3.12") [ + # TypeError: Could not determine schema for type hint Any. + "test_batching_beam_row_input" + "test_auto_convert" + "test_unbatching_series" + "test_batching_beam_row_to_dataframe" + + # AssertionError: Any != + "test_pycallable_map" + "testAlwaysReturnsEarly" + + # TypeError: Expected Iterator in return type annotatio + "test_get_output_batch_type" + ]; meta = { description = "Unified model for defining both batch and streaming data-parallel processing pipelines";