From 65b802a6fdf2d3f07ddf50b3b5ef88a21e021d62 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Thu, 27 Mar 2025 08:49:14 -0400 Subject: [PATCH] python312Packages.torch: build torch.distributed on darwin It's disabled on MacOS by default: https://pytorch.org/docs/stable/distributed.html#torch.distributed.is_available When distributed module build is enabled, a compilation error happens on fresh clang. Upstream repo (which is a submodule in pytorch) is archived so it's impossible to contribute the fix there. Instead, an inline patch is included that is enough to fix the error. Signed-off-by: Ihar Hrachyshka --- .../python-modules/colbert-ai/default.nix | 5 --- .../compressed-tensors/default.nix | 24 ++++++-------- .../docling-ibm-models/default.nix | 6 ---- .../python-modules/docling/default.nix | 20 ------------ .../finetuning-scheduler/default.nix | 4 --- .../python-modules/gradio/default.nix | 23 ++++++-------- .../python-modules/ignite/default.nix | 2 -- .../python-modules/peft/default.nix | 4 --- .../sentence-transformers/default.nix | 4 --- .../python-modules/smolagents/default.nix | 6 ---- .../python-modules/tensordict/default.nix | 13 +++----- .../torch/clang19-template-warning.patch | 31 +++++++++++++++++++ .../python-modules/torch/default.nix | 7 ++++- .../python-modules/torchsnapshot/default.nix | 4 --- 14 files changed, 59 insertions(+), 94 deletions(-) create mode 100644 pkgs/development/python-modules/torch/clang19-template-warning.patch diff --git a/pkgs/development/python-modules/colbert-ai/default.nix b/pkgs/development/python-modules/colbert-ai/default.nix index 8d949efde7e2..5b4ebfdeb2c4 100644 --- a/pkgs/development/python-modules/colbert-ai/default.nix +++ b/pkgs/development/python-modules/colbert-ai/default.nix @@ -74,10 +74,5 @@ buildPythonPackage rec { maintainers = with lib.maintainers; [ bachp ]; - badPlatforms = [ - # `import torch.distributed.tensor` fails on darwin: - # ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a packag - lib.systems.inspect.patterns.isDarwin - ]; }; } diff --git a/pkgs/development/python-modules/compressed-tensors/default.nix b/pkgs/development/python-modules/compressed-tensors/default.nix index 3dd16d37e9f7..e62a07a2ad0b 100644 --- a/pkgs/development/python-modules/compressed-tensors/default.nix +++ b/pkgs/development/python-modules/compressed-tensors/default.nix @@ -43,21 +43,15 @@ buildPythonPackage rec { pytestCheckHook ]; - disabledTests = - [ - # these try to download models from HF Hub - "test_get_observer_token_count" - "test_kv_cache_quantization" - "test_target_prioritization" - "test_load_compressed_sharded" - "test_save_compressed_model" - "test_apply_tinyllama_dynamic_activations" - ] - ++ lib.optionals stdenv.hostPlatform.isDarwin [ - # No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package - "test_composability" - "test_missing_and_unexpected_keys_on_compression" - ]; + disabledTests = [ + # these try to download models from HF Hub + "test_get_observer_token_count" + "test_kv_cache_quantization" + "test_target_prioritization" + "test_load_compressed_sharded" + "test_save_compressed_model" + "test_apply_tinyllama_dynamic_activations" + ]; disabledTestPaths = [ # these try to download models from HF Hub diff --git a/pkgs/development/python-modules/docling-ibm-models/default.nix b/pkgs/development/python-modules/docling-ibm-models/default.nix index 93bd42c78167..362e5c59ae8a 100644 --- a/pkgs/development/python-modules/docling-ibm-models/default.nix +++ b/pkgs/development/python-modules/docling-ibm-models/default.nix @@ -83,12 +83,6 @@ buildPythonPackage rec { "test_tf_predictor" ]; - disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [ - # No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package - "tests/test_code_formula_predictor.py" - "tests/test_layout_predictor.py" - ]; - meta = { changelog = "https://github.com/DS4SD/docling-ibm-models/blob/${src.tag}/CHANGELOG.md"; description = "Docling IBM models"; diff --git a/pkgs/development/python-modules/docling/default.nix b/pkgs/development/python-modules/docling/default.nix index 7e60149238a0..fddcf855bbbf 100644 --- a/pkgs/development/python-modules/docling/default.nix +++ b/pkgs/development/python-modules/docling/default.nix @@ -164,26 +164,6 @@ buildPythonPackage rec { "test_e2e_valid_csv_conversions" ]; - disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [ - # No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package - "tests/test_backend_csv.py" - "tests/test_backend_html.py" - "tests/test_backend_jats.py" - "tests/test_backend_msexcel.py" - "tests/test_backend_msword.py" - "tests/test_backend_pptx.py" - "tests/test_cli.py" - "tests/test_code_formula.py" - "tests/test_document_picture_classifier.py" - "tests/test_e2e_conversion.py" - "tests/test_e2e_ocr_conversion.py" - "tests/test_input_doc.py" - "tests/test_interfaces.py" - "tests/test_invalid_input.py" - "tests/test_legacy_format_transform.py" - "tests/test_options.py" - ]; - meta = { description = "Get your documents ready for gen AI"; homepage = "https://github.com/DS4SD/docling"; diff --git a/pkgs/development/python-modules/finetuning-scheduler/default.nix b/pkgs/development/python-modules/finetuning-scheduler/default.nix index ce1d1f858a30..a51ebd1f7a45 100644 --- a/pkgs/development/python-modules/finetuning-scheduler/default.nix +++ b/pkgs/development/python-modules/finetuning-scheduler/default.nix @@ -71,9 +71,5 @@ buildPythonPackage rec { changelog = "https://github.com/speediedan/finetuning-scheduler/blob/v${version}/CHANGELOG.md"; license = lib.licenses.asl20; maintainers = with lib.maintainers; [ bcdarwin ]; - badPlatforms = [ - # "No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package" at import time: - lib.systems.inspect.patterns.isDarwin - ]; }; } diff --git a/pkgs/development/python-modules/gradio/default.nix b/pkgs/development/python-modules/gradio/default.nix index 05c02ca38d2e..0184f539e946 100644 --- a/pkgs/development/python-modules/gradio/default.nix +++ b/pkgs/development/python-modules/gradio/default.nix @@ -301,21 +301,16 @@ buildPythonPackage rec { "test_updates_stored_up_to_capacity" "test_varying_output_forms_with_generators" ]; - disabledTestPaths = - [ - # 100% touches network - "test/test_networking.py" - "client/python/test/test_client.py" - # makes pytest freeze 50% of the time - "test/test_interfaces.py" + disabledTestPaths = [ + # 100% touches network + "test/test_networking.py" + "client/python/test/test_client.py" + # makes pytest freeze 50% of the time + "test/test_interfaces.py" - # Local network tests dependant on port availability (port 7860-7959) - "test/test_routes.py" - ] - ++ lib.optionals (stdenv.hostPlatform.isDarwin) [ - # No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package - "test/test_pipelines.py" - ]; + # Local network tests dependant on port availability (port 7860-7959) + "test/test_routes.py" + ]; pytestFlagsArray = [ "-x" # abort on first failure "-m 'not flaky'" diff --git a/pkgs/development/python-modules/ignite/default.nix b/pkgs/development/python-modules/ignite/default.nix index a34e5c5d6c24..6da0f8e7123b 100644 --- a/pkgs/development/python-modules/ignite/default.nix +++ b/pkgs/development/python-modules/ignite/default.nix @@ -96,7 +96,5 @@ buildPythonPackage rec { changelog = "https://github.com/pytorch/ignite/releases/tag/v${version}"; license = lib.licenses.bsd3; maintainers = [ lib.maintainers.bcdarwin ]; - # ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package - broken = stdenv.hostPlatform.isDarwin; }; } diff --git a/pkgs/development/python-modules/peft/default.nix b/pkgs/development/python-modules/peft/default.nix index 831970cdb825..ad2727b4de94 100644 --- a/pkgs/development/python-modules/peft/default.nix +++ b/pkgs/development/python-modules/peft/default.nix @@ -104,9 +104,5 @@ buildPythonPackage rec { changelog = "https://github.com/huggingface/peft/releases/tag/v${version}"; license = lib.licenses.asl20; maintainers = with lib.maintainers; [ bcdarwin ]; - badPlatforms = [ - # No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package - lib.systems.inspect.patterns.isDarwin - ]; }; } diff --git a/pkgs/development/python-modules/sentence-transformers/default.nix b/pkgs/development/python-modules/sentence-transformers/default.nix index 2ba0e892ac86..aa0f6a2a3a6d 100644 --- a/pkgs/development/python-modules/sentence-transformers/default.nix +++ b/pkgs/development/python-modules/sentence-transformers/default.nix @@ -113,9 +113,5 @@ buildPythonPackage rec { changelog = "https://github.com/UKPLab/sentence-transformers/releases/tag/v${version}"; license = lib.licenses.asl20; maintainers = with lib.maintainers; [ dit7ya ]; - badPlatforms = [ - # No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package - lib.systems.inspect.patterns.isDarwin - ]; }; } diff --git a/pkgs/development/python-modules/smolagents/default.nix b/pkgs/development/python-modules/smolagents/default.nix index b26a14f557c4..bcd0f8bd8a1c 100644 --- a/pkgs/development/python-modules/smolagents/default.nix +++ b/pkgs/development/python-modules/smolagents/default.nix @@ -136,12 +136,6 @@ buildPythonPackage rec { "test_new_instance" ]; - disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [ - # No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package - "tests/test_final_answer.py" - "tests/test_types.py" - ]; - __darwinAllowLocalNetworking = true; meta = { diff --git a/pkgs/development/python-modules/tensordict/default.nix b/pkgs/development/python-modules/tensordict/default.nix index 29c92e118b36..19bfaa01a116 100644 --- a/pkgs/development/python-modules/tensordict/default.nix +++ b/pkgs/development/python-modules/tensordict/default.nix @@ -66,15 +66,10 @@ buildPythonPackage rec { "test_map_iter_interrupt_early" ]; - disabledTestPaths = - [ - # torch._dynamo.exc.Unsupported: Graph break due to unsupported builtin None.ReferenceType.__new__. - "test/test_compile.py" - ] - ++ lib.optionals stdenv.hostPlatform.isDarwin [ - # ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package - "test/test_distributed.py" - ]; + disabledTestPaths = [ + # torch._dynamo.exc.Unsupported: Graph break due to unsupported builtin None.ReferenceType.__new__. + "test/test_compile.py" + ]; meta = { description = "Pytorch dedicated tensor container"; diff --git a/pkgs/development/python-modules/torch/clang19-template-warning.patch b/pkgs/development/python-modules/torch/clang19-template-warning.patch new file mode 100644 index 000000000000..deec511d011b --- /dev/null +++ b/pkgs/development/python-modules/torch/clang19-template-warning.patch @@ -0,0 +1,31 @@ +Submodule third_party/libnop contains modified content +diff --git a/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h b/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h +index cffbde1..b6ebfab 100644 +--- a/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h ++++ b/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h +@@ -238,7 +238,7 @@ class Variant { + // resulting type. + template + void Construct(Args&&... args) { +- index_ = value_.template Construct(std::forward(args)...); ++ index_ = value_.template Construct<>(std::forward(args)...); + } + void Construct(EmptyVariant) {} + +@@ -255,14 +255,14 @@ class Variant { + // multiple element types. + template + void Assign(TypeTag, U&& value) { +- if (!value_.template Assign(TypeTag{}, index_, std::forward(value))) { ++ if (!value_.template Assign<>(TypeTag{}, index_, std::forward(value))) { + Destruct(); + Construct(TypeTag{}, std::forward(value)); + } + } + template + void Assign(T&& value) { +- if (!value_.template Assign(index_, std::forward(value))) { ++ if (!value_.template Assign<>(index_, std::forward(value))) { + Destruct(); + Construct(std::forward(value)); + } diff --git a/pkgs/development/python-modules/torch/default.nix b/pkgs/development/python-modules/torch/default.nix index 2a3e54cab71b..6079bd70b43e 100644 --- a/pkgs/development/python-modules/torch/default.nix +++ b/pkgs/development/python-modules/torch/default.nix @@ -266,7 +266,8 @@ buildPythonPackage rec { }; patches = - lib.optionals cudaSupport [ ./fix-cmake-cuda-toolkit.patch ] + [ ./clang19-template-warning.patch ] + ++ lib.optionals cudaSupport [ ./fix-cmake-cuda-toolkit.patch ] ++ lib.optionals stdenv.hostPlatform.isLinux [ # Propagate CUPTI to Kineto by overriding the search path with environment variables. # https://github.com/pytorch/pytorch/pull/108847 @@ -381,6 +382,10 @@ buildPythonPackage rec { # Explicitly enable MPS for Darwin USE_MPS = setBool stdenv.hostPlatform.isDarwin; + # building torch.distributed on Darwin is disabled by default + # https://pytorch.org/docs/stable/distributed.html#torch.distributed.is_available + USE_DISTRIBUTED = setBool true; + cmakeFlags = [ # (lib.cmakeBool "CMAKE_FIND_DEBUG_MODE" true) diff --git a/pkgs/development/python-modules/torchsnapshot/default.nix b/pkgs/development/python-modules/torchsnapshot/default.nix index 37c882c9fd75..6b4ee0faf986 100644 --- a/pkgs/development/python-modules/torchsnapshot/default.nix +++ b/pkgs/development/python-modules/torchsnapshot/default.nix @@ -67,9 +67,5 @@ buildPythonPackage rec { changelog = "https://github.com/pytorch/torchsnapshot/releases/tag/${version}"; license = lib.licenses.bsd3; maintainers = with lib.maintainers; [ GaetanLepage ]; - badPlatforms = [ - # ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package - lib.systems.inspect.patterns.isDarwin - ]; }; }