gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

python312Packages.torch: build torch.distributed on darwin

Browse Source 
It's disabled on MacOS by default:
https://pytorch.org/docs/stable/distributed.html#torch.distributed.is_available

When distributed module build is enabled, a compilation error happens on
fresh clang.

Upstream repo (which is a submodule in pytorch) is archived so it's
impossible to contribute the fix there. Instead, an inline patch is
included that is enough to fix the error.

Signed-off-by: Ihar Hrachyshka <ihar.hrachyshka@gmail.com>
This commit is contained in:
Ihar Hrachyshka
2025-04-01 21:23:51 -04:00
parent 89624d61a6
commit 65b802a6fd
14 changed files with 59 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkgs/development/python-modules/colbert-ai/default.nix
-5
View File
@@ -74,10 +74,5 @@ buildPythonPackage rec {
maintainers = with lib.maintainers; [
bachp
];
badPlatforms = [
# `import torch.distributed.tensor` fails on darwin:
# ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a packag
lib.systems.inspect.patterns.isDarwin
];
};
}
pkgs/development/python-modules/compressed-tensors/default.nix
+9 -15
View File
@@ -43,21 +43,15 @@ buildPythonPackage rec {
pytestCheckHook
];
disabledTests =
[
# these try to download models from HF Hub
"test_get_observer_token_count"
"test_kv_cache_quantization"
"test_target_prioritization"
"test_load_compressed_sharded"
"test_save_compressed_model"
"test_apply_tinyllama_dynamic_activations"
]
++ lib.optionals stdenv.hostPlatform.isDarwin [
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
"test_composability"
"test_missing_and_unexpected_keys_on_compression"
];
disabledTests = [
# these try to download models from HF Hub
"test_get_observer_token_count"
"test_kv_cache_quantization"
"test_target_prioritization"
"test_load_compressed_sharded"
"test_save_compressed_model"
"test_apply_tinyllama_dynamic_activations"
];
disabledTestPaths = [
# these try to download models from HF Hub
pkgs/development/python-modules/docling-ibm-models/default.nix
-6
View File
@@ -83,12 +83,6 @@ buildPythonPackage rec {
"test_tf_predictor"
];
disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
"tests/test_code_formula_predictor.py"
"tests/test_layout_predictor.py"
];
meta = {
changelog = "https://github.com/DS4SD/docling-ibm-models/blob/${src.tag}/CHANGELOG.md";
description = "Docling IBM models";
pkgs/development/python-modules/docling/default.nix
-20
View File
@@ -164,26 +164,6 @@ buildPythonPackage rec {
"test_e2e_valid_csv_conversions"
];
disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
"tests/test_backend_csv.py"
"tests/test_backend_html.py"
"tests/test_backend_jats.py"
"tests/test_backend_msexcel.py"
"tests/test_backend_msword.py"
"tests/test_backend_pptx.py"
"tests/test_cli.py"
"tests/test_code_formula.py"
"tests/test_document_picture_classifier.py"
"tests/test_e2e_conversion.py"
"tests/test_e2e_ocr_conversion.py"
"tests/test_input_doc.py"
"tests/test_interfaces.py"
"tests/test_invalid_input.py"
"tests/test_legacy_format_transform.py"
"tests/test_options.py"
];
meta = {
description = "Get your documents ready for gen AI";
homepage = "https://github.com/DS4SD/docling";
pkgs/development/python-modules/finetuning-scheduler/default.nix
-4
View File
@@ -71,9 +71,5 @@ buildPythonPackage rec {
changelog = "https://github.com/speediedan/finetuning-scheduler/blob/v${version}/CHANGELOG.md";
license = lib.licenses.asl20;
maintainers = with lib.maintainers; [ bcdarwin ];
badPlatforms = [
# "No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package" at import time:
lib.systems.inspect.patterns.isDarwin
];
};
}
pkgs/development/python-modules/gradio/default.nix
+9 -14
View File
@@ -301,21 +301,16 @@ buildPythonPackage rec {
"test_updates_stored_up_to_capacity"
"test_varying_output_forms_with_generators"
];
disabledTestPaths =
[
# 100% touches network
"test/test_networking.py"
"client/python/test/test_client.py"
# makes pytest freeze 50% of the time
"test/test_interfaces.py"
disabledTestPaths = [
# 100% touches network
"test/test_networking.py"
"client/python/test/test_client.py"
# makes pytest freeze 50% of the time
"test/test_interfaces.py"
# Local network tests dependant on port availability (port 7860-7959)
"test/test_routes.py"
]
++ lib.optionals (stdenv.hostPlatform.isDarwin) [
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
"test/test_pipelines.py"
];
# Local network tests dependant on port availability (port 7860-7959)
"test/test_routes.py"
];
pytestFlagsArray = [
"-x" # abort on first failure
"-m 'not flaky'"
pkgs/development/python-modules/ignite/default.nix
-2
View File
@@ -96,7 +96,5 @@ buildPythonPackage rec {
changelog = "https://github.com/pytorch/ignite/releases/tag/v${version}";
license = lib.licenses.bsd3;
maintainers = [ lib.maintainers.bcdarwin ];
# ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
broken = stdenv.hostPlatform.isDarwin;
};
}
pkgs/development/python-modules/peft/default.nix
-4
View File
@@ -104,9 +104,5 @@ buildPythonPackage rec {
changelog = "https://github.com/huggingface/peft/releases/tag/v${version}";
license = lib.licenses.asl20;
maintainers = with lib.maintainers; [ bcdarwin ];
badPlatforms = [
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
lib.systems.inspect.patterns.isDarwin
];
};
}
pkgs/development/python-modules/sentence-transformers/default.nix
-4
View File
@@ -113,9 +113,5 @@ buildPythonPackage rec {
changelog = "https://github.com/UKPLab/sentence-transformers/releases/tag/v${version}";
license = lib.licenses.asl20;
maintainers = with lib.maintainers; [ dit7ya ];
badPlatforms = [
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
lib.systems.inspect.patterns.isDarwin
];
};
}
pkgs/development/python-modules/smolagents/default.nix
-6
View File
@@ -136,12 +136,6 @@ buildPythonPackage rec {
"test_new_instance"
];
disabledTestPaths = lib.optionals stdenv.hostPlatform.isDarwin [
# No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
"tests/test_final_answer.py"
"tests/test_types.py"
];
__darwinAllowLocalNetworking = true;
meta = {
pkgs/development/python-modules/tensordict/default.nix
+4 -9
View File
@@ -66,15 +66,10 @@ buildPythonPackage rec {
"test_map_iter_interrupt_early"
];
disabledTestPaths =
[
# torch._dynamo.exc.Unsupported: Graph break due to unsupported builtin None.ReferenceType.__new__.
"test/test_compile.py"
]
++ lib.optionals stdenv.hostPlatform.isDarwin [
# ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
"test/test_distributed.py"
];
disabledTestPaths = [
# torch._dynamo.exc.Unsupported: Graph break due to unsupported builtin None.ReferenceType.__new__.
"test/test_compile.py"
];
meta = {
description = "Pytorch dedicated tensor container";
pkgs/development/python-modules/torch/clang19-template-warning.patch
+31
View File
@@ -0,0 +1,31 @@
Submodule third_party/libnop contains modified content
diff --git a/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h b/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h
index cffbde1..b6ebfab 100644
--- a/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h
+++ b/third_party/tensorpipe/third_party/libnop/include/nop/types/variant.h
@@ -238,7 +238,7 @@ class Variant {
// resulting type.
template <typename... Args>
void Construct(Args&&... args) {
- index_ = value_.template Construct(std::forward<Args>(args)...);
+ index_ = value_.template Construct<>(std::forward<Args>(args)...);
}
void Construct(EmptyVariant) {}
@@ -255,14 +255,14 @@ class Variant {
// multiple element types.
template <typename T, typename U>
void Assign(TypeTag<T>, U&& value) {
- if (!value_.template Assign(TypeTag<T>{}, index_, std::forward<U>(value))) {
+ if (!value_.template Assign<>(TypeTag<T>{}, index_, std::forward<U>(value))) {
Destruct();
Construct(TypeTag<T>{}, std::forward<U>(value));
}
}
template <typename T>
void Assign(T&& value) {
- if (!value_.template Assign(index_, std::forward<T>(value))) {
+ if (!value_.template Assign<>(index_, std::forward<T>(value))) {
Destruct();
Construct(std::forward<T>(value));
}
pkgs/development/python-modules/torch/default.nix
+6 -1
View File
@@ -266,7 +266,8 @@ buildPythonPackage rec {
};
patches =
lib.optionals cudaSupport [ ./fix-cmake-cuda-toolkit.patch ]
[ ./clang19-template-warning.patch ]
++ lib.optionals cudaSupport [ ./fix-cmake-cuda-toolkit.patch ]
++ lib.optionals stdenv.hostPlatform.isLinux [
# Propagate CUPTI to Kineto by overriding the search path with environment variables.
# https://github.com/pytorch/pytorch/pull/108847
@@ -381,6 +382,10 @@ buildPythonPackage rec {
# Explicitly enable MPS for Darwin
USE_MPS = setBool stdenv.hostPlatform.isDarwin;
# building torch.distributed on Darwin is disabled by default
# https://pytorch.org/docs/stable/distributed.html#torch.distributed.is_available
USE_DISTRIBUTED = setBool true;
cmakeFlags =
[
# (lib.cmakeBool "CMAKE_FIND_DEBUG_MODE" true)
pkgs/development/python-modules/torchsnapshot/default.nix
-4
View File
@@ -67,9 +67,5 @@ buildPythonPackage rec {
changelog = "https://github.com/pytorch/torchsnapshot/releases/tag/${version}";
license = lib.licenses.bsd3;
maintainers = with lib.maintainers; [ GaetanLepage ];
badPlatforms = [
# ModuleNotFoundError: No module named 'torch._C._distributed_c10d'; 'torch._C' is not a package
lib.systems.inspect.patterns.isDarwin
];
};
}