From 56be58429ae69e7139d876c4b20dcaf60c072408 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Tue, 6 May 2025 09:33:24 -0700 Subject: [PATCH] cudaPackages: introduce and use lib.packagesFromDirectoryRecursive Signed-off-by: Connor Baker --- .../auto-add-cuda-compat-runpath.sh | 0 .../autoAddCudaCompatRunpath/package.nix | 27 +++++++++ .../backendStdenv.nix} | 8 +++ ...at-out-common-python-bindings-option.patch | 0 ...-so-headers-can-be-discovered-when-i.patch | 0 ...install-samples-and-tests-when-built.patch | 0 ...fix-instances-of-maybe-uninitialized.patch | 0 .../cudnn-frontend/CMakeLists.txt | 0 .../cudnn_frontend-config.cmake.in | 0 .../cudnn-frontend/package.nix} | 0 .../mark-for-cudatoolkit-root-hook.sh | 0 .../markForCudatoolkitRootHook/package.nix | 4 ++ .../default.nix => packages/nccl-tests.nix} | 0 .../{nccl/default.nix => packages/nccl.nix} | 0 .../saxpy/package.nix} | 0 .../{ => packages}/saxpy/src/CMakeLists.txt | 0 .../{ => packages}/saxpy/src/saxpy.cu | 0 .../packages/setupCudaHook/package.nix | 14 +++++ .../setupCudaHook}/setup-cuda-hook.sh | 0 .../writeGpuTestPython.nix} | 0 .../cuda-modules/setup-hooks/extension.nix | 55 ------------------- pkgs/top-level/cuda-packages.nix | 33 ++++------- 22 files changed, 65 insertions(+), 76 deletions(-) rename pkgs/development/cuda-modules/{setup-hooks => packages/autoAddCudaCompatRunpath}/auto-add-cuda-compat-runpath.sh (100%) create mode 100644 pkgs/development/cuda-modules/packages/autoAddCudaCompatRunpath/package.nix rename pkgs/development/cuda-modules/{backend-stdenv.nix => packages/backendStdenv.nix} (63%) rename pkgs/development/cuda-modules/{ => packages}/cudnn-frontend/0001-cmake-float-out-common-python-bindings-option.patch (100%) rename pkgs/development/cuda-modules/{ => packages}/cudnn-frontend/0002-cmake-add-config-so-headers-can-be-discovered-when-i.patch (100%) rename pkgs/development/cuda-modules/{ => packages}/cudnn-frontend/0003-cmake-install-samples-and-tests-when-built.patch (100%) rename pkgs/development/cuda-modules/{ => packages}/cudnn-frontend/0004-samples-fix-instances-of-maybe-uninitialized.patch (100%) rename pkgs/development/cuda-modules/{ => packages}/cudnn-frontend/CMakeLists.txt (100%) rename pkgs/development/cuda-modules/{ => packages}/cudnn-frontend/cudnn_frontend-config.cmake.in (100%) rename pkgs/development/cuda-modules/{cudnn-frontend/default.nix => packages/cudnn-frontend/package.nix} (100%) rename pkgs/development/cuda-modules/{setup-hooks => packages/markForCudatoolkitRootHook}/mark-for-cudatoolkit-root-hook.sh (100%) create mode 100644 pkgs/development/cuda-modules/packages/markForCudatoolkitRootHook/package.nix rename pkgs/development/cuda-modules/{nccl-tests/default.nix => packages/nccl-tests.nix} (100%) rename pkgs/development/cuda-modules/{nccl/default.nix => packages/nccl.nix} (100%) rename pkgs/development/cuda-modules/{saxpy/default.nix => packages/saxpy/package.nix} (100%) rename pkgs/development/cuda-modules/{ => packages}/saxpy/src/CMakeLists.txt (100%) rename pkgs/development/cuda-modules/{ => packages}/saxpy/src/saxpy.cu (100%) create mode 100644 pkgs/development/cuda-modules/packages/setupCudaHook/package.nix rename pkgs/development/cuda-modules/{setup-hooks => packages/setupCudaHook}/setup-cuda-hook.sh (100%) rename pkgs/development/cuda-modules/{write-gpu-test-python.nix => packages/writeGpuTestPython.nix} (100%) delete mode 100644 pkgs/development/cuda-modules/setup-hooks/extension.nix diff --git a/pkgs/development/cuda-modules/setup-hooks/auto-add-cuda-compat-runpath.sh b/pkgs/development/cuda-modules/packages/autoAddCudaCompatRunpath/auto-add-cuda-compat-runpath.sh similarity index 100% rename from pkgs/development/cuda-modules/setup-hooks/auto-add-cuda-compat-runpath.sh rename to pkgs/development/cuda-modules/packages/autoAddCudaCompatRunpath/auto-add-cuda-compat-runpath.sh diff --git a/pkgs/development/cuda-modules/packages/autoAddCudaCompatRunpath/package.nix b/pkgs/development/cuda-modules/packages/autoAddCudaCompatRunpath/package.nix new file mode 100644 index 000000000000..e94fbe485283 --- /dev/null +++ b/pkgs/development/cuda-modules/packages/autoAddCudaCompatRunpath/package.nix @@ -0,0 +1,27 @@ +# autoAddCudaCompatRunpath hook must be added AFTER `setupCudaHook`. Both +# hooks prepend a path with `libcuda.so` to the `DT_RUNPATH` section of +# patched elf files, but `cuda_compat` path must take precedence (otherwise, +# it doesn't have any effect) and thus appear first. Meaning this hook must be +# executed last. +{ + autoFixElfFiles, + cuda_compat ? null, + flags, + lib, + makeSetupHook, +}: +makeSetupHook { + name = "auto-add-cuda-compat-runpath-hook"; + propagatedBuildInputs = [ autoFixElfFiles ]; + + substitutions = { + # Hotfix Ofborg evaluation + libcudaPath = if flags.isJetsonBuild then "${cuda_compat}/compat" else null; + }; + + meta.broken = !flags.isJetsonBuild; + + # Pre-cuda_compat CUDA release: + meta.badPlatforms = lib.optionals (cuda_compat == null) lib.platforms.all; + meta.platforms = cuda_compat.meta.platforms or [ ]; +} ./auto-add-cuda-compat-runpath.sh diff --git a/pkgs/development/cuda-modules/backend-stdenv.nix b/pkgs/development/cuda-modules/packages/backendStdenv.nix similarity index 63% rename from pkgs/development/cuda-modules/backend-stdenv.nix rename to pkgs/development/cuda-modules/packages/backendStdenv.nix index 57219ad607c0..9fd705f98193 100644 --- a/pkgs/development/cuda-modules/backend-stdenv.nix +++ b/pkgs/development/cuda-modules/packages/backendStdenv.nix @@ -1,3 +1,11 @@ +# Exposed as cudaPackages.backendStdenv. +# This is what nvcc uses as a backend, +# and it has to be an officially supported one (e.g. gcc11 for cuda11). +# +# It, however, propagates current stdenv's libstdc++ to avoid "GLIBCXX_* not found errors" +# when linked with other C++ libraries. +# E.g. for cudaPackages_11_8 we use gcc11 with gcc12's libstdc++ +# Cf. https://github.com/NixOS/nixpkgs/pull/218265 for context { cudaVersion, lib, diff --git a/pkgs/development/cuda-modules/cudnn-frontend/0001-cmake-float-out-common-python-bindings-option.patch b/pkgs/development/cuda-modules/packages/cudnn-frontend/0001-cmake-float-out-common-python-bindings-option.patch similarity index 100% rename from pkgs/development/cuda-modules/cudnn-frontend/0001-cmake-float-out-common-python-bindings-option.patch rename to pkgs/development/cuda-modules/packages/cudnn-frontend/0001-cmake-float-out-common-python-bindings-option.patch diff --git a/pkgs/development/cuda-modules/cudnn-frontend/0002-cmake-add-config-so-headers-can-be-discovered-when-i.patch b/pkgs/development/cuda-modules/packages/cudnn-frontend/0002-cmake-add-config-so-headers-can-be-discovered-when-i.patch similarity index 100% rename from pkgs/development/cuda-modules/cudnn-frontend/0002-cmake-add-config-so-headers-can-be-discovered-when-i.patch rename to pkgs/development/cuda-modules/packages/cudnn-frontend/0002-cmake-add-config-so-headers-can-be-discovered-when-i.patch diff --git a/pkgs/development/cuda-modules/cudnn-frontend/0003-cmake-install-samples-and-tests-when-built.patch b/pkgs/development/cuda-modules/packages/cudnn-frontend/0003-cmake-install-samples-and-tests-when-built.patch similarity index 100% rename from pkgs/development/cuda-modules/cudnn-frontend/0003-cmake-install-samples-and-tests-when-built.patch rename to pkgs/development/cuda-modules/packages/cudnn-frontend/0003-cmake-install-samples-and-tests-when-built.patch diff --git a/pkgs/development/cuda-modules/cudnn-frontend/0004-samples-fix-instances-of-maybe-uninitialized.patch b/pkgs/development/cuda-modules/packages/cudnn-frontend/0004-samples-fix-instances-of-maybe-uninitialized.patch similarity index 100% rename from pkgs/development/cuda-modules/cudnn-frontend/0004-samples-fix-instances-of-maybe-uninitialized.patch rename to pkgs/development/cuda-modules/packages/cudnn-frontend/0004-samples-fix-instances-of-maybe-uninitialized.patch diff --git a/pkgs/development/cuda-modules/cudnn-frontend/CMakeLists.txt b/pkgs/development/cuda-modules/packages/cudnn-frontend/CMakeLists.txt similarity index 100% rename from pkgs/development/cuda-modules/cudnn-frontend/CMakeLists.txt rename to pkgs/development/cuda-modules/packages/cudnn-frontend/CMakeLists.txt diff --git a/pkgs/development/cuda-modules/cudnn-frontend/cudnn_frontend-config.cmake.in b/pkgs/development/cuda-modules/packages/cudnn-frontend/cudnn_frontend-config.cmake.in similarity index 100% rename from pkgs/development/cuda-modules/cudnn-frontend/cudnn_frontend-config.cmake.in rename to pkgs/development/cuda-modules/packages/cudnn-frontend/cudnn_frontend-config.cmake.in diff --git a/pkgs/development/cuda-modules/cudnn-frontend/default.nix b/pkgs/development/cuda-modules/packages/cudnn-frontend/package.nix similarity index 100% rename from pkgs/development/cuda-modules/cudnn-frontend/default.nix rename to pkgs/development/cuda-modules/packages/cudnn-frontend/package.nix diff --git a/pkgs/development/cuda-modules/setup-hooks/mark-for-cudatoolkit-root-hook.sh b/pkgs/development/cuda-modules/packages/markForCudatoolkitRootHook/mark-for-cudatoolkit-root-hook.sh similarity index 100% rename from pkgs/development/cuda-modules/setup-hooks/mark-for-cudatoolkit-root-hook.sh rename to pkgs/development/cuda-modules/packages/markForCudatoolkitRootHook/mark-for-cudatoolkit-root-hook.sh diff --git a/pkgs/development/cuda-modules/packages/markForCudatoolkitRootHook/package.nix b/pkgs/development/cuda-modules/packages/markForCudatoolkitRootHook/package.nix new file mode 100644 index 000000000000..86ff28d6c41a --- /dev/null +++ b/pkgs/development/cuda-modules/packages/markForCudatoolkitRootHook/package.nix @@ -0,0 +1,4 @@ +# Internal hook, used by cudatoolkit and cuda redist packages +# to accommodate automatic CUDAToolkit_ROOT construction +{ makeSetupHook }: +makeSetupHook { name = "mark-for-cudatoolkit-root-hook"; } ./mark-for-cudatoolkit-root-hook.sh diff --git a/pkgs/development/cuda-modules/nccl-tests/default.nix b/pkgs/development/cuda-modules/packages/nccl-tests.nix similarity index 100% rename from pkgs/development/cuda-modules/nccl-tests/default.nix rename to pkgs/development/cuda-modules/packages/nccl-tests.nix diff --git a/pkgs/development/cuda-modules/nccl/default.nix b/pkgs/development/cuda-modules/packages/nccl.nix similarity index 100% rename from pkgs/development/cuda-modules/nccl/default.nix rename to pkgs/development/cuda-modules/packages/nccl.nix diff --git a/pkgs/development/cuda-modules/saxpy/default.nix b/pkgs/development/cuda-modules/packages/saxpy/package.nix similarity index 100% rename from pkgs/development/cuda-modules/saxpy/default.nix rename to pkgs/development/cuda-modules/packages/saxpy/package.nix diff --git a/pkgs/development/cuda-modules/saxpy/src/CMakeLists.txt b/pkgs/development/cuda-modules/packages/saxpy/src/CMakeLists.txt similarity index 100% rename from pkgs/development/cuda-modules/saxpy/src/CMakeLists.txt rename to pkgs/development/cuda-modules/packages/saxpy/src/CMakeLists.txt diff --git a/pkgs/development/cuda-modules/saxpy/src/saxpy.cu b/pkgs/development/cuda-modules/packages/saxpy/src/saxpy.cu similarity index 100% rename from pkgs/development/cuda-modules/saxpy/src/saxpy.cu rename to pkgs/development/cuda-modules/packages/saxpy/src/saxpy.cu diff --git a/pkgs/development/cuda-modules/packages/setupCudaHook/package.nix b/pkgs/development/cuda-modules/packages/setupCudaHook/package.nix new file mode 100644 index 000000000000..d7560f132c4c --- /dev/null +++ b/pkgs/development/cuda-modules/packages/setupCudaHook/package.nix @@ -0,0 +1,14 @@ +# Currently propagated by cuda_nvcc or cudatoolkit, rather than used directly +{ makeSetupHook, backendStdenv }: +makeSetupHook { + name = "setup-cuda-hook"; + + substitutions.setupCudaHook = placeholder "out"; + + # Point NVCC at a compatible compiler + substitutions.ccRoot = "${backendStdenv.cc}"; + + # Required in addition to ccRoot as otherwise bin/gcc is looked up + # when building CMakeCUDACompilerId.cu + substitutions.ccFullPath = "${backendStdenv.cc}/bin/${backendStdenv.cc.targetPrefix}c++"; +} ./setup-cuda-hook.sh diff --git a/pkgs/development/cuda-modules/setup-hooks/setup-cuda-hook.sh b/pkgs/development/cuda-modules/packages/setupCudaHook/setup-cuda-hook.sh similarity index 100% rename from pkgs/development/cuda-modules/setup-hooks/setup-cuda-hook.sh rename to pkgs/development/cuda-modules/packages/setupCudaHook/setup-cuda-hook.sh diff --git a/pkgs/development/cuda-modules/write-gpu-test-python.nix b/pkgs/development/cuda-modules/packages/writeGpuTestPython.nix similarity index 100% rename from pkgs/development/cuda-modules/write-gpu-test-python.nix rename to pkgs/development/cuda-modules/packages/writeGpuTestPython.nix diff --git a/pkgs/development/cuda-modules/setup-hooks/extension.nix b/pkgs/development/cuda-modules/setup-hooks/extension.nix deleted file mode 100644 index 327f335f1890..000000000000 --- a/pkgs/development/cuda-modules/setup-hooks/extension.nix +++ /dev/null @@ -1,55 +0,0 @@ -final: _: { - # Internal hook, used by cudatoolkit and cuda redist packages - # to accommodate automatic CUDAToolkit_ROOT construction - markForCudatoolkitRootHook = final.callPackage ( - { makeSetupHook }: - makeSetupHook { name = "mark-for-cudatoolkit-root-hook"; } ./mark-for-cudatoolkit-root-hook.sh - ) { }; - - # Currently propagated by cuda_nvcc or cudatoolkit, rather than used directly - setupCudaHook = ( - final.callPackage ( - { makeSetupHook, backendStdenv }: - makeSetupHook { - name = "setup-cuda-hook"; - - substitutions.setupCudaHook = placeholder "out"; - - # Point NVCC at a compatible compiler - substitutions.ccRoot = "${backendStdenv.cc}"; - - # Required in addition to ccRoot as otherwise bin/gcc is looked up - # when building CMakeCUDACompilerId.cu - substitutions.ccFullPath = "${backendStdenv.cc}/bin/${backendStdenv.cc.targetPrefix}c++"; - } ./setup-cuda-hook.sh - ) { } - ); - - # autoAddCudaCompatRunpath hook must be added AFTER `setupCudaHook`. Both - # hooks prepend a path with `libcuda.so` to the `DT_RUNPATH` section of - # patched elf files, but `cuda_compat` path must take precedence (otherwise, - # it doesn't have any effect) and thus appear first. Meaning this hook must be - # executed last. - autoAddCudaCompatRunpath = final.callPackage ( - { - makeSetupHook, - autoFixElfFiles, - cuda_compat ? null, - }: - makeSetupHook { - name = "auto-add-cuda-compat-runpath-hook"; - propagatedBuildInputs = [ autoFixElfFiles ]; - - substitutions = { - # Hotfix Ofborg evaluation - libcudaPath = if final.flags.isJetsonBuild then "${cuda_compat}/compat" else null; - }; - - meta.broken = !final.flags.isJetsonBuild; - - # Pre-cuda_compat CUDA release: - meta.badPlatforms = final.lib.optionals (cuda_compat == null) final.lib.platforms.all; - meta.platforms = cuda_compat.meta.platforms or [ ]; - } ./auto-add-cuda-compat-runpath.sh - ) { }; -} diff --git a/pkgs/top-level/cuda-packages.nix b/pkgs/top-level/cuda-packages.nix index 3770aee3bae7..4f44a495fd52 100644 --- a/pkgs/top-level/cuda-packages.nix +++ b/pkgs/top-level/cuda-packages.nix @@ -35,14 +35,13 @@ let fixedPoints lists strings - trivial versions ; # Backbone gpus = builtins.import ../development/cuda-modules/gpus.nix; nvccCompatibilities = builtins.import ../development/cuda-modules/nvcc-compatibilities.nix; flags = callPackage ../development/cuda-modules/flags.nix { inherit cudaVersion gpus; }; - passthruFunction = final: ({ + passthruFunction = final: { inherit cudaVersion lib pkgs; inherit gpus nvccCompatibilities flags; cudaMajorVersion = versions.major cudaVersion; @@ -63,27 +62,15 @@ let # TODO(@connorbaker): `cudaFlags` is an alias for `flags` which should be removed in the future. cudaFlags = flags; - # Exposed as cudaPackages.backendStdenv. - # This is what nvcc uses as a backend, - # and it has to be an officially supported one (e.g. gcc11 for cuda11). - # - # It, however, propagates current stdenv's libstdc++ to avoid "GLIBCXX_* not found errors" - # when linked with other C++ libraries. - # E.g. for cudaPackages_11_8 we use gcc11 with gcc12's libstdc++ - # Cf. https://github.com/NixOS/nixpkgs/pull/218265 for context - backendStdenv = final.callPackage ../development/cuda-modules/backend-stdenv.nix { }; - # Loose packages + # Barring packages which share a home (e.g., cudatoolkit and cudatoolkit-legacy-runfile), new packages + # should be added to ../development/cuda-modules/packages in "by-name" style, where they will be automatically + # discovered and added to the package set. # TODO: Move to aliases.nix once all Nixpkgs has migrated to the splayed CUDA packages cudatoolkit = final.callPackage ../development/cuda-modules/cudatoolkit/redist-wrapper.nix { }; cudatoolkit-legacy-runfile = final.callPackage ../development/cuda-modules/cudatoolkit { }; - cudnn-frontend = final.callPackage ../development/cuda-modules/cudnn-frontend/default.nix { }; - saxpy = final.callPackage ../development/cuda-modules/saxpy { }; - nccl = final.callPackage ../development/cuda-modules/nccl { }; - nccl-tests = final.callPackage ../development/cuda-modules/nccl-tests { }; - tests = let bools = [ @@ -122,9 +109,7 @@ let }; in attrsets.listToAttrs (attrsets.mapCartesianProduct builder configs); - - writeGpuTestPython = final.callPackage ../development/cuda-modules/write-gpu-test-python.nix { }; - }); + }; mkVersionedPackageName = name: version: @@ -135,7 +120,13 @@ let composedExtension = fixedPoints.composeManyExtensions ( [ - (import ../development/cuda-modules/setup-hooks/extension.nix) + ( + final: _: + lib.packagesFromDirectoryRecursive { + inherit (final) callPackage; + directory = ../development/cuda-modules/packages; + } + ) (callPackage ../development/cuda-modules/cuda/extension.nix { inherit cudaVersion; }) (import ../development/cuda-modules/cuda/overrides.nix) (callPackage ../development/cuda-modules/generic-builders/multiplex.nix {