From 35b1e5803aa0cb8001fa80c14546a05f581c5e5c Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 22 Dec 2023 00:03:16 +0000 Subject: [PATCH 1/3] cudaPackages.cuda_nvcc: patch nvcc.profile let nvcc know about cudart and the stub drivers; this way we do not need to search for cudart in the setupCudaHook --- .../cuda-modules/cuda/overrides.nix | 25 ++++++++++++++++++- .../cuda-modules/saxpy/default.nix | 1 + .../setup-hooks/setup-cuda-hook.sh | 20 --------------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/pkgs/development/cuda-modules/cuda/overrides.nix b/pkgs/development/cuda-modules/cuda/overrides.nix index fd32978bfb59..ff462b4cc970 100644 --- a/pkgs/development/cuda-modules/cuda/overrides.nix +++ b/pkgs/development/cuda-modules/cuda/overrides.nix @@ -61,7 +61,30 @@ attrsets.filterAttrs (attr: _: (builtins.hasAttr attr prev)) { cuda_nvcc = prev.cuda_nvcc.overrideAttrs ( oldAttrs: { - propagatedBuildInputs = [final.setupCudaHook]; + + outputs = oldAttrs.outputs ++ [ "lib" ]; + + postPatch = + (oldAttrs.postPatch or "") + + '' + substituteInPlace bin/nvcc.profile \ + --replace \ + '$(TOP)/lib' \ + "''${!outputLib}/lib" + + cat << EOF >> bin/nvcc.profile + LIBRARIES += "-L''${!outputBin}/nvvm/lib" "-L$static/lib" "-L${final.cuda_cudart.lib}/lib" "-L${final.cuda_cudart.lib}/lib/stubs" + INCLUDES += "-I''${!outputDev}/include" "-I''${!outputBin}/nvvm/include" "-I${final.cuda_cudart.dev}/include" + EOF + ''; + + propagatedBuildInputs = [ final.setupCudaHook ]; + + postInstall = + (oldAttrs.postInstall or "") + + '' + moveToOutput "nvvm" "''${!outputBin}" + ''; meta = (oldAttrs.meta or {}) // { mainProgram = "nvcc"; diff --git a/pkgs/development/cuda-modules/saxpy/default.nix b/pkgs/development/cuda-modules/saxpy/default.nix index fff52801e3cc..73e17b28757b 100644 --- a/pkgs/development/cuda-modules/saxpy/default.nix +++ b/pkgs/development/cuda-modules/saxpy/default.nix @@ -14,6 +14,7 @@ let cudaVersion flags libcublas + setupCudaHook ; in backendStdenv.mkDerivation { diff --git a/pkgs/development/cuda-modules/setup-hooks/setup-cuda-hook.sh b/pkgs/development/cuda-modules/setup-hooks/setup-cuda-hook.sh index 7b7b3bdde80e..a4a444fcd241 100644 --- a/pkgs/development/cuda-modules/setup-hooks/setup-cuda-hook.sh +++ b/pkgs/development/cuda-modules/setup-hooks/setup-cuda-hook.sh @@ -93,26 +93,6 @@ setupCUDAToolkitCompilers() { if [[ -z "${dontCompressFatbin-}" ]]; then export NVCC_PREPEND_FLAGS+=" -Xfatbin=-compress-all" fi - - # CMake's enable_language(CUDA) runs a compiler test and it doesn't account for - # CUDAToolkit_ROOT. We have to help it locate libcudart - if [[ -z "${nvccDontPrependCudartFlags-}" ]] ; then - if [[ ! -v cudaOutputToPath["cuda_cudart-out"] ]] ; then - echo "setupCUDAToolkitCompilers: missing cudaPackages.cuda_cudart. This may become an an error in the future" >&2 - # exit 1 - fi - for pkg in "${!cudaOutputToPath[@]}" ; do - [[ ! "$pkg" = cuda_cudart* ]] && continue - - local path="${cudaOutputToPath[$pkg]}" - if [[ -d "$path/include" ]] ; then - export NVCC_PREPEND_FLAGS+=" -I$path/include" - fi - if [[ -d "$path/lib" ]] ; then - export NVCC_PREPEND_FLAGS+=" -L$path/lib" - fi - done - fi } preConfigureHooks+=(setupCUDAToolkitCompilers) From d6450daf5dcc078592231bcd9affd2b353a16e16 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 22 Dec 2023 03:24:33 +0000 Subject: [PATCH 2/3] cudaPackages.cuda_nvcc: explain nvcc.profile syntaax --- pkgs/development/cuda-modules/cuda/overrides.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkgs/development/cuda-modules/cuda/overrides.nix b/pkgs/development/cuda-modules/cuda/overrides.nix index ff462b4cc970..92b4a21fe772 100644 --- a/pkgs/development/cuda-modules/cuda/overrides.nix +++ b/pkgs/development/cuda-modules/cuda/overrides.nix @@ -64,6 +64,15 @@ attrsets.filterAttrs (attr: _: (builtins.hasAttr attr prev)) { outputs = oldAttrs.outputs ++ [ "lib" ]; + # Patch the nvcc.profile. + # Syntax: + # - `=` for assignment, + # - `?=` for conditional assignment, + # - `+=` to "prepend", + # - `=+` to "append". + + # Cf. https://web.archive.org/web/20230308044351/https://arcb.csc.ncsu.edu/~mueller/cluster/nvidia/2.0/nvcc_2.0.pdf + postPatch = (oldAttrs.postPatch or "") + '' From 6ac1e13d33f3e75b14d4a1aaca97c52517e6da11 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 22 Dec 2023 03:36:45 +0000 Subject: [PATCH 3/3] cudaPackages.cuda_nvcc: pin the backend cc This makes @ccRoot@ in setupCudaHook redundant when used with the redist cuda packages. We do not touch setupCudaHook yet, because we still have to address the deprecated cudaPackages.cudatoolkit --- .../cuda-modules/cuda/overrides.nix | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/pkgs/development/cuda-modules/cuda/overrides.nix b/pkgs/development/cuda-modules/cuda/overrides.nix index 92b4a21fe772..12e14ef9965b 100644 --- a/pkgs/development/cuda-modules/cuda/overrides.nix +++ b/pkgs/development/cuda-modules/cuda/overrides.nix @@ -73,17 +73,37 @@ attrsets.filterAttrs (attr: _: (builtins.hasAttr attr prev)) { # Cf. https://web.archive.org/web/20230308044351/https://arcb.csc.ncsu.edu/~mueller/cluster/nvidia/2.0/nvcc_2.0.pdf + # We set all variables with the lowest priority (=+), but we do force + # nvcc to use the fixed backend toolchain. Cf. comments in + # backend-stdenv.nix + postPatch = (oldAttrs.postPatch or "") + '' substituteInPlace bin/nvcc.profile \ --replace \ '$(TOP)/lib' \ - "''${!outputLib}/lib" + "''${!outputLib}/lib" \ + --replace \ + '$(TOP)/$(_NVVM_BRANCH_)' \ + "''${!outputBin}/nvvm" \ + --replace \ + '$(TOP)/$(_TARGET_DIR_)/include' \ + "''${!outputDev}/include" cat << EOF >> bin/nvcc.profile - LIBRARIES += "-L''${!outputBin}/nvvm/lib" "-L$static/lib" "-L${final.cuda_cudart.lib}/lib" "-L${final.cuda_cudart.lib}/lib/stubs" - INCLUDES += "-I''${!outputDev}/include" "-I''${!outputBin}/nvvm/include" "-I${final.cuda_cudart.dev}/include" + + # Fix a compatible backend compiler + PATH += ${lib.getBin final.backendStdenv.cc}/bin: + LIBRARIES += "-L${lib.getLib final.backendStdenv.nixpkgsCompatibleLibstdcxx}/lib" + + # Expose the split-out nvvm + LIBRARIES =+ -L''${!outputBin}/nvvm/lib + INCLUDES =+ -I''${!outputBin}/nvvm/include + + # Expose cudart and the libcuda stubs + LIBRARIES =+ -L$static/lib" "-L${final.cuda_cudart.lib}/lib -L${final.cuda_cudart.lib}/lib/stubs + INCLUDES =+ -I${final.cuda_cudart.dev}/include EOF '';