Gaetan Lepage
260 lines
10 KiB
Nix
260 lines
10 KiB
Nix
{
|
|
_cuda,
|
|
backendStdenv,
|
|
buildRedist,
|
|
setupCudaHook,
|
|
cudaAtLeast,
|
|
cudaOlder,
|
|
cccl,
|
|
glibc,
|
|
lib,
|
|
libnvvm,
|
|
makeBinaryWrapper,
|
|
}:
|
|
buildRedist (finalAttrs: {
|
|
redistName = "cuda";
|
|
pname = "cuda_nvcc";
|
|
|
|
# NOTE: We restrict cuda_nvcc to a single output to avoid breaking consumers which expect NVCC to be within a single
|
|
# directory structure. This happens partly because NVCC is also home to NVVM.
|
|
outputs = [
|
|
"out"
|
|
];
|
|
|
|
# The nvcc and cicc binaries contain hard-coded references to /usr
|
|
allowFHSReferences = true;
|
|
|
|
nativeBuildInputs = [
|
|
makeBinaryWrapper
|
|
];
|
|
|
|
# Entries here will be in nativeBuildInputs when cuda_nvcc is in nativeBuildInputs
|
|
propagatedBuildInputs = [ setupCudaHook ];
|
|
|
|
# Patch the nvcc.profile.
|
|
# Syntax:
|
|
# - `=` for assignment,
|
|
# - `?=` for conditional assignment,
|
|
# - `+=` to "prepend",
|
|
# - `=+` to "append".
|
|
|
|
# Cf. https://web.archive.org/web/20220912081901/https://developer.download.nvidia.com/compute/DevZone/docs/html/C/doc/nvcc.pdf
|
|
|
|
# We set all variables with the lowest priority (=+), but we do force
|
|
# nvcc to use the fixed backend toolchain. Cf. comments in
|
|
# backend-stdenv.nix
|
|
|
|
# As an example, here's the nvcc.profile for CUDA 11.8-12.4 (yes, that is a leading newline):
|
|
|
|
#
|
|
# TOP = $(_HERE_)/..
|
|
#
|
|
# NVVMIR_LIBRARY_DIR = $(TOP)/$(_NVVM_BRANCH_)/libdevice
|
|
#
|
|
# LD_LIBRARY_PATH += $(TOP)/lib:
|
|
# PATH += $(TOP)/$(_NVVM_BRANCH_)/bin:$(_HERE_):
|
|
#
|
|
# INCLUDES += "-I$(TOP)/$(_TARGET_DIR_)/include" $(_SPACE_)
|
|
#
|
|
# LIBRARIES =+ $(_SPACE_) "-L$(TOP)/$(_TARGET_DIR_)/lib$(_TARGET_SIZE_)/stubs" "-L$(TOP)/$(_TARGET_DIR_)/lib$(_TARGET_SIZE_)"
|
|
#
|
|
# CUDAFE_FLAGS +=
|
|
# PTXAS_FLAGS +=
|
|
|
|
# And here's the nvcc.profile for CUDA 12.5:
|
|
|
|
#
|
|
# TOP = $(_HERE_)/..
|
|
#
|
|
# CICC_PATH = $(TOP)/nvvm/bin
|
|
# CICC_NEXT_PATH = $(TOP)/nvvm-next/bin
|
|
# NVVMIR_LIBRARY_DIR = $(TOP)/nvvm/libdevice
|
|
#
|
|
# LD_LIBRARY_PATH += $(TOP)/lib:
|
|
# PATH += $(CICC_PATH):$(_HERE_):
|
|
#
|
|
# INCLUDES += "-I$(TOP)/$(_TARGET_DIR_)/include" $(_SPACE_)
|
|
#
|
|
# LIBRARIES =+ $(_SPACE_) "-L$(TOP)/$(_TARGET_DIR_)/lib$(_TARGET_SIZE_)/stubs" "-L$(TOP)/$(_TARGET_DIR_)/lib$(_TARGET_SIZE_)"
|
|
#
|
|
# CUDAFE_FLAGS +=
|
|
# PTXAS_FLAGS +=
|
|
|
|
postInstall =
|
|
let
|
|
# TODO: Should we also patch the LIBRARIES line's use of $(TOP)/$(_TARGET_DIR_)?
|
|
oldNvvmDir = lib.concatStringsSep "/" (
|
|
[ "$(TOP)" ]
|
|
++ lib.optionals (cudaOlder "12.5") [ "$(_NVVM_BRANCH_)" ]
|
|
++ lib.optionals (cudaAtLeast "12.5") [ "nvvm" ]
|
|
);
|
|
newNvvmDir = "\${!outputBin:?}/nvvm";
|
|
in
|
|
lib.optionalString finalAttrs.finalPackage.meta.available (
|
|
# From CUDA 13.0, NVVM is available as a separate library and not bundled in the NVCC redist.
|
|
lib.optionalString (cudaOlder "13.0") ''
|
|
nixLog "moving $PWD/nvvm to ''${!outputBin:?} and renaming lib64 to lib"
|
|
moveToOutput "nvvm" "''${!outputBin:?}"
|
|
mv --verbose --no-clobber "${newNvvmDir}/lib64" "${newNvvmDir}/lib"
|
|
''
|
|
# NVVM is unpacked and made top-level; we cannot make a symlink to it because build systems (like CMake)
|
|
# may take the target and do relative path operations to it.
|
|
+ lib.optionalString (cudaAtLeast "13.0") ''
|
|
nixLog "copying ${libnvvm} to ${newNvvmDir} and fixing permissions"
|
|
cp -rv "${libnvvm}" "${newNvvmDir}"
|
|
chmod -Rv u+w "${newNvvmDir}"
|
|
''
|
|
# Unconditional patching to remove the use of $(_TARGET_SIZE_) since we don't use lib64 in Nixpkgs
|
|
+ ''
|
|
nixLog 'removing $(_TARGET_SIZE_) from nvcc.profile'
|
|
substituteInPlace "''${!outputBin:?}/bin/nvcc.profile" \
|
|
--replace-fail \
|
|
'$(_TARGET_SIZE_)' \
|
|
""
|
|
''
|
|
# CUDA 13.0+ introduced
|
|
# SYSTEM_INCLUDES += "-isystem" "$(TOP)/$(_TARGET_DIR_)/include/cccl" $(_SPACE_)
|
|
# so we need to make sure to patch the reference to cccl.
|
|
+ lib.optionalString (cudaAtLeast "13.0") ''
|
|
nixLog "patching nvcc.profile to include correct path to cccl"
|
|
substituteInPlace "''${!outputBin:?}/bin/nvcc.profile" \
|
|
--replace-fail \
|
|
'$(TOP)/$(_TARGET_DIR_)/include/cccl' \
|
|
"${lib.getOutput "include" cccl}/include"
|
|
''
|
|
# Unconditional patching to switch to the correct include paths.
|
|
# NOTE: _TARGET_DIR_ appears to be used for the target architecture, which is relevant for cross-compilation.
|
|
+ ''
|
|
nixLog "patching nvcc.profile to use the correct include paths"
|
|
substituteInPlace "''${!outputBin:?}/bin/nvcc.profile" \
|
|
--replace-fail \
|
|
'$(TOP)/$(_TARGET_DIR_)/include' \
|
|
"''${!outputInclude:?}/include"
|
|
''
|
|
# Fixup the nvcc.profile to use the correct paths for NVVM.
|
|
# NOTE: In our replacement substitution, we use double quotes to allow for variable expansion.
|
|
# NOTE: We use a trailing slash only on the NVVM directory replacement to prevent partial matches.
|
|
+ ''
|
|
nixLog "patching nvcc.profile to use the correct NVVM paths"
|
|
substituteInPlace "''${!outputBin:?}/bin/nvcc.profile" \
|
|
--replace-fail \
|
|
'${oldNvvmDir}/' \
|
|
"${newNvvmDir}/"
|
|
|
|
nixLog "adding ${newNvvmDir} to nvcc.profile"
|
|
cat << EOF >> "''${!outputBin:?}/bin/nvcc.profile"
|
|
|
|
# Expose the split-out nvvm
|
|
LIBRARIES =+ \$(_SPACE_) "-L${newNvvmDir}/lib"
|
|
INCLUDES =+ \$(_SPACE_) "-I${newNvvmDir}/include"
|
|
EOF
|
|
''
|
|
# Add the dependency on backendStdenv.cc to the nvcc.profile.
|
|
# NOTE: NVCC explodes in horrifying fashion if GCC is not on PATH -- it fails even before
|
|
# reading nvcc.profile!
|
|
+ ''
|
|
nixLog "setting compiler-bindir to backendStdenv.cc in nvcc.profile"
|
|
cat << EOF >> "''${!outputBin:?}/bin/nvcc.profile"
|
|
# Fix a compatible backend compiler
|
|
compiler-bindir = ${backendStdenv.cc}/bin
|
|
EOF
|
|
|
|
nixLog "wrapping nvcc to add backendStdenv.cc to its PATH"
|
|
wrapProgramBinary \
|
|
"''${!outputBin:?}/bin/nvcc" \
|
|
--prefix PATH : ${lib.makeBinPath [ backendStdenv.cc ]}
|
|
''
|
|
# Fix compatibility with glibc 2.42:
|
|
# The cospi|sinpi|rsqrt function signatures in include/common/math_functions.h do not match
|
|
# glibc 2.42's.
|
|
# Indeed, there they are declared with noexcept(true) which is not the case in cuda_nvcc.
|
|
# - In CUDA < 13.0, sinpi/cospi/rsqrt all lack exception specifiers.
|
|
# - In CUDA >= 13.0, NVIDIA fixed sinpi/cospi (using __NV_IEC_60559_FUNCS_EXCEPTION_SPECIFIER)
|
|
# but rsqrt/rsqrtf still lack noexcept, so we only patch those.
|
|
# As the CRT headers (including math_functions.h) moved to the cuda_crt package, the glibc
|
|
# 2.42 compatibility patch is applied there instead.
|
|
+ lib.optionalString (cudaOlder "13.0" && lib.versionAtLeast glibc.version "2.42") ''
|
|
nixLog "Patching math_functions.h signatures to match glibc's ones"
|
|
substituteInPlace "''${!outputInclude:?}/include/crt/math_functions.h" \
|
|
--replace-fail \
|
|
"sinpi(double x);" \
|
|
"sinpi(double x) noexcept (true);" \
|
|
--replace-fail \
|
|
"sinpif(float x);" \
|
|
"sinpif(float x) noexcept (true);" \
|
|
--replace-fail \
|
|
"cospi(double x);" \
|
|
"cospi(double x) noexcept (true);" \
|
|
--replace-fail \
|
|
"cospif(float x);" \
|
|
"cospif(float x) noexcept (true);" \
|
|
--replace-fail \
|
|
"rsqrt(double x);" \
|
|
"rsqrt(double x) noexcept (true);" \
|
|
--replace-fail \
|
|
"rsqrtf(float x);" \
|
|
"rsqrtf(float x) noexcept (true);"
|
|
|
|
# math_functions.hpp has the same functions wrapped in __func__() macros.
|
|
# These also need throw() annotations to match glibc 2.42's declarations.
|
|
nixLog "Patching math_functions.hpp signatures to match glibc's ones"
|
|
substituteInPlace "''${!outputInclude:?}/include/crt/math_functions.hpp" \
|
|
--replace-fail \
|
|
"__func__(double rsqrt(const double a))" \
|
|
"__func__(double rsqrt(const double a) throw())" \
|
|
--replace-fail \
|
|
"__func__(double sinpi(double a))" \
|
|
"__func__(double sinpi(double a) throw())" \
|
|
--replace-fail \
|
|
"__func__(double cospi(double a))" \
|
|
"__func__(double cospi(double a) throw())" \
|
|
--replace-fail \
|
|
"__func__(float rsqrtf(const float a))" \
|
|
"__func__(float rsqrtf(const float a) throw())" \
|
|
--replace-fail \
|
|
"__func__(float sinpif(const float a))" \
|
|
"__func__(float sinpif(const float a) throw())" \
|
|
--replace-fail \
|
|
"__func__(float cospif(const float a))" \
|
|
"__func__(float cospif(const float a) throw())"
|
|
''
|
|
|
|
# Fix clang CUDA compilation: host_defines.h redefines __noinline__ as
|
|
# __attribute__((noinline)), which conflicts with libstdc++ >=12 using
|
|
# __attribute__((__noinline__)) — the macro expands to
|
|
# __attribute__((__attribute__((noinline)))) which is invalid.
|
|
# Clang natively understands __noinline__ as an attribute so the macro
|
|
# is unnecessary. Skip it when clang is the compiler.
|
|
+ lib.optionalString (cudaOlder "13.0") ''
|
|
nixLog "Patching host_defines.h to skip __noinline__ macro under clang"
|
|
substituteInPlace "''${!outputInclude:?}/include/crt/host_defines.h" \
|
|
--replace-fail \
|
|
'#if defined(__CUDACC__) || defined(__CUDA_ARCH__) || defined(__CUDA_LIBDEVICE__)' \
|
|
'#if (defined(__CUDACC__) || defined(__CUDA_ARCH__) || defined(__CUDA_LIBDEVICE__)) && !defined(__clang__)'
|
|
''
|
|
);
|
|
|
|
brokenAssertions = [
|
|
# TODO(@connorbaker): Build fails on x86 when using pkgsLLVM.
|
|
# .../include/crt/host_defines.h:67:2:
|
|
# error: "libc++ is not supported on x86 system"
|
|
#
|
|
# 67 | #error "libc++ is not supported on x86 system"
|
|
# | ^
|
|
#
|
|
# 1 error generated.
|
|
#
|
|
# # --error 0x1 --
|
|
{
|
|
message = "cannot use libc++ on x86_64-linux";
|
|
assertion = backendStdenv.hostNixSystem == "x86_64-linux" -> backendStdenv.cc.libcxx == null;
|
|
}
|
|
];
|
|
|
|
meta = {
|
|
description = "CUDA compiler driver";
|
|
homepage = "https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc";
|
|
mainProgram = "nvcc";
|
|
};
|
|
})
|