From 59331c3a174455ff68cdaa959dd98a5fd13a6930 Mon Sep 17 00:00:00 2001 From: Peter Waller Date: Fri, 6 Dec 2024 11:26:22 +0000 Subject: [PATCH] llvmPackages: Split tablegen into its own derivation Background: LLVM has some tools that run at build time. In native builds, these are built as a part of the usual build, but in cross builds they need to come from buildPackages. In many scenarios this is a small problem because LLVM from buildPackages is already available as a build; but if cross building a version of LLVM which is not available (e.g. a new git commit of LLVM) this results in two builds of LLVM and clang, one native and one for the cross. Full builds of LLVM are expensive; and unnecessary in this scenario. We don't need a native LLVM, only a native copy of the tools which run at build time. This is only tablegen and related tooling, which are cheap to build. Implementation-wise, we introduce a derivation llvmPackages.tblgen, which specifies the tablegen targets which need to be built and has a custom installPhase to copy them to the output. A previous attempt in https://github.com/NixOS/nixpkgs/pull/359967 dropped the use of LLVM_TABLEGEN_EXE and friends on the grounds that llvm can already cross build these things, but that is false since it's necessary in that case to wire in a cross compiler. This PR avoids that problem by allowing access to buildPackages.tblgen. Signed-off-by: Peter Waller --- .../compilers/llvm/common/bolt/default.nix | 8 +- .../compilers/llvm/common/clang/default.nix | 14 +-- .../compilers/llvm/common/default.nix | 2 + .../compilers/llvm/common/lld/default.nix | 4 +- .../compilers/llvm/common/llvm/default.nix | 2 +- .../compilers/llvm/common/mlir/default.nix | 5 +- .../compilers/llvm/common/tblgen.nix | 99 +++++++++++++++++++ 7 files changed, 117 insertions(+), 17 deletions(-) create mode 100644 pkgs/development/compilers/llvm/common/tblgen.nix diff --git a/pkgs/development/compilers/llvm/common/bolt/default.nix b/pkgs/development/compilers/llvm/common/bolt/default.nix index d6a7fae48161..1d8fc25f90ee 100644 --- a/pkgs/development/compilers/llvm/common/bolt/default.nix +++ b/pkgs/development/compilers/llvm/common/bolt/default.nix @@ -52,11 +52,9 @@ stdenv.mkDerivation (finalAttrs: { libxml2 ]; - cmakeFlags = - lib.optionals (!stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ - (lib.cmakeFeature "LLVM_TABLEGEN_EXE" "${buildLlvmTools.llvm}/bin/llvm-tblgen") - ] - ++ devExtraCmakeFlags; + cmakeFlags = [ + (lib.cmakeFeature "LLVM_TABLEGEN_EXE" "${buildLlvmTools.tblgen}/bin/llvm-tblgen") + ] ++ devExtraCmakeFlags; postUnpack = '' chmod -R u+w -- $sourceRoot/.. diff --git a/pkgs/development/compilers/llvm/common/clang/default.nix b/pkgs/development/compilers/llvm/common/clang/default.nix index db19f8b59ea2..c855562acc12 100644 --- a/pkgs/development/compilers/llvm/common/clang/default.nix +++ b/pkgs/development/compilers/llvm/common/clang/default.nix @@ -60,16 +60,18 @@ let "-DSPHINX_OUTPUT_MAN=ON" "-DSPHINX_OUTPUT_HTML=OFF" "-DSPHINX_WARNINGS_AS_ERRORS=OFF" - ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) ([ - "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.llvm}/bin/llvm-tblgen" - "-DCLANG_TABLEGEN=${buildLlvmTools.libclang.dev}/bin/clang-tblgen" + ] ++ [ + "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.tblgen}/bin/llvm-tblgen" + "-DCLANG_TABLEGEN=${buildLlvmTools.tblgen}/bin/clang-tblgen" ] ++ lib.optionals (lib.versionAtLeast release_version "15") [ # Added in LLVM15: # `clang-tidy-confusable-chars-gen`: https://github.com/llvm/llvm-project/commit/c3574ef739fbfcc59d405985a3a4fa6f4619ecdb # `clang-pseudo-gen`: https://github.com/llvm/llvm-project/commit/cd2292ef824591cc34cc299910a3098545c840c7 - "-DCLANG_TIDY_CONFUSABLE_CHARS_GEN=${buildLlvmTools.libclang.dev}/bin/clang-tidy-confusable-chars-gen" - "-DCLANG_PSEUDO_GEN=${buildLlvmTools.libclang.dev}/bin/clang-pseudo-gen" - ]) ++ lib.optional (lib.versionAtLeast release_version "20") "-DLLVM_DIR=${libllvm.dev}/lib/cmake/llvm" + "-DCLANG_TIDY_CONFUSABLE_CHARS_GEN=${buildLlvmTools.tblgen}/bin/clang-tidy-confusable-chars-gen" + ] ++ lib.optionals (lib.versionOlder release_version "20") [ + # clang-pseudo removed in LLVM20: https://github.com/llvm/llvm-project/commit/ed8f78827895050442f544edef2933a60d4a7935 + "-DCLANG_PSEUDO_GEN=${buildLlvmTools.tblgen}/bin/clang-pseudo-gen" + ] ++ lib.optional (lib.versionAtLeast release_version "20") "-DLLVM_DIR=${libllvm.dev}/lib/cmake/llvm" ++ devExtraCmakeFlags; postPatch = '' diff --git a/pkgs/development/compilers/llvm/common/default.nix b/pkgs/development/compilers/llvm/common/default.nix index 102a43161e3d..227f8db5f652 100644 --- a/pkgs/development/compilers/llvm/common/default.nix +++ b/pkgs/development/compilers/llvm/common/default.nix @@ -515,6 +515,8 @@ let # we need to reintroduce `outputSpecified` to get the expected behavior e.g. of lib.get* llvm = tools.libllvm; + tblgen = callPackage ./tblgen.nix { }; + libclang = callPackage ./clang { patches = [ diff --git a/pkgs/development/compilers/llvm/common/lld/default.nix b/pkgs/development/compilers/llvm/common/lld/default.nix index 4008f2076d73..8257971c29cc 100644 --- a/pkgs/development/compilers/llvm/common/lld/default.nix +++ b/pkgs/development/compilers/llvm/common/lld/default.nix @@ -48,8 +48,8 @@ stdenv.mkDerivation (rec { "-DLLVM_CONFIG_PATH=${libllvm.dev}/bin/llvm-config${lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) "-native"}" ] ++ lib.optionals (lib.versionAtLeast release_version "15") [ "-DLLD_INSTALL_PACKAGE_DIR=${placeholder "dev"}/lib/cmake/lld" - ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.llvm}/bin/llvm-tblgen" + ] ++ [ + "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.tblgen}/bin/llvm-tblgen" ] ++ devExtraCmakeFlags; # Musl's default stack size is too small for lld to be able to link Firefox. diff --git a/pkgs/development/compilers/llvm/common/llvm/default.nix b/pkgs/development/compilers/llvm/common/llvm/default.nix index ce35c039fc8c..637c47770c5f 100644 --- a/pkgs/development/compilers/llvm/common/llvm/default.nix +++ b/pkgs/development/compilers/llvm/common/llvm/default.nix @@ -346,6 +346,7 @@ stdenv.mkDerivation (finalAttrs: { "-DLLVM_LINK_LLVM_DYLIB=ON" ]; in flagsForLlvmConfig ++ [ + "-DLLVM_TABLEGEN=${buildLlvmTools.tblgen}/bin/llvm-tblgen" "-DLLVM_INSTALL_UTILS=ON" # Needed by rustc "-DLLVM_BUILD_TESTS=${if finalAttrs.finalPackage.doCheck then "ON" else "OFF"}" "-DLLVM_ENABLE_FFI=ON" @@ -382,7 +383,6 @@ stdenv.mkDerivation (finalAttrs: { "-DCAN_TARGET_i386=false" ] ++ optionals ((stdenv.hostPlatform != stdenv.buildPlatform) && !(stdenv.buildPlatform.canExecute stdenv.hostPlatform)) [ "-DCMAKE_CROSSCOMPILING=True" - "-DLLVM_TABLEGEN=${buildLlvmTools.llvm}/bin/llvm-tblgen" ( let nativeCC = pkgsBuildBuild.targetPackages.stdenv.cc; diff --git a/pkgs/development/compilers/llvm/common/mlir/default.nix b/pkgs/development/compilers/llvm/common/mlir/default.nix index 9c3c71c3b705..8dcf3937cd28 100644 --- a/pkgs/development/compilers/llvm/common/mlir/default.nix +++ b/pkgs/development/compilers/llvm/common/mlir/default.nix @@ -59,14 +59,13 @@ stdenv.mkDerivation rec { "-DLLVM_HOST_TRIPLE=${stdenv.hostPlatform.config}" "-DLLVM_DEFAULT_TARGET_TRIPLE=${stdenv.hostPlatform.config}" "-DLLVM_ENABLE_DUMP=ON" + "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.tblgen}/bin/llvm-tblgen" + "-DMLIR_TABLEGEN_EXE=${buildLlvmTools.tblgen}/bin/mlir-tblgen" ] ++ lib.optionals stdenv.hostPlatform.isStatic [ # Disables building of shared libs, -fPIC is still injected by cc-wrapper "-DLLVM_ENABLE_PIC=OFF" "-DLLVM_BUILD_STATIC=ON" "-DLLVM_LINK_LLVM_DYLIB=OFF" - ] ++ lib.optionals ((stdenv.hostPlatform != stdenv.buildPlatform) && !(stdenv.buildPlatform.canExecute stdenv.hostPlatform)) [ - "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.llvm}/bin/llvm-tblgen" - "-DMLIR_TABLEGEN_EXE=${buildLlvmTools.mlir}/bin/mlir-tblgen" ] ++ devExtraCmakeFlags; outputs = [ "out" "dev" ]; diff --git a/pkgs/development/compilers/llvm/common/tblgen.nix b/pkgs/development/compilers/llvm/common/tblgen.nix new file mode 100644 index 000000000000..fe213e00b847 --- /dev/null +++ b/pkgs/development/compilers/llvm/common/tblgen.nix @@ -0,0 +1,99 @@ +{ + cmake, + devExtraCmakeFlags ? [ ], + lib, + llvm_meta, + monorepoSrc ? null, + ninja, + patches ? [ ], + python3, + release_version, + runCommand, + src ? null, + stdenv, + version, +}: + +let + # This is a synthetic package which is not an official part of the llvm-project. + # See https://github.com/NixOS/nixpkgs/pull/362384 for discussion. + # + # LLVM has tools that run at build time. In native builds, these are + # built as a part of the usual build, but in cross builds they need to + # come from buildPackages. + # + # In many scenarios this is a small problem because LLVM from + # buildPackages is already available as a build; but if cross building a + # version of LLVM which is not available (e.g. a new git commit of LLVM) + # this results in two builds of LLVM and clang, one native and one for the + # cross. + # + # Full builds of LLVM are expensive; and unnecessary in this scenario. We + # don't need a native LLVM, only a native copy of the tools which run at + # build time. This is only tablegen and related tooling, which are cheap + # to build. + pname = "llvm-tblgen"; + + src' = + if monorepoSrc != null then + runCommand "${pname}-src-${version}" { } ( + '' + mkdir -p "$out" + '' + + lib.optionalString (lib.versionAtLeast release_version "14") '' + cp -r ${monorepoSrc}/cmake "$out" + cp -r ${monorepoSrc}/third-party "$out" + '' + + '' + cp -r ${monorepoSrc}/llvm "$out" + cp -r ${monorepoSrc}/clang "$out" + cp -r ${monorepoSrc}/clang-tools-extra "$out" + cp -r ${monorepoSrc}/mlir "$out" + '' + ) + else + src; + + self = stdenv.mkDerivation (finalAttrs: rec { + inherit pname version patches; + + src = src'; + sourceRoot = "${src.name}/llvm"; + + nativeBuildInputs = [ + cmake + ninja + python3 + ]; + + cmakeFlags = [ + # Projects with tablegen-like tools. + "-DLLVM_ENABLE_PROJECTS=${ + lib.concatStringsSep ";" [ + "llvm" + "clang" + "clang-tools-extra" + "mlir" + ] + }" + ] ++ devExtraCmakeFlags; + + # List of tablegen targets. + ninjaFlags = + [ + "clang-tblgen" + "clang-tidy-confusable-chars-gen" + "llvm-tblgen" + "mlir-tblgen" + ] + ++ lib.optionals (lib.versionOlder release_version "20") [ + "clang-pseudo-gen" # Removed in LLVM 20 @ ed8f78827895050442f544edef2933a60d4a7935. + ]; + + installPhase = '' + mkdir -p $out + cp -ar bin $out/bin + ''; + }); +in +self