llvmPackages: Split tablegen into its own derivation

Background:

LLVM has some tools that run at build time. In native builds, these are
built as a part of the usual build, but in cross builds they need to
come from buildPackages.

In many scenarios this is a small problem because LLVM from
buildPackages is already available as a build; but if cross building a
version of LLVM which is not available (e.g. a new git commit of LLVM)
this results in two builds of LLVM and clang, one native and one for the
cross.

Full builds of LLVM are expensive; and unnecessary in this scenario. We
don't need a native LLVM, only a native copy of the tools which run at
build time. This is only tablegen and related tooling, which are cheap
to build.

Implementation-wise, we introduce a derivation llvmPackages.tblgen,
which specifies the tablegen targets which need to be built and has a
custom installPhase to copy them to the output.

A previous attempt in https://github.com/NixOS/nixpkgs/pull/359967
dropped the use of LLVM_TABLEGEN_EXE and friends on the grounds that
llvm can already cross build these things, but that is false since it's
necessary in that case to wire in a cross compiler. This PR avoids that
problem by allowing access to buildPackages.tblgen.

Signed-off-by: Peter Waller <p@pwaller.net>

pkgs/development/compilers/llvm/common/bolt/default.nix

@@ -52,11 +52,9 @@ stdenv.mkDerivation (finalAttrs: {
     libxml2
   ];
 
-  cmakeFlags =
-    lib.optionals (!stdenv.buildPlatform.canExecute stdenv.hostPlatform) [
-      (lib.cmakeFeature "LLVM_TABLEGEN_EXE" "${buildLlvmTools.llvm}/bin/llvm-tblgen")
-    ]
-    ++ devExtraCmakeFlags;
+  cmakeFlags = [
+    (lib.cmakeFeature "LLVM_TABLEGEN_EXE" "${buildLlvmTools.tblgen}/bin/llvm-tblgen")
+  ] ++ devExtraCmakeFlags;
 
   postUnpack = ''
     chmod -R u+w -- $sourceRoot/..

pkgs/development/compilers/llvm/common/clang/default.nix

@@ -60,16 +60,18 @@ let
       "-DSPHINX_OUTPUT_MAN=ON"
       "-DSPHINX_OUTPUT_HTML=OFF"
       "-DSPHINX_WARNINGS_AS_ERRORS=OFF"
-    ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) ([
-      "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.llvm}/bin/llvm-tblgen"
-      "-DCLANG_TABLEGEN=${buildLlvmTools.libclang.dev}/bin/clang-tblgen"
+    ] ++ [
+      "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.tblgen}/bin/llvm-tblgen"
+      "-DCLANG_TABLEGEN=${buildLlvmTools.tblgen}/bin/clang-tblgen"
     ] ++ lib.optionals (lib.versionAtLeast release_version "15") [
       # Added in LLVM15:
       # `clang-tidy-confusable-chars-gen`: https://github.com/llvm/llvm-project/commit/c3574ef739fbfcc59d405985a3a4fa6f4619ecdb
       # `clang-pseudo-gen`: https://github.com/llvm/llvm-project/commit/cd2292ef824591cc34cc299910a3098545c840c7
-      "-DCLANG_TIDY_CONFUSABLE_CHARS_GEN=${buildLlvmTools.libclang.dev}/bin/clang-tidy-confusable-chars-gen"
-      "-DCLANG_PSEUDO_GEN=${buildLlvmTools.libclang.dev}/bin/clang-pseudo-gen"
-    ]) ++ lib.optional (lib.versionAtLeast release_version "20") "-DLLVM_DIR=${libllvm.dev}/lib/cmake/llvm"
+      "-DCLANG_TIDY_CONFUSABLE_CHARS_GEN=${buildLlvmTools.tblgen}/bin/clang-tidy-confusable-chars-gen"
+    ] ++ lib.optionals (lib.versionOlder release_version "20") [
+      # clang-pseudo removed in LLVM20: https://github.com/llvm/llvm-project/commit/ed8f78827895050442f544edef2933a60d4a7935
+      "-DCLANG_PSEUDO_GEN=${buildLlvmTools.tblgen}/bin/clang-pseudo-gen"
+    ] ++ lib.optional (lib.versionAtLeast release_version "20") "-DLLVM_DIR=${libllvm.dev}/lib/cmake/llvm"
       ++ devExtraCmakeFlags;
 
     postPatch = ''

pkgs/development/compilers/llvm/common/default.nix

@@ -515,6 +515,8 @@ let
       # we need to reintroduce `outputSpecified` to get the expected behavior e.g. of lib.get*
       llvm = tools.libllvm;
 
+      tblgen = callPackage ./tblgen.nix { };
+
       libclang = callPackage ./clang {
         patches =
           [

pkgs/development/compilers/llvm/common/lld/default.nix

@@ -48,8 +48,8 @@ stdenv.mkDerivation (rec {
     "-DLLVM_CONFIG_PATH=${libllvm.dev}/bin/llvm-config${lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) "-native"}"
   ] ++ lib.optionals (lib.versionAtLeast release_version "15") [
     "-DLLD_INSTALL_PACKAGE_DIR=${placeholder "dev"}/lib/cmake/lld"
-  ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
-    "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.llvm}/bin/llvm-tblgen"
+  ] ++ [
+    "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.tblgen}/bin/llvm-tblgen"
   ] ++ devExtraCmakeFlags;
 
   # Musl's default stack size is too small for lld to be able to link Firefox.

pkgs/development/compilers/llvm/common/llvm/default.nix

@@ -346,6 +346,7 @@ stdenv.mkDerivation (finalAttrs: {
       "-DLLVM_LINK_LLVM_DYLIB=ON"
     ];
   in flagsForLlvmConfig ++ [
+    "-DLLVM_TABLEGEN=${buildLlvmTools.tblgen}/bin/llvm-tblgen"
     "-DLLVM_INSTALL_UTILS=ON"  # Needed by rustc
     "-DLLVM_BUILD_TESTS=${if finalAttrs.finalPackage.doCheck then "ON" else "OFF"}"
     "-DLLVM_ENABLE_FFI=ON"
@@ -382,7 +383,6 @@ stdenv.mkDerivation (finalAttrs: {
     "-DCAN_TARGET_i386=false"
   ] ++ optionals ((stdenv.hostPlatform != stdenv.buildPlatform) && !(stdenv.buildPlatform.canExecute stdenv.hostPlatform)) [
     "-DCMAKE_CROSSCOMPILING=True"
-    "-DLLVM_TABLEGEN=${buildLlvmTools.llvm}/bin/llvm-tblgen"
     (
       let
         nativeCC = pkgsBuildBuild.targetPackages.stdenv.cc;

pkgs/development/compilers/llvm/common/mlir/default.nix

@@ -59,14 +59,13 @@ stdenv.mkDerivation rec {
     "-DLLVM_HOST_TRIPLE=${stdenv.hostPlatform.config}"
     "-DLLVM_DEFAULT_TARGET_TRIPLE=${stdenv.hostPlatform.config}"
     "-DLLVM_ENABLE_DUMP=ON"
+    "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.tblgen}/bin/llvm-tblgen"
+    "-DMLIR_TABLEGEN_EXE=${buildLlvmTools.tblgen}/bin/mlir-tblgen"
   ] ++ lib.optionals stdenv.hostPlatform.isStatic [
     # Disables building of shared libs, -fPIC is still injected by cc-wrapper
     "-DLLVM_ENABLE_PIC=OFF"
     "-DLLVM_BUILD_STATIC=ON"
     "-DLLVM_LINK_LLVM_DYLIB=OFF"
-  ] ++ lib.optionals ((stdenv.hostPlatform != stdenv.buildPlatform) && !(stdenv.buildPlatform.canExecute stdenv.hostPlatform)) [
-    "-DLLVM_TABLEGEN_EXE=${buildLlvmTools.llvm}/bin/llvm-tblgen"
-    "-DMLIR_TABLEGEN_EXE=${buildLlvmTools.mlir}/bin/mlir-tblgen"
   ] ++ devExtraCmakeFlags;
 
   outputs = [ "out" "dev" ];

pkgs/development/compilers/llvm/common/tblgen.nix

@@ -0,0 +1,99 @@
+{
+  cmake,
+  devExtraCmakeFlags ? [ ],
+  lib,
+  llvm_meta,
+  monorepoSrc ? null,
+  ninja,
+  patches ? [ ],
+  python3,
+  release_version,
+  runCommand,
+  src ? null,
+  stdenv,
+  version,
+}:
+
+let
+  # This is a synthetic package which is not an official part of the llvm-project.
+  # See https://github.com/NixOS/nixpkgs/pull/362384 for discussion.
+  #
+  # LLVM has tools that run at build time. In native builds, these are
+  # built as a part of the usual build, but in cross builds they need to
+  # come from buildPackages.
+  #
+  # In many scenarios this is a small problem because LLVM from
+  # buildPackages is already available as a build; but if cross building a
+  # version of LLVM which is not available (e.g. a new git commit of LLVM)
+  # this results in two builds of LLVM and clang, one native and one for the
+  # cross.
+  #
+  # Full builds of LLVM are expensive; and unnecessary in this scenario. We
+  # don't need a native LLVM, only a native copy of the tools which run at
+  # build time. This is only tablegen and related tooling, which are cheap
+  # to build.
+  pname = "llvm-tblgen";
+
+  src' =
+    if monorepoSrc != null then
+      runCommand "${pname}-src-${version}" { } (
+        ''
+          mkdir -p "$out"
+        ''
+        + lib.optionalString (lib.versionAtLeast release_version "14") ''
+          cp -r ${monorepoSrc}/cmake "$out"
+          cp -r ${monorepoSrc}/third-party "$out"
+        ''
+        + ''
+          cp -r ${monorepoSrc}/llvm "$out"
+          cp -r ${monorepoSrc}/clang "$out"
+          cp -r ${monorepoSrc}/clang-tools-extra "$out"
+          cp -r ${monorepoSrc}/mlir "$out"
+        ''
+      )
+    else
+      src;
+
+  self = stdenv.mkDerivation (finalAttrs: rec {
+    inherit pname version patches;
+
+    src = src';
+    sourceRoot = "${src.name}/llvm";
+
+    nativeBuildInputs = [
+      cmake
+      ninja
+      python3
+    ];
+
+    cmakeFlags = [
+      # Projects with tablegen-like tools.
+      "-DLLVM_ENABLE_PROJECTS=${
+        lib.concatStringsSep ";" [
+          "llvm"
+          "clang"
+          "clang-tools-extra"
+          "mlir"
+        ]
+      }"
+    ] ++ devExtraCmakeFlags;
+
+    # List of tablegen targets.
+    ninjaFlags =
+      [
+        "clang-tblgen"
+        "clang-tidy-confusable-chars-gen"
+        "llvm-tblgen"
+        "mlir-tblgen"
+      ]
+      ++ lib.optionals (lib.versionOlder release_version "20") [
+        "clang-pseudo-gen" # Removed in LLVM 20 @ ed8f78827895050442f544edef2933a60d4a7935.
+      ];
+
+    installPhase = ''
+      mkdir -p $out
+      cp -ar bin $out/bin
+    '';
+  });
+in
+self