From 022befe8ae00f679ee3cd991338b8acf50f9bb76 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Mon, 23 Oct 2023 21:05:21 +0200 Subject: [PATCH] Get bazel building on all major platforms, with tests --- .../bazel/bazel_7/bazel-repository-cache.nix | 55 ++-- .../build-managers/bazel/bazel_7/default.nix | 198 +++++++------ .../bazel_7/extra_toolchains_precedence.patch | 13 + .../bazel/bazel_7/java_toolchain.patch | 27 +- .../build-managers/bazel/bazel_7/no-arc.patch | 5 +- .../bazel/bazel_7/patched_sources.nix | 40 +++ .../build-managers/bazel/bazel_7/tests.nix | 3 +- .../bazel_7/toolchain_better_debug.patch | 157 ++++++++++ .../bazel/bazel_7/toolchain_group_debug.patch | 271 ++++++++++++++++++ .../tools/build-managers/bazel/java-test.nix | 16 +- 10 files changed, 641 insertions(+), 144 deletions(-) create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_7/extra_toolchains_precedence.patch create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_7/patched_sources.nix create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_7/toolchain_better_debug.patch create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_7/toolchain_group_debug.patch diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/bazel-repository-cache.nix b/pkgs/development/tools/build-managers/bazel/bazel_7/bazel-repository-cache.nix index 3fdc047aa08d..07405a9a5e05 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_7/bazel-repository-cache.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/bazel-repository-cache.nix @@ -1,5 +1,5 @@ { lib -, nix +, rnix-hashes , runCommand , fetchurl # The path to the right MODULE.bazel.lock @@ -29,7 +29,7 @@ let # like foldl', force evaluation of intermediate results builtins.seq acc' children; - extract_source = acc: value: + extract_source = f: acc: value: # We take any "attributes" object that has a "sha256" field. Every value # under "attributes" is assumed to be an object, and all the "attributes" # with a "sha256" field are assumed to have either a "urls" or "url" field. @@ -47,8 +47,6 @@ let # }; # } let - # remove the "--" prefix, abusing undocumented negative substring length - sanitize = builtins.substring 2 (-1); attrs = value.attributes; entry = hash: urls: { ${hash} = fetchurl { @@ -58,31 +56,26 @@ let passthru.sha256 = hash; }; }; - insert = acc: hash: urls: acc // entry (sanitize hash) (map sanitize urls); + insert = acc: hash: urls: + acc // entry (sanitize hash) (map sanitize urls); + accWithRemotePatches = lib.foldlAttrs + (acc: url: hash: insert acc hash [ url ]) + acc + (attrs.remote_patches or { }); + accWithNewSource = insert + accWithRemotePatches + (attrs.integrity or attrs.sha256) + (attrs.urls or [ attrs.url ]); in if builtins.isAttrs value && value ? attributes && (attrs ? sha256 || attrs ? integrity) - then - #builtins.trace attrs - ( - insert - acc - (attrs.integrity or attrs.sha256) - (attrs.urls or [ attrs.url ]) - ) - else if builtins.isAttrs value && value ? remote_patches - && builtins.isAttrs value.remote_patches - then - #builtins.trace value.remote_patches - ( - lib.foldlAttrs - (acc: url: hash: insert acc hash [ url ]) - acc - value.remote_patches - ) - else acc; + && f attrs.name + then accWithNewSource else acc; - inputs = foldlJSON extract_source { } modules; + # remove the "--" prefix, abusing undocumented negative substring length + sanitize = builtins.substring 2 (-1); + + inputs = foldlJSON (extract_source (n: builtins.trace (sanitize n) true)) { } modules; requiredHashes = builtins.fromJSON (builtins.readFile requiredDeps); requiredAttrs = lib.genAttrs requiredHashes throw; @@ -94,16 +87,16 @@ let command = '' mkdir -p $out/content_addressable/sha256 - cd $out/content_addressable/sha256 + cd $out '' + lib.concatMapStrings # TODO: Do not re-hash. Use nix-hash to convert hashes (drv: '' filename=$(basename "${lib.head drv.urls}") - echo Caching $filename - hash=$(${nix}/bin/nix-hash --type sha256 --to-base16 ${drv.sha256}) - mkdir -p $hash - ln -sfn ${drv} $hash/file - ln -sfn ${drv} $filename + hash=$(${rnix-hashes}/bin/rnix-hashes --encoding BASE16 ${drv.sha256} | cut -f 2) + mkdir -p content_addressable/sha256/$hash + ln -sfn ${drv} content_addressable/sha256/$hash/file + # Expect file name conflicts + ln -sn ${drv} $filename || true '') (builtins.attrValues requiredInputs ++ extraInputs) ; diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_7/default.nix index ebd726bd4953..9b144544a42f 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_7/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/default.nix @@ -54,7 +54,7 @@ # Always assume all markers valid (this is needed because we remove markers; they are non-deterministic). # Also, don't clean up environment variables (so that NIX_ environment variables are passed to compilers). , enableNixHacks ? false -}: +}@args: let version = "7.0.0-pre.20230917.3"; @@ -136,71 +136,75 @@ let # on aarch64 Darwin, `uname -m` returns "arm64" arch = with stdenv.hostPlatform; if isDarwin && isAarch64 then "arm64" else parsed.cpu.name; + #build --extra_toolchains=@local_jdk//:all + #build --tool_java_runtime_version=local_jdk + #build --java_runtime_version=local_jdk + #build --repo_env=JAVA_HOME=${buildJdk}${if isDarwin then "/zulu-11.jdk/Contents/Home" else "/lib/openjdk"} + bazelRC = writeTextFile { name = "bazel-rc"; text = '' startup --server_javabase=${buildJdk} - build --extra_toolchains=@local_jdk//:all + # Register nix-specific nonprebuilt java toolchains + build --extra_toolchains=@bazel_tools//tools/jdk:all + # and set bazel to use them by default build --tool_java_runtime_version=local_jdk build --java_runtime_version=local_jdk - build --repo_env=JAVA_HOME=${buildJdk}${if isDarwin then "/zulu-11.jdk/Contents/Home" else "/lib/openjdk"} # load default location for the system wide configuration try-import /etc/bazel.bazelrc ''; }; - bazelNixFlagsScript = writeScript "bazel-nix-flags" '' - cat << EOF - common --announce_rc - build --toolchain_resolution_debug=".*" - build --local_ram_resources=HOST_RAM*.5 - build --local_cpu_resources=HOST_CPUS*.75 - build --copt=$(echo $NIX_CFLAGS_COMPILE | sed -e 's/ / --copt=/g') - build --host_copt=$(echo $NIX_CFLAGS_COMPILE | sed -e 's/ / --host_copt=/g') - build --linkopt=$(echo $(< ${stdenv.cc}/nix-support/libcxx-ldflags) | sed -e 's/ / --linkopt=/g') - build --host_linkopt=$(echo $(< ${stdenv.cc}/nix-support/libcxx-ldflags) | sed -e 's/ / --host_linkopt=/g') - build --linkopt=-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ / --linkopt=-Wl,/g') - build --host_linkopt=-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ / --host_linkopt=-Wl,/g') - build --extra_toolchains=@bazel_tools//tools/jdk:nonprebuilt_toolchain_definition - build --verbose_failures - build --curses=no - build --features=-layering_check - build --experimental_strict_java_deps=off - build --strict_proto_deps=off - build --extra_toolchains=@bazel_tools//tools/jdk:nonprebuilt_toolchain_java11_definition - build --extra_toolchains=@local_jdk//:all - build --tool_java_runtime_version=local_jdk_11 - build --java_runtime_version=local_jdk_11 - build --repo_env=JAVA_HOME=${buildJdk}${if isDarwin then "/zulu-11.jdk/Contents/Home" else "/lib/openjdk"} - EOF - ''; in stdenv.mkDerivation rec { pname = "bazel"; - inherit version; - - meta = with lib; { - homepage = "https://github.com/bazelbuild/bazel/"; - description = "Build tool that builds code quickly and reliably"; - sourceProvenance = with sourceTypes; [ - fromSource - binaryBytecode # source bundles dependencies as jars - ]; - license = licenses.asl20; - maintainers = lib.teams.bazel.members; - inherit platforms; - }; - - inherit src; + inherit version src; inherit sourceRoot; + patches = [ # TODO: Make GSON work, # In particular, our bazel build cannot emit MODULE.bazel.lock # it only produces an empty json object `{ }`. ./serialize_nulls.patch + # --extra_toolchains defined later should come before the ones defined earlier. + # As-is, this patch also inverts the order of extra_toolchains lists, but it's just a hack + ./extra_toolchains_precedence.patch + + #./toolchain_better_debug.patch + ./toolchain_group_debug.patch + + # Remote java toolchains do not work on NixOS because they download binaries, + # so we need to use the @local_jdk//:jdk + # It could in theory be done by registering @local_jdk//:all toolchains, + # but these java toolchains still bundle binaries for ijar and stuff. So we + # need a nonprebult java toolchain (where ijar and stuff is built from + # sources). + # There is no such java toolchain, so we introduce one here. + # By providing no version information, the toolchain will set itself to the + # version of $JAVA_HOME/bin/java, just like the local_jdk does. + # To ensure this toolchain gets used, we can set + # --{,tool_}java_runtime_version=local_jdk and rely on the fact no java + # toolchain registered by default uses the local_jdk, making the selection + # unambiguous. + # This toolchain has the advantage that it can use any ambiant java jdk, + # not only a given, fixed version. It allows bazel to work correctly in any + # environment where JAVA_HOME is set to the right java version, like inside + # nix derivations. + # However, this patch breaks bazel hermeticity, by picking the ambiant java + # version instead of the more hermetic remote_jdk prebuilt binaries that + # rules_java provide by default. It also requires the user to have a + # JAVA_HOME set to the exact version required by the project. + # With more code, we could define java toolchains for all the java versions + # supported by the jdk as in rules_java's + # toolchains/local_java_repository.bzl, but this is not implemented here. + # To recover vanilla behavior, non NixOS users can set + # --{,tool_}java_runtime_version=remote_jdk, effectively reverting the + # effect of this patch and the fake system bazelrc. + ./java_toolchain.patch + # Bazel integrates with apple IOKit to inhibit and track system sleep. # Inside the darwin sandbox, these API calls are blocked, and bazel # crashes. It seems possible to allow these APIs inside the sandbox, but it @@ -239,10 +243,7 @@ stdenv.mkDerivation rec { # This patch removes using the -fobjc-arc compiler option and makes the code # compile without automatic reference counting. Caveat: this leaks memory, but # we accept this fact because xcode_locator is only a short-lived process used during the build. - (substituteAll { - src = ./no-arc.patch; - multiBinPatch = if stdenv.hostPlatform.system == "aarch64-darwin" then "arm64" else "x86_64"; - }) + ./no-arc.patch # --experimental_strict_action_env (which may one day become the default # see bazelbuild/bazel#2574) hardcodes the default @@ -270,13 +271,10 @@ stdenv.mkDerivation rec { # See enableNixHacks argument above. ++ lib.optional enableNixHacks ./nix-hacks.patch; - - # Bazel starts a local server and needs to bind a local address. - __darwinAllowLocalNetworking = true; - postPatch = let darwinPatches = '' + bazelLinkFlags () { eval set -- "$NIX_LDFLAGS" local flag @@ -285,10 +283,6 @@ stdenv.mkDerivation rec { done } - # Disable Bazel's Xcode toolchain detection which would configure compilers - # and linkers from Xcode instead of from PATH - export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 - # Explicitly configure gcov since we don't have it on Darwin, so autodetection fails export GCOV=${coreutils}/bin/false @@ -300,6 +294,10 @@ stdenv.mkDerivation rec { # https://github.com/NixOS/nixpkgs/pull/41589 export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -isystem ${lib.getDev libcxx}/include/c++/v1" + # This variable is used by bazel to propagate env vars for homebrew, + # which is exactly what we need too. + export HOMEBREW_RUBY_PATH="foo" + # don't use system installed Xcode to run clang, use Nix clang instead sed -i -E \ -e "s;/usr/bin/xcrun (--sdk macosx )?clang;${stdenv.cc}/bin/clang $NIX_CFLAGS_COMPILE $(bazelLinkFlags) -framework CoreFoundation;g" \ @@ -307,8 +305,6 @@ stdenv.mkDerivation rec { scripts/bootstrap/compile.sh \ tools/osx/BUILD - substituteInPlace scripts/bootstrap/compile.sh --replace ' -mmacosx-version-min=10.9' "" - # nixpkgs's libSystem cannot use pthread headers directly, must import GCD headers instead sed -i -e "/#include /i #include " src/main/cpp/blaze_util_darwin.cc @@ -316,23 +312,26 @@ stdenv.mkDerivation rec { # invocations of gcc to clang, but vanilla clang doesn't sed -i -e 's;_find_generic(repository_ctx, "gcc", "CC", overriden_tools);_find_generic(repository_ctx, "clang", "CC", overriden_tools);g' tools/cpp/unix_cc_configure.bzl + # This is necessary to avoid: + # "error: no visible @interface for 'NSDictionary' declares the selector + # 'initWithContentsOfURL:error:'" + # This can be removed when the apple_sdk is upgraded beyond 10.13+ + sedVerbose tools/osx/xcode_locator.m \ + -e '/initWithContentsOfURL:versionPlistUrl/ { + N + s/error:nil\];/\];/ + }' + sed -i -e 's;"/usr/bin/libtool";_find_generic(repository_ctx, "libtool", "LIBTOOL", overriden_tools);g' tools/cpp/unix_cc_configure.bzl wrappers=( tools/cpp/osx_cc_wrapper.sh.tpl ) for wrapper in "''${wrappers[@]}"; do - sed -i -e "s,/usr/bin/gcc,${stdenv.cc}/bin/clang,g" $wrapper - sed -i -e "s,/usr/bin/install_name_tool,${cctools}/bin/install_name_tool,g" $wrapper - sed -i -e "s,/usr/bin/xcrun install_name_tool,${cctools}/bin/install_name_tool,g" $wrapper + sedVerbose $wrapper \ + -e "s,/usr/bin/xcrun install_name_tool,${cctools}/bin/install_name_tool,g" done ''; + # -e "s,%{cc},${stdenv.cc}/bin/clang,g" \ genericPatches = '' - function sedVerbose() { - local path=$1; shift; - sed -i".bak-nix" "$path" "$@" - diff -U0 "$path.bak-nix" "$path" | sed "s/^/ /" || true - rm -f "$path.bak-nix" - } - # unzip builtins_bzl.zip so the contents get patched builtins_bzl=src/main/java/com/google/devtools/build/lib/bazel/rules/builtins_bzl unzip ''${builtins_bzl}.zip -d ''${builtins_bzl}_zip >/dev/null @@ -348,10 +347,7 @@ stdenv.mkDerivation rec { grep -rlZ /bin/ \ src/main/java/com/google/devtools \ src/main/starlark/builtins_bzl/common/python \ - tools/python \ - tools/cpp \ - tools/build_rules \ - tools/osx/BUILD \ + tools \ | while IFS="" read -r -d "" path; do # If you add more replacements here, you must change the grep above! # Only files containing /bin are taken into account. @@ -377,23 +373,24 @@ stdenv.mkDerivation rec { # Passing EXTRA_BAZEL_ARGS is tricky due to quoting. sedVerbose compile.sh \ - -e "/bazel_build /a\ --copt=\"$(echo $NIX_CFLAGS_COMPILE | sed -e 's/ /" --copt=\"/g')\" \\\\" \ - -e "/bazel_build /a\ --host_copt=\"$(echo $NIX_CFLAGS_COMPILE | sed -e 's/ /" --host_copt=\"/g')\" \\\\" \ - -e "/bazel_build /a\ --linkopt=\"$(echo $(< ${stdenv.cc}/nix-support/libcxx-ldflags) | sed -e 's/ /" --linkopt=\"/g')\" \\\\" \ - -e "/bazel_build /a\ --host_linkopt=\"$(echo $(< ${stdenv.cc}/nix-support/libcxx-ldflags) | sed -e 's/ /" --host_linkopt=\"/g')\" \\\\" \ - -e "/bazel_build /a\ --linkopt=\"-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ /" --linkopt=\"-Wl,/g')\" \\\\" \ - -e "/bazel_build /a\ --host_linkopt=\"-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ /" --host_linkopt=\"-Wl,/g')\" \\\\" \ -e "/bazel_build /a\ --verbose_failures \\\\" \ -e "/bazel_build /a\ --curses=no \\\\" \ -e "/bazel_build /a\ --features=-layering_check \\\\" \ -e "/bazel_build /a\ --experimental_strict_java_deps=off \\\\" \ -e "/bazel_build /a\ --strict_proto_deps=off \\\\" \ + -e "/bazel_build /a\ --action_env=NIX_CFLAGS_COMPILE=\"$NIX_CFLAGS_COMPILE\" \\\\" \ + -e "/bazel_build /a\ --action_env=NIX_LDFLAGS=\"$NIX_LDFLAGS\" \\\\" \ + -e "/bazel_build /a\ --host_action_env=NIX_CFLAGS_COMPILE=\"$NIX_CFLAGS_COMPILE\" \\\\" \ + -e "/bazel_build /a\ --host_action_env=NIX_LDFLAGS=\"$NIX_LDFLAGS\" \\\\" \ + -e "/bazel_build /a\ --toolchain_resolution_debug='@bazel_tools//tools/jdk:(runtime_)?toolchain_type' \\\\" \ -e "/bazel_build /a\ --tool_java_runtime_version=local_jdk \\\\" \ -e "/bazel_build /a\ --java_runtime_version=local_jdk \\\\" \ - -e "/bazel_build /a\ --repo_env=JAVA_HOME=${buildJdk}/${if isDarwin then "/zulu-11.jdk/Contents/Home" else "/lib/openjdk"} \\\\" \ - -e "/bazel_build /a\ --extra_toolchains=@local_jdk//:all \\\\" \ - -e "/bazel_build /a\ --toolchain_resolution_debug=@bazel_tools//tools/jdk:runtime_toolchain_type \\\\" \ - -e "/bazel_build /a\ --sandbox_debug --verbose_failures \\\\" \ + -e "/bazel_build /a\ --extra_toolchains=@bazel_tools//tools/jdk:all \\\\" \ + -e "/bazel_build /a\ --distdir=${distDir} \\\\" \ + + #-e "/bazel_build /a\ --action_env=NIX_BINTOOLS=\"$NIX_BINTOOLS\" \\\\" \ + #-e "/bazel_build /a\ --action_env=NIX_CC=\"$NIX_CC\" \\\\" \ + #-e "/bazel_build /a\ --action_env=nativeBuildInputs=\"$nativeBuildInputs\" \\\\" \ # Also build parser_deploy.jar with bootstrap bazel # TODO: Turn into a proper patch @@ -401,17 +398,6 @@ stdenv.mkDerivation rec { -e 's!bazel_build !bazel_build src/tools/execlog:parser_deploy.jar !' \ -e 's!clear_log!cp $(get_bazel_bin_path)/src/tools/execlog/parser_deploy.jar output\nclear_log!' - - # This is necessary to avoid: - # "error: no visible @interface for 'NSDictionary' declares the selector - # 'initWithContentsOfURL:error:'" - # This can be removed when the apple_sdk is upgraded beyond 10.13+ - sedVerbose tools/osx/xcode_locator.m \ - -e '/initWithContentsOfURL:versionPlistUrl/ { - N - s/error:nil\];/\];/ - }' - # append the PATH with defaultShellPath in tools/bash/runfiles/runfiles.bash echo "PATH=\$PATH:${defaultShellPath}" >> runfiles.bash.tmp cat tools/bash/runfiles/runfiles.bash >> runfiles.bash.tmp @@ -427,7 +413,31 @@ stdenv.mkDerivation rec { patchShebangs . >/dev/null ''; in - lib.optionalString isDarwin darwinPatches + genericPatches; + '' + function sedVerbose() { + local path=$1; shift; + sed -i".bak-nix" "$path" "$@" + diff -U0 "$path.bak-nix" "$path" | sed "s/^/ /" || true + rm -f "$path.bak-nix" + } + '' + + lib.optionalString stdenv.hostPlatform.isDarwin darwinPatches + + genericPatches; + + meta = with lib; { + homepage = "https://github.com/bazelbuild/bazel/"; + description = "Build tool that builds code quickly and reliably"; + sourceProvenance = with sourceTypes; [ + fromSource + binaryBytecode # source bundles dependencies as jars + ]; + license = licenses.asl20; + maintainers = lib.teams.bazel.members; + inherit platforms; + }; + + # Bazel starts a local server and needs to bind a local address. + __darwinAllowLocalNetworking = true; buildInputs = [ buildJdk ] ++ defaultShellUtils; @@ -444,9 +454,9 @@ stdenv.mkDerivation rec { ] ++ lib.optionals (stdenv.isDarwin) [ cctools libcxx + Foundation CoreFoundation CoreServices - Foundation ]; # Bazel makes extensive use of symlinks in the WORKSPACE. diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/extra_toolchains_precedence.patch b/pkgs/development/tools/build-managers/bazel/bazel_7/extra_toolchains_precedence.patch new file mode 100644 index 000000000000..9ad6f38c0249 --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/extra_toolchains_precedence.patch @@ -0,0 +1,13 @@ +diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunction.java +index 2b407202a3..7591e78f85 100644 +--- a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunction.java ++++ b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunction.java +@@ -88,7 +88,7 @@ public class RegisteredToolchainsFunction implements SkyFunction { + try { + targetPatternBuilder.addAll( + TargetPatternUtil.parseAllSigned( +- platformConfiguration.getExtraToolchains(), mainRepoParser)); ++ platformConfiguration.getExtraToolchains().reverse(), mainRepoParser)); + } catch (InvalidTargetPatternException e) { + throw new RegisteredToolchainsFunctionException( + new InvalidToolchainLabelException(e), Transience.PERSISTENT); diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/java_toolchain.patch b/pkgs/development/tools/build-managers/bazel/bazel_7/java_toolchain.patch index d9c41c36b53a..fa7bed8ac151 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_7/java_toolchain.patch +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/java_toolchain.patch @@ -1,20 +1,29 @@ diff --git a/tools/jdk/BUILD.tools b/tools/jdk/BUILD.tools -index a8af76e90c..af2540f838 100644 +index a8af76e90c..7f8b030f63 100644 --- a/tools/jdk/BUILD.tools +++ b/tools/jdk/BUILD.tools -@@ -146,6 +146,16 @@ py_test( +@@ -146,6 +146,25 @@ py_test( ], ) ++##### Nonprebuilt toolchains definitions for NixOS and nix build sandboxes #### ++ +load("@rules_java//toolchains:default_java_toolchain.bzl", "default_java_toolchain", "NONPREBUILT_TOOLCHAIN_CONFIGURATION") + -+default_java_toolchain( -+ name = "nonprebuilt_toolchain_java11", -+ configuration = NONPREBUILT_TOOLCHAIN_CONFIGURATION, -+ java_runtime = "@local_jdk//:jdk", -+ source_version = "11", -+ target_version = "11", -+) ++[ ++ default_java_toolchain( ++ name = "nonprebuilt_toolchain_java" + str(version), ++ configuration = NONPREBUILT_TOOLCHAIN_CONFIGURATION, ++ java_runtime = "@local_jdk//:jdk", ++ source_version = str(version), ++ target_version = str(version), ++ ) ++ # Ideally we would only define toolchains for the java versions that the ++ # local jdk supports. But we cannot access this information in a BUILD ++ # file, and this is a hack anyway, so just pick a large enough upper bound. ++ # At the current pace, java <= 30 should cover all realeases until 2028. ++ for version in range(8, 31) ++] + #### Aliases to rules_java to keep backward-compatibility (begin) #### diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/no-arc.patch b/pkgs/development/tools/build-managers/bazel/bazel_7/no-arc.patch index 69d7c47ffd00..5712c12f2bed 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_7/no-arc.patch +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/no-arc.patch @@ -2,15 +2,14 @@ diff --git a/tools/osx/BUILD b/tools/osx/BUILD index 0358fb0ffe..baae1bf65b 100644 --- a/tools/osx/BUILD +++ b/tools/osx/BUILD -@@ -27,9 +27,9 @@ exports_files([ +@@ -27,9 +27,8 @@ exports_files([ ]) DARWIN_XCODE_LOCATOR_COMPILE_COMMAND = """ - /usr/bin/xcrun --sdk macosx clang -mmacosx-version-min=10.13 -fobjc-arc -framework CoreServices \ - -framework Foundation -arch arm64 -arch x86_64 -Wl,-no_adhoc_codesign -Wl,-no_uuid -o $@ $< && \ - env -i codesign --identifier $@ --force --sign - $@ -+ /usr/bin/xcrun --sdk macosx clang -mmacosx-version-min=10.13 -framework CoreServices \ -+ -framework Foundation -Wl,-no_uuid -o $@ $< && \ ++ /usr/bin/xcrun --sdk macosx clang -framework CoreServices -framework Foundation -Wl,-no_uuid -o $@ $< && \ + /usr/bin/env -i /usr/bin/codesign --identifier $@ --force --sign - $@ """ diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/patched_sources.nix b/pkgs/development/tools/build-managers/bazel/bazel_7/patched_sources.nix new file mode 100644 index 000000000000..84cf2a08fad4 --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/patched_sources.nix @@ -0,0 +1,40 @@ +{ lib +, stdenv +, version +, src +, sourceRoot +, substituteAll +, defaultShellPath +, bazelRC +, enableNixHacks +, coreutils +, CoreFoundation +, CoreServices +, Foundation +, IOKit +, libcxx +, cctools +, sigtool +, bash +, python3 +, buildJdk +, unzip +, zip +, ... +}: + +stdenv.mkDerivation { + name = "bazel-patched-sources"; + inherit version; + inherit src; + + dontBuild = true; + + nativeBuildInputs = [ zip unzip ]; + + installPhase = '' + mkdir $out + cp -r . $out + ''; + +} diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/tests.nix b/pkgs/development/tools/build-managers/bazel/bazel_7/tests.nix index af5eed6b1531..495b7611c19f 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_7/tests.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/tests.nix @@ -2,6 +2,7 @@ # tooling , fetchFromGitHub , newScope +, recurseIntoAttrs , runCommandCC , stdenv # inputs @@ -12,6 +13,7 @@ , repoCache , runJdk , xe +, bazel-watcher }: let inherit (stdenv.hostPlatform) isDarwin; @@ -112,7 +114,6 @@ let inherit Foundation; extraBazelArgs = '' --repository_cache=${repoCache} \ - --repo_env=JAVA_HOME=${runJdk}${if isDarwin then "/zulu-17.jdk/Contents/Home" else "/lib/openjdk"} \ ''; bazel = bazel_self; }; diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/toolchain_better_debug.patch b/pkgs/development/tools/build-managers/bazel/bazel_7/toolchain_better_debug.patch new file mode 100644 index 000000000000..7f0420609871 --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/toolchain_better_debug.patch @@ -0,0 +1,157 @@ +diff --git a/src/main/java/com/google/devtools/build/lib/analysis/platform/DeclaredToolchainInfo.java b/src/main/java/com/google/devtools/build/lib/analysis/platform/DeclaredToolchainInfo.java +index fa322cf6f0..4a94087a3c 100644 +--- a/src/main/java/com/google/devtools/build/lib/analysis/platform/DeclaredToolchainInfo.java ++++ b/src/main/java/com/google/devtools/build/lib/analysis/platform/DeclaredToolchainInfo.java +@@ -47,6 +47,9 @@ public abstract class DeclaredToolchainInfo implements TransitiveInfoProvider { + /** The label of the toolchain to resolve for use in toolchain-aware rules. */ + public abstract Label toolchainLabel(); + ++ /** The label of the toolchain native rule invocation, for debugging. */ ++ public abstract Label label(); ++ + /** Builder class to assist in creating {@link DeclaredToolchainInfo} instances. */ + public static class Builder { + private ToolchainTypeInfo toolchainType; +@@ -55,6 +58,7 @@ public abstract class DeclaredToolchainInfo implements TransitiveInfoProvider { + private ImmutableList.Builder targetSettings = + new ImmutableList.Builder<>(); + private Label toolchainLabel; ++ private Label label; + + /** Sets the type of the toolchain being declared. */ + @CanIgnoreReturnValue +@@ -100,6 +104,13 @@ public abstract class DeclaredToolchainInfo implements TransitiveInfoProvider { + return this; + } + ++ /** Sets the label of the toolchain to resolve for use in toolchain-aware rules. */ ++ @CanIgnoreReturnValue ++ public Builder label(Label label) { ++ this.label = label; ++ return this; ++ } ++ + /** Returns the newly created {@link DeclaredToolchainInfo} instance. */ + public DeclaredToolchainInfo build() throws DuplicateConstraintException { + // Handle constraint duplication in attributes separately, so they can be reported correctly. +@@ -128,7 +139,8 @@ public abstract class DeclaredToolchainInfo implements TransitiveInfoProvider { + execConstraints, + targetConstraints, + targetSettings.build(), +- toolchainLabel); ++ toolchainLabel, ++ label); + } + } + +diff --git a/src/main/java/com/google/devtools/build/lib/rules/platform/Toolchain.java b/src/main/java/com/google/devtools/build/lib/rules/platform/Toolchain.java +index 16d14a5edd..93ad4e47ec 100644 +--- a/src/main/java/com/google/devtools/build/lib/rules/platform/Toolchain.java ++++ b/src/main/java/com/google/devtools/build/lib/rules/platform/Toolchain.java +@@ -57,6 +57,7 @@ public class Toolchain implements RuleConfiguredTargetFactory { + .collect(toImmutableList()); + Label toolchainLabel = + ruleContext.attributes().get(ToolchainRule.TOOLCHAIN_ATTR, BuildType.NODEP_LABEL); ++ Label label = ruleContext.getLabel(); + + DeclaredToolchainInfo registeredToolchain; + try { +@@ -67,6 +68,7 @@ public class Toolchain implements RuleConfiguredTargetFactory { + .addTargetConstraints(targetConstraints) + .addTargetSettings(targetSettings) + .toolchainLabel(toolchainLabel) ++ .label(label) + .build(); + } catch (DeclaredToolchainInfo.DuplicateConstraintException e) { + if (e.execConstraintsException() != null) { +diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java +index 933c200471..5c61ebcb12 100644 +--- a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java ++++ b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java +@@ -182,11 +182,12 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + if (!nonmatchingSettings.isEmpty()) { + debugMessage( + eventHandler, +- " Type %s: %s platform %s: Rejected toolchain %s; mismatching config settings: %s", ++ " Type %s: %s platform %s: Rejected toolchain %s (from %s); mismatching config settings: %s", + toolchainType.toolchainType(), + "target", + targetPlatform.label(), + toolchain.toolchainLabel(), ++ toolchain.label(), + String.join(", ", nonmatchingSettings)); + continue; + } +@@ -198,7 +199,8 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + "target", + targetPlatform, + toolchainType, +- toolchain.toolchainLabel())) { ++ toolchain.toolchainLabel(), ++ toolchain.label())) { + continue; + } + +@@ -208,12 +210,13 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + if (platformKeysSeen.contains(executionPlatformKey)) { + debugMessage( + eventHandler, +- " Type %s: target platform %s: execution platform %s: Skipping toolchain %s;" ++ " Type %s: target platform %s: execution platform %s: Skipping toolchain %s (from %s);" + + " execution platform already has selected toolchain", + toolchainType.toolchainType(), + targetPlatform.label(), + executionPlatformKey.getLabel(), +- toolchain.toolchainLabel()); ++ toolchain.toolchainLabel(), ++ toolchain.label()); + continue; + } + +@@ -224,17 +227,19 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + "execution", + executionPlatform, + toolchainType, +- toolchain.toolchainLabel())) { ++ toolchain.toolchainLabel(), ++ toolchain.label())) { + continue; + } + + debugMessage( + eventHandler, +- " Type %s: target platform %s: execution %s: Selected toolchain %s", ++ " Type %s: target platform %s: execution %s: Selected toolchain %s (from %s)", + toolchainType.toolchainType(), + targetPlatform.label(), + executionPlatformKey.getLabel(), +- toolchain.toolchainLabel()); ++ toolchain.toolchainLabel(), ++ toolchain.label()); + builder.put(executionPlatformKey, toolchain.toolchainLabel()); + platformKeysSeen.add(executionPlatformKey); + } +@@ -276,7 +281,8 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + String platformType, + PlatformInfo platform, + ToolchainTypeRequirement toolchainType, +- Label toolchainLabel) { ++ Label toolchainLabel, ++ Label label) { + + // Check every constraint_setting in either the toolchain or the platform. + ImmutableSet mismatchSettings = +@@ -315,11 +321,12 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + } + debugMessage( + eventHandler, +- " Type %s: %s platform %s: Rejected toolchain %s%s%s", ++ " Type %s: %s platform %s: Rejected toolchain %s (from %s)%s%s", + toolchainType.toolchainType(), + platformType, + platform.label(), + toolchainLabel, ++ label, + mismatchValues, + missingSettings); + } diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/toolchain_group_debug.patch b/pkgs/development/tools/build-managers/bazel/bazel_7/toolchain_group_debug.patch new file mode 100644 index 000000000000..2af9f4924eff --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/toolchain_group_debug.patch @@ -0,0 +1,271 @@ +diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java +index 933c200471..1e5bbaa44b 100644 +--- a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java ++++ b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java +@@ -32,7 +32,6 @@ import com.google.devtools.build.lib.analysis.platform.PlatformInfo; + import com.google.devtools.build.lib.analysis.platform.ToolchainTypeInfo; + import com.google.devtools.build.lib.cmdline.Label; + import com.google.devtools.build.lib.events.Event; +-import com.google.devtools.build.lib.events.EventHandler; + import com.google.devtools.build.lib.server.FailureDetails.Toolchain.Code; + import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; + import com.google.devtools.build.lib.skyframe.toolchains.PlatformLookupUtil.InvalidPlatformException; +@@ -91,14 +90,21 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + .debugToolchainResolution(key.toolchainType().toolchainType()); + + // Find the right one. +- return resolveConstraints( ++ ArrayList trace = debug ? new ArrayList<>() : null; ++ SingleToolchainResolutionValue toolchainResolution = resolveConstraints( + key.toolchainType(), + key.toolchainTypeInfo(), + key.availableExecutionPlatformKeys(), + key.targetPlatformKey(), + toolchains.registeredToolchains(), + env, +- debug ? env.getListener() : null); ++ trace); ++ ++ if (debug) { ++ env.getListener().handle(Event.info(String.join("\n", trace))); ++ } ++ ++ return toolchainResolution; + } + + /** +@@ -114,7 +120,7 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + ConfiguredTargetKey targetPlatformKey, + ImmutableList toolchains, + Environment env, +- @Nullable EventHandler eventHandler) ++ @Nullable List trace) + throws ToolchainResolutionFunctionException, InterruptedException { + + // Load the PlatformInfo needed to check constraints. +@@ -151,10 +157,20 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + toolchain.toolchainType().typeLabel().equals(toolchainType.toolchainType())) + .collect(toImmutableList()); + ++ debugMessage( ++ trace, ++ "Performing resolution of %s for target platform %s", ++ toolchainType.toolchainType(), ++ targetPlatform.label()); ++ + for (DeclaredToolchainInfo toolchain : filteredToolchains) { + // Make sure the target setting matches but watch out for resolution errors. + ArrayList nonmatchingSettings = new ArrayList<>(); + ArrayList errors = new ArrayList<>(); ++ debugMessage( ++ trace, ++ " toolchain %s", ++ toolchain.toolchainLabel()); + + // TODO(blaze-configurability-team): If this pattern comes up more often, add a central + // facility for merging multiple MatchResult +@@ -181,11 +197,8 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + } + if (!nonmatchingSettings.isEmpty()) { + debugMessage( +- eventHandler, +- " Type %s: %s platform %s: Rejected toolchain %s; mismatching config settings: %s", +- toolchainType.toolchainType(), +- "target", +- targetPlatform.label(), ++ trace, ++ " Rejected toolchain %s; mismatching config settings: %s", + toolchain.toolchainLabel(), + String.join(", ", nonmatchingSettings)); + continue; +@@ -193,35 +206,36 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + + // Make sure the target platform matches. + if (!checkConstraints( +- eventHandler, ++ trace, + toolchain.targetConstraints(), +- "target", ++ " Rejected toolchain %1s%3s", + targetPlatform, + toolchainType, + toolchain.toolchainLabel())) { + continue; + } ++ ++ debugMessage( ++ trace, ++ " Toolchain %s is compatible with target plaform, searching for execution platforms:", ++ toolchain.toolchainLabel()); + + // Find the matching execution platforms. + for (ConfiguredTargetKey executionPlatformKey : availableExecutionPlatformKeys) { + // Only check the toolchains if this is a new platform. + if (platformKeysSeen.contains(executionPlatformKey)) { + debugMessage( +- eventHandler, +- " Type %s: target platform %s: execution platform %s: Skipping toolchain %s;" +- + " execution platform already has selected toolchain", +- toolchainType.toolchainType(), +- targetPlatform.label(), +- executionPlatformKey.getLabel(), +- toolchain.toolchainLabel()); ++ trace, ++ " Skipping execution platform %s; it has already selected a toolchain", ++ executionPlatformKey.getLabel()); + continue; + } + + PlatformInfo executionPlatform = platforms.get(executionPlatformKey); + if (!checkConstraints( +- eventHandler, ++ trace, + toolchain.execConstraints(), +- "execution", ++ " Incompatible execution platform %2s%3s", + executionPlatform, + toolchainType, + toolchain.toolchainLabel())) { +@@ -229,51 +243,64 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + } + + debugMessage( +- eventHandler, +- " Type %s: target platform %s: execution %s: Selected toolchain %s", +- toolchainType.toolchainType(), +- targetPlatform.label(), +- executionPlatformKey.getLabel(), +- toolchain.toolchainLabel()); ++ trace, ++ " Compatible execution platform %s", ++ executionPlatformKey.getLabel()); + builder.put(executionPlatformKey, toolchain.toolchainLabel()); + platformKeysSeen.add(executionPlatformKey); + } + } + + ImmutableMap resolvedToolchainLabels = builder.buildOrThrow(); +- if (resolvedToolchainLabels.isEmpty()) { +- debugMessage( +- eventHandler, +- " Type %s: target platform %s: No toolchains found.", +- toolchainType.toolchainType(), +- targetPlatform.label()); ++ if (trace != null) { ++ if (resolvedToolchainLabels.isEmpty()) { ++ debugMessage( ++ trace, ++ "No %s toolchain found for target platform %s.", ++ toolchainType.toolchainType(), ++ targetPlatform.label()); ++ } else { ++ debugMessage( ++ trace, ++ "Recap of selected %s toolchains for target platform %s:", ++ toolchainType.toolchainType(), ++ targetPlatform.label()); ++ resolvedToolchainLabels.forEach((executionPlatformKey, toolchainLabel) -> ++ debugMessage( ++ trace, ++ " Selected toolchain %s to run on exec platform %s", ++ toolchainLabel, ++ executionPlatformKey.getLabel()) ++ ); ++ } + } + + return SingleToolchainResolutionValue.create(toolchainTypeInfo, resolvedToolchainLabels); + } + + /** +- * Helper method to print a debugging message, if the given {@link EventHandler} is not {@code ++ * Helper method to print a debugging message, if the given {@link trace} is not {@code + * null}. + */ + @FormatMethod + private static void debugMessage( +- @Nullable EventHandler eventHandler, @FormatString String template, Object... args) { +- if (eventHandler == null) { ++ @Nullable List trace, @FormatString String template, Object... args) { ++ if (trace == null) { + return; + } + +- eventHandler.handle(Event.info("ToolchainResolution: " + String.format(template, args))); ++ trace.add(String.format(template, args)); + } + + /** + * Returns {@code true} iff all constraints set by the toolchain and in the {@link PlatformInfo} + * match. + */ ++ @SuppressWarnings("FormatStringAnnotation") + private static boolean checkConstraints( +- @Nullable EventHandler eventHandler, ++ @Nullable List trace, + ConstraintCollection toolchainConstraints, +- String platformType, ++ String debugMessage, + PlatformInfo platform, + ToolchainTypeRequirement toolchainType, + Label toolchainLabel) { +@@ -295,7 +322,7 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + .filter(toolchainConstraints::hasWithoutDefault) + .collect(ImmutableSet.toImmutableSet()); + +- if (!mismatchSettingsWithDefault.isEmpty()) { ++ if (trace != null && !mismatchSettingsWithDefault.isEmpty()) { + String mismatchValues = + mismatchSettingsWithDefault.stream() + .filter(toolchainConstraints::has) +@@ -314,14 +341,11 @@ public class SingleToolchainResolutionFunction implements SkyFunction { + missingSettings = "; missing: " + missingSettings; + } + debugMessage( +- eventHandler, +- " Type %s: %s platform %s: Rejected toolchain %s%s%s", +- toolchainType.toolchainType(), +- platformType, +- platform.label(), ++ trace, ++ debugMessage, + toolchainLabel, +- mismatchValues, +- missingSettings); ++ platform.label(), ++ mismatchValues + missingSettings); + } + + return mismatchSettingsWithDefault.isEmpty(); +diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/ToolchainResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/ToolchainResolutionFunction.java +index 3bdf2d0b23..fea848eca0 100644 +--- a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/ToolchainResolutionFunction.java ++++ b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/ToolchainResolutionFunction.java +@@ -80,14 +80,7 @@ public class ToolchainResolutionFunction implements SkyFunction { + Preconditions.checkNotNull(configuration.getFragment(PlatformConfiguration.class)); + + // Check if debug output should be generated. +- boolean debug = +- key.debugTarget() +- || configuration +- .getFragment(PlatformConfiguration.class) +- .debugToolchainResolution( +- key.toolchainTypes().stream() +- .map(ToolchainTypeRequirement::toolchainType) +- .collect(toImmutableSet())); ++ boolean debug = key.debugTarget(); + + // Load the configured target for the toolchain types to ensure that they are valid and + // resolve aliases. +@@ -138,9 +131,10 @@ public class ToolchainResolutionFunction implements SkyFunction { + .handle( + Event.info( + String.format( +- "ToolchainResolution: Target platform %s: Selected execution platform %s," ++ "ToolchainResolution: Target platform %s (%s): Selected execution platform %s, " + + " %s", + unloadedToolchainContext.targetPlatform().label(), ++ configuration, + unloadedToolchainContext.executionPlatform().label(), + selectedToolchains))); + } diff --git a/pkgs/development/tools/build-managers/bazel/java-test.nix b/pkgs/development/tools/build-managers/bazel/java-test.nix index f07952ac8c1f..59235ee88611 100644 --- a/pkgs/development/tools/build-managers/bazel/java-test.nix +++ b/pkgs/development/tools/build-managers/bazel/java-test.nix @@ -7,7 +7,6 @@ , lib , openjdk8 , jdk11_headless -, jdk17_headless , runLocal , runtimeShell , writeScript @@ -46,19 +45,16 @@ let inherit workspaceDir; bazelPkg = bazel; buildInputs = [ - #(if lib.strings.versionOlder bazel.version "5.0.0" then openjdk8 - #else if lib.strings.versionOlder bazel.version "7.0.0" then jdk11_headless - #else jdk17_headless) - jdk17_headless + (if lib.strings.versionOlder bazel.version "5.0.0" then openjdk8 else jdk11_headless) ]; bazelScript = '' ${bazel}/bin/bazel \ run \ --announce_rc \ + --toolchain_resolution_debug='@bazel_tools//tools/jdk:(runtime_)?toolchain_type' \ --distdir=${distDir} \ --verbose_failures \ --curses=no \ - --sandbox_debug \ --strict_java_deps=off \ //:ProjectRunner \ '' + lib.optionalString (lib.strings.versionOlder bazel.version "5.0.0") '' @@ -68,6 +64,14 @@ let '' + extraBazelArgs; }; + # --repo_env=JAVA_HOME=${jdk11_headless}/${if stdenv.hostPlatform.isDarwin then "/zulu-17.jdk/Contents/Home" else "/lib/openjdk"} \ + #--java_language_version=17 \ + #--java_language_version=17 \ + #--java_runtime_version=local_jdk \ + # --java_language_version=11 \ + # --tool_java_runtime_version=local_jdk_17 \ + # --tool_java_language_version=17 \ + #--java_runtime_version=local_jdk_11 \ in testBazel