From ba327a54b67732a715cc4237f5de20411b635319 Mon Sep 17 00:00:00 2001 From: Guillaume Bouchard Date: Wed, 19 Jun 2019 20:11:07 +0200 Subject: [PATCH 1/4] bazel: 0.26.1 -> 0.27.0 - Fixs for newly introduced bin/bash hardcoded reference - Bazel now references `remote_java_tools_xxx` which contains prebuilt binaries. We prefetch them, fix them, and force bazel to use the fixed repository. It also closes #63096 --- .../tools/build-managers/bazel/default.nix | 64 +++++++++++++++++-- .../tools/build-managers/bazel/src-deps.json | 39 ++++++----- 2 files changed, 81 insertions(+), 22 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/default.nix b/pkgs/development/tools/build-managers/bazel/default.nix index f8ed157e693b..ec36ca3e6d3f 100644 --- a/pkgs/development/tools/build-managers/bazel/default.nix +++ b/pkgs/development/tools/build-managers/bazel/default.nix @@ -14,14 +14,16 @@ # Always assume all markers valid (don't redownload dependencies). # Also, don't clean up environment variables. , enableNixHacks ? false +, gcc-unwrapped +, autoPatchelfHook }: let - version = "0.26.1"; + version = "0.27.0"; src = fetchurl { url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-dist.zip"; - sha256 = "000ny51hwnjyizm1md4w8q7m832jhf3c767pgbvg6nc7h67lzsf0"; + sha256 = "0yn662dzgfr8ls4avfl12k5sr4f210bab12wml18bh4sjlxhs263"; }; # Update with `eval $(nix-build -A bazel.updater)`, @@ -40,12 +42,12 @@ let srcs.bazel_skylib srcs.io_bazel_rules_sass (if stdenv.hostPlatform.isDarwin - then srcs.${"java_tools_javac10_darwin-v3.2.zip"} - else srcs.${"java_tools_javac10_linux-v3.2.zip"}) + then srcs.${"java_tools_javac11_darwin-v2.0.zip"} + else srcs.${"java_tools_javac11_linux-v2.0.zip"}) srcs.${"coverage_output_generator-v1.0.zip"} srcs.build_bazel_rules_nodejs - srcs.${"android_tools_pkg-0.2.tar.gz"} - ]; + srcs.${"android_tools_pkg-0.4.tar.gz"} + ]; distDir = runCommand "bazel-deps" {} '' mkdir -p $out @@ -87,6 +89,36 @@ let platforms = lib.platforms.linux ++ lib.platforms.darwin; + # This repository is fetched by bazel at runtime + # however it contains prebuilt java binaries, with wrong interpreter + # and libraries path. + # We prefetch it, patch it, and override it in a global bazelrc. + system = if stdenv.hostPlatform.isDarwin + then "darwin" else "linux"; + + remote_java_tools = stdenv.mkDerivation { + name = "remote_java_tools_${system}"; + + src = fetchurl { + url = "https://mirror.bazel.build/bazel_java_tools/releases/javac11/v2.0/java_tools_javac11_${system}-v2.0.zip"; + sha256 = "074d624fb34441df369afdfd454e75dba821d5d54932fcfee5ba598d17dc1b99"; + }; + + nativeBuildInputs = [ autoPatchelfHook unzip ]; + buildInputs = [ gcc-unwrapped ]; + + sourceRoot = "."; + + buildPhase = '' + mkdir $out; + ''; + + installPhase = '' + cp -Ra * $out/ + touch $out/WORKSPACE + ''; + }; + in stdenv.mkDerivation rec { name = "bazel-${version}"; @@ -117,6 +149,7 @@ stdenv.mkDerivation rec { runLocal = name: attrs: script: runCommandCC name ({ preferLocalBuild = true; meta.platforms = platforms; + buildInputs = [ python3 ]; } // attrs) script; # bazel wants to extract itself into $install_dir/install every time it runs, @@ -275,6 +308,11 @@ stdenv.mkDerivation rec { --replace /bin/true ${coreutils}/bin/true done + # bazel test runner include references to /bin/bash + substituteInPlace tools/build_rules/test_rules.bzl \ + --replace /bin/bash ${customBash}/bin/bash + + # Fixup scripts that generate scripts. Not fixed up by patchShebangs below. substituteInPlace scripts/bootstrap/compile.sh \ --replace /bin/bash ${customBash}/bin/bash @@ -323,12 +361,20 @@ stdenv.mkDerivation rec { mv runfiles.bash.tmp tools/bash/runfiles/runfiles.bash patchShebangs . + + # bazel reads its system bazelrc in /etc + # override this path to a builtin one + substituteInPlace \ + src/main/cpp/option_processor.cc \ + --replace BAZEL_SYSTEM_BAZELRC_PATH "\"$out/etc/bazelrc\"" ''; in lib.optionalString stdenv.hostPlatform.isDarwin darwinPatches + genericPatches; buildInputs = [ buildJdk + python3 # bazel build requires python3. However we still use python2 for most of the other tasks + # This will have to be refactored later. ]; # when a command can’t be found in a bazel build, you might also @@ -375,6 +421,12 @@ stdenv.mkDerivation rec { wrapProgram "$out/bin/bazel" --add-flags --server_javabase="${runJdk}" + # generates the system bazelrc + # warning: the name of the repository depends on the system, hence + # the reference to .name + mkdir $out/etc + echo "build --override_repository=${remote_java_tools.name}=${remote_java_tools}" > $out/etc/bazelrc + # shell completion files mkdir -p $out/share/bash-completion/completions $out/share/zsh/site-functions mv ./bazel_src/output/bazel-complete.bash $out/share/bash-completion/completions/bazel diff --git a/pkgs/development/tools/build-managers/bazel/src-deps.json b/pkgs/development/tools/build-managers/bazel/src-deps.json index 752f0a4c9909..a0d07f1a68d4 100644 --- a/pkgs/development/tools/build-managers/bazel/src-deps.json +++ b/pkgs/development/tools/build-managers/bazel/src-deps.json @@ -23,11 +23,11 @@ "https://github.com/bazelbuild/rules_sass/archive/8ccf4f1c351928b55d5dddf3672e3667f6978d60.tar.gz" ] }, - "android_tools_pkg-0.2.tar.gz": { - "name": "android_tools_pkg-0.2.tar.gz", - "sha256": "04f85f2dd049e87805511e3babc5cea3f5e72332b1627e34f3a5461cc38e815f", + "android_tools_pkg-0.4.tar.gz": { + "name": "android_tools_pkg-0.4.tar.gz", + "sha256": "331e7706f2bcae8a68057d8ddd3e3f1574bca26c67c65802fc4a8ac6164fa912", "urls": [ - "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.2.tar.gz" + "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.4.tar.gz" ] }, "bazel_j2objc": { @@ -134,25 +134,25 @@ "https://github.com/bazelbuild/skydoc/archive/2d9566b21fbe405acf5f7bf77eda30df72a4744c.tar.gz" ] }, - "java_tools_javac10_darwin-v3.2.zip": { - "name": "java_tools_javac10_darwin-v3.2.zip", - "sha256": "1437327179b4284f7082cee0bdc3328f040e62fc5cc59c32f6824b8c520e2b7b", + "java_tools_javac11_darwin-v2.0.zip": { + "name": "java_tools_javac11_darwin-v2.0.zip", + "sha256": "0ceb0c9ff91256fe33508306bc9cd9e188dcca38df78e70839d426bdaef67a38", "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/javac10/v3.2/java_tools_javac10_darwin-v3.2.zip" + "https://mirror.bazel.build/bazel_java_tools/releases/javac11/v2.0/java_tools_javac11_darwin-v2.0.zip" ] }, - "java_tools_javac10_linux-v3.2.zip": { - "name": "java_tools_javac10_linux-v3.2.zip", - "sha256": "b93e7c556b01815afb6c248aa73f06b7ec912805bde8898eedac1e20d08f2e67", + "java_tools_javac11_linux-v2.0.zip": { + "name": "java_tools_javac11_linux-v2.0.zip", + "sha256": "074d624fb34441df369afdfd454e75dba821d5d54932fcfee5ba598d17dc1b99", "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/javac10/v3.2/java_tools_javac10_linux-v3.2.zip" + "https://mirror.bazel.build/bazel_java_tools/releases/javac11/v2.0/java_tools_javac11_linux-v2.0.zip" ] }, - "java_tools_javac10_windows-v3.2.zip": { - "name": "java_tools_javac10_windows-v3.2.zip", - "sha256": "86d3cc7fa0dc91ccb8f78ae3af8440fe459177e22062043ee4b83d55e6b7dfb0", + "java_tools_javac11_windows-v2.0.zip": { + "name": "java_tools_javac11_windows-v2.0.zip", + "sha256": "2c3fc0ce7d30d60e26f4b8a36e2eadcf9e6a9d5a51b667d3d13b78db53b24251", "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/javac10/v3.2/java_tools_javac10_windows-v3.2.zip" + "https://mirror.bazel.build/bazel_java_tools/releases/javac11/v2.0/java_tools_javac11_windows-v2.0.zip" ] }, "java_tools_langtools_javac10": { @@ -162,6 +162,13 @@ "https://mirror.bazel.build/bazel_java_tools/jdk_langtools/langtools_jdk10.zip" ] }, + "java_tools_langtools_javac11": { + "name": "java_tools_langtools_javac11", + "sha256": "128a63f39d3f828a761f6afcfe3c6115279336a72ea77f60d7b3acf1841c9acb", + "urls": [ + "https://mirror.bazel.build/bazel_java_tools/jdk_langtools/langtools_jdk11.zip" + ] + }, "java_tools_langtools_javac9": { "name": "java_tools_langtools_javac9", "sha256": "3b6bbc47256acf2f61883901e2d4e3f9b292f5fe154a6912b928805de24cb864", From 7fed6eabcf76bd3bec4d8e5d384cd162c68ebfce Mon Sep 17 00:00:00 2001 From: Guillaume Bouchard Date: Thu, 20 Jun 2019 17:13:06 +0200 Subject: [PATCH 2/4] bazel: restore installCheckPhase All the dependencies of this phase are prefetched and provided to the bazel environment using --override_repository. --- .../tools/build-managers/bazel/default.nix | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/default.nix b/pkgs/development/tools/build-managers/bazel/default.nix index ec36ca3e6d3f..2cb00be63a90 100644 --- a/pkgs/development/tools/build-managers/bazel/default.nix +++ b/pkgs/development/tools/build-managers/bazel/default.nix @@ -28,15 +28,16 @@ let # Update with `eval $(nix-build -A bazel.updater)`, # then add new dependencies from the dict in ./src-deps.json as required. - srcDeps = + srcDeps = lib.attrsets.attrValues srcDepsSet; + srcDepsSet = let srcs = (builtins.fromJSON (builtins.readFile ./src-deps.json)); - toFetchurl = d: fetchurl { + toFetchurl = d: lib.attrsets.nameValuePair d.name (fetchurl { name = d.name; urls = d.urls; sha256 = d.sha256; - }; - in map toFetchurl [ + }); + in builtins.listToAttrs (map toFetchurl [ srcs.desugar_jdk_libs srcs.io_bazel_skydoc srcs.bazel_skylib @@ -47,7 +48,7 @@ let srcs.${"coverage_output_generator-v1.0.zip"} srcs.build_bazel_rules_nodejs srcs.${"android_tools_pkg-0.4.tar.gz"} - ]; + ]); distDir = runCommand "bazel-deps" {} '' mkdir -p $out @@ -433,14 +434,28 @@ stdenv.mkDerivation rec { cp ./bazel_src/scripts/zsh_completion/_bazel $out/share/zsh/site-functions/ ''; - # Temporarily disabling for now. A new approach is needed for this derivation as Bazel - # accesses the internet during the tests which fails in a sandbox. - doInstallCheck = false; + doInstallCheck = true; installCheckPhase = '' export TEST_TMPDIR=$(pwd) + tar xf ${srcDepsSet.io_bazel_skydoc} -C $TEST_TMPDIR + mv $(ls | grep skydoc-) io_bazel_skydoc + + tar xf ${srcDepsSet.bazel_skylib} -C $TEST_TMPDIR + mv $(ls | grep bazel-skylib-) bazel_skylib + + tar xf ${srcDepsSet.io_bazel_rules_sass} -C $TEST_TMPDIR + mv $(ls | grep rules_sass-) rules_sass + + unzip ${srcDepsSet.build_bazel_rules_nodejs} -d $TEST_TMPDIR + mv rules_nodejs-0.16.2 build_bazel_rules_nodejs + hello_test () { $out/bin/bazel test \ + --override_repository=io_bazel_skydoc=$TEST_TMPDIR/io_bazel_skydoc \ + --override_repository=bazel_skylib=$TEST_TMPDIR/bazel_skylib \ + --override_repository=io_bazel_rules_sass=$TEST_TMPDIR/rules_sass \ + --override_repository=build_bazel_rules_nodejs=$TEST_TMPDIR/build_bazel_rules_nodejs \ --test_output=errors \ --java_toolchain='${javaToolchain}' \ examples/cpp:hello-success_test \ From 27d4ce7ce08613ab124d160ff269c48c86eb3d83 Mon Sep 17 00:00:00 2001 From: Guillaume Bouchard Date: Thu, 20 Jun 2019 17:20:58 +0200 Subject: [PATCH 3/4] bazel: Full switch to python3 --- .../tools/build-managers/bazel/default.nix | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/default.nix b/pkgs/development/tools/build-managers/bazel/default.nix index 2cb00be63a90..257cc185b2e2 100644 --- a/pkgs/development/tools/build-managers/bazel/default.nix +++ b/pkgs/development/tools/build-managers/bazel/default.nix @@ -2,7 +2,7 @@ # this package (through the fixpoint glass) , bazel , lr, xe, zip, unzip, bash, writeCBin, coreutils -, which, python, gawk, gnused, gnutar, gnugrep, gzip, findutils +, which, gawk, gnused, gnutar, gnugrep, gzip, findutils # updater , python3, writeScript # Apple dependencies @@ -292,8 +292,8 @@ stdenv.mkDerivation rec { # Substitute python's stub shebang to plain python path. (see TODO add pr URL) # See also `postFixup` where python is added to $out/nix-support substituteInPlace src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt\ - --replace "/usr/bin/env python" "${python}/bin/python" \ - --replace "NIX_STORE_PYTHON_PATH" "${python}/bin/python" \ + --replace "/usr/bin/env python" "${python3}/bin/python" \ + --replace "NIX_STORE_PYTHON_PATH" "${python3}/bin/python" \ # md5sum is part of coreutils sed -i 's|/sbin/md5|md5sum|' \ @@ -374,15 +374,14 @@ stdenv.mkDerivation rec { buildInputs = [ buildJdk - python3 # bazel build requires python3. However we still use python2 for most of the other tasks - # This will have to be refactored later. + python3 ]; # when a command can’t be found in a bazel build, you might also # need to add it to `defaultShellPath`. nativeBuildInputs = [ zip - python + python3 unzip makeWrapper which @@ -491,7 +490,7 @@ stdenv.mkDerivation rec { echo "${customBash} ${defaultShellPath}" >> $out/nix-support/depends # The templates get tar’d up into a .jar, # so nix can’t detect python is needed in the runtime closure - echo "${python}" >> $out/nix-support/depends + echo "${python3}" >> $out/nix-support/depends ''; dontStrip = true; From d086d45952149dd87edd826f27f11e4a46d168f7 Mon Sep 17 00:00:00 2001 From: Guillaume Bouchard Date: Mon, 24 Jun 2019 11:40:53 +0200 Subject: [PATCH 4/4] Fix source url for java tools on darwin --- pkgs/development/tools/build-managers/bazel/default.nix | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/default.nix b/pkgs/development/tools/build-managers/bazel/default.nix index 257cc185b2e2..8edc7fe9a8e2 100644 --- a/pkgs/development/tools/build-managers/bazel/default.nix +++ b/pkgs/development/tools/build-managers/bazel/default.nix @@ -100,10 +100,7 @@ let remote_java_tools = stdenv.mkDerivation { name = "remote_java_tools_${system}"; - src = fetchurl { - url = "https://mirror.bazel.build/bazel_java_tools/releases/javac11/v2.0/java_tools_javac11_${system}-v2.0.zip"; - sha256 = "074d624fb34441df369afdfd454e75dba821d5d54932fcfee5ba598d17dc1b99"; - }; + src = srcDepsSet."java_tools_javac11_${system}-v2.0.zip"; nativeBuildInputs = [ autoPatchelfHook unzip ]; buildInputs = [ gcc-unwrapped ];