From 2a4cbf783777e93c8f181da338bb765d623608dc Mon Sep 17 00:00:00 2001 From: Philip Taron Date: Sat, 27 Jul 2024 10:28:17 -0700 Subject: [PATCH] freshBootstrapTools.build: extract as a package --- pkgs/stdenv/darwin/make-bootstrap-tools.nix | 261 +-------------- pkgs/stdenv/darwin/stdenv-bootstrap-tools.nix | 296 ++++++++++++++++++ 2 files changed, 297 insertions(+), 260 deletions(-) create mode 100644 pkgs/stdenv/darwin/stdenv-bootstrap-tools.nix diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix index b921333f1bf7..075e3aeec1d1 100644 --- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix +++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix @@ -21,269 +21,10 @@ let else {}; pkgs = import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap); - inherit (pkgs) lib stdenv darwin; in rec { - build = stdenv.mkDerivation { - name = "stdenv-bootstrap-tools"; - - nativeBuildInputs = [ pkgs.dumpnar pkgs.nukeReferences ]; - - buildCommand = let - inherit (lib) - getBin - getDev - getLib - ; - - coreutils_ = (pkgs.coreutils.override (args: { - # We want coreutils without ACL support. - aclSupport = false; - # Cannot use a single binary build, or it gets dynamically linked against gmp. - singleBinary = false; - })).overrideAttrs (oa: { - # Increase header size to be able to inject extra RPATHs. Otherwise - # x86_64-darwin build fails as: - # https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv - NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names"; - }); - - # Avoid messing with libkrb5 and libnghttp2. - curl_ = pkgs.curlMinimal.override (args: { - gssSupport = false; - http2Support = false; - scpSupport = false; - }); - - unpackScript = pkgs.writeText "bootstrap-tools-unpack.sh" '' - set -euo pipefail - - echo Unpacking the bootstrap tools... >&2 - mkdir $out - tar xf "$1" -C $out - - updateInstallName() { - local path="$1" - - cp "$path" "$path.new" - install_name_tool -id "$path" "$path.new" - codesign -f -i "$(basename "$path")" -s - "$path.new" - mv -f "$path.new" "$path" - } - - find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do - updateInstallName "$lib" - done - - # as is a wrapper around clang. need to replace the nuked store paths - sed -i 's|/.*/bin/|'"$out"'/bin/|' $out/bin/as - - # Provide a gunzip script. - cat > $out/bin/gunzip < $out/bin/egrep - echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep - echo "#! $out/bin/sh" > $out/bin/fgrep - echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep - - cat >$out/bin/dsymutil << EOF - #!$out/bin/sh - EOF - - chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil - ''; - - in - '' - mkdir -p $out/bin $out/lib $out/lib/darwin - - ${lib.optionalString stdenv.targetPlatform.isx86_64 '' - # Copy libSystem's .o files for various low-level boot stuff. - cp -d ${getLib darwin.Libsystem}/lib/*.o $out/lib - - # Resolv is actually a link to another package, so let's copy it properly - cp -L ${getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib - ''} - - cp -rL ${getDev darwin.Libsystem}/include $out - chmod -R u+w $out/include - cp -rL ${getDev pkgs.libiconv}/include/* $out/include - cp -rL ${getDev pkgs.gnugrep.pcre2}/include/* $out/include - mv $out/include $out/include-Libsystem - - # Copy binutils. - for i in as ld ar ranlib nm strip otool install_name_tool lipo codesign_allocate; do - cp ${getBin darwin.binutils-unwrapped}/bin/$i $out/bin - done - cp -d ${getLib ld64}/lib/libcodedirectory*.dylib $out/lib - - # Copy coreutils, bash, etc. - cp ${getBin coreutils_}/bin/* $out/bin - (cd $out/bin && rm vdir dir sha*sum pinky factor pathchk runcon shuf who whoami shred users) - - cp -d ${getBin pkgs.bash}/bin/{ba,}sh $out/bin - cp -d ${getBin pkgs.diffutils}/bin/* $out/bin - cp ${getBin pkgs.findutils}/bin/{find,xargs} $out/bin - cp -d ${getBin pkgs.gawk}/bin/{g,}awk $out/bin - cp -d ${getBin pkgs.gnugrep}/bin/grep $out/bin - cp -d ${getBin pkgs.gnumake}/bin/* $out/bin - cp -d ${getBin pkgs.gnused}/bin/* $out/bin - cp -d ${getBin pkgs.patch}/bin/* $out/bin - - cp -d ${getLib pkgs.gettext}/lib/libintl*.dylib $out/lib - cp -d ${getLib pkgs.gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib - cp -d ${getLib pkgs.libiconv}/lib/lib*.dylib $out/lib - cp -d ${getLib pkgs.libxml2}/lib/libxml2*.dylib $out/lib - cp -d ${getLib pkgs.ncurses}/lib/libncurses*.dylib $out/lib - - # copy package extraction tools - cp -d ${getBin pkgs.bzip2}/bin/b{,un}zip2 $out/bin - cp ${getBin pkgs.cpio}/bin/cpio $out/bin - cp ${getBin pkgs.gnutar}/bin/tar $out/bin - cp ${getBin pkgs.gzip}/bin/.gzip-wrapped $out/bin/gzip - cp ${getBin pkgs.pbzx}/bin/pbzx $out/bin - cp ${getBin pkgs.xz}/bin/xz $out/bin - cp -d ${getLib pkgs.bzip2}/lib/libbz2*.dylib $out/lib - cp -d ${getLib pkgs.gmpxx}/lib/libgmp*.dylib $out/lib - cp -d ${getLib pkgs.xar}/lib/libxar*.dylib $out/lib - cp -d ${getLib pkgs.xz}/lib/liblzma*.dylib $out/lib - cp -d ${getLib pkgs.zlib}/lib/libz*.dylib $out/lib - - # This used to be in-nixpkgs, but now is in the bundle - # because I can't be bothered to make it partially static - cp ${getBin curl_}/bin/curl $out/bin - cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib - cp -d ${getLib pkgs.openssl}/lib/*.dylib $out/lib - - # Copy what we need of clang - cp -d ${getBin pkgs.llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin - cp -d ${getLib pkgs.llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib - cp -rd ${getLib pkgs.llvmPackages.clang-unwrapped}/lib/clang $out/lib - - cp -d ${getLib pkgs.llvmPackages.libcxx}/lib/libc++*.dylib $out/lib - mkdir -p $out/lib/darwin - cp -d ${getLib pkgs.llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a $out/lib/darwin - cp -d ${getLib pkgs.llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib - cp -d ${getLib pkgs.llvmPackages.llvm}/lib/libLLVM.dylib $out/lib - cp -d ${getLib pkgs.libffi}/lib/libffi*.dylib $out/lib - - mkdir $out/include - cp -rd ${getDev pkgs.llvmPackages.libcxx}/include/c++ $out/include - - # copy .tbd assembly utils - cp ${getBin darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin - cp -d ${getLib pkgs.libyaml}/lib/libyaml*.dylib $out/lib - - # copy sigtool - cp -d ${getBin darwin.sigtool}/bin/{codesign,sigtool} $out/bin - - cp -d ${getLib darwin.libtapi}/lib/libtapi*.dylib $out/lib - - # tools needed to unpack bootstrap archive - mkdir -p unpack/bin unpack/lib - cp -d ${getBin pkgs.bash}/bin/{bash,sh} unpack/bin - cp ${getBin coreutils_}/bin/mkdir unpack/bin - cp ${getBin pkgs.gnutar}/bin/tar unpack/bin - cp ${getBin pkgs.xz}/bin/xz unpack/bin - cp -d ${getLib pkgs.gettext}/lib/libintl*.dylib unpack/lib - cp -d ${getLib pkgs.libiconv}/lib/lib*.dylib unpack/lib - cp -d ${getLib pkgs.xz}/lib/liblzma*.dylib unpack/lib - cp ${unpackScript} unpack/bootstrap-tools-unpack.sh - - # - # All files copied. Perform processing to update references to point into - # the archive - # - - chmod -R u+w $out unpack - - # - change nix store library paths to use @rpath/library - # - if needed add an rpath containing lib/ - # - strip executable - rpathify() { - local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true) - local lib rpath - for lib in $libs; do - ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1" - done - - case "$(dirname "$1")" in - */bin) - # Strip executables even further - ${stdenv.cc.targetPrefix}strip "$i" - rpath='@executable_path/../lib' - ;; - */lib) - # the '/.' suffix is required - rpath='@loader_path/.' - ;; - */lib/darwin) - rpath='@loader_path/..' - ;; - *) - echo unkown executable $1 >&2 - exit 1 - ;; - esac - - # if shared object contains references add an rpath to lib/ - if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then - ${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1" - fi - } - - # check that linked library paths exist in lib - # must be run after rpathify is performed - checkDeps() { - local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^ ]*' || true) - local lib - shopt -s extglob - for lib in $deps; do - local root="''${1/\/@(lib|bin)\/*}" - if [[ ! -e $root/''${lib/@rpath/lib} ]]; then - echo "error: $1 missing lib for $lib" >&2 - exit 1 - fi - done - shopt -u extglob - } - - for i in {unpack,$out}/bin/* {unpack,$out}/lib{,/darwin}/*.dylib; do - if [[ ! -L $i ]] && isMachO "$i"; then - rpathify "$i" - checkDeps "$i" - fi - done - - nuke-refs {unpack,$out}/bin/* - nuke-refs {unpack,$out}/lib/* - nuke-refs $out/lib/darwin/* - - mkdir $out/.pack - mv $out/* $out/.pack - mv $out/.pack $out/pack - - mkdir $out/on-server - cp -r unpack $out - - XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \ - --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack . - dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz - ''; - - allowedReferences = []; - - meta = { - maintainers = [ lib.maintainers.copumpkin ]; - }; - }; - + build = pkgs.callPackage ./stdenv-bootstrap-tools.nix { }; bootstrapFiles = { bootstrapTools = "${build}/on-server/bootstrap-tools.tar.xz"; unpack = pkgs.runCommand "unpack" { allowedReferences = []; } '' diff --git a/pkgs/stdenv/darwin/stdenv-bootstrap-tools.nix b/pkgs/stdenv/darwin/stdenv-bootstrap-tools.nix new file mode 100644 index 000000000000..0aa15d526f9f --- /dev/null +++ b/pkgs/stdenv/darwin/stdenv-bootstrap-tools.nix @@ -0,0 +1,296 @@ +{ + lib, + stdenv, + bash, + bzip2, + coreutils, + cpio, + curlMinimal, + darwin, + diffutils, + dumpnar, + findutils, + gawk, + gettext, + gmpxx, + gnugrep, + gnumake, + gnused, + gnutar, + gzip, + ld64, + libffi, + libiconv, + libxml2, + libyaml, + llvmPackages, + ncurses, + nukeReferences, + openssl, + patch, + pbzx, + writeText, + xar, + xz, + zlib, +}: +stdenv.mkDerivation { + name = "stdenv-bootstrap-tools"; + + nativeBuildInputs = [ + dumpnar + nukeReferences + ]; + + buildCommand = + let + inherit (lib) getBin getDev getLib; + + coreutils_ = + (coreutils.override (prevArgs: { + # We want coreutils without ACL support. + aclSupport = false; + # Cannot use a single binary build, or it gets dynamically linked against gmp. + singleBinary = false; + })).overrideAttrs + (prevAttrs: { + # Increase header size to be able to inject extra RPATHs. Otherwise + # x86_64-darwin build fails as: + # https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv + NIX_LDFLAGS = (prevAttrs.NIX_LDFLAGS or "") + " -headerpad_max_install_names"; + }); + + # Avoid messing with libkrb5 and libnghttp2. + curl_ = curlMinimal.override (prevArgs: { + gssSupport = false; + http2Support = false; + scpSupport = false; + }); + + unpackScript = writeText "bootstrap-tools-unpack.sh" '' + set -euo pipefail + + echo Unpacking the bootstrap tools... >&2 + mkdir $out + tar xf "$1" -C $out + + updateInstallName() { + local path="$1" + + cp "$path" "$path.new" + install_name_tool -id "$path" "$path.new" + codesign -f -i "$(basename "$path")" -s - "$path.new" + mv -f "$path.new" "$path" + } + + find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do + updateInstallName "$lib" + done + + # as is a wrapper around clang. need to replace the nuked store paths + sed -i 's|/.*/bin/|'"$out"'/bin/|' $out/bin/as + + # Provide a gunzip script. + cat > $out/bin/gunzip < $out/bin/egrep + echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep + echo "#! $out/bin/sh" > $out/bin/fgrep + echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep + + cat >$out/bin/dsymutil << EOF + #!$out/bin/sh + EOF + + chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil + ''; + + in + '' + mkdir -p $out/bin $out/lib $out/lib/darwin + + ${lib.optionalString stdenv.targetPlatform.isx86_64 '' + # Copy libSystem's .o files for various low-level boot stuff. + cp -d ${getLib darwin.Libsystem}/lib/*.o $out/lib + + # Resolv is actually a link to another package, so let's copy it properly + cp -L ${getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib + ''} + + cp -rL ${getDev darwin.Libsystem}/include $out + chmod -R u+w $out/include + cp -rL ${getDev libiconv}/include/* $out/include + cp -rL ${getDev gnugrep.pcre2}/include/* $out/include + mv $out/include $out/include-Libsystem + + # Copy binutils. + for i in as ld ar ranlib nm strip otool install_name_tool lipo codesign_allocate; do + cp ${getBin darwin.binutils-unwrapped}/bin/$i $out/bin + done + cp -d ${getLib ld64}/lib/libcodedirectory*.dylib $out/lib + + # Copy coreutils, bash, etc. + cp ${getBin coreutils_}/bin/* $out/bin + (cd $out/bin && rm vdir dir sha*sum pinky factor pathchk runcon shuf who whoami shred users) + + cp -d ${getBin bash}/bin/{ba,}sh $out/bin + cp -d ${getBin diffutils}/bin/* $out/bin + cp ${getBin findutils}/bin/{find,xargs} $out/bin + cp -d ${getBin gawk}/bin/{g,}awk $out/bin + cp -d ${getBin gnugrep}/bin/grep $out/bin + cp -d ${getBin gnumake}/bin/* $out/bin + cp -d ${getBin gnused}/bin/* $out/bin + cp -d ${getBin patch}/bin/* $out/bin + + cp -d ${getLib gettext}/lib/libintl*.dylib $out/lib + cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib + cp -d ${getLib libiconv}/lib/lib*.dylib $out/lib + cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib + cp -d ${getLib ncurses}/lib/libncurses*.dylib $out/lib + + # copy package extraction tools + cp -d ${getBin bzip2}/bin/b{,un}zip2 $out/bin + cp ${getBin cpio}/bin/cpio $out/bin + cp ${getBin gnutar}/bin/tar $out/bin + cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip + cp ${getBin pbzx}/bin/pbzx $out/bin + cp ${getBin xz}/bin/xz $out/bin + cp -d ${getLib bzip2}/lib/libbz2*.dylib $out/lib + cp -d ${getLib gmpxx}/lib/libgmp*.dylib $out/lib + cp -d ${getLib xar}/lib/libxar*.dylib $out/lib + cp -d ${getLib xz}/lib/liblzma*.dylib $out/lib + cp -d ${getLib zlib}/lib/libz*.dylib $out/lib + + # This used to be in-nixpkgs, but now is in the bundle + # because I can't be bothered to make it partially static + cp ${getBin curl_}/bin/curl $out/bin + cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib + cp -d ${getLib openssl}/lib/*.dylib $out/lib + + # Copy what we need of clang + cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin + cp -d ${getLib llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib + cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/clang $out/lib + + cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib + mkdir -p $out/lib/darwin + cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a $out/lib/darwin + cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib + cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib + cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib + + mkdir $out/include + cp -rd ${getDev llvmPackages.libcxx}/include/c++ $out/include + + # copy .tbd assembly utils + cp ${getBin darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin + cp -d ${getLib libyaml}/lib/libyaml*.dylib $out/lib + + # copy sigtool + cp -d ${getBin darwin.sigtool}/bin/{codesign,sigtool} $out/bin + + cp -d ${getLib darwin.libtapi}/lib/libtapi*.dylib $out/lib + + # tools needed to unpack bootstrap archive + mkdir -p unpack/bin unpack/lib + cp -d ${getBin bash}/bin/{bash,sh} unpack/bin + cp ${getBin coreutils_}/bin/mkdir unpack/bin + cp ${getBin gnutar}/bin/tar unpack/bin + cp ${getBin xz}/bin/xz unpack/bin + cp -d ${getLib gettext}/lib/libintl*.dylib unpack/lib + cp -d ${getLib libiconv}/lib/lib*.dylib unpack/lib + cp -d ${getLib xz}/lib/liblzma*.dylib unpack/lib + cp ${unpackScript} unpack/bootstrap-tools-unpack.sh + + # + # All files copied. Perform processing to update references to point into + # the archive + # + + chmod -R u+w $out unpack + + # - change nix store library paths to use @rpath/library + # - if needed add an rpath containing lib/ + # - strip executable + rpathify() { + local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true) + local lib rpath + for lib in $libs; do + ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1" + done + + case "$(dirname "$1")" in + */bin) + # Strip executables even further + ${stdenv.cc.targetPrefix}strip "$i" + rpath='@executable_path/../lib' + ;; + */lib) + # the '/.' suffix is required + rpath='@loader_path/.' + ;; + */lib/darwin) + rpath='@loader_path/..' + ;; + *) + echo unkown executable $1 >&2 + exit 1 + ;; + esac + + # if shared object contains references add an rpath to lib/ + if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then + ${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1" + fi + } + + # check that linked library paths exist in lib + # must be run after rpathify is performed + checkDeps() { + local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^ ]*' || true) + local lib + shopt -s extglob + for lib in $deps; do + local root="''${1/\/@(lib|bin)\/*}" + if [[ ! -e $root/''${lib/@rpath/lib} ]]; then + echo "error: $1 missing lib for $lib" >&2 + exit 1 + fi + done + shopt -u extglob + } + + for i in {unpack,$out}/bin/* {unpack,$out}/lib{,/darwin}/*.dylib; do + if [[ ! -L $i ]] && isMachO "$i"; then + rpathify "$i" + checkDeps "$i" + fi + done + + nuke-refs {unpack,$out}/bin/* + nuke-refs {unpack,$out}/lib/* + nuke-refs $out/lib/darwin/* + + mkdir $out/.pack + mv $out/* $out/.pack + mv $out/.pack $out/pack + + mkdir $out/on-server + cp -r unpack $out + + XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \ + --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack . + dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz + ''; + + allowedReferences = [ ]; + + meta = { + maintainers = [ lib.maintainers.copumpkin ]; + }; +}