From f3a42b27e3bb9187d7496cee65572be2dafe21a3 Mon Sep 17 00:00:00 2001 From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org> Date: Tue, 3 Aug 2021 21:35:28 +0200 Subject: [PATCH 1/3] boost-build: allow supplying a boost version to build b2 for The `useBoost` argument expects an attribute set with an src and version attribute (so a boost derivation works) and builds b2 for the given version of boost. This is useful when bootstrapping boost: We can override boost-build to get a boost-build derivation matching the current boost version we want to build (b2 is not backwards compatible enough to build older boost versions) without the need of having one boost-build${version} attribute for every boost version we have which is not very useful for nixpkgs' users. --- .../development/tools/boost-build/default.nix | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/pkgs/development/tools/boost-build/default.nix b/pkgs/development/tools/boost-build/default.nix index 5016aa590b5f..c6c66d4d212d 100644 --- a/pkgs/development/tools/boost-build/default.nix +++ b/pkgs/development/tools/boost-build/default.nix @@ -2,18 +2,36 @@ , stdenv , fetchFromGitHub , bison +# boost derivation to use for the src and version. +# This is used by the boost derivation to build +# a b2 matching their version (by overriding this +# argument). Infinite recursion is not an issue +# since we only look at src and version of boost. +, useBoost ? {} }: -stdenv.mkDerivation rec { - pname = "boost-build"; - version = "4.4.1"; +let + defaultVersion = "4.4.1"; +in - src = fetchFromGitHub { +stdenv.mkDerivation { + pname = "boost-build"; + version = + if useBoost ? version + then "boost-${useBoost.version}" + else defaultVersion; + + src = useBoost.src or (fetchFromGitHub { owner = "boostorg"; repo = "build"; - rev = version; + rev = defaultVersion; sha256 = "1r4rwlq87ydmsdqrik4ly5iai796qalvw7603mridg2nwcbbnf54"; - }; + }); + + # b2 is in a subdirectory of boost source tarballs + postUnpack = lib.optionalString (useBoost ? src) '' + sourceRoot="$sourceRoot/tools/build" + ''; patches = [ # Upstream defaults to gcc on darwin, but we use clang. @@ -32,8 +50,13 @@ stdenv.mkDerivation rec { installPhase = '' runHook preInstall + ./b2 install --prefix="$out" - ln -s b2 "$out/bin/bjam" + + # older versions of b2 created this symlink, + # which we want to support building via useBoost. + test -e "$out/bin/bjam" || ln -s b2 "$out/bin/bjam" + runHook postInstall ''; From d20aa4955d4c31d8393753ddf088e1ee82c5d12e Mon Sep 17 00:00:00 2001 From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org> Date: Tue, 3 Aug 2021 23:05:12 +0200 Subject: [PATCH 2/3] boost*: use packaged b2 instead it of building in configurePhase If we build two things in one derivation, it becomes more complicated and its build time is extended. Therefore we should avoid this if possible. There's a good opportunity for this with boost: We have boost-build packaged already. This has the additional benefit that we can get rid of $CC_FOR_BUILD entirely in boost, meaning we don't need to rely on (as many) hacks to make boost understand our way of cross compiling. Unfortunately boost-build is not backwards compatible, so we need to build a specific boost-build for each boost derivation (the number could probably be reduced, but I'm not interested in testing a lot of boost builds at the moment). Additionally we fix a few cross compilation problems: - boost couldn't cope with different types of compilers for native and cross (as happens if useLLVM is true). Since we only use one of them per derivation, this is no longer an issue. - boost didn't find the cross ar and ranlib for compilation (since it doesn't check $AR or $RANLIB). Instead it used plain ar and ranlib form $PATH which were the native ones before. This is now fixed by setting these tools explicitly in user-config.jam (and no longer providing the native tools). With these changes, pkgsLLVM.boost builds. On darwin, instead of patching the clang-darwin.jam definition, we instead supply -rpath $out/lib via which causes the correct directory to be added to the libraries' rpaths, so that they find each other. --- pkgs/development/libraries/boost/default.nix | 47 ++++++++++++++++ pkgs/development/libraries/boost/generic.nix | 56 +++++++++++++------- pkgs/top-level/all-packages.nix | 31 ++++++----- 3 files changed, 101 insertions(+), 33 deletions(-) create mode 100644 pkgs/development/libraries/boost/default.nix diff --git a/pkgs/development/libraries/boost/default.nix b/pkgs/development/libraries/boost/default.nix new file mode 100644 index 000000000000..3b79378605f2 --- /dev/null +++ b/pkgs/development/libraries/boost/default.nix @@ -0,0 +1,47 @@ +{ lib +, callPackage +, boost-build +, fetchurl +}: + +let + # for boost 1.55 we need to use 1.56's b2 + # since 1.55's build system is not working + # with our derivation + useBoost156 = rec { + version = "1.56.0"; + src = fetchurl { + url = "mirror://sourceforge/boost/boost_${lib.replaceStrings ["."] ["_"] version}.tar.bz2"; + sha256 = "07gz62nj767qzwqm3xjh11znpyph8gcii0cqhnx7wvismyn34iqk"; + }; + }; + + makeBoost = file: + lib.fix (self: + callPackage file { + boost-build = boost-build.override { + # useBoost allows us passing in src and version from + # the derivation we are building to get a matching b2 version. + useBoost = + if lib.versionAtLeast self.version "1.56" + then self + else useBoost156; # see above + }; + } + ); +in { + boost155 = makeBoost ./1.55.nix; + boost159 = makeBoost ./1.59.nix; + boost160 = makeBoost ./1.60.nix; + boost165 = makeBoost ./1.65.nix; + boost166 = makeBoost ./1.66.nix; + boost167 = makeBoost ./1.67.nix; + boost168 = makeBoost ./1.68.nix; + boost169 = makeBoost ./1.69.nix; + boost170 = makeBoost ./1.70.nix; + boost171 = makeBoost ./1.71.nix; + boost172 = makeBoost ./1.72.nix; + boost173 = makeBoost ./1.73.nix; + boost174 = makeBoost ./1.74.nix; + boost175 = makeBoost ./1.75.nix; +} diff --git a/pkgs/development/libraries/boost/generic.nix b/pkgs/development/libraries/boost/generic.nix index 1f9bbe5ffcad..f1a39c3d8cdb 100644 --- a/pkgs/development/libraries/boost/generic.nix +++ b/pkgs/development/libraries/boost/generic.nix @@ -1,7 +1,7 @@ { lib, stdenv, icu, expat, zlib, bzip2, python ? null, fixDarwinDylibNames, libiconv +, boost-build , fetchpatch , which -, buildPackages , toolset ? /**/ if stdenv.cc.isClang then "clang" else null , enableRelease ? true @@ -67,6 +67,8 @@ let else "$NIX_BUILD_CORES"; + needUserConfig = stdenv.hostPlatform != stdenv.buildPlatform || useMpi || stdenv.isDarwin; + b2Args = concatStringsSep " " ([ "--includedir=$dev/include" "--libdir=$out/lib" @@ -95,7 +97,7 @@ let ++ optional (variant == "release") "debug-symbols=off" ++ optional (toolset != null) "toolset=${toolset}" ++ optional (!enablePython) "--without-python" - ++ optional (useMpi || stdenv.hostPlatform != stdenv.buildPlatform) "--user-config=user-config.jam" + ++ optional needUserConfig "--user-config=user-config.jam" ++ optionals (stdenv.hostPlatform.libc == "msvcrt") [ "threadapi=win32" ] ++ extraB2Args @@ -137,22 +139,39 @@ stdenv.mkDerivation { maintainers = with maintainers; [ peti ]; }; - preConfigure = '' - if test -f tools/build/src/tools/clang-darwin.jam ; then - substituteInPlace tools/build/src/tools/clang-darwin.jam \ - --replace '@rpath/$(<[1]:D=)' "$out/lib/\$(<[1]:D=)"; - fi; - '' + optionalString useMpi '' + preConfigure = optionalString useMpi '' cat << EOF >> user-config.jam using mpi : ${mpi}/bin/mpiCC ; EOF - '' + optionalString (stdenv.hostPlatform != stdenv.buildPlatform) '' + '' + # On darwin we need to add the `$out/lib` to the libraries' rpath explicitly, + # otherwise the dynamic linker is unable to resolve the reference to @rpath + # when the boost libraries want to load each other at runtime. + + optionalString (stdenv.isDarwin && enableShared) '' cat << EOF >> user-config.jam - using gcc : cross : ${stdenv.cc.targetPrefix}c++ ; + using clang-darwin : : ${stdenv.cc.targetPrefix}c++ + : "-rpath $out/lib/" + ; + EOF + '' + # b2 has trouble finding the correct compiler and tools for cross compilation + # since it apparently ignores $CC, $AR etc. Thus we need to set everything + # in user-config.jam. To keep things simple we just set everything in an + # uniform way for clang and gcc (which works thanks to our cc-wrapper). + # We pass toolset later which will make b2 invoke everything in the right + # way -- the other toolset in user-config.jam will be ignored. + + optionalString (stdenv.hostPlatform != stdenv.buildPlatform) '' + cat << EOF >> user-config.jam + using gcc : cross : ${stdenv.cc.targetPrefix}c++ + : $AR + $RANLIB + ; + + using clang : cross : ${stdenv.cc.targetPrefix}c++ + : $AR + $RANLIB + ; EOF - # Build b2 with buildPlatform CC/CXX. - sed '2i export CC=$CC_FOR_BUILD; export CXX=$CXX_FOR_BUILD' \ - -i ./tools/build/src/engine/build.sh ''; NIX_CFLAGS_LINK = lib.optionalString stdenv.isDarwin @@ -160,9 +179,8 @@ stdenv.mkDerivation { enableParallelBuilding = true; - nativeBuildInputs = [ which ] + nativeBuildInputs = [ which boost-build ] ++ optional stdenv.hostPlatform.isDarwin fixDarwinDylibNames; - depsBuildBuild = [ buildPackages.stdenv.cc ]; buildInputs = [ expat zlib bzip2 libiconv ] ++ optional (stdenv.hostPlatform == stdenv.buildPlatform) icu ++ optional enablePython python @@ -173,13 +191,13 @@ stdenv.mkDerivation { configureFlags = [ "--includedir=$(dev)/include" "--libdir=$(out)/lib" + "--with-bjam=b2" # prevent bootstrapping b2 in configurePhase ] ++ optional enablePython "--with-python=${python.interpreter}" - ++ [ (if stdenv.hostPlatform == stdenv.buildPlatform then "--with-icu=${icu.dev}" else "--without-icu") ] - ++ optional (toolset != null) "--with-toolset=${toolset}"; + ++ [ (if stdenv.hostPlatform == stdenv.buildPlatform then "--with-icu=${icu.dev}" else "--without-icu") ]; buildPhase = '' runHook preBuild - ./b2 ${b2Args} + b2 ${b2Args} runHook postBuild ''; @@ -191,7 +209,7 @@ stdenv.mkDerivation { cp -a tools/boostbook/{xsl,dtd} $dev/share/boostbook/ # Let boost install everything else - ./b2 ${b2Args} install + b2 ${b2Args} install runHook postInstall ''; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 412985e644a5..9330e230b97d 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -14815,22 +14815,25 @@ in boolstuff = callPackage ../development/libraries/boolstuff { }; - boost155 = callPackage ../development/libraries/boost/1.55.nix { }; - boost159 = callPackage ../development/libraries/boost/1.59.nix { }; + inherit (callPackage ../development/libraries/boost { inherit (buildPackages) boost-build; }) + boost155 + boost159 + boost160 + boost165 + boost166 + boost167 + boost168 + boost169 + boost170 + boost171 + boost172 + boost173 + boost174 + boost175 + ; + boost15x = boost159; - boost160 = callPackage ../development/libraries/boost/1.60.nix { }; - boost165 = callPackage ../development/libraries/boost/1.65.nix { }; - boost166 = callPackage ../development/libraries/boost/1.66.nix { }; - boost167 = callPackage ../development/libraries/boost/1.67.nix { }; - boost168 = callPackage ../development/libraries/boost/1.68.nix { }; - boost169 = callPackage ../development/libraries/boost/1.69.nix { }; boost16x = boost169; - boost170 = callPackage ../development/libraries/boost/1.70.nix { }; - boost171 = callPackage ../development/libraries/boost/1.71.nix { }; - boost172 = callPackage ../development/libraries/boost/1.72.nix { }; - boost173 = callPackage ../development/libraries/boost/1.73.nix { }; - boost174 = callPackage ../development/libraries/boost/1.74.nix { }; - boost175 = callPackage ../development/libraries/boost/1.75.nix { }; boost17x = boost175; boost = boost16x; From 43e0f0688a19bbd88cdf898efb7b14ecf09717c4 Mon Sep 17 00:00:00 2001 From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org> Date: Wed, 4 Aug 2021 23:48:22 +0200 Subject: [PATCH 3/3] boost: always set toolset for known compilers There is an edge case when cross compiling where the auto detection script would not correctly recognize the compiler (as it is only good at recognizing native compilers, really, which we don't have anymore since we don't need the build->build one anymore). If bootstrap.sh doesn't detect the compiler correctly, it'll generate a project-config.jam with a syntax error which breaks the build in a hard to spot way: only a warning is displayed after configuring, the build will appear to run normally until it fails quietly just before installing. By providing it explicitly, we can prevent this. --- pkgs/development/libraries/boost/generic.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/development/libraries/boost/generic.nix b/pkgs/development/libraries/boost/generic.nix index f1a39c3d8cdb..7ea7902e0873 100644 --- a/pkgs/development/libraries/boost/generic.nix +++ b/pkgs/development/libraries/boost/generic.nix @@ -3,6 +3,7 @@ , fetchpatch , which , toolset ? /**/ if stdenv.cc.isClang then "clang" + else if stdenv.cc.isGNU then "gcc" else null , enableRelease ? true , enableDebug ? false @@ -193,6 +194,7 @@ stdenv.mkDerivation { "--libdir=$(out)/lib" "--with-bjam=b2" # prevent bootstrapping b2 in configurePhase ] ++ optional enablePython "--with-python=${python.interpreter}" + ++ optional (toolset != null) "--with-toolset=${toolset}" ++ [ (if stdenv.hostPlatform == stdenv.buildPlatform then "--with-icu=${icu.dev}" else "--without-icu") ]; buildPhase = ''