From 3029bd758b4ff3a240bead21c3491a51759a6ccd Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Wed, 19 Mar 2025 17:59:53 -0700 Subject: [PATCH 1/2] lixPackageSets.lix_2_92: init This adapts #375030 now that #391402 is merged, as are the `editline` (#391425, #391426) and `capnproto` (#391424) packaging updates. This introduces https://lix.systems/blog/2025-01-18-lix-2.92-release/ in nixpkgs. This new release introduces a bunch of changes at the packaging level, reliant on new features of Meson, new features of some of our dependencies which are done in previous commits (editline & capnproto). The approach taken here is that it's OK to add more dependencies than required for older versions but not OK to break them. Moreover, Lix is now load-bearing dependent on the Clang stdenv, it cannot be compiled with GCC anymore. An escape hatch is included but if this escape hatch is used, the user is on its own and should probably work with GCC upstream to fix miscompilations. --- .../package-management/lix/common-lix.nix | 88 +++++++++++++++++-- .../lix/common-nix-eval-jobs.nix | 5 +- pkgs/tools/package-management/lix/default.nix | 56 +++++++++++- 3 files changed, 136 insertions(+), 13 deletions(-) diff --git a/pkgs/tools/package-management/lix/common-lix.nix b/pkgs/tools/package-management/lix/common-lix.nix index 2e23c9c9e049..b9077e7672d4 100644 --- a/pkgs/tools/package-management/lix/common-lix.nix +++ b/pkgs/tools/package-management/lix/common-lix.nix @@ -3,11 +3,22 @@ suffix ? "", version, src, - docCargoDeps, + # For Lix versions >= 2.92, Rust sources are in the Lix repository root. + cargoDeps ? null, + # For previous versions, Rust sources are only in a subdirectory for + # `lix-doc`. + docCargoDeps ? null, patches ? [ ], maintainers ? lib.teams.lix.members, }@args: +assert lib.assertMsg ( + lib.versionOlder version "2.92" -> docCargoDeps != null +) "`lix-doc` `cargoDeps` must be set for Lix < 2.92"; +assert lib.assertMsg ( + lib.versionAtLeast version "2.92" -> cargoDeps != null +) "`cargoDeps` must be set for Lix ≥ 2.92"; + { stdenv, meson, @@ -18,6 +29,8 @@ busybox-sandbox-shell, bzip2, callPackage, + capnproto, + cargo, curl, cmake, doxygen, @@ -30,6 +43,7 @@ libarchive, libcpuid, libsodium, + llvmPackages, lowdown, lowdown-unsandboxed, lsof, @@ -39,6 +53,7 @@ nlohmann_json, ninja, openssl, + rustc, toml11, pegtl, python3, @@ -47,8 +62,11 @@ Security, sqlite, util-linuxMinimal, + removeReferencesTo, xz, nixosTests, + rustPlatform, + # Only used for versions before 2.92. lix-doc ? callPackage ./doc { inherit src; version = "${version}${suffix}"; @@ -57,6 +75,7 @@ enableDocumentation ? stdenv.hostPlatform == stdenv.buildPlatform, enableStatic ? stdenv.hostPlatform.isStatic, + enableStrictLLVMChecks ? true, withAWS ? !enableStatic && (stdenv.hostPlatform.isLinux || stdenv.hostPlatform.isDarwin), aws-sdk-cpp, # RISC-V support in progress https://github.com/seccomp/libseccomp/pull/50 @@ -67,11 +86,16 @@ stateDir, storeDir, }: - let + isLLVMOnly = lib.versionAtLeast version "2.92"; + hasExternalLixDoc = lib.versionOlder version "2.92"; isLegacyParser = lib.versionOlder version "2.91"; in -stdenv.mkDerivation { +# gcc miscompiles coroutines at least until 13.2, possibly longer +# do not remove this check unless you are sure you (or your users) will not report bugs to Lix upstream about GCC miscompilations. +assert lib.assertMsg (enableStrictLLVMChecks && isLLVMOnly -> stdenv.cc.isClang) + "Lix upstream strongly discourage the usage of GCC to compile Lix as there's known miscompilations in important places. If you are a compiler developer, please get in touch with us."; +stdenv.mkDerivation (finalAttrs: { pname = "lix"; version = "${version}${suffix}"; @@ -91,9 +115,24 @@ stdenv.mkDerivation { ]; strictDeps = true; + disallowedReferences = lib.optionals isLLVMOnly [ + # We don't want the Clang. + stdenv.cc.cc + # We don't want the underlying GCC neither! + stdenv.cc.cc.stdenv.cc.cc + ]; + + # We only include CMake so that Meson can locate toml11, which only ships CMake dependency metadata. + dontUseCmakeConfigure = true; nativeBuildInputs = [ + # python3.withPackages does not splice properly, see https://github.com/NixOS/nixpkgs/issues/305858 + (python3.pythonOnBuildForHost.withPackages (p: [ + p.pytest + p.pytest-xdist + p.python-frontmatter + ])) pkg-config flex jq @@ -101,6 +140,8 @@ stdenv.mkDerivation { ninja cmake python3 + # Required for libstd++ assertions that leaks inside of the final binary. + removeReferencesTo # Tests git @@ -108,6 +149,11 @@ stdenv.mkDerivation { jq lsof ] + ++ lib.optionals isLLVMOnly [ + rustc + cargo + rustPlatform.cargoSetupHook + ] ++ lib.optionals isLegacyParser [ bison ] ++ lib.optionals enableDocumentation [ (lib.getBin lowdown-unsandboxed) @@ -123,6 +169,7 @@ stdenv.mkDerivation { brotli bzip2 curl + capnproto editline libsodium openssl @@ -133,14 +180,31 @@ stdenv.mkDerivation { lowdown rapidcheck toml11 - lix-doc ] + ++ lib.optionals hasExternalLixDoc [ lix-doc ] ++ lib.optionals (!isLegacyParser) [ pegtl ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ Security ] + # NOTE(Raito): I'd have expected that the LLVM packaging would inject the + # libunwind library path directly in the wrappers, but it does inject + # -lunwind without injecting the library path... + ++ lib.optionals stdenv.hostPlatform.isStatic [ llvmPackages.libunwind ] ++ lib.optionals (stdenv.hostPlatform.isx86_64) [ libcpuid ] ++ lib.optionals withLibseccomp [ libseccomp ] ++ lib.optionals withAWS [ aws-sdk-cpp ]; + inherit cargoDeps; + + env = { + # Meson allows referencing a /usr/share/cargo/registry shaped thing for subproject sources. + # Turns out the Nix-generated Cargo dependencies are named the same as they + # would be in a Cargo registry cache. + MESON_PACKAGE_CACHE_DIR = + if finalAttrs.cargoDeps != null then + finalAttrs.cargoDeps + else + "lix: no `MESON_PACKAGE_CACHE_DIR`, set `cargoDeps`"; + }; + propagatedBuildInputs = [ boehmgc nlohmann_json @@ -179,8 +243,9 @@ stdenv.mkDerivation { mesonFlags = [ # Enable LTO, since it improves eval performance a fair amount - # LTO is disabled on static due to strange linking errors - (lib.mesonBool "b_lto" (!stdenv.hostPlatform.isStatic && stdenv.cc.isGNU)) + # LTO is disabled on: + # - static builds (strange linkage errors) + (lib.mesonBool "b_lto" (!stdenv.hostPlatform.isStatic && (isLLVMOnly || stdenv.cc.isGNU))) (lib.mesonEnable "gc" true) (lib.mesonBool "enable-tests" true) (lib.mesonBool "enable-docs" enableDocumentation) @@ -208,6 +273,11 @@ stdenv.mkDerivation { mkdir -p $devdoc/nix-support echo "devdoc internal-api $devdoc/share/doc/nix/internal-api" >> $devdoc/nix-support/hydra-build-products '' + + lib.optionalString (!hasExternalLixDoc) '' + # We do not need static archives. + # FIXME(Raito): why are they getting installed _at all_ ? + rm $out/lib/liblix_doc.a + '' + lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products @@ -219,6 +289,10 @@ stdenv.mkDerivation { "$out/lib/libboost_context.dylib" \ "$out/lib/$lib" done + '' + + '' + # Drop all references to libstd++ include files due to `__FILE__` leaking in libstd++ assertions. + find "$out" -type f -exec remove-references-to -t ${stdenv.cc.cc.stdenv.cc.cc} '{}' + ''; # This needs to run after _multioutDocs moves the docs to $doc @@ -299,4 +373,4 @@ stdenv.mkDerivation { outputsToInstall = [ "out" ] ++ lib.optional enableDocumentation "man"; mainProgram = "nix"; }; -} +}) diff --git a/pkgs/tools/package-management/lix/common-nix-eval-jobs.nix b/pkgs/tools/package-management/lix/common-nix-eval-jobs.nix index f80353d1754f..d3be7a626d97 100644 --- a/pkgs/tools/package-management/lix/common-nix-eval-jobs.nix +++ b/pkgs/tools/package-management/lix/common-nix-eval-jobs.nix @@ -17,7 +17,7 @@ pkg-config, ninja, cmake, - clang-tools, + buildPackages, }: stdenv.mkDerivation { @@ -35,7 +35,7 @@ stdenv.mkDerivation { ninja # nlohmann_json can be only discovered via cmake files cmake - ] ++ (lib.optional stdenv.cc.isClang [ clang-tools ]); + ] ++ (lib.optional stdenv.cc.isClang [ buildPackages.clang-tools ]); # point 'nix edit' and ofborg at the file that defines the attribute, # not this common file. @@ -52,5 +52,6 @@ stdenv.mkDerivation { license = lib.licenses.gpl3; inherit maintainers; platforms = lib.platforms.unix; + broken = stdenv.hostPlatform.isStatic; }; } diff --git a/pkgs/tools/package-management/lix/default.nix b/pkgs/tools/package-management/lix/default.nix index 5c351b2542a2..30025e74b3bb 100644 --- a/pkgs/tools/package-management/lix/default.nix +++ b/pkgs/tools/package-management/lix/default.nix @@ -8,6 +8,10 @@ rustPlatform, Security, newScope, + editline, + ncurses, + stdenv, + clangStdenv, storeDir ? "/nix/store", stateDir ? "/nix/var", @@ -19,6 +23,10 @@ let lix-args, nix-eval-jobs-args, }: + let + # GCC 13.2 is known to miscompile Lix coroutines (introduced in 2.92). + lixStdenv = if lib.versionAtLeast lix-args.version "2.92" then clangStdenv else stdenv; + in lib.makeScope newScope ( self: lib.recurseIntoAttrs { @@ -58,6 +66,11 @@ let requiredSystemFeatures = [ ]; }; + editline = editline.override { + inherit ncurses; + enableTermcap = true; + }; + # NOTE: The `common-*.nix` helpers contain a top-level function which # takes the Lix source to build and version information. We use the # outer `callPackage` for that. @@ -65,12 +78,15 @@ let # That *returns* another function which takes the actual build # dependencies, and that uses the new scope's `self.callPackage` so # that `nix-eval-jobs` can be built against the correct `lix` version. - lix = self.callPackage (callPackage ./common-lix.nix lix-args) { }; + lix = self.callPackage (callPackage ./common-lix.nix lix-args) { + stdenv = lixStdenv; + }; - nix-eval-jobs = self.callPackage (callPackage ./common-nix-eval-jobs.nix nix-eval-jobs-args) { }; + nix-eval-jobs = self.callPackage (callPackage ./common-nix-eval-jobs.nix nix-eval-jobs-args) { + stdenv = lixStdenv; + }; } ); - in lib.makeExtensible (self: { inherit makeLixScope; @@ -137,7 +153,39 @@ lib.makeExtensible (self: { }; }; - latest = self.lix_2_91; + lix_2_92 = self.makeLixScope { + lix-args = rec { + version = "2.92.0"; + + src = fetchFromGitHub { + owner = "lix-project"; + repo = "lix"; + rev = version; + hash = "sha256-CCKIAE84dzkrnlxJCKFyffAxP3yfsOAbdvydUGqq24g="; + }; + + cargoDeps = rustPlatform.fetchCargoVendor { + name = "lix-${version}"; + inherit src; + allowGitDependencies = false; + hash = "sha256-YMyNOXdlx0I30SkcmdW/6DU0BYc3ZOa2FMJSKMkr7I8="; + }; + }; + + nix-eval-jobs-args = rec { + version = "2.92.0"; + src = fetchgit { + url = "https://git.lix.systems/lix-project/nix-eval-jobs.git"; + rev = version; + hash = "sha256-tPr61X9v/OMVt7VXOs1RRStciwN8gDGxEKx+h0/Fg48="; + }; + }; + }; + + latest = self.lix_2_92; + + # Note: This is not yet 2.92 because of a non-deterministic `curl` error. + # See: https://git.lix.systems/lix-project/lix/issues/662 stable = self.lix_2_91; # Previously, `nix-eval-jobs` was not packaged here, so we export an From 0d079338b2f5e681304215e901eebf9e9cb1dd58 Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Sun, 16 Feb 2025 17:52:58 +0100 Subject: [PATCH 2/2] nixos/tests/installer: introduce `lix-simple` variant An installer tests exacerbates the distribution packaging but in the case of NixOS: the Nix package manager implementation. As part of our classical release management process, the Lix team tests whether a NixOS system installs just fine with Lix or not. To avoid bloating the CI needlessly and keeping it simple, we only introduce it on the simple variant and give a general way to pipe a Nix implementation inside of an installer test. Change-Id: I781da14475867dc2d946b740bad10af5de79ec5a Signed-off-by: Raito Bezarius --- nixos/tests/installer.nix | 5 +++++ pkgs/tools/package-management/lix/common-lix.nix | 1 + 2 files changed, 6 insertions(+) diff --git a/nixos/tests/installer.nix b/nixos/tests/installer.nix index 6be3346d9850..db9fccaa2bb4 100644 --- a/nixos/tests/installer.nix +++ b/nixos/tests/installer.nix @@ -638,6 +638,7 @@ let clevisTest ? false, clevisFallbackTest ? false, disableFileSystems ? false, + selectNixPackage ? pkgs: pkgs.nixStable, }: let isEfi = bootLoader == "systemd-boot" || (bootLoader == "grub" && grubUseEfi); @@ -701,6 +702,7 @@ let virtualisation.rootDevice = "/dev/vdb"; hardware.enableAllFirmware = mkForce false; + nix.package = selectNixPackage pkgs; # The test cannot access the network, so any packages we # need must be included in the VM. @@ -1101,6 +1103,9 @@ in # The (almost) simplest partitioning scheme: a swap partition and # one big filesystem partition. simple = makeInstallerTest "simple" simple-test-config; + lix-simple = makeInstallerTest "simple" simple-test-config // { + selectNixPackage = pkgs: pkgs.lix; + }; switchToFlake = makeInstallerTest "switch-to-flake" simple-test-config-flake; diff --git a/pkgs/tools/package-management/lix/common-lix.nix b/pkgs/tools/package-management/lix/common-lix.nix index b9077e7672d4..6f819fb734ef 100644 --- a/pkgs/tools/package-management/lix/common-lix.nix +++ b/pkgs/tools/package-management/lix/common-lix.nix @@ -351,6 +351,7 @@ stdenv.mkDerivation (finalAttrs: { inherit aws-sdk-cpp boehmgc; tests = { misc = nixosTests.nix-misc.lix; + installer = nixosTests.installer.lix-simple; }; };