From bbbd6b711cda065023eb7caa0bf4462e8fdb2cdf Mon Sep 17 00:00:00 2001 From: Talha Can Havadar Date: Tue, 5 Nov 2024 15:45:16 +0100 Subject: [PATCH 01/78] maintainers: add talhaHavadar --- maintainers/maintainer-list.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix index f53e59971f2a..8bf842d77677 100644 --- a/maintainers/maintainer-list.nix +++ b/maintainers/maintainer-list.nix @@ -21325,6 +21325,17 @@ githubId = 6064962; name = "TakWolf"; }; + talhaHavadar = { + email = "havadartalha@gmail.com"; + github = "talhaHavadar"; + githubId = 6908462; + name = "Talha Can Havadar"; + keys = [ + { + fingerprint = "1E13 12DF 4B71 58B6 EBF9 DE78 2574 3879 62FE B0D1"; + } + ]; + }; talkara = { email = "taito.horiuchi@relexsolutions.com"; github = "talkara"; From ff0d3ad153794f6244360b15da7b6ad0a3a9da3c Mon Sep 17 00:00:00 2001 From: Flameopathic Date: Mon, 11 Nov 2024 09:44:15 -0500 Subject: [PATCH 02/78] wl-clicker: init at v0.3.1 --- pkgs/by-name/wl/wl-clicker/package.nix | 50 ++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 pkgs/by-name/wl/wl-clicker/package.nix diff --git a/pkgs/by-name/wl/wl-clicker/package.nix b/pkgs/by-name/wl/wl-clicker/package.nix new file mode 100644 index 000000000000..aa54cf4d6db7 --- /dev/null +++ b/pkgs/by-name/wl/wl-clicker/package.nix @@ -0,0 +1,50 @@ +{ + lib, + fetchFromGitHub, + stdenv, + wayland-scanner, + wlr-protocols, + wayland, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "wl-clicker"; + version = "0.3.1"; + + nativeBuildInputs = [ wayland-scanner ]; + buildInputs = [ + wlr-protocols + wayland + ]; + + src = fetchFromGitHub { + owner = "phonetic112"; + repo = "wl-clicker"; + rev = "v${finalAttrs.version}"; + hash = "sha256-HuUK1kkqIdB5SdcewJ1J8elzqrFaFWC3BRV3GgcasTU="; + }; + + postPatch = '' + sed -i 's|/usr|${wlr-protocols}|g' Makefile + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/bin + install build/wl-clicker $out/bin/wl-clicker + + runHook postInstall + ''; + + meta = { + description = "Wayland autoclicker"; + longDescription = "Script for auto clicking at incredibly high speeds - user must be a part of `input` group to run."; + homepage = "https://github.com/phonetic112/wl-clicker"; + license = lib.licenses.mit; + maintainers = [ lib.maintainers.Flameopathic ]; + mainProgram = "wl-clicker"; + platforms = lib.platforms.linux; + broken = stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64; + }; +}) From f2a0dc5399da4388068d615244ca9ac0e07c5516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Romildo?= Date: Tue, 26 Nov 2024 07:44:37 -0300 Subject: [PATCH 03/78] lxqt.lxqt-wayland-session: 0.1.0 -> 0.1.1 Diff: https://github.com/lxqt/lxqt-wayland-session/compare/0.1.0...0.1.1 --- .../lxqt/lxqt-wayland-session/default.nix | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pkgs/desktops/lxqt/lxqt-wayland-session/default.nix b/pkgs/desktops/lxqt/lxqt-wayland-session/default.nix index 21f4ef2548d9..430cde7762c5 100644 --- a/pkgs/desktops/lxqt/lxqt-wayland-session/default.nix +++ b/pkgs/desktops/lxqt/lxqt-wayland-session/default.nix @@ -4,12 +4,15 @@ fetchFromGitHub, cmake, kwindowsystem, + layer-shell-qt, liblxqt, libqtxdg, lxqt-build-tools, + lxqt-session, pkg-config, qtsvg, qttools, + qtxdg-tools, xdg-user-dirs, xkeyboard_config, gitUpdater, @@ -17,13 +20,13 @@ stdenv.mkDerivation rec { pname = "lxqt-wayland-session"; - version = "0.1.0"; + version = "0.1.1"; src = fetchFromGitHub { owner = "lxqt"; repo = "lxqt-wayland-session"; rev = version; - hash = "sha256-5WdfwJ89HWlXL6y9Lpgs7H3mbN/wbf+9VbP9ERPasBM="; + hash = "sha256-UMlV8LqUXM2+3ZSLj30FFgC+ZVPmt2W8uE2RrZKqCJE="; }; nativeBuildInputs = [ @@ -35,15 +38,19 @@ stdenv.mkDerivation rec { buildInputs = [ kwindowsystem + layer-shell-qt # for applications that need layer-shell-qt (ex: lxqt-panel) liblxqt libqtxdg + lxqt-session qtsvg - xdg-user-dirs + qtxdg-tools # allow to use xdg-utils under LXQt, similar to https://github.com/lxqt/lxqt-session/blob/2.0.0/CHANGELOG#L27 + xdg-user-dirs # startlxqtwayland sets XDG_CURRENT_DESKTOP ]; postPatch = '' substituteInPlace startlxqtwayland.in \ - --replace-fail /usr/share/X11/xkb/rules ${xkeyboard_config}/share/X11/xkb/rules + --replace-fail /usr/share/X11/xkb/rules ${xkeyboard_config}/share/X11/xkb/rules \ + --replace-fail "cp -av " "cp -av --no-preserve=mode " substituteInPlace configurations/{labwc/autostart,lxqt-hyprland.conf,lxqt-wayfire.ini} \ --replace-fail /usr/share/lxqt/wallpapers $out/share/lxqt/wallpapers From b65739e061fbf58aed6ef6145685b154c65ddc97 Mon Sep 17 00:00:00 2001 From: Samuel Tardieu Date: Thu, 28 Nov 2024 14:39:53 +0100 Subject: [PATCH 04/78] =?UTF-8?q?zotero:=207.0.8=20=E2=86=92=207.0.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diff: https://github.com/zotero/zotero/compare/7.0.8...7.0.10 --- pkgs/by-name/zo/zotero/darwin.nix | 2 +- pkgs/by-name/zo/zotero/linux.nix | 2 +- pkgs/by-name/zo/zotero/package.nix | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/zo/zotero/darwin.nix b/pkgs/by-name/zo/zotero/darwin.nix index ad60ee9d409e..12697c31a085 100644 --- a/pkgs/by-name/zo/zotero/darwin.nix +++ b/pkgs/by-name/zo/zotero/darwin.nix @@ -12,7 +12,7 @@ stdenv.mkDerivation rec { src = fetchurl { url = "https://download.zotero.org/client/release/${version}/Zotero-${version}.dmg"; - hash = "sha256-z1HAkHz4JGRfQJna2cINZTmYPzyj9ofp7mJHN/mNyOo="; + hash = "sha256-JP9zRPsEDICw6Q7VHRd7IB0J9+v0qU7QuLf7eTPafBo="; }; sourceRoot = "."; diff --git a/pkgs/by-name/zo/zotero/linux.nix b/pkgs/by-name/zo/zotero/linux.nix index 1b7d58f86883..5a3d3570fc3f 100644 --- a/pkgs/by-name/zo/zotero/linux.nix +++ b/pkgs/by-name/zo/zotero/linux.nix @@ -27,7 +27,7 @@ stdenv.mkDerivation rec { src = fetchurl { url = "https://download.zotero.org/client/release/${version}/Zotero-${version}_linux-x86_64.tar.bz2"; - hash = "sha256-utiqS4/PvlkQesqTOyEvj8uwW1sDxlqjnhv6GFfY5uk="; + hash = "sha256-t0LApaU13tT/14nvRpnWZwFyWiJq+WfZNgVyhNayMcs="; }; dontPatchELF = true; diff --git a/pkgs/by-name/zo/zotero/package.nix b/pkgs/by-name/zo/zotero/package.nix index f6117050c97f..a56aef7d3ae7 100644 --- a/pkgs/by-name/zo/zotero/package.nix +++ b/pkgs/by-name/zo/zotero/package.nix @@ -6,7 +6,7 @@ let pname = "zotero"; - version = "7.0.8"; + version = "7.0.10"; meta = { homepage = "https://www.zotero.org"; description = "Collect, organize, cite, and share your research sources"; From a2e747664b7005a36a367c8acde256a3a4dfee65 Mon Sep 17 00:00:00 2001 From: Yaroslav Chvanov Date: Fri, 29 Nov 2024 00:35:00 +0300 Subject: [PATCH 05/78] strawberry: format with nixfmt --- .../applications/audio/strawberry/default.nix | 172 +++++++++--------- 1 file changed, 91 insertions(+), 81 deletions(-) diff --git a/pkgs/applications/audio/strawberry/default.nix b/pkgs/applications/audio/strawberry/default.nix index 58ac8f60ff1c..3e30c90b755c 100644 --- a/pkgs/applications/audio/strawberry/default.nix +++ b/pkgs/applications/audio/strawberry/default.nix @@ -1,41 +1,42 @@ -{ stdenv -, lib -, fetchFromGitHub -, cmake -, pkg-config -, wrapQtAppsHook -, alsa-lib -, boost -, chromaprint -, fftw -, gnutls -, libcdio -, libebur128 -, libmtp -, libpthreadstubs -, libtasn1 -, libXdmcp -, ninja -, pcre -, protobuf -, sqlite -, taglib -, libgpod -, libidn2 -, libpulseaudio -, libselinux -, libsepol -, p11-kit -, util-linux -, qtbase -, qtx11extras ? null # doesn't exist in qt6 -, qttools -, withGstreamer ? true -, glib-networking -, gst_all_1 -, withVlc ? true -, libvlc -, nix-update-script +{ + alsa-lib, + boost, + chromaprint, + cmake, + fetchFromGitHub, + fftw, + glib-networking, + gnutls, + gst_all_1, + lib, + libXdmcp, + libcdio, + libebur128, + libgpod, + libidn2, + libmtp, + libpthreadstubs, + libpulseaudio, + libselinux, + libsepol, + libtasn1, + libvlc, + ninja, + nix-update-script, + p11-kit, + pcre, + pkg-config, + protobuf, + qtbase, + qttools, + qtx11extras ? null, # doesn't exist in qt6 + sqlite, + stdenv, + taglib, + util-linux, + withGstreamer ? true, + withVlc ? true, + wrapQtAppsHook, }: let @@ -60,50 +61,59 @@ stdenv.mkDerivation rec { --replace pictures/strawberry.png pictures/strawberry-grey.png ''; - buildInputs = [ - alsa-lib - boost - chromaprint - fftw - gnutls - libcdio - libebur128 - libidn2 - libmtp - libpthreadstubs - libtasn1 - libXdmcp - pcre - protobuf - sqlite - taglib - qtbase - qtx11extras - ] ++ optionals stdenv.hostPlatform.isLinux [ - libgpod - libpulseaudio - libselinux - libsepol - p11-kit - ] ++ optionals withGstreamer (with gst_all_1; [ - glib-networking - gstreamer - gst-libav - gst-plugins-base - gst-plugins-good - gst-plugins-bad - gst-plugins-ugly - ]) ++ optionals withVlc [ libvlc ]; + buildInputs = + [ + alsa-lib + boost + chromaprint + fftw + gnutls + libXdmcp + libcdio + libebur128 + libidn2 + libmtp + libpthreadstubs + libtasn1 + pcre + protobuf + qtbase + qtx11extras + sqlite + taglib + ] + ++ optionals stdenv.hostPlatform.isLinux [ + libgpod + libpulseaudio + libselinux + libsepol + p11-kit + ] + ++ optionals withGstreamer ( + with gst_all_1; + [ + glib-networking + gst-libav + gst-plugins-bad + gst-plugins-base + gst-plugins-good + gst-plugins-ugly + gstreamer + ] + ) + ++ optionals withVlc [ libvlc ]; - nativeBuildInputs = [ - cmake - ninja - pkg-config - qttools - wrapQtAppsHook - ] ++ optionals stdenv.hostPlatform.isLinux [ - util-linux - ]; + nativeBuildInputs = + [ + cmake + ninja + pkg-config + qttools + wrapQtAppsHook + ] + ++ optionals stdenv.hostPlatform.isLinux [ + util-linux + ]; postInstall = optionalString withGstreamer '' qtWrapperArgs+=( From cc90b03c71a9988c0f87bc90479fa2dbfaa259d2 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Fri, 29 Nov 2024 17:54:22 +0100 Subject: [PATCH 06/78] python312Packages.jupyterlab-server: remove jupyterlab_server.pytest_plugin from import checks Allows to override the package with `doCheck = false` --- .../development/python-modules/jupyterlab-server/default.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/jupyterlab-server/default.nix b/pkgs/development/python-modules/jupyterlab-server/default.nix index 2461e5ea0de3..357b2b292595 100644 --- a/pkgs/development/python-modules/jupyterlab-server/default.nix +++ b/pkgs/development/python-modules/jupyterlab-server/default.nix @@ -79,16 +79,15 @@ buildPythonPackage rec { pythonImportsCheck = [ "jupyterlab_server" - "jupyterlab_server.pytest_plugin" ]; __darwinAllowLocalNetworking = true; - meta = with lib; { + meta = { description = "Set of server components for JupyterLab and JupyterLab like applications"; homepage = "https://github.com/jupyterlab/jupyterlab_server"; changelog = "https://github.com/jupyterlab/jupyterlab_server/blob/v${version}/CHANGELOG.md"; - license = licenses.bsd3; + license = lib.licenses.bsd3; maintainers = lib.teams.jupyter.members; }; } From 8f28ab10cfe39674cffe01a939d1f21b0c34349d Mon Sep 17 00:00:00 2001 From: Cosmic Horror Date: Fri, 29 Nov 2024 02:19:43 -0700 Subject: [PATCH 07/78] to-html: 0.1.4 -> 0.1.6 --- pkgs/by-name/to/to-html/package.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/to/to-html/package.nix b/pkgs/by-name/to/to-html/package.nix index 5ddba61276d4..493ceb60a054 100644 --- a/pkgs/by-name/to/to-html/package.nix +++ b/pkgs/by-name/to/to-html/package.nix @@ -4,17 +4,17 @@ }: rustPlatform.buildRustPackage rec { - version = "0.1.4"; + version = "0.1.6"; pname = "to-html"; src = fetchFromGitHub { owner = "Aloso"; repo = "to-html"; rev = "v${version}"; - hash = "sha256-zkTBjsMFhRz7lVRh8i+XkaJ/qWmTAMPnkH5aDhbHf8U="; + hash = "sha256-eNFt9/yK4oHOspNM8PMTewhx8APaHzmgNdrWqrUuQSU="; }; - cargoHash = "sha256-hXc+lB3DKnRZkp1U5wW/vPKSZ0c1UknQCAxDfE7Eubg="; + cargoHash = "sha256-UxDyk7m6eWvuWPHWUY2OJI59p43/bag8pcZyeGJr/QA="; # Requires external resources doCheck = false; From 88f32cf694a7f6fc16c2e5b80d615b8b317fa9c4 Mon Sep 17 00:00:00 2001 From: Cosmic Horror Date: Fri, 29 Nov 2024 03:10:07 -0700 Subject: [PATCH 08/78] to-html: Add shell completions --- pkgs/by-name/to/to-html/package.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkgs/by-name/to/to-html/package.nix b/pkgs/by-name/to/to-html/package.nix index 493ceb60a054..cfa80570f50c 100644 --- a/pkgs/by-name/to/to-html/package.nix +++ b/pkgs/by-name/to/to-html/package.nix @@ -1,5 +1,6 @@ { lib , fetchFromGitHub +, installShellFiles , rustPlatform }: @@ -19,6 +20,14 @@ rustPlatform.buildRustPackage rec { # Requires external resources doCheck = false; + nativeBuildInputs = [ installShellFiles ]; + + postInstall = '' + installShellCompletion \ + $releaseDir/build/to-html-*/out/to-html.{bash,fish} \ + --zsh $releaseDir/build/to-html-*/out/_to-html + ''; + meta = { description = "Terminal wrapper for rendering a terminal on a website by converting ANSI escape sequences to HTML"; mainProgram = "to-html"; From e3a9c19e86661ce0d5799b20c91a5156eaf34db8 Mon Sep 17 00:00:00 2001 From: Yaroslav Chvanov Date: Fri, 29 Nov 2024 00:36:03 +0300 Subject: [PATCH 09/78] strawberry: drop strawberry-qt5 in favor of strawberry-qt6 --- nixos/doc/manual/release-notes/rl-2505.section.md | 3 +++ pkgs/top-level/aliases.nix | 1 + pkgs/top-level/all-packages.nix | 4 +--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-2505.section.md b/nixos/doc/manual/release-notes/rl-2505.section.md index be4351684fdb..f51130a227bd 100644 --- a/nixos/doc/manual/release-notes/rl-2505.section.md +++ b/nixos/doc/manual/release-notes/rl-2505.section.md @@ -27,6 +27,9 @@ - `buildGoPackage` has been removed. Use `buildGoModule` instead. See the [Go section in the nixpkgs manual](https://nixos.org/manual/nixpkgs/unstable/#sec-language-go) for details. +- `strawberry` has been updated to 1.2, which drops support for the VLC backend and Qt 5. The `strawberry-qt5` package + and `withGstreamer`/`withVlc` override options have been removed due to this. + - `timescaledb` requires manual upgrade steps. After you run ALTER EXTENSION, you must run [this SQL script](https://github.com/timescale/timescaledb-extras/blob/master/utils/2.15.X-fix_hypertable_foreign_keys.sql). For more details, see the following pull requests [#6797](https://github.com/timescale/timescaledb/pull/6797). PostgreSQL 13 is no longer supported in TimescaleDB v2.16. diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index e4ff2d475eeb..1189328802e1 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -1193,6 +1193,7 @@ mapAliases { steam-small = steam; # Added 2024-09-12 steam-run-native = steam-run; # added 2022-02-21 StormLib = stormlib; # Added 2024-01-21 + strawberry-qt5 = throw "strawberry-qt5 has been replaced by strawberry-qt6"; # Added 2024-11-22 sumneko-lua-language-server = lua-language-server; # Added 2023-02-07 sumokoin = throw "sumokoin has been removed as it was abandoned upstream"; # Added 2024-11-23 swiProlog = lib.warn "swiProlog has been renamed to swi-prolog" swi-prolog; # Added 2024-09-07 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7e806f98b32b..ffa8c6f275b1 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -3228,11 +3228,9 @@ with pkgs; sonobuoy = callPackage ../applications/networking/cluster/sonobuoy { }; - strawberry-qt5 = libsForQt5.callPackage ../applications/audio/strawberry { }; - strawberry-qt6 = qt6Packages.callPackage ../applications/audio/strawberry { }; - strawberry = strawberry-qt5; + strawberry = strawberry-qt6; schleuder = callPackage ../tools/security/schleuder { }; From 10ce0b2de85389ae66966eb2f947b94c23feb683 Mon Sep 17 00:00:00 2001 From: Yaroslav Chvanov Date: Fri, 29 Nov 2024 00:45:39 +0300 Subject: [PATCH 10/78] strawberry: 1.1.3 -> 1.2.2 VLC backend and Qt 5 build are no longer supported. protobuf is no longer required. Use packaged kdsingleapplication instead of building it from submodule. --- .../applications/audio/strawberry/default.nix | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/pkgs/applications/audio/strawberry/default.nix b/pkgs/applications/audio/strawberry/default.nix index 3e30c90b755c..94072c6ae69a 100644 --- a/pkgs/applications/audio/strawberry/default.nix +++ b/pkgs/applications/audio/strawberry/default.nix @@ -8,6 +8,7 @@ glib-networking, gnutls, gst_all_1, + kdsingleapplication, lib, libXdmcp, libcdio, @@ -20,39 +21,33 @@ libselinux, libsepol, libtasn1, - libvlc, ninja, nix-update-script, p11-kit, pcre, pkg-config, - protobuf, qtbase, qttools, - qtx11extras ? null, # doesn't exist in qt6 sqlite, stdenv, taglib, util-linux, - withGstreamer ? true, - withVlc ? true, wrapQtAppsHook, }: let - inherit (lib) optionals optionalString; + inherit (lib) optionals; in stdenv.mkDerivation rec { pname = "strawberry"; - version = "1.1.3"; + version = "1.2.2"; src = fetchFromGitHub { owner = "jonaski"; repo = pname; rev = version; - hash = "sha256-yca1BJWhSUVamqSKfvEzU3xbzdR+kwfSs0pyS08oUR0="; - fetchSubmodules = true; + hash = "sha256-X752GsP2b7rumQHzw52zI7PeE8tdM9Scgl3nHVcpO/s="; }; # the big strawberry shown in the context menu is *very* much in your face, so use the grey version instead @@ -68,6 +63,7 @@ stdenv.mkDerivation rec { chromaprint fftw gnutls + kdsingleapplication libXdmcp libcdio libebur128 @@ -76,9 +72,7 @@ stdenv.mkDerivation rec { libpthreadstubs libtasn1 pcre - protobuf qtbase - qtx11extras sqlite taglib ] @@ -89,19 +83,15 @@ stdenv.mkDerivation rec { libsepol p11-kit ] - ++ optionals withGstreamer ( - with gst_all_1; - [ - glib-networking - gst-libav - gst-plugins-bad - gst-plugins-base - gst-plugins-good - gst-plugins-ugly - gstreamer - ] - ) - ++ optionals withVlc [ libvlc ]; + ++ (with gst_all_1; [ + glib-networking + gst-libav + gst-plugins-bad + gst-plugins-base + gst-plugins-good + gst-plugins-ugly + gstreamer + ]); nativeBuildInputs = [ @@ -115,7 +105,7 @@ stdenv.mkDerivation rec { util-linux ]; - postInstall = optionalString withGstreamer '' + postInstall = '' qtWrapperArgs+=( --prefix GST_PLUGIN_SYSTEM_PATH_1_0 : "$GST_PLUGIN_SYSTEM_PATH_1_0" --prefix GIO_EXTRA_MODULES : "${glib-networking.out}/lib/gio/modules" From 6f860d8ef6d6d67ea4d82fb4458b7d50ab037d78 Mon Sep 17 00:00:00 2001 From: Yaroslav Chvanov Date: Sat, 30 Nov 2024 10:51:50 +0300 Subject: [PATCH 11/78] strawberry: drop PCRE dependency While strawberry seems to bundle PCRE2 on MacOS and Windows, it doesn't seem to use it directly. --- pkgs/applications/audio/strawberry/default.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkgs/applications/audio/strawberry/default.nix b/pkgs/applications/audio/strawberry/default.nix index 94072c6ae69a..bd85eef99f50 100644 --- a/pkgs/applications/audio/strawberry/default.nix +++ b/pkgs/applications/audio/strawberry/default.nix @@ -24,7 +24,6 @@ ninja, nix-update-script, p11-kit, - pcre, pkg-config, qtbase, qttools, @@ -71,7 +70,6 @@ stdenv.mkDerivation rec { libmtp libpthreadstubs libtasn1 - pcre qtbase sqlite taglib From 44c70837465d73f661f0e69fefe9648b2b535255 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Sat, 30 Nov 2024 13:55:46 +0000 Subject: [PATCH 12/78] highs: 1.8.0 -> 1.8.1 --- pkgs/by-name/hi/highs/package.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/hi/highs/package.nix b/pkgs/by-name/hi/highs/package.nix index 836e947706e5..73460ab54164 100644 --- a/pkgs/by-name/hi/highs/package.nix +++ b/pkgs/by-name/hi/highs/package.nix @@ -7,13 +7,13 @@ stdenv.mkDerivation (finalAttrs: { pname = "highs"; - version = "1.8.0"; + version = "1.8.1"; src = fetchFromGitHub { owner = "ERGO-Code"; repo = "HiGHS"; rev = "v${finalAttrs.version}"; - hash = "sha256-3ACo6V6Y99XPe4dvurWflNGPjHy3irURv4jYHMrgSms="; + hash = "sha256-JuUPnsCNImWpkeh0B//7yM1Wix9SpnfApp8cxX9WeYk="; }; strictDeps = true; From d325edd6272a18fa5e7f2e5b5c611badeb58e4a2 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Wed, 27 Nov 2024 19:01:57 +0000 Subject: [PATCH 13/78] nixos-rebuild-ng: introduce models.BuildingAttr --- .../src/nixos_rebuild/__init__.py | 22 ++++++++----------- .../src/nixos_rebuild/models.py | 12 ++++++++++ .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 15 +++++-------- .../nixos-rebuild-ng/src/tests/test_models.py | 13 +++++++++++ .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 16 +++++--------- 5 files changed, 45 insertions(+), 33 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 25519dac6240..a83718c89587 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory from typing import assert_never from . import nix -from .models import Action, Flake, NRError, Profile +from .models import Action, BuildAttr, Flake, NRError, Profile from .process import Remote, cleanup_ssh from .utils import info @@ -170,6 +170,7 @@ def execute(argv: list[str]) -> None: profile = Profile.from_name(args.profile_name) target_host = Remote.from_arg(args.target_host, args.ask_sudo_password, tmpdir_path) + build_attr = BuildAttr.from_arg(args.attr, args.file) flake = Flake.from_arg(args.flake, target_host) if args.upgrade or args.upgrade_all: @@ -204,8 +205,7 @@ def execute(argv: list[str]) -> None: else: path_to_config = nix.nixos_build( "system", - args.attr, - args.file, + build_attr, no_out_link=True, **build_flags, ) @@ -244,8 +244,7 @@ def execute(argv: list[str]) -> None: else: path_to_config = nix.nixos_build( "system", - args.attr, - args.file, + build_attr, dry_run=dry_run, **build_flags, ) @@ -269,8 +268,7 @@ def execute(argv: list[str]) -> None: else: path_to_config = nix.nixos_build( attr, - args.attr, - args.file, + build_attr, **build_flags, ) vm_path = next(path_to_config.glob("bin/run-*-vm"), "./result/bin/run-*-vm") @@ -307,12 +305,10 @@ def execute(argv: list[str]) -> None: ) print(table) case Action.REPL: - # For now just redirect it to `nixos-rebuild` instead of - # duplicating the code - os.execv( - "@nixos_rebuild@", - argv, - ) + if flake: + nix.repl_flake("toplevel", flake, **flake_build_flags) + else: + nix.repl("system", build_attr, **build_flags) case _: assert_never(action) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py index 51cee3b7c517..59d99230061c 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py @@ -45,6 +45,18 @@ class Action(Enum): return [a.value for a in Action] +@dataclass(frozen=True) +class BuildAttr: + path: Path + attr: str | None + + @classmethod + def from_arg(cls, attr: str | None, file: str | None) -> Self | None: + if not (attr or file): + return None + return cls(Path(file or "default.nix"), attr) + + @dataclass(frozen=True) class Flake: path: Path diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index f0c328cff4c0..4009996609c7 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -6,6 +6,7 @@ from typing import Final from .models import ( Action, + BuildAttr, Flake, Generation, GenerationJson, @@ -235,24 +236,20 @@ def list_generations(profile: Profile) -> list[GenerationJson]: def nixos_build( attr: str, - pre_attr: str | None, - file: str | None, + build_attr: BuildAttr | None, **nix_flags: Args, ) -> Path: """Build NixOS attribute using classic Nix. - It will by default build `` with `attr`, however it - optionally supports building from an external file and custom attributes - paths. - Returns the built attribute as path. """ - if pre_attr or file: + run_args: list[str | Path] + if build_attr: run_args = [ "nix-build", - file or "default.nix", + build_attr.path, "--attr", - f"{'.'.join(x for x in [pre_attr, attr] if x)}", + f"{'.'.join(x for x in [build_attr.attr, attr] if x)}", ] else: run_args = ["nix-build", "", "--attr", attr] diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py index f30ef4059310..a2646eaec416 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py @@ -9,6 +9,19 @@ import nixos_rebuild.models as m from .helpers import get_qualified_name +def test_building_attr_from_arg() -> None: + assert m.BuildAttr.from_arg(None, None) is None + assert m.BuildAttr.from_arg("attr", None) == m.BuildAttr( + Path("default.nix"), "attr" + ) + assert m.BuildAttr.from_arg("attr", "file.nix") == m.BuildAttr( + Path("file.nix"), "attr" + ) + assert m.BuildAttr.from_arg(None, "file.nix") == m.BuildAttr( + Path("file.nix"), None + ) + + def test_flake_parse() -> None: assert m.Flake.parse("/path/to/flake#attr") == m.Flake( Path("/path/to/flake"), "nixosConfigurations.attr" diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index 8234f0ff98f9..0d6079e8eb5b 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -229,27 +229,21 @@ def test_nixos_build_flake(mock_run: Any) -> None: return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), ) def test_nixos_build(mock_run: Any, monkeypatch: Any) -> None: - assert n.nixos_build("attr", None, None, nix_flag="foo") == Path("/path/to/file") + assert n.nixos_build("attr", None, nix_flag="foo") == Path("/path/to/file") mock_run.assert_called_with( ["nix-build", "", "--attr", "attr", "--nix-flag", "foo"], stdout=PIPE, ) - n.nixos_build("attr", "preAttr", "file") + n.nixos_build("attr", m.BuildAttr(Path("file"), "preAttr")) mock_run.assert_called_with( - ["nix-build", "file", "--attr", "preAttr.attr"], + ["nix-build", Path("file"), "--attr", "preAttr.attr"], stdout=PIPE, ) - n.nixos_build("attr", None, "file", no_out_link=True) + n.nixos_build("attr", m.BuildAttr(Path("file"), None)) mock_run.assert_called_with( - ["nix-build", "file", "--attr", "attr", "--no-out-link"], - stdout=PIPE, - ) - - n.nixos_build("attr", "preAttr", None, no_out_link=False, keep_going=True) - mock_run.assert_called_with( - ["nix-build", "default.nix", "--attr", "preAttr.attr", "--keep-going"], + ["nix-build", Path("file"), "--attr", "attr"], stdout=PIPE, ) From 88b4eb3aeb0f24a1055dd83bd977b19175cd9d88 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Wed, 27 Nov 2024 19:07:16 +0000 Subject: [PATCH 14/78] nixos-rebuild-ng: add repl --- pkgs/by-name/ni/nixos-rebuild-ng/README.md | 2 +- pkgs/by-name/ni/nixos-rebuild-ng/package.nix | 5 --- .../src/nixos_rebuild/__init__.py | 1 - .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 38 ++++++++++++++++++ .../src/nixos_rebuild/repl.template.nix | 40 +++++++++++++++++++ .../ni/nixos-rebuild-ng/src/pyproject.toml | 3 ++ .../nixos-rebuild-ng/src/tests/test_models.py | 4 +- .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 19 +++++++++ 8 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/repl.template.nix diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/README.md b/pkgs/by-name/ni/nixos-rebuild-ng/README.md index a6199e873060..f34c3d4996ef 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/README.md +++ b/pkgs/by-name/ni/nixos-rebuild-ng/README.md @@ -126,7 +126,7 @@ ruff format . - [ ] Change module system to allow easier opt-in, like `system.switch.enableNg` for `switch-to-configuration-ng` - [ ] Improve documentation -- [ ] `nixos-rebuild repl` (calling old `nixos-rebuild` for now) +- [x] `nixos-rebuild repl` - [ ] `nix` build/bootstrap - [ ] Generate tab completion via [`shtab`](https://docs.iterative.ai/shtab/) - [x] Reduce build closure diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/package.nix b/pkgs/by-name/ni/nixos-rebuild-ng/package.nix index b111c7cb108a..4ef9d29807f5 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/package.nix +++ b/pkgs/by-name/ni/nixos-rebuild-ng/package.nix @@ -39,11 +39,6 @@ python3Packages.buildPythonApplication rec { nix ]; - preBuild = '' - substituteInPlace nixos_rebuild/__init__.py \ - --subst-var-by nixos_rebuild ${lib.getExe nixos-rebuild} - ''; - postInstall = '' installManPage ${nixos-rebuild}/share/man/man8/nixos-rebuild.8 diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index a83718c89587..f57991800404 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -1,7 +1,6 @@ import argparse import atexit import json -import os import sys from pathlib import Path from subprocess import run diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index 4009996609c7..aeaea3bf03c9 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -1,6 +1,8 @@ import os from datetime import datetime +from importlib.resources import files from pathlib import Path +from string import Template from subprocess import PIPE, CalledProcessError from typing import Final @@ -18,6 +20,7 @@ from .process import run_wrapper from .utils import Args, dict_to_flags, info FLAKE_FLAGS: Final = ["--extra-experimental-features", "nix-command flakes"] +FLAKE_REPL_TEMPLATE: Final = "repl.template.nix" def copy_closure( @@ -279,6 +282,41 @@ def nixos_build_flake( return Path(r.stdout.strip()) +def repl(attr: str, build_attr: BuildAttr | None, **nix_flags: Args) -> None: + run_args: list[str | Path] = ["nix", "repl", "--file"] + if build_attr: + run_args.append(build_attr.path) + if build_attr.attr: + run_args.append(build_attr.attr) + run_wrapper([*run_args, *dict_to_flags(nix_flags)]) + else: + run_wrapper([*run_args, "", *dict_to_flags(nix_flags)]) + + +def repl_flake(attr: str, flake: Flake, **flake_flags: Args) -> None: + expr = Template( + files(__package__).joinpath(FLAKE_REPL_TEMPLATE).read_text() + ).substitute( + flake_path=flake.path, + flake_attr=flake.attr, + bold="\033[1m", + blue="\033[34;1m", + attention="\033[35;1m", + reset="\033[0m", + ) + run_wrapper( + [ + "nix", + *FLAKE_FLAGS, + "repl", + "--impure", + "--expr", + expr, + *dict_to_flags(flake_flags), + ] + ) + + def rollback(profile: Profile, target_host: Remote | None, sudo: bool) -> Path: "Rollback Nix profile, like one created by `nixos-rebuild switch`." run_wrapper( diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/repl.template.nix b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/repl.template.nix new file mode 100644 index 000000000000..0fffd6ecc746 --- /dev/null +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/repl.template.nix @@ -0,0 +1,40 @@ +let + flake = builtins.getFlake ''${flake_path}''; + configuration = flake.${flake_attr}; + motd = '' + $${"\n"} + Hello and welcome to the NixOS configuration + ${flake_attr} + in ${flake_path} + + The following is loaded into nix repl's scope: + + - ${blue}config${reset} All option values + - ${blue}options${reset} Option data and metadata + - ${blue}pkgs${reset} Nixpkgs package set + - ${blue}lib${reset} Nixpkgs library functions + - other module arguments + + - ${blue}flake${reset} Flake outputs, inputs and source info of ${flake_path} + + Use tab completion to browse around ${blue}config${reset}. + + Use ${bold}:r${reset} to ${bold}reload${reset} everything after making a change in the flake. + (assuming ${flake_path} is a mutable flake ref) + + See ${bold}:?${reset} for more repl commands. + + ${attention}warning:${reset} nixos-rebuild repl does not currently enforce pure evaluation. + ''; + scope = + assert configuration._type or null == ''configuration''; + assert configuration.class or ''nixos'' == ''nixos''; + configuration._module.args + // configuration._module.specialArgs + // { + inherit (configuration) config options; + lib = configuration.lib or configuration.pkgs.lib; + inherit flake; + }; +in +builtins.seq scope builtins.trace motd scope diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml b/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml index f26b3cba7056..0dc662814c33 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml @@ -9,6 +9,9 @@ version = "0.0.0" [project.scripts] nixos-rebuild = "nixos_rebuild:main" +[tool.setuptools.package-data] +nixos_rebuild = ["*.template.nix"] + [tool.mypy] # `--strict` config, but explicit options to avoid breaking build when mypy is # updated diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py index a2646eaec416..1a604ad53164 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py @@ -17,9 +17,7 @@ def test_building_attr_from_arg() -> None: assert m.BuildAttr.from_arg("attr", "file.nix") == m.BuildAttr( Path("file.nix"), "attr" ) - assert m.BuildAttr.from_arg(None, "file.nix") == m.BuildAttr( - Path("file.nix"), None - ) + assert m.BuildAttr.from_arg(None, "file.nix") == m.BuildAttr(Path("file.nix"), None) def test_flake_parse() -> None: diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index 0d6079e8eb5b..92411a0f5488 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -248,6 +248,25 @@ def test_nixos_build(mock_run: Any, monkeypatch: Any) -> None: ) +@patch(get_qualified_name(n.run_wrapper, n), autospec=True) +def test_repl(mock_run: Any) -> None: + n.repl("attr", None, nix_flag=True) + mock_run.assert_called_with( + ["nix", "repl", "--file", "", "--nix-flag"] + ) + + n.repl("attr", m.BuildAttr(Path("file.nix"), "myAttr")) + mock_run.assert_called_with(["nix", "repl", "--file", Path("file.nix"), "myAttr"]) + + +@patch(get_qualified_name(n.run_wrapper, n), autospec=True) +def test_repl_flake(mock_run: Any) -> None: + n.repl_flake("attr", m.Flake(Path("flake.nix"), "myAttr"), nix_flag=True) + # This method would be really annoying to test, and it is not that important + # So just check that we are at least calling it + assert mock_run.called + + @patch(get_qualified_name(n.run_wrapper, n), autospec=True) def test_rollback(mock_run: Any, tmp_path: Path) -> None: path = tmp_path / "test" From 0774b36546976a2a0454f51bec02d1b695c5f9dd Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Wed, 27 Nov 2024 23:14:22 +0000 Subject: [PATCH 15/78] nixos-rebuild-ng: move default path logic to BuildAttr --- .../src/nixos_rebuild/models.py | 6 ++-- .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 34 +++++++------------ .../nixos-rebuild-ng/src/tests/test_models.py | 4 +-- .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 12 +++---- 4 files changed, 22 insertions(+), 34 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py index 59d99230061c..3039a3eafc7f 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py @@ -47,13 +47,13 @@ class Action(Enum): @dataclass(frozen=True) class BuildAttr: - path: Path + path: str | Path attr: str | None @classmethod - def from_arg(cls, attr: str | None, file: str | None) -> Self | None: + def from_arg(cls, attr: str | None, file: str | None) -> Self: if not (attr or file): - return None + return cls("", None) return cls(Path(file or "default.nix"), attr) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index aeaea3bf03c9..f56044451f02 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -239,24 +239,20 @@ def list_generations(profile: Profile) -> list[GenerationJson]: def nixos_build( attr: str, - build_attr: BuildAttr | None, + build_attr: BuildAttr, **nix_flags: Args, ) -> Path: """Build NixOS attribute using classic Nix. Returns the built attribute as path. """ - run_args: list[str | Path] - if build_attr: - run_args = [ - "nix-build", - build_attr.path, - "--attr", - f"{'.'.join(x for x in [build_attr.attr, attr] if x)}", - ] - else: - run_args = ["nix-build", "", "--attr", attr] - run_args += dict_to_flags(nix_flags) + run_args = [ + "nix-build", + build_attr.path, + "--attr", + f"{'.'.join(x for x in [build_attr.attr, attr] if x)}", + *dict_to_flags(nix_flags), + ] r = run_wrapper(run_args, stdout=PIPE) return Path(r.stdout.strip()) @@ -282,15 +278,11 @@ def nixos_build_flake( return Path(r.stdout.strip()) -def repl(attr: str, build_attr: BuildAttr | None, **nix_flags: Args) -> None: - run_args: list[str | Path] = ["nix", "repl", "--file"] - if build_attr: - run_args.append(build_attr.path) - if build_attr.attr: - run_args.append(build_attr.attr) - run_wrapper([*run_args, *dict_to_flags(nix_flags)]) - else: - run_wrapper([*run_args, "", *dict_to_flags(nix_flags)]) +def repl(attr: str, build_attr: BuildAttr, **nix_flags: Args) -> None: + run_args = ["nix", "repl", "--file", build_attr.path] + if build_attr.attr: + run_args.append(build_attr.attr) + run_wrapper([*run_args, *dict_to_flags(nix_flags)]) def repl_flake(attr: str, flake: Flake, **flake_flags: Args) -> None: diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py index 1a604ad53164..31920e97a25c 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py @@ -9,8 +9,8 @@ import nixos_rebuild.models as m from .helpers import get_qualified_name -def test_building_attr_from_arg() -> None: - assert m.BuildAttr.from_arg(None, None) is None +def test_build_attr_from_arg() -> None: + assert m.BuildAttr.from_arg(None, None) == m.BuildAttr("", None) assert m.BuildAttr.from_arg("attr", None) == m.BuildAttr( Path("default.nix"), "attr" ) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index 92411a0f5488..e47869867a7e 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -229,7 +229,9 @@ def test_nixos_build_flake(mock_run: Any) -> None: return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), ) def test_nixos_build(mock_run: Any, monkeypatch: Any) -> None: - assert n.nixos_build("attr", None, nix_flag="foo") == Path("/path/to/file") + assert n.nixos_build( + "attr", m.BuildAttr("", None), nix_flag="foo" + ) == Path("/path/to/file") mock_run.assert_called_with( ["nix-build", "", "--attr", "attr", "--nix-flag", "foo"], stdout=PIPE, @@ -241,16 +243,10 @@ def test_nixos_build(mock_run: Any, monkeypatch: Any) -> None: stdout=PIPE, ) - n.nixos_build("attr", m.BuildAttr(Path("file"), None)) - mock_run.assert_called_with( - ["nix-build", Path("file"), "--attr", "attr"], - stdout=PIPE, - ) - @patch(get_qualified_name(n.run_wrapper, n), autospec=True) def test_repl(mock_run: Any) -> None: - n.repl("attr", None, nix_flag=True) + n.repl("attr", m.BuildAttr("", None), nix_flag=True) mock_run.assert_called_with( ["nix", "repl", "--file", "", "--nix-flag"] ) From 5cc71a346a16620c721ac08be73efe1361a60c97 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Wed, 27 Nov 2024 23:51:27 +0000 Subject: [PATCH 16/78] nixos-rebuild-ng: Profile.from_name -> Profile.from_arg --- .../ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py | 2 +- .../by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py | 2 +- pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index f57991800404..e37f50ef7611 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -167,7 +167,7 @@ def execute(argv: list[str]) -> None: tmpdir_path = Path(tmpdir.name) atexit.register(cleanup_ssh, tmpdir_path) - profile = Profile.from_name(args.profile_name) + profile = Profile.from_arg(args.profile_name) target_host = Remote.from_arg(args.target_host, args.ask_sudo_password, tmpdir_path) build_attr = BuildAttr.from_arg(args.attr, args.file) flake = Flake.from_arg(args.flake, target_host) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py index 3039a3eafc7f..235ce9424e3a 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py @@ -137,7 +137,7 @@ class Profile: path: Path @classmethod - def from_name(cls, name: str = "system") -> Self: + def from_arg(cls, name: str) -> Self: match name: case "system": return cls(name, Path("/nix/var/nix/profiles/system")) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py index 31920e97a25c..baeb5977b84f 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py @@ -111,13 +111,13 @@ def test_flake_from_arg(mock_node: Any) -> None: @patch(get_qualified_name(m.Path.mkdir, m), autospec=True) -def test_profile_from_name(mock_mkdir: Any) -> None: - assert m.Profile.from_name("system") == m.Profile( +def test_profile_from_arg(mock_mkdir: Any) -> None: + assert m.Profile.from_arg("system") == m.Profile( "system", Path("/nix/var/nix/profiles/system"), ) - assert m.Profile.from_name("something") == m.Profile( + assert m.Profile.from_arg("something") == m.Profile( "something", Path("/nix/var/nix/profiles/system-profiles/something"), ) From 2db09d7f77305e4b6283cf2adb3c7151ecc7b570 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 10:20:41 +0000 Subject: [PATCH 17/78] nixos-rebuild-ng: configure logging --- .../src/nixos_rebuild/__init__.py | 24 ++++++++++++------- .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 15 ++++++++---- .../src/nixos_rebuild/process.py | 15 +++++++----- .../src/nixos_rebuild/utils.py | 20 ++++++++++++---- .../nixos-rebuild-ng/src/tests/test_main.py | 5 ++-- 5 files changed, 53 insertions(+), 26 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index e37f50ef7611..51f2c79e7702 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -1,6 +1,7 @@ import argparse import atexit import json +import logging import sys from pathlib import Path from subprocess import run @@ -10,9 +11,10 @@ from typing import assert_never from . import nix from .models import Action, BuildAttr, Flake, NRError, Profile from .process import Remote, cleanup_ssh -from .utils import info +from .utils import LogFormatter -VERBOSE = 0 +logger = logging.getLogger() +logger.setLevel(logging.INFO) def get_parser() -> tuple[argparse.ArgumentParser, dict[str, argparse.ArgumentParser]]: @@ -103,11 +105,11 @@ def parse_args( } def parser_warn(msg: str) -> None: - info(f"{parser.prog}: warning: {msg}") + print(f"{parser.prog}: warning: {msg}", file=sys.stderr) - global VERBOSE # This flag affects both nix and this script - VERBOSE = args.verbose + if args.verbose: + logger.setLevel(logging.DEBUG) # https://github.com/NixOS/nixpkgs/blob/master/pkgs/os-specific/linux/nixos-rebuild/nixos-rebuild.sh#L56 if args.action == Action.DRY_RUN.value: @@ -190,7 +192,7 @@ def execute(argv: list[str]) -> None: match action: case Action.SWITCH | Action.BOOT: - info("building the system configuration...") + logger.info("building the system configuration...") if args.rollback: path_to_config = nix.rollback(profile, target_host, sudo=args.sudo) else: @@ -219,7 +221,7 @@ def execute(argv: list[str]) -> None: install_bootloader=args.install_bootloader, ) case Action.TEST | Action.BUILD | Action.DRY_BUILD | Action.DRY_ACTIVATE: - info("building the system configuration...") + logger.info("building the system configuration...") dry_run = action == Action.DRY_BUILD if args.rollback: if action not in (Action.TEST, Action.BUILD): @@ -256,7 +258,7 @@ def execute(argv: list[str]) -> None: specialisation=args.specialisation, ) case Action.BUILD_VM | Action.BUILD_VM_WITH_BOOTLOADER: - info("building the system configuration...") + logger.info("building the system configuration...") attr = "vm" if action == Action.BUILD_VM else "vmWithBootLoader" if flake: path_to_config = nix.nixos_build_flake( @@ -313,10 +315,14 @@ def execute(argv: list[str]) -> None: def main() -> None: + ch = logging.StreamHandler() + ch.setFormatter(LogFormatter("%(levelname)s: %(message)s")) + logger.addHandler(ch) + try: execute(sys.argv) except (Exception, KeyboardInterrupt) as ex: - if VERBOSE: + if logger.level == logging.DEBUG: raise ex else: sys.exit(str(ex)) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index f56044451f02..ab34b0896ffc 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -1,3 +1,4 @@ +import logging import os from datetime import datetime from importlib.resources import files @@ -17,10 +18,11 @@ from .models import ( Remote, ) from .process import run_wrapper -from .utils import Args, dict_to_flags, info +from .utils import Args, dict_to_flags FLAKE_FLAGS: Final = ["--extra-experimental-features", "nix-command flakes"] FLAKE_REPL_TEMPLATE: Final = "repl.template.nix" +logger = logging.getLogger(__name__) def copy_closure( @@ -107,7 +109,7 @@ def get_nixpkgs_rev(nixpkgs_path: Path | None) -> str | None: ) except FileNotFoundError: # Git is not included in the closure so we need to check - info(f"warning: Git not found; cannot figure out revision of '{nixpkgs_path}'") + logger.warning(f"Git not found; cannot figure out revision of '{nixpkgs_path}'") return None if rev := r.stdout.strip(): @@ -203,13 +205,15 @@ def list_generations(profile: Profile) -> list[GenerationJson]: ) try: nixos_version = (generation_path / "nixos-version").read_text().strip() - except IOError: + except IOError as ex: + logger.debug("could not get nixos-version: %s", ex) nixos_version = "Unknown" try: kernel_version = next( (generation_path / "kernel-modules/lib/modules").iterdir() ).name - except IOError: + except IOError as ex: + logger.debug("could not get kernel version: %s", ex) kernel_version = "Unknown" specialisations = [ s.name for s in (generation_path / "specialisation").glob("*") if s.is_dir() @@ -219,7 +223,8 @@ def list_generations(profile: Profile) -> list[GenerationJson]: [generation_path / "sw/bin/nixos-version", "--configuration-revision"], capture_output=True, ).stdout.strip() - except (CalledProcessError, IOError): + except (CalledProcessError, IOError) as ex: + logger.debug("could not get configuration revision: %s", ex) configuration_revision = "Unknown" result.append( diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index db470ee5753c..91a9f04593c8 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os import subprocess from dataclasses import dataclass @@ -7,7 +8,7 @@ from getpass import getpass from pathlib import Path from typing import Self, Sequence, TypedDict, Unpack -from .utils import info +logger = logging.getLogger(__name__) @dataclass(frozen=True) @@ -46,13 +47,13 @@ class Remote: def _validate_opts(opts: list[str], ask_sudo_password: bool | None) -> None: for o in opts: if o in ["-t", "-tt", "RequestTTY=yes", "RequestTTY=force"]: - info( - f"warning: detected option '{o}' in NIX_SSHOPTS. SSH's TTY " - + "may cause issues, it is recommended to remove this option" + logger.warning( + f"detected option '{o}' in NIX_SSHOPTS. SSH's TTY may " + + "cause issues, it is recommended to remove this option" ) if not ask_sudo_password: - info( - "If you want to prompt for sudo password use " + logger.warning( + "if you want to prompt for sudo password use " + "'--ask-sudo-password' option instead" ) @@ -99,6 +100,8 @@ def run_wrapper( if sudo: args = ["sudo", *args] + logger.debug("calling run with args=%s, extra_env=%s", args, extra_env) + return subprocess.run( args, check=check, diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/utils.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/utils.py index 8d4f128075d4..dc8d899ab056 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/utils.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/utils.py @@ -1,11 +1,23 @@ -import sys -from functools import partial -from typing import TypeAlias +import logging +from typing import TypeAlias, override -info = partial(print, file=sys.stderr) Args: TypeAlias = bool | str | list[str] | int | None +class LogFormatter(logging.Formatter): + @override + def format(self, record: logging.LogRecord) -> str: + record.levelname = record.levelname.lower() + match record.levelno: + case logging.INFO: + self._style._fmt = "%(message)s" + case logging.DEBUG: + self._style._fmt = "%(levelname)s: %(name)s: %(message)s" + case _: + self._style._fmt = "%(levelname)s: %(message)s" + return super().format(record) + + def dict_to_flags(d: dict[str, Args]) -> list[str]: flags = [] for key, value in d.items(): diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py index e3bb1ba8f6e7..8bb624a44c30 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py @@ -1,3 +1,4 @@ +import logging import textwrap from pathlib import Path from subprocess import PIPE, CompletedProcess @@ -43,7 +44,7 @@ def test_parse_args() -> None: "bar", ] ) - assert nr.VERBOSE == 0 + assert nr.logger.level == logging.INFO assert r1.flake == "/etc/nixos" assert r1.install_bootloader is True assert r1.install_grub is True @@ -65,7 +66,7 @@ def test_parse_args() -> None: "-vvv", ] ) - assert nr.VERBOSE == 3 + assert nr.logger.level == logging.DEBUG assert r2.verbose == 3 assert r2.flake is False assert r2.action == "dry-build" From 1e34a97f9f793110ecfc9f5f08a9e86961ad0d57 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 13:12:47 +0000 Subject: [PATCH 18/78] nixos-rebuild-ng: add message to help if the user forgot --ask-sudo-password --- .../src/nixos_rebuild/process.py | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index 91a9f04593c8..a64905d7d3bc 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -102,14 +102,22 @@ def run_wrapper( logger.debug("calling run with args=%s, extra_env=%s", args, extra_env) - return subprocess.run( - args, - check=check, - env=env, - input=input, - # Hope nobody is using NixOS with non-UTF8 encodings, but "surrogateescape" - # should still work in those systems. - text=True, - errors="surrogateescape", - **kwargs, - ) + try: + return subprocess.run( + args, + check=check, + env=env, + input=input, + # Hope nobody is using NixOS with non-UTF8 encodings, but "surrogateescape" + # should still work in those systems. + text=True, + errors="surrogateescape", + **kwargs, + ) + except subprocess.CalledProcessError as ex: + if sudo and remote: + logger.error( + "while running command with remote sudo, did you forget to use " + + "--ask-sudo-password?" + ) + raise ex From 7a01349f792840731bbc6e410c118b857dd01e87 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 13:28:46 +0000 Subject: [PATCH 19/78] nixos-rebuild-ng: add TRY201 check for ruff --- .../by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py | 2 +- pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py | 4 ++-- pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 51f2c79e7702..90572f51b1db 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -323,6 +323,6 @@ def main() -> None: execute(sys.argv) except (Exception, KeyboardInterrupt) as ex: if logger.level == logging.DEBUG: - raise ex + raise else: sys.exit(str(ex)) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index a64905d7d3bc..2770a5a7d969 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -114,10 +114,10 @@ def run_wrapper( errors="surrogateescape", **kwargs, ) - except subprocess.CalledProcessError as ex: + except subprocess.CalledProcessError: if sudo and remote: logger.error( "while running command with remote sudo, did you forget to use " + "--ask-sudo-password?" ) - raise ex + raise diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml b/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml index 0dc662814c33..d773e8888923 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml @@ -43,6 +43,8 @@ extend-select = [ "I", # require `check` argument for `subprocess.run` "PLW1510", + # check for needless exception names in raise statements + "TRY201", ] [tool.ruff.lint.per-file-ignores] From 3a0c0975a8b623129c7d1132934e4f3e35f1ce67 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 15:23:23 +0000 Subject: [PATCH 20/78] nixos-rebuild-ng: remove unnecessary "from __future__ import annotations" --- pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py | 2 -- pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py | 2 -- pkgs/by-name/ni/nixos-rebuild-ng/src/tests/helpers.py | 2 -- 3 files changed, 6 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py index 235ce9424e3a..ffd8c5bde454 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import platform import re import subprocess diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index 2770a5a7d969..945a1154ee1b 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import logging import os import subprocess diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/helpers.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/helpers.py index 0474c6671edd..77ddad740865 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/helpers.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/helpers.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from types import ModuleType from typing import Any, Callable From 73567536e149a08acac4204aceae4ff4843a6037 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 15:32:28 +0000 Subject: [PATCH 21/78] nixos-rebuild-ng: add from_host in nix.copy_closure --- .../src/nixos_rebuild/__init__.py | 4 ++-- .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 11 ++++++--- .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 23 ++++++++++++++++--- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 90572f51b1db..b53d59f037f7 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -210,7 +210,7 @@ def execute(argv: list[str]) -> None: no_out_link=True, **build_flags, ) - nix.copy_closure(path_to_config, target_host, **copy_flags) + nix.copy_closure(path_to_config, target_host, None, **copy_flags) nix.set_profile(profile, path_to_config, target_host, sudo=args.sudo) nix.switch_to_configuration( path_to_config, @@ -253,7 +253,7 @@ def execute(argv: list[str]) -> None: nix.switch_to_configuration( path_to_config, action, - target_host, + target_host=None, sudo=args.sudo, specialisation=args.specialisation, ) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index ab34b0896ffc..3921629edd1e 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -27,10 +27,14 @@ logger = logging.getLogger(__name__) def copy_closure( closure: Path, - target_host: Remote | None, + to_host: Remote | None, + from_host: Remote | None = None, **copy_flags: Args, ) -> None: - host = target_host + """Copy a nix closure to or from host to localhost. + + Also supports copying a closure from a remote to another remote.""" + host = to_host or from_host if not host: return @@ -38,11 +42,12 @@ def copy_closure( [ "nix-copy-closure", *dict_to_flags(copy_flags), - "--to", + "--to" if to_host else "--from", host.host, closure, ], extra_env={"NIX_SSHOPTS": " ".join(host.opts)}, + remote=from_host if to_host else None, ) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index e47869867a7e..3d748732775b 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -18,11 +18,28 @@ def test_copy_closure(mock_run: Any) -> None: n.copy_closure(closure, None) mock_run.assert_not_called() - target_host = m.Remote("user@host", ["--ssh", "opt"], None) + target_host = m.Remote("user@target.host", ["--ssh", "target-opt"], None) + build_host = m.Remote("user@build.host", ["--ssh", "build-opt"], None) + n.copy_closure(closure, target_host) mock_run.assert_called_with( - ["nix-copy-closure", "--to", "user@host", closure], - extra_env={"NIX_SSHOPTS": "--ssh opt"}, + ["nix-copy-closure", "--to", "user@target.host", closure], + extra_env={"NIX_SSHOPTS": "--ssh target-opt"}, + remote=None, + ) + + n.copy_closure(closure, None, build_host) + mock_run.assert_called_with( + ["nix-copy-closure", "--from", "user@build.host", closure], + extra_env={"NIX_SSHOPTS": "--ssh build-opt"}, + remote=None, + ) + + n.copy_closure(closure, target_host, build_host) + mock_run.assert_called_with( + ["nix-copy-closure", "--to", "user@target.host", closure], + extra_env={"NIX_SSHOPTS": "--ssh target-opt"}, + remote=build_host, ) From 02b943d57ff972d21df41acca540e7bee7f0d9f5 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 15:42:27 +0000 Subject: [PATCH 22/78] nixos-rebuild-ng: use find_file in edit --- .../ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index 3921629edd1e..b129a65d1cf3 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -69,13 +69,7 @@ def edit(flake: Flake | None, **flake_flags: Args) -> None: if flake_flags: raise NRError("'edit' does not support extra Nix flags") nixos_config = Path( - os.getenv("NIXOS_CONFIG") - or run_wrapper( - ["nix-instantiate", "--find-file", "nixos-config"], - stdout=PIPE, - check=False, - ).stdout.strip() - or "/etc/nixos/default.nix" + os.getenv("NIXOS_CONFIG") or find_file("nixos-config") or "/etc/nixos" ) if nixos_config.is_dir(): nixos_config /= "default.nix" @@ -87,7 +81,7 @@ def edit(flake: Flake | None, **flake_flags: Args) -> None: def find_file(file: str, **nix_flags: Args) -> Path | None: - "Find classic Nixpkgs location." + "Find classic Nix file location." r = run_wrapper( ["nix-instantiate", "--find-file", file, *dict_to_flags(nix_flags)], stdout=PIPE, From f7266986d371f814f9f5ab16b228d242aa05e51e Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 16:27:38 +0000 Subject: [PATCH 23/78] nixos-rebuild-ng: implement --build-host --- .../src/nixos_rebuild/__init__.py | 71 ++++++++---- .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 66 ++++++++++- .../src/nixos_rebuild/process.py | 7 +- .../nixos-rebuild-ng/src/tests/test_main.py | 99 ++++++++++++++++- .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 103 +++++++++++++++++- 5 files changed, 317 insertions(+), 29 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index b53d59f037f7..cff4f4f7c945 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -87,7 +87,7 @@ def get_parser() -> tuple[argparse.ArgumentParser, dict[str, argparse.ArgumentPa main_parser.add_argument("--ask-sudo-password", action="store_true") main_parser.add_argument("--use-remote-sudo", action="store_true") # deprecated main_parser.add_argument("--no-ssh-tty", action="store_true") # deprecated - # parser.add_argument("--build-host") # TODO + main_parser.add_argument("--build-host") main_parser.add_argument("--target-host") main_parser.add_argument("action", choices=Action.values(), nargs="?") @@ -135,15 +135,11 @@ def parse_args( if args.action == Action.EDIT.value and (args.file or args.attr): parser.error("--file and --attr are not supported with 'edit'") - if args.target_host and args.action not in ( + if (args.target_host or args.build_host) and args.action not in ( Action.SWITCH.value, Action.BOOT.value, - Action.TEST.value, - Action.BUILD.value, - Action.DRY_BUILD.value, - Action.DRY_ACTIVATE.value, ): - parser.error(f"--target-host is not supported with '{args.action}'") + parser.error(f"--target-host/--build-host is not supported with '{args.action}'") if args.flake and (args.file or args.attr): parser.error("--flake cannot be used with --file or --attr") @@ -170,6 +166,7 @@ def execute(argv: list[str]) -> None: atexit.register(cleanup_ssh, tmpdir_path) profile = Profile.from_arg(args.profile_name) + build_host = Remote.from_arg(args.build_host, False, tmpdir_path) target_host = Remote.from_arg(args.target_host, args.ask_sudo_password, tmpdir_path) build_attr = BuildAttr.from_arg(args.attr, args.file) flake = Flake.from_arg(args.flake, target_host) @@ -197,25 +194,55 @@ def execute(argv: list[str]) -> None: path_to_config = nix.rollback(profile, target_host, sudo=args.sudo) else: if flake: - path_to_config = nix.nixos_build_flake( - "toplevel", - flake, - no_link=True, - **flake_build_flags, - ) + if build_host: + path_to_config = nix.nixos_remote_build_flake( + "toplevel", + flake, + build_host, + flake_build_flags=flake_build_flags, + copy_flags=copy_flags, + build_flags=build_flags, + ) + else: + path_to_config = nix.nixos_build_flake( + "toplevel", + flake, + no_link=True, + **flake_build_flags, + ) else: - path_to_config = nix.nixos_build( - "system", - build_attr, - no_out_link=True, - **build_flags, - ) - nix.copy_closure(path_to_config, target_host, None, **copy_flags) - nix.set_profile(profile, path_to_config, target_host, sudo=args.sudo) + if build_host: + path_to_config = nix.nixos_remote_build( + "system", + build_attr, + build_host, + instantiate_flags=common_flags, + copy_flags=copy_flags, + build_flags=build_flags, + ) + else: + path_to_config = nix.nixos_build( + "system", + build_attr, + no_out_link=True, + **build_flags, + ) + nix.copy_closure( + path_to_config, + to_host=target_host, + from_host=None, + **copy_flags, + ) + nix.set_profile( + profile, + path_to_config, + target_host=target_host, + sudo=args.sudo, + ) nix.switch_to_configuration( path_to_config, action, - target_host, + target_host=target_host, sudo=args.sudo, specialisation=args.specialisation, install_bootloader=args.install_bootloader, diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index b129a65d1cf3..2f86bf02faa6 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -244,7 +244,7 @@ def list_generations(profile: Profile) -> list[GenerationJson]: def nixos_build( attr: str, build_attr: BuildAttr, - **nix_flags: Args, + **build_flags: Args, ) -> Path: """Build NixOS attribute using classic Nix. @@ -255,7 +255,7 @@ def nixos_build( build_attr.path, "--attr", f"{'.'.join(x for x in [build_attr.attr, attr] if x)}", - *dict_to_flags(nix_flags), + *dict_to_flags(build_flags), ] r = run_wrapper(run_args, stdout=PIPE) return Path(r.stdout.strip()) @@ -264,7 +264,7 @@ def nixos_build( def nixos_build_flake( attr: str, flake: Flake, - **flake_flags: Args, + **flake_build_flags: Args, ) -> Path: """Build NixOS attribute using Flakes. @@ -276,12 +276,70 @@ def nixos_build_flake( "build", "--print-out-paths", f"{flake}.config.system.build.{attr}", - *dict_to_flags(flake_flags), + *dict_to_flags(flake_build_flags), ] r = run_wrapper(run_args, stdout=PIPE) return Path(r.stdout.strip()) +def nixos_remote_build( + attr: str, + build_attr: BuildAttr, + build_host: Remote | None, + build_flags: dict[str, Args] | None = None, + instantiate_flags: dict[str, Args] | None = None, + copy_flags: dict[str, Args] | None = None, +) -> Path: + r = run_wrapper( + [ + "nix-instantiate", + "--raw", + build_attr.path, + "--attr", + f"{'.'.join(x for x in [build_attr.attr, attr] if x)}", + *dict_to_flags(instantiate_flags or {}), + ], + stdout=PIPE, + ) + drv = Path(r.stdout.strip()) + copy_closure(drv, to_host=build_host, from_host=None, **(copy_flags or {})) + r = run_wrapper( + ["nix-store", "--realise", drv, *dict_to_flags(build_flags or {})], + remote=build_host, + stdout=PIPE, + ) + return Path(r.stdout.strip()) + + +def nixos_remote_build_flake( + attr: str, + flake: Flake, + build_host: Remote, + flake_build_flags: dict[str, Args] | None = None, + copy_flags: dict[str, Args] | None = None, + build_flags: dict[str, Args] | None = None, +) -> Path: + r = run_wrapper( + [ + "nix", + *FLAKE_FLAGS, + "eval", + "--raw", + f"{flake}.config.system.build.{attr}.drvPath", + *dict_to_flags(flake_build_flags or {}), + ], + stdout=PIPE, + ) + drv = Path(r.stdout.strip()) + copy_closure(drv, to_host=build_host, from_host=None, **(copy_flags or {})) + r = run_wrapper( + ["nix-store", "--realise", drv, *dict_to_flags(build_flags or {})], + remote=build_host, + stdout=PIPE, + ) + return Path(r.stdout.strip()) + + def repl(attr: str, build_attr: BuildAttr, **nix_flags: Args) -> None: run_args = ["nix", "repl", "--file", build_attr.path] if build_attr.attr: diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index 945a1154ee1b..ed51f1e87937 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -98,7 +98,12 @@ def run_wrapper( if sudo: args = ["sudo", *args] - logger.debug("calling run with args=%s, extra_env=%s", args, extra_env) + logger.debug( + "calling run with args=%s, kwargs=%s, extra_env=%s", + args, + kwargs, + extra_env, + ) try: return subprocess.run( diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py index 8bb624a44c30..020fbe0a8cd3 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py @@ -218,7 +218,7 @@ def test_execute_nix_switch_flake(mock_run: Any, tmp_path: Path) -> None: @patch.dict(nr.process.os.environ, {}, clear=True) @patch(get_qualified_name(nr.process.subprocess.run), autospec=True) @patch(get_qualified_name(nr.TemporaryDirectory, nr)) # can't autospec -def test_execute_nix_switch_flake_remote( +def test_execute_nix_switch_flake_target_host( mock_tmpdir: Any, mock_run: Any, tmp_path: Path, @@ -316,6 +316,103 @@ def test_execute_nix_switch_flake_remote( ) +@patch.dict(nr.process.os.environ, {}, clear=True) +@patch(get_qualified_name(nr.process.subprocess.run), autospec=True) +@patch(get_qualified_name(nr.TemporaryDirectory, nr)) # can't autospec +def test_execute_nix_switch_flake_build_host( + mock_tmpdir: Any, + mock_run: Any, + tmp_path: Path, +) -> None: + config_path = tmp_path / "test" + config_path.touch() + mock_run.side_effect = [ + # nixos_build_flake + CompletedProcess([], 0, str(config_path)), + CompletedProcess([], 0), + CompletedProcess([], 0, str(config_path)), + # set_profile + CompletedProcess([], 0), + # copy_closure + CompletedProcess([], 0), + # switch_to_configuration + CompletedProcess([], 0), + ] + mock_tmpdir.return_value.name = "/tmp/test" + + nr.execute( + [ + "nixos-rebuild", + "switch", + "--flake", + "/path/to/config#hostname", + "--use-remote-sudo", + "--build-host", + "user@localhost", + ] + ) + + assert mock_run.call_count == 5 + mock_run.assert_has_calls( + [ + call( + [ + "nix", + "--extra-experimental-features", + "nix-command flakes", + "eval", + "--raw", + "/path/to/config#nixosConfigurations.hostname.config.system.build.toplevel.drvPath", + ], + check=True, + stdout=PIPE, + **DEFAULT_RUN_KWARGS, + ), + call( + ["nix-copy-closure", "--to", "user@localhost", config_path], + check=True, + **DEFAULT_RUN_KWARGS, + ), + call( + [ + "ssh", + "-o", + "ControlMaster=auto", + "-o", + "ControlPath=/tmp/test/ssh-%n", + "-o", + "ControlPersist=60", + "user@localhost", + "--", + "nix-store", + "--realise", + config_path, + ], + check=True, + stdout=PIPE, + **DEFAULT_RUN_KWARGS, + ), + call( + [ + "sudo", + "nix-env", + "-p", + Path("/nix/var/nix/profiles/system"), + "--set", + config_path, + ], + check=True, + **DEFAULT_RUN_KWARGS, + ), + call( + ["sudo", config_path / "bin/switch-to-configuration", "switch"], + check=True, + **DEFAULT_RUN_KWARGS, + ), + ] + ) + + @patch(get_qualified_name(nr.process.subprocess.run), autospec=True) def test_execute_switch_rollback(mock_run: Any, tmp_path: Path) -> None: nixpkgs_path = tmp_path / "nixpkgs" diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index 3d748732775b..c10aa7c28c5f 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -254,13 +254,114 @@ def test_nixos_build(mock_run: Any, monkeypatch: Any) -> None: stdout=PIPE, ) - n.nixos_build("attr", m.BuildAttr(Path("file"), "preAttr")) + assert n.nixos_build("attr", m.BuildAttr(Path("file"), "preAttr")) == Path( + "/path/to/file" + ) mock_run.assert_called_with( ["nix-build", Path("file"), "--attr", "preAttr.attr"], stdout=PIPE, ) +@patch( + get_qualified_name(n.run_wrapper, n), + autospec=True, + return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), +) +def test_nixos_remote_build_flake(mock_run: Any) -> None: + flake = m.Flake.parse(".#hostname") + build_host = m.Remote("user@host", ["--ssh", "opts"], None) + + assert n.nixos_remote_build_flake( + "toplevel", + flake, + build_host, + flake_build_flags={"flake": True}, + copy_flags={"copy": True}, + build_flags={"build": True}, + ) == Path("/path/to/file") + mock_run.assert_has_calls( + [ + call( + [ + "nix", + "--extra-experimental-features", + "nix-command flakes", + "eval", + "--raw", + ".#nixosConfigurations.hostname.config.system.build.toplevel.drvPath", + "--flake", + ], + stdout=PIPE, + ), + call( + [ + "nix-copy-closure", + "--copy", + "--to", + "user@host", + Path("/path/to/file"), + ], + extra_env={"NIX_SSHOPTS": "--ssh opts"}, + remote=None, + ), + call( + ["nix-store", "--realise", Path("/path/to/file"), "--build"], + remote=build_host, + stdout=PIPE, + ), + ] + ) + + +@patch( + get_qualified_name(n.run_wrapper, n), + autospec=True, + return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), +) +def test_nixos_remote_build(mock_run: Any, monkeypatch: Any) -> None: + build_host = m.Remote("user@host", ["--ssh", "opts"], None) + assert n.nixos_remote_build( + "attr", + m.BuildAttr("", None), + build_host, + build_flags={"build": True}, + instantiate_flags={"inst": True}, + copy_flags={"copy": True}, + ) == Path("/path/to/file") + mock_run.assert_has_calls( + [ + call( + [ + "nix-instantiate", + "--raw", + "", + "--attr", + "attr", + "--inst", + ], + stdout=PIPE, + ), + call( + [ + "nix-copy-closure", + "--copy", + "--to", + "user@host", + Path("/path/to/file"), + ], + extra_env={"NIX_SSHOPTS": "--ssh opts"}, + remote=None, + ), + call( + ["nix-store", "--realise", Path("/path/to/file"), "--build"], + remote=build_host, + stdout=PIPE, + ), + ] + ) + + @patch(get_qualified_name(n.run_wrapper, n), autospec=True) def test_repl(mock_run: Any) -> None: n.repl("attr", m.BuildAttr("", None), nix_flag=True) From 8a4105cfd7a6f58bf88a5f9e71dd1754f0996b36 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 18:08:43 +0000 Subject: [PATCH 24/78] nixos-rebuild-ng: update README.md --- pkgs/by-name/ni/nixos-rebuild-ng/README.md | 27 +++++++++++----------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/README.md b/pkgs/by-name/ni/nixos-rebuild-ng/README.md index f34c3d4996ef..c52d1c06d920 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/README.md +++ b/pkgs/by-name/ni/nixos-rebuild-ng/README.md @@ -97,18 +97,8 @@ ruff format . version (the one that will be switched) and re-exec to it instead. This means that in case of bugs in `nixos-rebuild-ng`, the only way that you will get them fixed is **after** you switch to a new version -- `nix` bootstrap is also **not** implemented yet, so this means that you will - eval with an old version of Nix instead of a newer one. This is unlikely to - cause issues, because the build will happen in the daemon anyway (that is - only changed after the switch), and unless you are using bleeding edge `nix` - features you will probably have zero problems here. You can basically think - that using `nixos-rebuild-ng` is similar to running `nixos-rebuild --fast` - right now - Ignore any performance advantages of the rewrite right now, because of the 2 caveats above -- `--target-host` and `--build-host` are not implemented yet and this is - probably the thing that will be most difficult to implement. Help here is - welcome - Bugs in the profile manipulation can cause corruption of your profile that may be difficult to fix, so right now I only recommend using `nixos-rebuild-ng` if you are testing in a VM or in a filesystem with @@ -118,7 +108,7 @@ ruff format . ## TODO -- [ ] Remote host/builders (via SSH) +- [x] Remote host/builders (via SSH) - [x] Improve nix arguments handling (e.g.: `nixFlags` vs `copyFlags` in the old `nixos-rebuild`) - [ ] `_NIXOS_REBUILD_EXEC` @@ -127,11 +117,20 @@ ruff format . `system.switch.enableNg` for `switch-to-configuration-ng` - [ ] Improve documentation - [x] `nixos-rebuild repl` -- [ ] `nix` build/bootstrap - [ ] Generate tab completion via [`shtab`](https://docs.iterative.ai/shtab/) - [x] Reduce build closure ## TODON'T -- Reimplement `systemd-run` logic (will be moved to the new - [`apply`](https://github.com/NixOS/nixpkgs/pull/344407) script) +- Reimplement `systemd-run` logic: will be moved to the new + [`apply`](https://github.com/NixOS/nixpkgs/pull/344407) script +- Nix bootstrap: it is only used for non-Flake paths and it is basically + useless nowadays. It was created at a time when Nix was changing frequently + and there was a need to bootstrap a new version of Nix before evaluating the + configuration (otherwise the new Nixpkgs version may have code that is only + compatible with a newer version of Nix). Nixpkgs now has a policy to be + compatible with Nix 2.3, and even if this is bumped as long we don't do + drastic minimum version changes this should not be an issue. Also, the daemon + itself always run with the previous version since even we can replace Nix in + `PATH` (so Nix client), but we can't replace the daemon without switching to + a new version. From 34cda442082f6ac50ef3196149dd5e8343789e59 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 19:57:03 +0000 Subject: [PATCH 25/78] nixos-rebuild-ng: use `nix build` for remote builds in Flakes, fix remote args --- .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 9 +++++++- .../src/nixos_rebuild/process.py | 12 ++++++++++- .../nixos-rebuild-ng/src/tests/test_main.py | 21 ++++--------------- .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 10 ++++++++- .../src/tests/test_process.py | 13 +++--------- 5 files changed, 35 insertions(+), 30 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index 2f86bf02faa6..11d13502e13c 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -333,7 +333,14 @@ def nixos_remote_build_flake( drv = Path(r.stdout.strip()) copy_closure(drv, to_host=build_host, from_host=None, **(copy_flags or {})) r = run_wrapper( - ["nix-store", "--realise", drv, *dict_to_flags(build_flags or {})], + [ + "nix", + *FLAKE_FLAGS, + "build", + f"{drv}^*", + "--print-out-paths", + *dict_to_flags(build_flags or {}), + ], remote=build_host, stdout=PIPE, ) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index ed51f1e87937..05d2840529a1 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -1,5 +1,6 @@ import logging import os +import shlex import subprocess from dataclasses import dataclass from getpass import getpass @@ -91,7 +92,16 @@ def run_wrapper( input = remote.sudo_password + "\n" else: args = ["sudo", *args] - args = ["ssh", *remote.opts, remote.host, "--", *args] + args = [ + "ssh", + *remote.opts, + remote.host, + "--", + # sadly SSH just join all remaining parameters, expanding glob and + # ignoring quotes + # so we need to use shlex.join() to safely join the arguments + shlex.join(str(a) for a in args), + ] else: if extra_env: env = os.environ | extra_env diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py index 020fbe0a8cd3..70fc0e7045ea 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py @@ -282,12 +282,7 @@ def test_execute_nix_switch_flake_target_host( "ControlPersist=60", "user@localhost", "--", - "sudo", - "nix-env", - "-p", - Path("/nix/var/nix/profiles/system"), - "--set", - config_path, + f"sudo nix-env -p /nix/var/nix/profiles/system --set {config_path}", ], check=True, **DEFAULT_RUN_KWARGS, @@ -303,11 +298,7 @@ def test_execute_nix_switch_flake_target_host( "ControlPersist=60", "user@localhost", "--", - "sudo", - "env", - "NIXOS_INSTALL_BOOTLOADER=0", - config_path / "bin/switch-to-configuration", - "switch", + f"sudo env NIXOS_INSTALL_BOOTLOADER=0 {config_path / 'bin/switch-to-configuration'} switch", ], check=True, **DEFAULT_RUN_KWARGS, @@ -346,7 +337,6 @@ def test_execute_nix_switch_flake_build_host( "switch", "--flake", "/path/to/config#hostname", - "--use-remote-sudo", "--build-host", "user@localhost", ] @@ -384,9 +374,7 @@ def test_execute_nix_switch_flake_build_host( "ControlPersist=60", "user@localhost", "--", - "nix-store", - "--realise", - config_path, + f"nix --extra-experimental-features 'nix-command flakes' build '{config_path}^*' --print-out-paths", ], check=True, stdout=PIPE, @@ -394,7 +382,6 @@ def test_execute_nix_switch_flake_build_host( ), call( [ - "sudo", "nix-env", "-p", Path("/nix/var/nix/profiles/system"), @@ -405,7 +392,7 @@ def test_execute_nix_switch_flake_build_host( **DEFAULT_RUN_KWARGS, ), call( - ["sudo", config_path / "bin/switch-to-configuration", "switch"], + [config_path / "bin/switch-to-configuration", "switch"], check=True, **DEFAULT_RUN_KWARGS, ), diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index c10aa7c28c5f..5a5ee01cb974 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -306,7 +306,15 @@ def test_nixos_remote_build_flake(mock_run: Any) -> None: remote=None, ), call( - ["nix-store", "--realise", Path("/path/to/file"), "--build"], + [ + "nix", + "--extra-experimental-features", + "nix-command flakes", + "build", + "/path/to/file^*", + "--print-out-paths", + "--build", + ], remote=build_host, stdout=PIPE, ), diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py index 6987aac9283b..57d3865bcef4 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py @@ -40,12 +40,12 @@ def test_run(mock_run: Any) -> None: ) p.run_wrapper( - ["test", "--with", "flags"], + ["test", "--with", "some flags"], check=True, remote=m.Remote("user@localhost", ["--ssh", "opt"], "password"), ) mock_run.assert_called_with( - ["ssh", "--ssh", "opt", "user@localhost", "--", "test", "--with", "flags"], + ["ssh", "--ssh", "opt", "user@localhost", "--", "test --with 'some flags'"], check=True, text=True, errors="surrogateescape", @@ -67,14 +67,7 @@ def test_run(mock_run: Any) -> None: "opt", "user@localhost", "--", - "sudo", - "--prompt=", - "--stdin", - "env", - "FOO=bar", - "test", - "--with", - "flags", + "sudo --prompt= --stdin env FOO=bar test --with flags", ], check=True, env=None, From 7d58c66881eba7fdc4701664b6393c4bf035d585 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 20:02:53 +0000 Subject: [PATCH 26/78] nixos-rebuild-ng: fix --build-host --- .../nixos-rebuild-ng/src/nixos_rebuild/__init__.py | 2 +- .../ni/nixos-rebuild-ng/src/tests/test_main.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index cff4f4f7c945..ad4b757878dd 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -230,7 +230,7 @@ def execute(argv: list[str]) -> None: nix.copy_closure( path_to_config, to_host=target_host, - from_host=None, + from_host=build_host, **copy_flags, ) nix.set_profile( diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py index 70fc0e7045ea..eeecf9445978 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py @@ -342,7 +342,7 @@ def test_execute_nix_switch_flake_build_host( ] ) - assert mock_run.call_count == 5 + assert mock_run.call_count == 6 mock_run.assert_has_calls( [ call( @@ -380,6 +380,16 @@ def test_execute_nix_switch_flake_build_host( stdout=PIPE, **DEFAULT_RUN_KWARGS, ), + call( + [ + "nix-copy-closure", + "--from", + "user@localhost", + config_path, + ], + check=True, + **DEFAULT_RUN_KWARGS, + ), call( [ "nix-env", From 2ac1f78a110332353d1e5b6020c608830d4358bf Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Thu, 28 Nov 2024 20:07:35 +0000 Subject: [PATCH 27/78] nixos-rebuild-ng: validate NIX_SSHOPTS only once --- .../ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py | 8 ++++++-- .../ni/nixos-rebuild-ng/src/nixos_rebuild/process.py | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index ad4b757878dd..99391ecf0c98 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -139,7 +139,9 @@ def parse_args( Action.SWITCH.value, Action.BOOT.value, ): - parser.error(f"--target-host/--build-host is not supported with '{args.action}'") + parser.error( + f"--target-host/--build-host is not supported with '{args.action}'" + ) if args.flake and (args.file or args.attr): parser.error("--flake cannot be used with --file or --attr") @@ -166,7 +168,9 @@ def execute(argv: list[str]) -> None: atexit.register(cleanup_ssh, tmpdir_path) profile = Profile.from_arg(args.profile_name) - build_host = Remote.from_arg(args.build_host, False, tmpdir_path) + build_host = Remote.from_arg( + args.build_host, False, tmpdir_path, validate_opts=False + ) target_host = Remote.from_arg(args.target_host, args.ask_sudo_password, tmpdir_path) build_attr = BuildAttr.from_arg(args.attr, args.file) flake = Flake.from_arg(args.flake, target_host) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index 05d2840529a1..6fa5be980c16 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -22,12 +22,14 @@ class Remote: host: str | None, ask_sudo_password: bool | None, tmp_dir: Path, + validate_opts: bool = True, ) -> Self | None: if not host: return None opts = os.getenv("NIX_SSHOPTS", "").split() - cls._validate_opts(opts, ask_sudo_password) + if validate_opts: + cls._validate_opts(opts, ask_sudo_password) opts += [ # SSH ControlMaster flags, allow for faster re-connection "-o", From 29e9b42022b6cf20b2ca7bc0b6b841c51428fc96 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 10:14:43 +0000 Subject: [PATCH 28/78] nixos-rebuild-ng: fix --build-host and --target-host case --- .../by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py | 9 ++++++++- pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py | 5 +++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index 11d13502e13c..317c04467a5d 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -46,7 +46,14 @@ def copy_closure( host.host, closure, ], - extra_env={"NIX_SSHOPTS": " ".join(host.opts)}, + extra_env={ + # for the remote to remote case, we can't use host.opts because it + # includes the ControlPane opts that will not work in the remote, + # because the temporary directory that we created will not exist + "NIX_SSHOPTS": os.environ.get("NIX_SSHOPTS", "") + if from_host and to_host + else " ".join(host.opts) + }, remote=from_host if to_host else None, ) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index 5a5ee01cb974..712bd09b8704 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -13,7 +13,7 @@ from .helpers import get_qualified_name @patch(get_qualified_name(n.run_wrapper, n), autospec=True) -def test_copy_closure(mock_run: Any) -> None: +def test_copy_closure(mock_run: Any, monkeypatch: Any) -> None: closure = Path("/path/to/closure") n.copy_closure(closure, None) mock_run.assert_not_called() @@ -35,11 +35,12 @@ def test_copy_closure(mock_run: Any) -> None: remote=None, ) + monkeypatch.setenv("NIX_SSHOPTS", "--ssh build-target-opt") n.copy_closure(closure, target_host, build_host) mock_run.assert_called_with( ["nix-copy-closure", "--to", "user@target.host", closure], - extra_env={"NIX_SSHOPTS": "--ssh target-opt"}, remote=build_host, + extra_env={"NIX_SSHOPTS": "--ssh build-target-opt"}, ) From c859df048f77690fdf8e1fab5c45513423596af8 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 11:07:44 +0000 Subject: [PATCH 29/78] nixos-rebuild-ng: refactor classic Nix to simplify logic --- .../src/nixos_rebuild/__init__.py | 6 +++--- .../ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py | 4 ++-- .../ni/nixos-rebuild-ng/src/tests/test_main.py | 2 +- .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 15 +++++++++++---- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 99391ecf0c98..0ee3c6db0fd7 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -217,7 +217,7 @@ def execute(argv: list[str]) -> None: else: if build_host: path_to_config = nix.nixos_remote_build( - "system", + "toplevel", build_attr, build_host, instantiate_flags=common_flags, @@ -226,7 +226,7 @@ def execute(argv: list[str]) -> None: ) else: path_to_config = nix.nixos_build( - "system", + "toplevel", build_attr, no_out_link=True, **build_flags, @@ -275,7 +275,7 @@ def execute(argv: list[str]) -> None: ) else: path_to_config = nix.nixos_build( - "system", + "toplevel", build_attr, dry_run=dry_run, **build_flags, diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index 317c04467a5d..a48dc18d9bf2 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -261,7 +261,7 @@ def nixos_build( "nix-build", build_attr.path, "--attr", - f"{'.'.join(x for x in [build_attr.attr, attr] if x)}", + f"{build_attr.attr + '.' if build_attr.attr else ''}config.system.build.{attr}", *dict_to_flags(build_flags), ] r = run_wrapper(run_args, stdout=PIPE) @@ -303,7 +303,7 @@ def nixos_remote_build( "--raw", build_attr.path, "--attr", - f"{'.'.join(x for x in [build_attr.attr, attr] if x)}", + f"{build_attr.attr + '.' if build_attr.attr else ''}config.system.build.{attr}", *dict_to_flags(instantiate_flags or {}), ], stdout=PIPE, diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py index eeecf9445978..b5c93a2dd217 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py @@ -122,7 +122,7 @@ def test_execute_nix_boot(mock_run: Any, tmp_path: Path) -> None: "nix-build", "", "--attr", - "system", + "config.system.build.toplevel", "--no-out-link", "-vvv", ], diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index 712bd09b8704..dce98cf6b939 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -251,7 +251,14 @@ def test_nixos_build(mock_run: Any, monkeypatch: Any) -> None: "attr", m.BuildAttr("", None), nix_flag="foo" ) == Path("/path/to/file") mock_run.assert_called_with( - ["nix-build", "", "--attr", "attr", "--nix-flag", "foo"], + [ + "nix-build", + "", + "--attr", + "config.system.build.attr", + "--nix-flag", + "foo", + ], stdout=PIPE, ) @@ -259,7 +266,7 @@ def test_nixos_build(mock_run: Any, monkeypatch: Any) -> None: "/path/to/file" ) mock_run.assert_called_with( - ["nix-build", Path("file"), "--attr", "preAttr.attr"], + ["nix-build", Path("file"), "--attr", "preAttr.config.system.build.attr"], stdout=PIPE, ) @@ -332,7 +339,7 @@ def test_nixos_remote_build(mock_run: Any, monkeypatch: Any) -> None: build_host = m.Remote("user@host", ["--ssh", "opts"], None) assert n.nixos_remote_build( "attr", - m.BuildAttr("", None), + m.BuildAttr("", "preAttr"), build_host, build_flags={"build": True}, instantiate_flags={"inst": True}, @@ -346,7 +353,7 @@ def test_nixos_remote_build(mock_run: Any, monkeypatch: Any) -> None: "--raw", "", "--attr", - "attr", + "preAttr.config.system.build.attr", "--inst", ], stdout=PIPE, From 359d3415359066c2200013cde0ee672c8ac7a569 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 11:22:00 +0000 Subject: [PATCH 30/78] nixos-rebuild-ng: add {BuildAttr,Flake}.to_attr() --- .../src/nixos_rebuild/models.py | 6 ++++++ .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 8 ++++---- .../nixos-rebuild-ng/src/tests/test_models.py | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py index ffd8c5bde454..47ce0f5ca91d 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py @@ -48,6 +48,9 @@ class BuildAttr: path: str | Path attr: str | None + def to_attr(self, *attrs: str) -> str: + return f"{self.attr + '.' if self.attr else ''}{".".join(attrs)}" + @classmethod def from_arg(cls, attr: str | None, file: str | None) -> Self: if not (attr or file): @@ -61,6 +64,9 @@ class Flake: attr: str _re: ClassVar = re.compile(r"^(?P[^\#]*)\#?(?P[^\#\"]*)$") + def to_attr(self, *attrs: str) -> str: + return f"{self}.{".".join(attrs)}" + @override def __str__(self) -> str: return f"{self.path}#{self.attr}" diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index a48dc18d9bf2..7a42645d904b 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -261,7 +261,7 @@ def nixos_build( "nix-build", build_attr.path, "--attr", - f"{build_attr.attr + '.' if build_attr.attr else ''}config.system.build.{attr}", + build_attr.to_attr("config", "system", "build", attr), *dict_to_flags(build_flags), ] r = run_wrapper(run_args, stdout=PIPE) @@ -282,7 +282,7 @@ def nixos_build_flake( *FLAKE_FLAGS, "build", "--print-out-paths", - f"{flake}.config.system.build.{attr}", + flake.to_attr("config", "system", "build", attr), *dict_to_flags(flake_build_flags), ] r = run_wrapper(run_args, stdout=PIPE) @@ -303,7 +303,7 @@ def nixos_remote_build( "--raw", build_attr.path, "--attr", - f"{build_attr.attr + '.' if build_attr.attr else ''}config.system.build.{attr}", + build_attr.to_attr("config", "system", "build", attr), *dict_to_flags(instantiate_flags or {}), ], stdout=PIPE, @@ -332,7 +332,7 @@ def nixos_remote_build_flake( *FLAKE_FLAGS, "eval", "--raw", - f"{flake}.config.system.build.{attr}.drvPath", + flake.to_attr("config", "system", "build", attr, "drvPath"), *dict_to_flags(flake_build_flags or {}), ], stdout=PIPE, diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py index baeb5977b84f..4da2d6ed7356 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_models.py @@ -20,6 +20,16 @@ def test_build_attr_from_arg() -> None: assert m.BuildAttr.from_arg(None, "file.nix") == m.BuildAttr(Path("file.nix"), None) +def test_build_attr_to_attr() -> None: + assert ( + m.BuildAttr("", None).to_attr("attr1", "attr2") == "attr1.attr2" + ) + assert ( + m.BuildAttr("", "preAttr").to_attr("attr1", "attr2") + == "preAttr.attr1.attr2" + ) + + def test_flake_parse() -> None: assert m.Flake.parse("/path/to/flake#attr") == m.Flake( Path("/path/to/flake"), "nixosConfigurations.attr" @@ -35,6 +45,15 @@ def test_flake_parse() -> None: assert m.Flake.parse(".") == m.Flake(Path("."), "nixosConfigurations.default") +def test_flake_to_attr() -> None: + assert ( + m.Flake(Path("/path/to/flake"), "nixosConfigurations.preAttr").to_attr( + "attr1", "attr2" + ) + == "/path/to/flake#nixosConfigurations.preAttr.attr1.attr2" + ) + + @patch(get_qualified_name(platform.node), autospec=True) def test_flake_from_arg(mock_node: Any) -> None: mock_node.return_value = "hostname" From 3cadcd1653e49a2b9a96ebb4a79bef513cfbe248 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 11:36:39 +0000 Subject: [PATCH 31/78] nixos-rebuild-ng: make build functions more generic --- .../src/nixos_rebuild/__init__.py | 34 +- .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 212 +++++------ .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 334 +++++++++--------- 3 files changed, 291 insertions(+), 289 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 0ee3c6db0fd7..ec10db85fcc5 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -194,13 +194,14 @@ def execute(argv: list[str]) -> None: match action: case Action.SWITCH | Action.BOOT: logger.info("building the system configuration...") + attr = "config.system.build.toplevel" if args.rollback: path_to_config = nix.rollback(profile, target_host, sudo=args.sudo) else: if flake: if build_host: - path_to_config = nix.nixos_remote_build_flake( - "toplevel", + path_to_config = nix.remote_build_flake( + attr, flake, build_host, flake_build_flags=flake_build_flags, @@ -208,16 +209,16 @@ def execute(argv: list[str]) -> None: build_flags=build_flags, ) else: - path_to_config = nix.nixos_build_flake( - "toplevel", + path_to_config = nix.build_flake( + attr, flake, no_link=True, **flake_build_flags, ) else: if build_host: - path_to_config = nix.nixos_remote_build( - "toplevel", + path_to_config = nix.remote_build( + attr, build_attr, build_host, instantiate_flags=common_flags, @@ -225,8 +226,8 @@ def execute(argv: list[str]) -> None: build_flags=build_flags, ) else: - path_to_config = nix.nixos_build( - "toplevel", + path_to_config = nix.build( + attr, build_attr, no_out_link=True, **build_flags, @@ -253,6 +254,7 @@ def execute(argv: list[str]) -> None: ) case Action.TEST | Action.BUILD | Action.DRY_BUILD | Action.DRY_ACTIVATE: logger.info("building the system configuration...") + attr = "config.system.build.toplevel" dry_run = action == Action.DRY_BUILD if args.rollback: if action not in (Action.TEST, Action.BUILD): @@ -267,15 +269,15 @@ def execute(argv: list[str]) -> None: else: raise NRError("could not find previous generation") elif flake: - path_to_config = nix.nixos_build_flake( - "toplevel", + path_to_config = nix.build_flake( + attr, flake, dry_run=dry_run, **flake_build_flags, ) else: - path_to_config = nix.nixos_build( - "toplevel", + path_to_config = nix.build( + attr, build_attr, dry_run=dry_run, **build_flags, @@ -292,14 +294,14 @@ def execute(argv: list[str]) -> None: logger.info("building the system configuration...") attr = "vm" if action == Action.BUILD_VM else "vmWithBootLoader" if flake: - path_to_config = nix.nixos_build_flake( - attr, + path_to_config = nix.build_flake( + f"config.system.build.{attr}", flake, **flake_build_flags, ) else: - path_to_config = nix.nixos_build( - attr, + path_to_config = nix.build( + f"config.system.build.{attr}", build_attr, **build_flags, ) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index 7a42645d904b..ccf927a5e59f 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -25,6 +25,112 @@ FLAKE_REPL_TEMPLATE: Final = "repl.template.nix" logger = logging.getLogger(__name__) +def build( + attr: str, + build_attr: BuildAttr, + **build_flags: Args, +) -> Path: + """Build NixOS attribute using classic Nix. + + Returns the built attribute as path. + """ + run_args = [ + "nix-build", + build_attr.path, + "--attr", + build_attr.to_attr(attr), + *dict_to_flags(build_flags), + ] + r = run_wrapper(run_args, stdout=PIPE) + return Path(r.stdout.strip()) + + +def build_flake( + attr: str, + flake: Flake, + **flake_build_flags: Args, +) -> Path: + """Build NixOS attribute using Flakes. + + Returns the built attribute as path. + """ + run_args = [ + "nix", + *FLAKE_FLAGS, + "build", + "--print-out-paths", + flake.to_attr(attr), + *dict_to_flags(flake_build_flags), + ] + r = run_wrapper(run_args, stdout=PIPE) + return Path(r.stdout.strip()) + + +def remote_build( + attr: str, + build_attr: BuildAttr, + build_host: Remote | None, + build_flags: dict[str, Args] | None = None, + instantiate_flags: dict[str, Args] | None = None, + copy_flags: dict[str, Args] | None = None, +) -> Path: + r = run_wrapper( + [ + "nix-instantiate", + "--raw", + build_attr.path, + "--attr", + build_attr.to_attr(attr), + *dict_to_flags(instantiate_flags or {}), + ], + stdout=PIPE, + ) + drv = Path(r.stdout.strip()) + copy_closure(drv, to_host=build_host, from_host=None, **(copy_flags or {})) + r = run_wrapper( + ["nix-store", "--realise", drv, *dict_to_flags(build_flags or {})], + remote=build_host, + stdout=PIPE, + ) + return Path(r.stdout.strip()) + + +def remote_build_flake( + attr: str, + flake: Flake, + build_host: Remote, + flake_build_flags: dict[str, Args] | None = None, + copy_flags: dict[str, Args] | None = None, + build_flags: dict[str, Args] | None = None, +) -> Path: + r = run_wrapper( + [ + "nix", + *FLAKE_FLAGS, + "eval", + "--raw", + flake.to_attr(attr, "drvPath"), + *dict_to_flags(flake_build_flags or {}), + ], + stdout=PIPE, + ) + drv = Path(r.stdout.strip()) + copy_closure(drv, to_host=build_host, from_host=None, **(copy_flags or {})) + r = run_wrapper( + [ + "nix", + *FLAKE_FLAGS, + "build", + f"{drv}^*", + "--print-out-paths", + *dict_to_flags(build_flags or {}), + ], + remote=build_host, + stdout=PIPE, + ) + return Path(r.stdout.strip()) + + def copy_closure( closure: Path, to_host: Remote | None, @@ -248,112 +354,6 @@ def list_generations(profile: Profile) -> list[GenerationJson]: return result -def nixos_build( - attr: str, - build_attr: BuildAttr, - **build_flags: Args, -) -> Path: - """Build NixOS attribute using classic Nix. - - Returns the built attribute as path. - """ - run_args = [ - "nix-build", - build_attr.path, - "--attr", - build_attr.to_attr("config", "system", "build", attr), - *dict_to_flags(build_flags), - ] - r = run_wrapper(run_args, stdout=PIPE) - return Path(r.stdout.strip()) - - -def nixos_build_flake( - attr: str, - flake: Flake, - **flake_build_flags: Args, -) -> Path: - """Build NixOS attribute using Flakes. - - Returns the built attribute as path. - """ - run_args = [ - "nix", - *FLAKE_FLAGS, - "build", - "--print-out-paths", - flake.to_attr("config", "system", "build", attr), - *dict_to_flags(flake_build_flags), - ] - r = run_wrapper(run_args, stdout=PIPE) - return Path(r.stdout.strip()) - - -def nixos_remote_build( - attr: str, - build_attr: BuildAttr, - build_host: Remote | None, - build_flags: dict[str, Args] | None = None, - instantiate_flags: dict[str, Args] | None = None, - copy_flags: dict[str, Args] | None = None, -) -> Path: - r = run_wrapper( - [ - "nix-instantiate", - "--raw", - build_attr.path, - "--attr", - build_attr.to_attr("config", "system", "build", attr), - *dict_to_flags(instantiate_flags or {}), - ], - stdout=PIPE, - ) - drv = Path(r.stdout.strip()) - copy_closure(drv, to_host=build_host, from_host=None, **(copy_flags or {})) - r = run_wrapper( - ["nix-store", "--realise", drv, *dict_to_flags(build_flags or {})], - remote=build_host, - stdout=PIPE, - ) - return Path(r.stdout.strip()) - - -def nixos_remote_build_flake( - attr: str, - flake: Flake, - build_host: Remote, - flake_build_flags: dict[str, Args] | None = None, - copy_flags: dict[str, Args] | None = None, - build_flags: dict[str, Args] | None = None, -) -> Path: - r = run_wrapper( - [ - "nix", - *FLAKE_FLAGS, - "eval", - "--raw", - flake.to_attr("config", "system", "build", attr, "drvPath"), - *dict_to_flags(flake_build_flags or {}), - ], - stdout=PIPE, - ) - drv = Path(r.stdout.strip()) - copy_closure(drv, to_host=build_host, from_host=None, **(copy_flags or {})) - r = run_wrapper( - [ - "nix", - *FLAKE_FLAGS, - "build", - f"{drv}^*", - "--print-out-paths", - *dict_to_flags(build_flags or {}), - ], - remote=build_host, - stdout=PIPE, - ) - return Path(r.stdout.strip()) - - def repl(attr: str, build_attr: BuildAttr, **nix_flags: Args) -> None: run_args = ["nix", "repl", "--file", build_attr.path] if build_attr.attr: diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index dce98cf6b939..317821d5fea0 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -12,6 +12,173 @@ import nixos_rebuild.nix as n from .helpers import get_qualified_name +@patch( + get_qualified_name(n.run_wrapper, n), + autospec=True, + return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), +) +def test_build(mock_run: Any, monkeypatch: Any) -> None: + assert n.build( + "config.system.build.attr", m.BuildAttr("", None), nix_flag="foo" + ) == Path("/path/to/file") + mock_run.assert_called_with( + [ + "nix-build", + "", + "--attr", + "config.system.build.attr", + "--nix-flag", + "foo", + ], + stdout=PIPE, + ) + + assert n.build( + "config.system.build.attr", m.BuildAttr(Path("file"), "preAttr") + ) == Path("/path/to/file") + mock_run.assert_called_with( + ["nix-build", Path("file"), "--attr", "preAttr.config.system.build.attr"], + stdout=PIPE, + ) + + +@patch( + get_qualified_name(n.run_wrapper, n), + autospec=True, + return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), +) +def test_build_flake(mock_run: Any) -> None: + flake = m.Flake.parse(".#hostname") + + assert n.build_flake( + "config.system.build.toplevel", + flake, + no_link=True, + nix_flag="foo", + ) == Path("/path/to/file") + mock_run.assert_called_with( + [ + "nix", + "--extra-experimental-features", + "nix-command flakes", + "build", + "--print-out-paths", + ".#nixosConfigurations.hostname.config.system.build.toplevel", + "--no-link", + "--nix-flag", + "foo", + ], + stdout=PIPE, + ) + + +@patch( + get_qualified_name(n.run_wrapper, n), + autospec=True, + return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), +) +def test_remote_build(mock_run: Any, monkeypatch: Any) -> None: + build_host = m.Remote("user@host", ["--ssh", "opts"], None) + assert n.remote_build( + "config.system.build.toplevel", + m.BuildAttr("", "preAttr"), + build_host, + build_flags={"build": True}, + instantiate_flags={"inst": True}, + copy_flags={"copy": True}, + ) == Path("/path/to/file") + mock_run.assert_has_calls( + [ + call( + [ + "nix-instantiate", + "--raw", + "", + "--attr", + "preAttr.config.system.build.toplevel", + "--inst", + ], + stdout=PIPE, + ), + call( + [ + "nix-copy-closure", + "--copy", + "--to", + "user@host", + Path("/path/to/file"), + ], + extra_env={"NIX_SSHOPTS": "--ssh opts"}, + remote=None, + ), + call( + ["nix-store", "--realise", Path("/path/to/file"), "--build"], + remote=build_host, + stdout=PIPE, + ), + ] + ) + + +@patch( + get_qualified_name(n.run_wrapper, n), + autospec=True, + return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), +) +def test_remote_build_flake(mock_run: Any) -> None: + flake = m.Flake.parse(".#hostname") + build_host = m.Remote("user@host", ["--ssh", "opts"], None) + + assert n.remote_build_flake( + "config.system.build.toplevel", + flake, + build_host, + flake_build_flags={"flake": True}, + copy_flags={"copy": True}, + build_flags={"build": True}, + ) == Path("/path/to/file") + mock_run.assert_has_calls( + [ + call( + [ + "nix", + "--extra-experimental-features", + "nix-command flakes", + "eval", + "--raw", + ".#nixosConfigurations.hostname.config.system.build.toplevel.drvPath", + "--flake", + ], + stdout=PIPE, + ), + call( + [ + "nix-copy-closure", + "--copy", + "--to", + "user@host", + Path("/path/to/file"), + ], + extra_env={"NIX_SSHOPTS": "--ssh opts"}, + remote=None, + ), + call( + [ + "nix", + "--extra-experimental-features", + "nix-command flakes", + "build", + "/path/to/file^*", + "--print-out-paths", + "--build", + ], + remote=build_host, + stdout=PIPE, + ), + ] + ) + + @patch(get_qualified_name(n.run_wrapper, n), autospec=True) def test_copy_closure(mock_run: Any, monkeypatch: Any) -> None: closure = Path("/path/to/closure") @@ -211,173 +378,6 @@ def test_list_generations(mock_get_generations: Any, tmp_path: Path) -> None: ] -@patch( - get_qualified_name(n.run_wrapper, n), - autospec=True, - return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), -) -def test_nixos_build_flake(mock_run: Any) -> None: - flake = m.Flake.parse(".#hostname") - - assert n.nixos_build_flake( - "toplevel", - flake, - no_link=True, - nix_flag="foo", - ) == Path("/path/to/file") - mock_run.assert_called_with( - [ - "nix", - "--extra-experimental-features", - "nix-command flakes", - "build", - "--print-out-paths", - ".#nixosConfigurations.hostname.config.system.build.toplevel", - "--no-link", - "--nix-flag", - "foo", - ], - stdout=PIPE, - ) - - -@patch( - get_qualified_name(n.run_wrapper, n), - autospec=True, - return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), -) -def test_nixos_build(mock_run: Any, monkeypatch: Any) -> None: - assert n.nixos_build( - "attr", m.BuildAttr("", None), nix_flag="foo" - ) == Path("/path/to/file") - mock_run.assert_called_with( - [ - "nix-build", - "", - "--attr", - "config.system.build.attr", - "--nix-flag", - "foo", - ], - stdout=PIPE, - ) - - assert n.nixos_build("attr", m.BuildAttr(Path("file"), "preAttr")) == Path( - "/path/to/file" - ) - mock_run.assert_called_with( - ["nix-build", Path("file"), "--attr", "preAttr.config.system.build.attr"], - stdout=PIPE, - ) - - -@patch( - get_qualified_name(n.run_wrapper, n), - autospec=True, - return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), -) -def test_nixos_remote_build_flake(mock_run: Any) -> None: - flake = m.Flake.parse(".#hostname") - build_host = m.Remote("user@host", ["--ssh", "opts"], None) - - assert n.nixos_remote_build_flake( - "toplevel", - flake, - build_host, - flake_build_flags={"flake": True}, - copy_flags={"copy": True}, - build_flags={"build": True}, - ) == Path("/path/to/file") - mock_run.assert_has_calls( - [ - call( - [ - "nix", - "--extra-experimental-features", - "nix-command flakes", - "eval", - "--raw", - ".#nixosConfigurations.hostname.config.system.build.toplevel.drvPath", - "--flake", - ], - stdout=PIPE, - ), - call( - [ - "nix-copy-closure", - "--copy", - "--to", - "user@host", - Path("/path/to/file"), - ], - extra_env={"NIX_SSHOPTS": "--ssh opts"}, - remote=None, - ), - call( - [ - "nix", - "--extra-experimental-features", - "nix-command flakes", - "build", - "/path/to/file^*", - "--print-out-paths", - "--build", - ], - remote=build_host, - stdout=PIPE, - ), - ] - ) - - -@patch( - get_qualified_name(n.run_wrapper, n), - autospec=True, - return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), -) -def test_nixos_remote_build(mock_run: Any, monkeypatch: Any) -> None: - build_host = m.Remote("user@host", ["--ssh", "opts"], None) - assert n.nixos_remote_build( - "attr", - m.BuildAttr("", "preAttr"), - build_host, - build_flags={"build": True}, - instantiate_flags={"inst": True}, - copy_flags={"copy": True}, - ) == Path("/path/to/file") - mock_run.assert_has_calls( - [ - call( - [ - "nix-instantiate", - "--raw", - "", - "--attr", - "preAttr.config.system.build.attr", - "--inst", - ], - stdout=PIPE, - ), - call( - [ - "nix-copy-closure", - "--copy", - "--to", - "user@host", - Path("/path/to/file"), - ], - extra_env={"NIX_SSHOPTS": "--ssh opts"}, - remote=None, - ), - call( - ["nix-store", "--realise", Path("/path/to/file"), "--build"], - remote=build_host, - stdout=PIPE, - ), - ] - ) - - @patch(get_qualified_name(n.run_wrapper, n), autospec=True) def test_repl(mock_run: Any) -> None: n.repl("attr", m.BuildAttr("", None), nix_flag=True) From 3fd384af80c6be1c7159acbacb4ccefce3b1c1e6 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 12:49:09 +0000 Subject: [PATCH 32/78] nixos-rebuild-ng: implement _NIXOS_REBUILD_REEXEC --- pkgs/by-name/ni/nixos-rebuild-ng/README.md | 10 +------ .../src/nixos_rebuild/__init__.py | 30 ++++++++++++++++--- .../nixos-rebuild-ng/src/tests/test_main.py | 17 +++++------ 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/README.md b/pkgs/by-name/ni/nixos-rebuild-ng/README.md index c52d1c06d920..9a565de2dd18 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/README.md +++ b/pkgs/by-name/ni/nixos-rebuild-ng/README.md @@ -91,14 +91,6 @@ ruff format . conflicting with the current `nixos-rebuild`. This means you can keep both in your system at the same time, but it also means that a few things like bash completion are broken right now (since it looks at `nixos-rebuild` binary) -- `_NIXOS_REBUILD_EXEC` is **not** implemented yet, so different from - `nixos-rebuild`, this will use the current version of `nixos-rebuild-ng` in - your `PATH` to build/set profile/switch, while `nixos-rebuild` builds the new - version (the one that will be switched) and re-exec to it instead. This means - that in case of bugs in `nixos-rebuild-ng`, the only way that you will get - them fixed is **after** you switch to a new version -- Ignore any performance advantages of the rewrite right now, because of the 2 - caveats above - Bugs in the profile manipulation can cause corruption of your profile that may be difficult to fix, so right now I only recommend using `nixos-rebuild-ng` if you are testing in a VM or in a filesystem with @@ -111,7 +103,7 @@ ruff format . - [x] Remote host/builders (via SSH) - [x] Improve nix arguments handling (e.g.: `nixFlags` vs `copyFlags` in the old `nixos-rebuild`) -- [ ] `_NIXOS_REBUILD_EXEC` +- [x] `_NIXOS_REBUILD_REEXEC` - [ ] Port `nixos-rebuild.passthru.tests` - [ ] Change module system to allow easier opt-in, like `system.switch.enableNg` for `switch-to-configuration-ng` diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index ec10db85fcc5..71e630262608 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -2,6 +2,7 @@ import argparse import atexit import json import logging +import os import sys from pathlib import Path from subprocess import run @@ -87,6 +88,7 @@ def get_parser() -> tuple[argparse.ArgumentParser, dict[str, argparse.ArgumentPa main_parser.add_argument("--ask-sudo-password", action="store_true") main_parser.add_argument("--use-remote-sudo", action="store_true") # deprecated main_parser.add_argument("--no-ssh-tty", action="store_true") # deprecated + main_parser.add_argument("--fast", action="store_true") main_parser.add_argument("--build-host") main_parser.add_argument("--target-host") main_parser.add_argument("action", choices=Action.values(), nargs="?") @@ -174,10 +176,6 @@ def execute(argv: list[str]) -> None: target_host = Remote.from_arg(args.target_host, args.ask_sudo_password, tmpdir_path) build_attr = BuildAttr.from_arg(args.attr, args.file) flake = Flake.from_arg(args.flake, target_host) - - if args.upgrade or args.upgrade_all: - nix.upgrade_channels(bool(args.upgrade_all)) - action = Action(args.action) # Only run shell scripts from the Nixpkgs tree if the action is # "switch", "boot", or "test". With other actions (such as "build"), @@ -185,6 +183,30 @@ def execute(argv: list[str]) -> None: # executed, so it's safe to run nixos-rebuild against a potentially # untrusted tree. can_run = action in (Action.SWITCH, Action.BOOT, Action.TEST) + + # Re-exec to a newer version of the script before building to ensure we get + # the latest fixes + if can_run and not args.fast and not os.environ.get("_NIXOS_REBUILD_REEXEC"): + if flake: + drv = nix.build_flake("pkgs.nixos-rebuild-ng", flake, **flake_build_flags) + else: + drv = nix.build("pkgs.nixos-rebuild-ng", build_attr, **build_flags) + new = drv / "bin/nixos-rebuild-ng" + current = Path(argv[0]) + # Disable re-exec during development + if current.name != "__main__.py" and new != current: + logging.debug( + "detected newer version of script, re-exec'ing, current=%s, new=%s", + argv[0], + new, + ) + cleanup_ssh(tmpdir_path) + tmpdir.cleanup() + os.execve(new, argv, os.environ | {"_NIXOS_REBUILD_REEXEC": "1"}) + + if args.upgrade or args.upgrade_all: + nix.upgrade_channels(bool(args.upgrade_all)) + if can_run and not flake: nixpkgs_path = nix.find_file("nixpkgs", **build_flags) rev = nix.get_nixpkgs_rev(nixpkgs_path) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py index b5c93a2dd217..d647f78460b3 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py @@ -95,7 +95,7 @@ def test_execute_nix_boot(mock_run: Any, tmp_path: Path) -> None: CompletedProcess([], 0), ] - nr.execute(["nixos-rebuild", "boot", "--no-flake", "-vvv"]) + nr.execute(["nixos-rebuild", "boot", "--no-flake", "-vvv", "--fast"]) assert mock_run.call_count == 6 mock_run.assert_has_calls( @@ -173,6 +173,7 @@ def test_execute_nix_switch_flake(mock_run: Any, tmp_path: Path) -> None: "--install-bootloader", "--sudo", "--verbose", + "--fast", ] ) @@ -246,6 +247,7 @@ def test_execute_nix_switch_flake_target_host( "--use-remote-sudo", "--target-host", "user@localhost", + "--fast", ] ) @@ -339,6 +341,7 @@ def test_execute_nix_switch_flake_build_host( "/path/to/config#hostname", "--build-host", "user@localhost", + "--fast", ] ) @@ -415,7 +418,9 @@ def test_execute_switch_rollback(mock_run: Any, tmp_path: Path) -> None: nixpkgs_path = tmp_path / "nixpkgs" nixpkgs_path.touch() - nr.execute(["nixos-rebuild", "switch", "--rollback", "--install-bootloader"]) + nr.execute( + ["nixos-rebuild", "switch", "--rollback", "--install-bootloader", "--fast"] + ) assert mock_run.call_count >= 2 # ignoring update_nixpkgs_rev calls @@ -467,13 +472,7 @@ def test_execute_test_rollback( ] nr.execute( - [ - "nixos-rebuild", - "test", - "--rollback", - "--profile-name", - "foo", - ] + ["nixos-rebuild", "test", "--rollback", "--profile-name", "foo", "--fast"] ) assert mock_run.call_count == 2 From 776c21be0fc89962e04a0b44dd65ec2ef6d5adc5 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 13:06:51 +0000 Subject: [PATCH 33/78] nixos-rebuild-ng: fix cleanup_ssh --- .../src/nixos_rebuild/process.py | 4 +++- .../src/tests/test_process.py | 24 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index 6fa5be980c16..1c118d48b850 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -69,7 +69,9 @@ class RunKwargs(TypedDict, total=False): def cleanup_ssh(tmp_dir: Path) -> None: "Close SSH ControlMaster connection." for ctrl in tmp_dir.glob("ssh-*"): - subprocess.run(["ssh", "-o", f"ControlPath={ctrl}", "exit"], check=False) + subprocess.run( + ["ssh", "-o", f"ControlPath={ctrl}", "-O", "exit", "dummyhost"], check=False + ) def run_wrapper( diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py index 57d3865bcef4..998ad5dedc83 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py @@ -1,6 +1,6 @@ from pathlib import Path from typing import Any -from unittest.mock import patch +from unittest.mock import call, patch import nixos_rebuild.models as m import nixos_rebuild.process as p @@ -8,6 +8,28 @@ import nixos_rebuild.process as p from .helpers import get_qualified_name +@patch(get_qualified_name(p.subprocess.run)) +def test_cleanup_ssh(mock_run: Any, tmp_path: Path) -> None: + (tmp_path / "ssh-conn").touch() + + p.cleanup_ssh(tmp_path) + mock_run.assert_has_calls( + [ + call( + [ + "ssh", + "-o", + f"ControlPath={tmp_path}/ssh-conn", + "-O", + "exit", + "dummyhost", + ], + check=False, + ) + ] + ) + + @patch(get_qualified_name(p.subprocess.run)) def test_run(mock_run: Any) -> None: p.run_wrapper(["test", "--with", "flags"], check=True) From fed6778da3a41b30973b054ada0daf9d2b1d334a Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 20:04:38 +0000 Subject: [PATCH 34/78] nixos-rebuild-ng: move temporary directory to process --- .../src/nixos_rebuild/__init__.py | 15 ++-- .../src/nixos_rebuild/process.py | 28 ++++---- .../nixos-rebuild-ng/src/tests/test_main.py | 31 ++------- .../src/tests/test_process.py | 68 +++++-------------- 4 files changed, 45 insertions(+), 97 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 71e630262608..f0bbabdb66ab 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -6,7 +6,6 @@ import os import sys from pathlib import Path from subprocess import run -from tempfile import TemporaryDirectory from typing import assert_never from . import nix @@ -164,16 +163,11 @@ def execute(argv: list[str]) -> None: flake_build_flags = common_build_flags | vars(args_groups["flake_build_flags"]) copy_flags = common_flags | vars(args_groups["copy_flags"]) - # Will be cleaned up on exit automatically. - tmpdir = TemporaryDirectory(prefix="nixos-rebuild.") - tmpdir_path = Path(tmpdir.name) - atexit.register(cleanup_ssh, tmpdir_path) + atexit.register(cleanup_ssh) profile = Profile.from_arg(args.profile_name) - build_host = Remote.from_arg( - args.build_host, False, tmpdir_path, validate_opts=False - ) - target_host = Remote.from_arg(args.target_host, args.ask_sudo_password, tmpdir_path) + build_host = Remote.from_arg(args.build_host, False, validate_opts=False) + target_host = Remote.from_arg(args.target_host, args.ask_sudo_password) build_attr = BuildAttr.from_arg(args.attr, args.file) flake = Flake.from_arg(args.flake, target_host) action = Action(args.action) @@ -200,8 +194,7 @@ def execute(argv: list[str]) -> None: argv[0], new, ) - cleanup_ssh(tmpdir_path) - tmpdir.cleanup() + cleanup_ssh() os.execve(new, argv, os.environ | {"_NIXOS_REBUILD_REEXEC": "1"}) if args.upgrade or args.upgrade_all: diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index 1c118d48b850..54200f14d85a 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -5,10 +5,22 @@ import subprocess from dataclasses import dataclass from getpass import getpass from pathlib import Path +from tempfile import TemporaryDirectory from typing import Self, Sequence, TypedDict, Unpack logger = logging.getLogger(__name__) +TMPDIR = TemporaryDirectory(prefix="nixos-rebuild.") +TMPDIR_PATH = Path(TMPDIR.name) +SSH_DEFAULT_OPTS = [ + "-o", + "ControlMaster=auto", + "-o", + f"ControlPath={TMPDIR_PATH / "ssh-%n"}", + "-o", + "ControlPersist=60", +] + @dataclass(frozen=True) class Remote: @@ -21,7 +33,6 @@ class Remote: cls, host: str | None, ask_sudo_password: bool | None, - tmp_dir: Path, validate_opts: bool = True, ) -> Self | None: if not host: @@ -30,15 +41,6 @@ class Remote: opts = os.getenv("NIX_SSHOPTS", "").split() if validate_opts: cls._validate_opts(opts, ask_sudo_password) - opts += [ - # SSH ControlMaster flags, allow for faster re-connection - "-o", - "ControlMaster=auto", - "-o", - f"ControlPath={tmp_dir / "ssh-%n"}", - "-o", - "ControlPersist=60", - ] sudo_password = None if ask_sudo_password: sudo_password = getpass(f"[sudo] password for {host}: ") @@ -66,12 +68,13 @@ class RunKwargs(TypedDict, total=False): stdout: int | None -def cleanup_ssh(tmp_dir: Path) -> None: +def cleanup_ssh() -> None: "Close SSH ControlMaster connection." - for ctrl in tmp_dir.glob("ssh-*"): + for ctrl in TMPDIR_PATH.glob("ssh-*"): subprocess.run( ["ssh", "-o", f"ControlPath={ctrl}", "-O", "exit", "dummyhost"], check=False ) + TMPDIR.cleanup() def run_wrapper( @@ -99,6 +102,7 @@ def run_wrapper( args = [ "ssh", *remote.opts, + *SSH_DEFAULT_OPTS, remote.host, "--", # sadly SSH just join all remaining parameters, expanding glob and diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py index d647f78460b3..97ae0072a829 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py @@ -218,9 +218,9 @@ def test_execute_nix_switch_flake(mock_run: Any, tmp_path: Path) -> None: @patch.dict(nr.process.os.environ, {}, clear=True) @patch(get_qualified_name(nr.process.subprocess.run), autospec=True) -@patch(get_qualified_name(nr.TemporaryDirectory, nr)) # can't autospec +@patch(get_qualified_name(nr.cleanup_ssh, nr), autospec=True) def test_execute_nix_switch_flake_target_host( - mock_tmpdir: Any, + mock_cleanup_ssh: Any, mock_run: Any, tmp_path: Path, ) -> None: @@ -236,7 +236,6 @@ def test_execute_nix_switch_flake_target_host( # switch_to_configuration CompletedProcess([], 0), ] - mock_tmpdir.return_value.name = "/tmp/test" nr.execute( [ @@ -276,12 +275,7 @@ def test_execute_nix_switch_flake_target_host( call( [ "ssh", - "-o", - "ControlMaster=auto", - "-o", - "ControlPath=/tmp/test/ssh-%n", - "-o", - "ControlPersist=60", + *nr.process.SSH_DEFAULT_OPTS, "user@localhost", "--", f"sudo nix-env -p /nix/var/nix/profiles/system --set {config_path}", @@ -292,12 +286,7 @@ def test_execute_nix_switch_flake_target_host( call( [ "ssh", - "-o", - "ControlMaster=auto", - "-o", - "ControlPath=/tmp/test/ssh-%n", - "-o", - "ControlPersist=60", + *nr.process.SSH_DEFAULT_OPTS, "user@localhost", "--", f"sudo env NIXOS_INSTALL_BOOTLOADER=0 {config_path / 'bin/switch-to-configuration'} switch", @@ -311,9 +300,9 @@ def test_execute_nix_switch_flake_target_host( @patch.dict(nr.process.os.environ, {}, clear=True) @patch(get_qualified_name(nr.process.subprocess.run), autospec=True) -@patch(get_qualified_name(nr.TemporaryDirectory, nr)) # can't autospec +@patch(get_qualified_name(nr.cleanup_ssh, nr), autospec=True) def test_execute_nix_switch_flake_build_host( - mock_tmpdir: Any, + mock_cleanup_ssh: Any, mock_run: Any, tmp_path: Path, ) -> None: @@ -331,7 +320,6 @@ def test_execute_nix_switch_flake_build_host( # switch_to_configuration CompletedProcess([], 0), ] - mock_tmpdir.return_value.name = "/tmp/test" nr.execute( [ @@ -369,12 +357,7 @@ def test_execute_nix_switch_flake_build_host( call( [ "ssh", - "-o", - "ControlMaster=auto", - "-o", - "ControlPath=/tmp/test/ssh-%n", - "-o", - "ControlPersist=60", + *nr.process.SSH_DEFAULT_OPTS, "user@localhost", "--", f"nix --extra-experimental-features 'nix-command flakes' build '{config_path}^*' --print-out-paths", diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py index 998ad5dedc83..53ed6824e39f 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py @@ -1,6 +1,5 @@ -from pathlib import Path from typing import Any -from unittest.mock import call, patch +from unittest.mock import patch import nixos_rebuild.models as m import nixos_rebuild.process as p @@ -8,29 +7,7 @@ import nixos_rebuild.process as p from .helpers import get_qualified_name -@patch(get_qualified_name(p.subprocess.run)) -def test_cleanup_ssh(mock_run: Any, tmp_path: Path) -> None: - (tmp_path / "ssh-conn").touch() - - p.cleanup_ssh(tmp_path) - mock_run.assert_has_calls( - [ - call( - [ - "ssh", - "-o", - f"ControlPath={tmp_path}/ssh-conn", - "-O", - "exit", - "dummyhost", - ], - check=False, - ) - ] - ) - - -@patch(get_qualified_name(p.subprocess.run)) +@patch(get_qualified_name(p.subprocess.run), autospec=True) def test_run(mock_run: Any) -> None: p.run_wrapper(["test", "--with", "flags"], check=True) mock_run.assert_called_with( @@ -67,7 +44,15 @@ def test_run(mock_run: Any) -> None: remote=m.Remote("user@localhost", ["--ssh", "opt"], "password"), ) mock_run.assert_called_with( - ["ssh", "--ssh", "opt", "user@localhost", "--", "test --with 'some flags'"], + [ + "ssh", + "--ssh", + "opt", + *p.SSH_DEFAULT_OPTS, + "user@localhost", + "--", + "test --with 'some flags'", + ], check=True, text=True, errors="surrogateescape", @@ -87,6 +72,7 @@ def test_run(mock_run: Any) -> None: "ssh", "--ssh", "opt", + *p.SSH_DEFAULT_OPTS, "user@localhost", "--", "sudo --prompt= --stdin env FOO=bar test --with flags", @@ -99,38 +85,20 @@ def test_run(mock_run: Any) -> None: ) -def test_remote_from_name(monkeypatch: Any, tmpdir: Path) -> None: +def test_remote_from_name(monkeypatch: Any) -> None: monkeypatch.setenv("NIX_SSHOPTS", "") - assert m.Remote.from_arg("user@localhost", None, tmpdir) == m.Remote( + assert m.Remote.from_arg("user@localhost", None, False) == m.Remote( "user@localhost", - opts=[ - "-o", - "ControlMaster=auto", - "-o", - f"ControlPath={tmpdir / "ssh-%n"}", - "-o", - "ControlPersist=60", - ], + opts=[], sudo_password=None, ) # get_qualified_name doesn't work because getpass is aliased to another # function with patch(f"{p.__name__}.getpass", return_value="password"): - monkeypatch.setenv("NIX_SSHOPTS", "-f foo -b bar") - assert m.Remote.from_arg("user@localhost", True, tmpdir) == m.Remote( + monkeypatch.setenv("NIX_SSHOPTS", "-f foo -b bar -t") + assert m.Remote.from_arg("user@localhost", True, True) == m.Remote( "user@localhost", - opts=[ - "-f", - "foo", - "-b", - "bar", - "-o", - "ControlMaster=auto", - "-o", - f"ControlPath={tmpdir / "ssh-%n"}", - "-o", - "ControlPersist=60", - ], + opts=["-f", "foo", "-b", "bar", "-t"], sudo_password="password", ) From 6c3ba91ce46f66c455aa895e8e249bf5ee2a7a3f Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 20:18:42 +0000 Subject: [PATCH 35/78] nixos-rebuild-ng: rename template file to not trigger CI --- pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py | 2 +- .../src/nixos_rebuild/{repl.template.nix => repl.nix.template} | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) rename pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/{repl.template.nix => repl.nix.template} (98%) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index ccf927a5e59f..5247a2991b1b 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -21,7 +21,7 @@ from .process import run_wrapper from .utils import Args, dict_to_flags FLAKE_FLAGS: Final = ["--extra-experimental-features", "nix-command flakes"] -FLAKE_REPL_TEMPLATE: Final = "repl.template.nix" +FLAKE_REPL_TEMPLATE: Final = "repl.nix.template" logger = logging.getLogger(__name__) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/repl.template.nix b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/repl.nix.template similarity index 98% rename from pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/repl.template.nix rename to pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/repl.nix.template index 0fffd6ecc746..17db87423711 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/repl.template.nix +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/repl.nix.template @@ -1,3 +1,4 @@ +# vim: set syntax=nix: let flake = builtins.getFlake ''${flake_path}''; configuration = flake.${flake_attr}; From bebec2668b43941ec833c81c57e519f328c57862 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 20:29:26 +0000 Subject: [PATCH 36/78] nixos-rebuild-ng: add missing flags from nixos-rebuild-ng --- .../ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index f0bbabdb66ab..e7124f93001b 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -53,7 +53,9 @@ def get_parser() -> tuple[argparse.ArgumentParser, dict[str, argparse.ArgumentPa classic_build_flags.add_argument("--no-build-output", "-Q", action="store_true") copy_flags = argparse.ArgumentParser(add_help=False) - copy_flags.add_argument("--use-substitutes", "-s", action="store_true") + copy_flags.add_argument( + "--use-substitutes", "--substitute-on-destination", "-s", action="store_true" + ) sub_parsers = { "common_flags": common_flags, @@ -90,6 +92,7 @@ def get_parser() -> tuple[argparse.ArgumentParser, dict[str, argparse.ArgumentPa main_parser.add_argument("--fast", action="store_true") main_parser.add_argument("--build-host") main_parser.add_argument("--target-host") + main_parser.add_argument("--no-build-nix", action="store_true") # deprecated main_parser.add_argument("action", choices=Action.values(), nargs="?") return main_parser, sub_parsers @@ -133,6 +136,10 @@ def parse_args( if args.no_ssh_tty: parser_warn("--no-ssh-tty deprecated, SSH's TTY is never used anymore") + # TODO: use deprecated=True in Python >=3.13 + if args.no_build_nix: + parser_warn("--no-build-nix deprecated, we do not build nix anymore") + if args.action == Action.EDIT.value and (args.file or args.attr): parser.error("--file and --attr are not supported with 'edit'") From cfe42fba1c5c0f8dde09946d122ee02f4d67f997 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 20:49:44 +0000 Subject: [PATCH 37/78] nixos-rebuild-ng: do not fail if re-exec fails --- .../src/nixos_rebuild/__init__.py | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index e7124f93001b..d51a3c273c33 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -5,7 +5,7 @@ import logging import os import sys from pathlib import Path -from subprocess import run +from subprocess import CalledProcessError, run from typing import assert_never from . import nix @@ -185,27 +185,32 @@ def execute(argv: list[str]) -> None: # untrusted tree. can_run = action in (Action.SWITCH, Action.BOOT, Action.TEST) + if args.upgrade or args.upgrade_all: + nix.upgrade_channels(bool(args.upgrade_all)) + # Re-exec to a newer version of the script before building to ensure we get # the latest fixes if can_run and not args.fast and not os.environ.get("_NIXOS_REBUILD_REEXEC"): - if flake: - drv = nix.build_flake("pkgs.nixos-rebuild-ng", flake, **flake_build_flags) - else: - drv = nix.build("pkgs.nixos-rebuild-ng", build_attr, **build_flags) - new = drv / "bin/nixos-rebuild-ng" - current = Path(argv[0]) - # Disable re-exec during development - if current.name != "__main__.py" and new != current: - logging.debug( - "detected newer version of script, re-exec'ing, current=%s, new=%s", - argv[0], - new, - ) - cleanup_ssh() - os.execve(new, argv, os.environ | {"_NIXOS_REBUILD_REEXEC": "1"}) - - if args.upgrade or args.upgrade_all: - nix.upgrade_channels(bool(args.upgrade_all)) + try: + if flake: + drv = nix.build_flake( + "pkgs.nixos-rebuild-ng", flake, **flake_build_flags + ) + else: + drv = nix.build("pkgs.nixos-rebuild-ng", build_attr, **build_flags) + new = drv / "bin/nixos-rebuild-ng" + current = Path(argv[0]) + # Disable re-exec during development + if current.name != "__main__.py" and new != current: + logging.debug( + "detected newer version of script, re-exec'ing, current=%s, new=%s", + argv[0], + new, + ) + cleanup_ssh() + os.execve(new, argv, os.environ | {"_NIXOS_REBUILD_REEXEC": "1"}) + except CalledProcessError: + logger.warning("could not find a newer version of nixos-rebuild") if can_run and not flake: nixpkgs_path = nix.find_file("nixpkgs", **build_flags) From f72572c1470019513b4bf5a5d31ee8640dd5b99e Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 20:57:28 +0000 Subject: [PATCH 38/78] nixos-rebuild-ng: ignore non-directories in upgrade_channels --- pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py | 2 +- pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index 5247a2991b1b..7cc606561404 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -471,7 +471,7 @@ def upgrade_channels(all: bool = False) -> None: that has a `.update-on-nixos-rebuild` file) or all. """ for channel_path in Path("/nix/var/nix/profiles/per-user/root/channels/").glob("*"): - if ( + if channel_path.is_dir() and ( all or channel_path.name == "nixos" or (channel_path / ".update-on-nixos-rebuild").exists() diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index 317821d5fea0..093cba93ef36 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -561,7 +561,8 @@ def test_switch_to_configuration(mock_run: Any, monkeypatch: Any) -> None: Path("/nix/var/nix/profiles/per-user/root/channels/home-manager"), ], ) -def test_upgrade_channels(mock_glob: Any) -> None: +@patch(get_qualified_name(n.Path.is_dir, n), autospec=True, return_value=True) +def test_upgrade_channels(mock_is_dir: Any, mock_glob: Any) -> None: with patch(get_qualified_name(n.run_wrapper, n), autospec=True) as mock_run: n.upgrade_channels(False) mock_run.assert_called_with(["nix-channel", "--update", "nixos"], check=False) From c50144ab79b9a5cbeea195f032be2f8bb5a73147 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Fri, 29 Nov 2024 23:42:34 +0000 Subject: [PATCH 39/78] nixos-rebuild-ng: move reexec earlier --- .../src/nixos_rebuild/__init__.py | 82 ++++++++++++------- 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index d51a3c273c33..a6153ae974f8 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -11,7 +11,7 @@ from typing import assert_never from . import nix from .models import Action, BuildAttr, Flake, NRError, Profile from .process import Remote, cleanup_ssh -from .utils import LogFormatter +from .utils import Args, LogFormatter logger = logging.getLogger() logger.setLevel(logging.INFO) @@ -161,22 +161,60 @@ def parse_args( return args, args_groups +def reexec( + argv: list[str], + args: argparse.Namespace, + build_flags: dict[str, Args], + flake_build_flags: dict[str, Args], +) -> None: + drv = None + try: + # Need to set target_host=None, to avoid connecting to remote + if flake := Flake.from_arg(args.flake, None): + drv = nix.build_flake( + "pkgs.nixos-rebuild-ng", + flake, + **flake_build_flags, + no_link=True, + ) + else: + drv = nix.build( + "pkgs.nixos-rebuild-ng", + BuildAttr.from_arg(args.attr, args.file), + **build_flags, + no_out_link=True, + ) + except CalledProcessError: + logger.warning("could not find a newer version of nixos-rebuild") + + if drv: + new = drv / "bin/nixos-rebuild-ng" + current = Path(argv[0]) + # Disable re-exec during development + if current.name != "__main__.py" and new != current: + logging.debug( + "detected newer version of script, re-exec'ing, current=%s, new=%s", + argv[0], + new, + ) + cleanup_ssh() + os.execve(new, argv, os.environ | {"_NIXOS_REBUILD_REEXEC": "1"}) + + def execute(argv: list[str]) -> None: args, args_groups = parse_args(argv) + atexit.register(cleanup_ssh) + common_flags = vars(args_groups["common_flags"]) common_build_flags = common_flags | vars(args_groups["common_build_flags"]) build_flags = common_build_flags | vars(args_groups["classic_build_flags"]) flake_build_flags = common_build_flags | vars(args_groups["flake_build_flags"]) copy_flags = common_flags | vars(args_groups["copy_flags"]) - atexit.register(cleanup_ssh) + if args.upgrade or args.upgrade_all: + nix.upgrade_channels(bool(args.upgrade_all)) - profile = Profile.from_arg(args.profile_name) - build_host = Remote.from_arg(args.build_host, False, validate_opts=False) - target_host = Remote.from_arg(args.target_host, args.ask_sudo_password) - build_attr = BuildAttr.from_arg(args.attr, args.file) - flake = Flake.from_arg(args.flake, target_host) action = Action(args.action) # Only run shell scripts from the Nixpkgs tree if the action is # "switch", "boot", or "test". With other actions (such as "build"), @@ -185,32 +223,16 @@ def execute(argv: list[str]) -> None: # untrusted tree. can_run = action in (Action.SWITCH, Action.BOOT, Action.TEST) - if args.upgrade or args.upgrade_all: - nix.upgrade_channels(bool(args.upgrade_all)) - # Re-exec to a newer version of the script before building to ensure we get # the latest fixes if can_run and not args.fast and not os.environ.get("_NIXOS_REBUILD_REEXEC"): - try: - if flake: - drv = nix.build_flake( - "pkgs.nixos-rebuild-ng", flake, **flake_build_flags - ) - else: - drv = nix.build("pkgs.nixos-rebuild-ng", build_attr, **build_flags) - new = drv / "bin/nixos-rebuild-ng" - current = Path(argv[0]) - # Disable re-exec during development - if current.name != "__main__.py" and new != current: - logging.debug( - "detected newer version of script, re-exec'ing, current=%s, new=%s", - argv[0], - new, - ) - cleanup_ssh() - os.execve(new, argv, os.environ | {"_NIXOS_REBUILD_REEXEC": "1"}) - except CalledProcessError: - logger.warning("could not find a newer version of nixos-rebuild") + reexec(argv, args, build_flags, flake_build_flags) + + profile = Profile.from_arg(args.profile_name) + target_host = Remote.from_arg(args.target_host, args.ask_sudo_password) + build_host = Remote.from_arg(args.build_host, False, validate_opts=False) + build_attr = BuildAttr.from_arg(args.attr, args.file) + flake = Flake.from_arg(args.flake, target_host) if can_run and not flake: nixpkgs_path = nix.find_file("nixpkgs", **build_flags) From b118371ebb1fe22a0723f72832b9062dc1597deb Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 30 Nov 2024 17:45:30 +0000 Subject: [PATCH 40/78] nixos-rebuild-ng: add SSH_DEFAULT_OPTS to copy-closure --- .../ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py | 14 ++++++++------ .../src/nixos_rebuild/process.py | 2 +- .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 16 ++++++++++------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index 7cc606561404..e4e1c5663fcb 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -1,5 +1,6 @@ import logging import os +import shlex from datetime import datetime from importlib.resources import files from pathlib import Path @@ -17,7 +18,7 @@ from .models import ( Profile, Remote, ) -from .process import run_wrapper +from .process import SSH_DEFAULT_OPTS, run_wrapper from .utils import Args, dict_to_flags FLAKE_FLAGS: Final = ["--extra-experimental-features", "nix-command flakes"] @@ -153,12 +154,13 @@ def copy_closure( closure, ], extra_env={ - # for the remote to remote case, we can't use host.opts because it - # includes the ControlPane opts that will not work in the remote, - # because the temporary directory that we created will not exist - "NIX_SSHOPTS": os.environ.get("NIX_SSHOPTS", "") + # for the remote to remote case, we can't add SSH_DEFAULT_OPTS to + # host.opts because it includes the ControlPane opts that will not + # work in the remote, because the temporary directory that we + # created will not exist + "NIX_SSHOPTS": shlex.join(host.opts) if from_host and to_host - else " ".join(host.opts) + else shlex.join(SSH_DEFAULT_OPTS + host.opts) }, remote=from_host if to_host else None, ) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index 54200f14d85a..a9e8d7dff295 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -38,7 +38,7 @@ class Remote: if not host: return None - opts = os.getenv("NIX_SSHOPTS", "").split() + opts = shlex.split(os.getenv("NIX_SSHOPTS", "")) if validate_opts: cls._validate_opts(opts, ask_sudo_password) sudo_password = None diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index 093cba93ef36..0a98270490f9 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -8,6 +8,7 @@ import pytest import nixos_rebuild.models as m import nixos_rebuild.nix as n +import nixos_rebuild.process as p from .helpers import get_qualified_name @@ -108,7 +109,9 @@ def test_remote_build(mock_run: Any, monkeypatch: Any) -> None: "user@host", Path("/path/to/file"), ], - extra_env={"NIX_SSHOPTS": "--ssh opts"}, + extra_env={ + "NIX_SSHOPTS": " ".join(p.SSH_DEFAULT_OPTS + ["--ssh opts"]) + }, remote=None, ), call( @@ -159,7 +162,9 @@ def test_remote_build_flake(mock_run: Any) -> None: "user@host", Path("/path/to/file"), ], - extra_env={"NIX_SSHOPTS": "--ssh opts"}, + extra_env={ + "NIX_SSHOPTS": " ".join(p.SSH_DEFAULT_OPTS + ["--ssh opts"]) + }, remote=None, ), call( @@ -191,23 +196,22 @@ def test_copy_closure(mock_run: Any, monkeypatch: Any) -> None: n.copy_closure(closure, target_host) mock_run.assert_called_with( ["nix-copy-closure", "--to", "user@target.host", closure], - extra_env={"NIX_SSHOPTS": "--ssh target-opt"}, + extra_env={"NIX_SSHOPTS": " ".join(p.SSH_DEFAULT_OPTS + ["--ssh target-opt"])}, remote=None, ) n.copy_closure(closure, None, build_host) mock_run.assert_called_with( ["nix-copy-closure", "--from", "user@build.host", closure], - extra_env={"NIX_SSHOPTS": "--ssh build-opt"}, + extra_env={"NIX_SSHOPTS": " ".join(p.SSH_DEFAULT_OPTS + ["--ssh build-opt"])}, remote=None, ) - monkeypatch.setenv("NIX_SSHOPTS", "--ssh build-target-opt") n.copy_closure(closure, target_host, build_host) mock_run.assert_called_with( ["nix-copy-closure", "--to", "user@target.host", closure], remote=build_host, - extra_env={"NIX_SSHOPTS": "--ssh build-target-opt"}, + extra_env={"NIX_SSHOPTS": "--ssh target-opt"}, ) From cab7882bf5215adc8b0187bcbd1c9454f68e493a Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 30 Nov 2024 17:46:23 +0000 Subject: [PATCH 41/78] nixos-rebuild-ng: move check for missing action argument earlier --- .../ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index a6153ae974f8..92394197110a 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -122,6 +122,10 @@ def parse_args( if args.ask_sudo_password: args.sudo = True + if args.help or args.action is None: + r = run(["man", "8", "nixos-rebuild"], check=False) + parser.exit(r.returncode) + # TODO: use deprecated=True in Python >=3.13 if args.install_grub: parser_warn("--install-grub deprecated, use --install-bootloader instead") @@ -154,10 +158,6 @@ def parse_args( if args.flake and (args.file or args.attr): parser.error("--flake cannot be used with --file or --attr") - if args.help or args.action is None: - r = run(["man", "8", "nixos-rebuild"], check=False) - parser.exit(r.returncode) - return args, args_groups From a61aede3688d97cd8dd47f311c7e898cc006d8f7 Mon Sep 17 00:00:00 2001 From: r-vdp Date: Sat, 30 Nov 2024 17:57:06 +0100 Subject: [PATCH 42/78] nixos-rebuild-ng: make sure to copy the new closure when doing test or build --- pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 92394197110a..3371424ea214 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -331,6 +331,9 @@ def execute(argv: list[str]) -> None: dry_run=dry_run, **build_flags, ) + # If we are rolling back, the generation that we roll back to, should already be present on the remote + if not args.rollback: + nix.copy_closure(path_to_config, target_host, **copy_flags) if action in (Action.TEST, Action.DRY_ACTIVATE): nix.switch_to_configuration( path_to_config, From 4431a47fcb4a2a97cc40e5e5e931a1d06fd2d08f Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 30 Nov 2024 18:04:31 +0000 Subject: [PATCH 43/78] nixos-rebuild-ng: add --build-host/--target-host to TEST/BUILD/DRY_BUILD/DRY_ACTIVATE --- .../src/nixos_rebuild/__init__.py | 60 ++++++++++++++----- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 3371424ea214..25f6889314ad 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -150,6 +150,10 @@ def parse_args( if (args.target_host or args.build_host) and args.action not in ( Action.SWITCH.value, Action.BOOT.value, + Action.TEST.value, + Action.BUILD.value, + Action.DRY_BUILD.value, + Action.DRY_ACTIVATE.value, ): parser.error( f"--target-host/--build-host is not supported with '{args.action}'" @@ -318,27 +322,53 @@ def execute(argv: list[str]) -> None: else: raise NRError("could not find previous generation") elif flake: - path_to_config = nix.build_flake( - attr, - flake, - dry_run=dry_run, - **flake_build_flags, - ) + if build_host: + path_to_config = nix.remote_build_flake( + attr, + flake, + build_host, + flake_build_flags=flake_build_flags, + copy_flags=copy_flags, + build_flags=build_flags, + ) + else: + path_to_config = nix.build_flake( + attr, + flake, + dry_run=dry_run, + **flake_build_flags, + ) else: - path_to_config = nix.build( - attr, - build_attr, - dry_run=dry_run, - **build_flags, - ) - # If we are rolling back, the generation that we roll back to, should already be present on the remote + if build_host: + path_to_config = nix.remote_build( + attr, + build_attr, + build_host, + instantiate_flags=common_flags, + copy_flags=copy_flags, + build_flags=build_flags, + ) + else: + path_to_config = nix.build( + attr, + build_attr, + dry_run=dry_run, + **build_flags, + ) + # If we are rolling back, the generation that we roll back to, + # should already be present on the remote if not args.rollback: - nix.copy_closure(path_to_config, target_host, **copy_flags) + nix.copy_closure( + path_to_config, + to_host=target_host, + from_host=build_host, + **copy_flags, + ) if action in (Action.TEST, Action.DRY_ACTIVATE): nix.switch_to_configuration( path_to_config, action, - target_host=None, + target_host=target_host, sudo=args.sudo, specialisation=args.specialisation, ) From 4bc3ac552d0dc20d80333f329bac840ce0445a11 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 30 Nov 2024 18:28:36 +0000 Subject: [PATCH 44/78] nixos-rebuild-ng: merge actions --- .../src/nixos_rebuild/__init__.py | 104 ++++++------------ 1 file changed, 32 insertions(+), 72 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 25f6889314ad..14236c289709 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -245,11 +245,35 @@ def execute(argv: list[str]) -> None: (nixpkgs_path / ".version-suffix").write_text(rev) match action: - case Action.SWITCH | Action.BOOT: + case ( + Action.SWITCH + | Action.BOOT + | Action.TEST + | Action.BUILD + | Action.DRY_BUILD + | Action.DRY_ACTIVATE + ): logger.info("building the system configuration...") + attr = "config.system.build.toplevel" + dry_run = action == Action.DRY_BUILD + no_link = action in (Action.SWITCH, Action.BOOT) + if args.rollback: - path_to_config = nix.rollback(profile, target_host, sudo=args.sudo) + if action in (Action.SWITCH, Action.BOOT): + path_to_config = nix.rollback(profile, target_host, sudo=args.sudo) + elif action in (Action.TEST, Action.BUILD): + maybe_path_to_config = nix.rollback_temporary_profile( + profile, + target_host, + sudo=args.sudo, + ) + if maybe_path_to_config: # kinda silly but this makes mypy happy + path_to_config = maybe_path_to_config + else: + raise NRError("could not find previous generation") + else: + raise NRError(f"--rollback is incompatible with '{action}'") else: if flake: if build_host: @@ -265,7 +289,8 @@ def execute(argv: list[str]) -> None: path_to_config = nix.build_flake( attr, flake, - no_link=True, + no_link=no_link, + dry_run=dry_run, **flake_build_flags, ) else: @@ -282,7 +307,8 @@ def execute(argv: list[str]) -> None: path_to_config = nix.build( attr, build_attr, - no_out_link=True, + no_out_link=no_link, + dry_run=dry_run, **build_flags, ) nix.copy_closure( @@ -297,80 +323,14 @@ def execute(argv: list[str]) -> None: target_host=target_host, sudo=args.sudo, ) - nix.switch_to_configuration( - path_to_config, - action, - target_host=target_host, - sudo=args.sudo, - specialisation=args.specialisation, - install_bootloader=args.install_bootloader, - ) - case Action.TEST | Action.BUILD | Action.DRY_BUILD | Action.DRY_ACTIVATE: - logger.info("building the system configuration...") - attr = "config.system.build.toplevel" - dry_run = action == Action.DRY_BUILD - if args.rollback: - if action not in (Action.TEST, Action.BUILD): - raise NRError(f"--rollback is incompatible with '{action}'") - maybe_path_to_config = nix.rollback_temporary_profile( - profile, - target_host, - sudo=args.sudo, - ) - if maybe_path_to_config: # kinda silly but this makes mypy happy - path_to_config = maybe_path_to_config - else: - raise NRError("could not find previous generation") - elif flake: - if build_host: - path_to_config = nix.remote_build_flake( - attr, - flake, - build_host, - flake_build_flags=flake_build_flags, - copy_flags=copy_flags, - build_flags=build_flags, - ) - else: - path_to_config = nix.build_flake( - attr, - flake, - dry_run=dry_run, - **flake_build_flags, - ) - else: - if build_host: - path_to_config = nix.remote_build( - attr, - build_attr, - build_host, - instantiate_flags=common_flags, - copy_flags=copy_flags, - build_flags=build_flags, - ) - else: - path_to_config = nix.build( - attr, - build_attr, - dry_run=dry_run, - **build_flags, - ) - # If we are rolling back, the generation that we roll back to, - # should already be present on the remote - if not args.rollback: - nix.copy_closure( - path_to_config, - to_host=target_host, - from_host=build_host, - **copy_flags, - ) - if action in (Action.TEST, Action.DRY_ACTIVATE): + if action in (Action.SWITCH, Action.BOOT, Action.TEST, Action.DRY_ACTIVATE): nix.switch_to_configuration( path_to_config, action, target_host=target_host, sudo=args.sudo, specialisation=args.specialisation, + install_bootloader=args.install_bootloader, ) case Action.BUILD_VM | Action.BUILD_VM_WITH_BOOTLOADER: logger.info("building the system configuration...") From 6ea9eae476d5f325275aa7b604cf3bccf883c576 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sat, 30 Nov 2024 20:19:32 +0000 Subject: [PATCH 45/78] nixos-rebuild-ng: avoid usage of implementation details in LogFormatter --- .../src/nixos_rebuild/__init__.py | 2 +- .../nixos-rebuild-ng/src/nixos_rebuild/utils.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 14236c289709..5b90caf545a2 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -391,7 +391,7 @@ def execute(argv: list[str]) -> None: def main() -> None: ch = logging.StreamHandler() - ch.setFormatter(LogFormatter("%(levelname)s: %(message)s")) + ch.setFormatter(LogFormatter()) logger.addHandler(ch) try: diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/utils.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/utils.py index dc8d899ab056..cd89435f2d4f 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/utils.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/utils.py @@ -5,17 +5,17 @@ Args: TypeAlias = bool | str | list[str] | int | None class LogFormatter(logging.Formatter): + formatters = { + logging.INFO: logging.Formatter("%(message)s"), + logging.DEBUG: logging.Formatter("%(levelname)s: %(name)s: %(message)s"), + "DEFAULT": logging.Formatter("%(levelname)s: %(message)s"), + } + @override def format(self, record: logging.LogRecord) -> str: record.levelname = record.levelname.lower() - match record.levelno: - case logging.INFO: - self._style._fmt = "%(message)s" - case logging.DEBUG: - self._style._fmt = "%(levelname)s: %(name)s: %(message)s" - case _: - self._style._fmt = "%(levelname)s: %(message)s" - return super().format(record) + formatter = self.formatters.get(record.levelno, self.formatters["DEFAULT"]) + return formatter.format(record) def dict_to_flags(d: dict[str, Args]) -> list[str]: From c4902dad75f98479b5ca13be0d6a5abfc6d3f22a Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 1 Dec 2024 12:15:54 +0000 Subject: [PATCH 46/78] nixos-rebuild-ng: use raw NIX_SSHOPTS in copy_closure --- .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 14 +++++++------- .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 18 +++++++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index e4e1c5663fcb..7c9338b3ef92 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -1,6 +1,5 @@ import logging import os -import shlex from datetime import datetime from importlib.resources import files from pathlib import Path @@ -145,6 +144,7 @@ def copy_closure( if not host: return + sshopts = os.getenv("NIX_SSHOPTS", "") run_wrapper( [ "nix-copy-closure", @@ -154,13 +154,13 @@ def copy_closure( closure, ], extra_env={ - # for the remote to remote case, we can't add SSH_DEFAULT_OPTS to - # host.opts because it includes the ControlPane opts that will not - # work in the remote, because the temporary directory that we - # created will not exist - "NIX_SSHOPTS": shlex.join(host.opts) + # Using raw NIX_SSHOPTS here to avoid messing up with the passed + # parameters, and we do not add the SSH_DEFAULT_OPTS in the remote + # to remote case, otherwise it will fail because of ControlPath + # will not exist in remote + "NIX_SSHOPTS": sshopts if from_host and to_host - else shlex.join(SSH_DEFAULT_OPTS + host.opts) + else " ".join(filter(lambda x: x, [*SSH_DEFAULT_OPTS, sshopts])) }, remote=from_host if to_host else None, ) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index 0a98270490f9..6a34f98bd066 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -79,7 +79,8 @@ def test_build_flake(mock_run: Any) -> None: return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), ) def test_remote_build(mock_run: Any, monkeypatch: Any) -> None: - build_host = m.Remote("user@host", ["--ssh", "opts"], None) + build_host = m.Remote("user@host", [], None) + monkeypatch.setenv("NIX_SSHOPTS", "--ssh opts") assert n.remote_build( "config.system.build.toplevel", m.BuildAttr("", "preAttr"), @@ -128,9 +129,10 @@ def test_remote_build(mock_run: Any, monkeypatch: Any) -> None: autospec=True, return_value=CompletedProcess([], 0, stdout=" \n/path/to/file\n "), ) -def test_remote_build_flake(mock_run: Any) -> None: +def test_remote_build_flake(mock_run: Any, monkeypatch: Any) -> None: flake = m.Flake.parse(".#hostname") - build_host = m.Remote("user@host", ["--ssh", "opts"], None) + build_host = m.Remote("user@host", [], None) + monkeypatch.setenv("NIX_SSHOPTS", "--ssh opts") assert n.remote_build_flake( "config.system.build.toplevel", @@ -190,16 +192,17 @@ def test_copy_closure(mock_run: Any, monkeypatch: Any) -> None: n.copy_closure(closure, None) mock_run.assert_not_called() - target_host = m.Remote("user@target.host", ["--ssh", "target-opt"], None) - build_host = m.Remote("user@build.host", ["--ssh", "build-opt"], None) + target_host = m.Remote("user@target.host", [], None) + build_host = m.Remote("user@build.host", [], None) n.copy_closure(closure, target_host) mock_run.assert_called_with( ["nix-copy-closure", "--to", "user@target.host", closure], - extra_env={"NIX_SSHOPTS": " ".join(p.SSH_DEFAULT_OPTS + ["--ssh target-opt"])}, + extra_env={"NIX_SSHOPTS": " ".join(p.SSH_DEFAULT_OPTS)}, remote=None, ) + monkeypatch.setenv("NIX_SSHOPTS", "--ssh build-opt") n.copy_closure(closure, None, build_host) mock_run.assert_called_with( ["nix-copy-closure", "--from", "user@build.host", closure], @@ -207,11 +210,12 @@ def test_copy_closure(mock_run: Any, monkeypatch: Any) -> None: remote=None, ) + monkeypatch.setenv("NIX_SSHOPTS", "--ssh build-target-opt") n.copy_closure(closure, target_host, build_host) mock_run.assert_called_with( ["nix-copy-closure", "--to", "user@target.host", closure], remote=build_host, - extra_env={"NIX_SSHOPTS": "--ssh target-opt"}, + extra_env={"NIX_SSHOPTS": "--ssh build-target-opt"}, ) From 752c092c47a8ed8d10dcfe4f1e24ab963cc59ae8 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 1 Dec 2024 12:47:20 +0000 Subject: [PATCH 47/78] nixos-rebuild-ng: use shlex.quote instead of join in run_wrapper --- .../src/nixos_rebuild/process.py | 9 +++++---- .../nixos-rebuild-ng/src/tests/test_main.py | 20 ++++++++++++++++--- .../src/tests/test_process.py | 13 ++++++++++-- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index a9e8d7dff295..1232275d23c3 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -105,10 +105,11 @@ def run_wrapper( *SSH_DEFAULT_OPTS, remote.host, "--", - # sadly SSH just join all remaining parameters, expanding glob and - # ignoring quotes - # so we need to use shlex.join() to safely join the arguments - shlex.join(str(a) for a in args), + # SSH will join the parameters here and pass it to the shell, so we + # need to quote it to avoid issues. + # We can't use `shlex.join`, otherwise we will hit MAX_ARG_STRLEN + # limits when the command becomes too big. + *[shlex.quote(str(a)) for a in args], ] else: if extra_env: diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py index 97ae0072a829..9337fda44dc0 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py @@ -278,7 +278,12 @@ def test_execute_nix_switch_flake_target_host( *nr.process.SSH_DEFAULT_OPTS, "user@localhost", "--", - f"sudo nix-env -p /nix/var/nix/profiles/system --set {config_path}", + "sudo", + "nix-env", + "-p", + "/nix/var/nix/profiles/system", + "--set", + str(config_path), ], check=True, **DEFAULT_RUN_KWARGS, @@ -289,7 +294,11 @@ def test_execute_nix_switch_flake_target_host( *nr.process.SSH_DEFAULT_OPTS, "user@localhost", "--", - f"sudo env NIXOS_INSTALL_BOOTLOADER=0 {config_path / 'bin/switch-to-configuration'} switch", + "sudo", + "env", + "NIXOS_INSTALL_BOOTLOADER=0", + f"{config_path / 'bin/switch-to-configuration'}", + "switch", ], check=True, **DEFAULT_RUN_KWARGS, @@ -360,7 +369,12 @@ def test_execute_nix_switch_flake_build_host( *nr.process.SSH_DEFAULT_OPTS, "user@localhost", "--", - f"nix --extra-experimental-features 'nix-command flakes' build '{config_path}^*' --print-out-paths", + "nix", + "--extra-experimental-features", + "'nix-command flakes'", + "build", + f"'{config_path}^*'", + "--print-out-paths", ], check=True, stdout=PIPE, diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py index 53ed6824e39f..6c2ca2289245 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_process.py @@ -51,7 +51,9 @@ def test_run(mock_run: Any) -> None: *p.SSH_DEFAULT_OPTS, "user@localhost", "--", - "test --with 'some flags'", + "test", + "--with", + "'some flags'", ], check=True, text=True, @@ -75,7 +77,14 @@ def test_run(mock_run: Any) -> None: *p.SSH_DEFAULT_OPTS, "user@localhost", "--", - "sudo --prompt= --stdin env FOO=bar test --with flags", + "sudo", + "--prompt=", + "--stdin", + "env", + "FOO=bar", + "test", + "--with", + "flags", ], check=True, env=None, From d704aae2cff68bb6dd409c7467dca78f956d713a Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 1 Dec 2024 13:10:47 +0000 Subject: [PATCH 48/78] nixos-rebuild-ng: add logging for captured output values --- .../ni/nixos-rebuild-ng/src/nixos_rebuild/process.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index 1232275d23c3..946cc70f6a69 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -118,14 +118,14 @@ def run_wrapper( args = ["sudo", *args] logger.debug( - "calling run with args=%s, kwargs=%s, extra_env=%s", + "calling run with args=%r, kwargs=%r, extra_env=%r", args, kwargs, extra_env, ) try: - return subprocess.run( + r = subprocess.run( args, check=check, env=env, @@ -136,6 +136,11 @@ def run_wrapper( errors="surrogateescape", **kwargs, ) + + if kwargs.get("capture_output") or kwargs.get("stderr") or kwargs.get("stdout"): + logger.debug("captured output stdout=%r, stderr=%r", r.stdout, r.stderr) + + return r except subprocess.CalledProcessError: if sudo and remote: logger.error( From fb5178c3c512b3f29d35548b62cce5b74e4651c1 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 1 Dec 2024 13:54:53 +0000 Subject: [PATCH 49/78] nixos-rebuild-ng: set capture_output=True to cleanup_ssh --- .../ni/nixos-rebuild-ng/src/nixos_rebuild/process.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py index 946cc70f6a69..07d01c6f121b 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/process.py @@ -71,8 +71,10 @@ class RunKwargs(TypedDict, total=False): def cleanup_ssh() -> None: "Close SSH ControlMaster connection." for ctrl in TMPDIR_PATH.glob("ssh-*"): - subprocess.run( - ["ssh", "-o", f"ControlPath={ctrl}", "-O", "exit", "dummyhost"], check=False + run_wrapper( + ["ssh", "-o", f"ControlPath={ctrl}", "-O", "exit", "dummyhost"], + check=False, + capture_output=True, ) TMPDIR.cleanup() From d34056b2189d60118b5914db0d5d349efd25a3e0 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 1 Dec 2024 20:21:58 +0000 Subject: [PATCH 50/78] nixos-rebuild-ng: disable _NIXOS_REBUILD_REEXEC for now --- .../ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 5b90caf545a2..043aa9a3e157 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -229,7 +229,12 @@ def execute(argv: list[str]) -> None: # Re-exec to a newer version of the script before building to ensure we get # the latest fixes - if can_run and not args.fast and not os.environ.get("_NIXOS_REBUILD_REEXEC"): + if ( + False # disabled until we introduce `config.system.build.nixos-rebuild-ng` + and can_run + and not args.fast + and not os.environ.get("_NIXOS_REBUILD_REEXEC") + ): reexec(argv, args, build_flags, flake_build_flags) profile = Profile.from_arg(args.profile_name) From 9673dfc9770257cc8b48fed660fda689fd7c1bd2 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Mon, 2 Dec 2024 09:16:02 +0000 Subject: [PATCH 51/78] seatd: 0.8.0 -> 0.9.1 --- pkgs/by-name/se/seatd/package.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/se/seatd/package.nix b/pkgs/by-name/se/seatd/package.nix index c10d5085dfc8..b47e6bc46199 100644 --- a/pkgs/by-name/se/seatd/package.nix +++ b/pkgs/by-name/se/seatd/package.nix @@ -11,13 +11,13 @@ stdenv.mkDerivation (finalAttrs: { pname = "seatd"; - version = "0.8.0"; + version = "0.9.1"; src = fetchFromSourcehut { owner = "~kennylevinsen"; repo = "seatd"; rev = finalAttrs.version; - hash = "sha256-YaR4VuY+wrzbnhER4bkwdm0rTY1OVMtixdDEhu7Lnws="; + hash = "sha256-6F+TPSpcEwX4wW5GPy5Ck2AeHyH/Ba2oBUSUnVyAMeA="; }; outputs = [ "bin" "out" "dev" "man" ]; From 4fef32823db684e164c3194549676b1864d0ff04 Mon Sep 17 00:00:00 2001 From: Arsalan Date: Mon, 2 Dec 2024 05:08:45 -0500 Subject: [PATCH 52/78] linuxKernel.kernels.linux_zen: 6.11.5-zen1 -> v6.12.1-zen1 --- pkgs/os-specific/linux/kernel/zen-kernels.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/os-specific/linux/kernel/zen-kernels.nix b/pkgs/os-specific/linux/kernel/zen-kernels.nix index f4eda83e6f64..876b622f2562 100644 --- a/pkgs/os-specific/linux/kernel/zen-kernels.nix +++ b/pkgs/os-specific/linux/kernel/zen-kernels.nix @@ -5,9 +5,9 @@ let variants = { # ./update-zen.py zen zen = { - version = "6.11.5"; #zen + version = "6.12.1"; #zen suffix = "zen1"; #zen - sha256 = "1w342k54ns6rwkk13l9f7h0qzikn6hbnb2ydxyqalrmll8n2g237"; #zen + sha256 = "18aws41zlayv4xd6489jzrhr8b3kwmrx2q1b50g67v4rsp02sb4p"; #zen isLqx = false; }; # ./update-zen.py lqx From c1d0a025aab8609c00258b163a9d9ac3498118ea Mon Sep 17 00:00:00 2001 From: Arsalan Date: Mon, 2 Dec 2024 05:10:07 -0500 Subject: [PATCH 53/78] linuxKernel.kernels.linux_lqx: 6.11.5-lqx1 -> v6.12.1-lqx1 --- pkgs/os-specific/linux/kernel/zen-kernels.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/os-specific/linux/kernel/zen-kernels.nix b/pkgs/os-specific/linux/kernel/zen-kernels.nix index 876b622f2562..47a8ac0151a8 100644 --- a/pkgs/os-specific/linux/kernel/zen-kernels.nix +++ b/pkgs/os-specific/linux/kernel/zen-kernels.nix @@ -12,9 +12,9 @@ let }; # ./update-zen.py lqx lqx = { - version = "6.11.5"; #lqx + version = "6.12.1"; #lqx suffix = "lqx1"; #lqx - sha256 = "0ql7nw6sph4ai44n66c7l06aj4ibhdy45415yzgibin1jg9rqa6d"; #lqx + sha256 = "1q8rdghkgq0kn530pxncwsrfqlf3xfn4mdvxysdizyfn71vmrz8f"; #lqx isLqx = true; }; }; From debf187c9a087c5db69c98dde29ea98a4ef990fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Romildo?= Date: Mon, 2 Dec 2024 18:47:36 -0300 Subject: [PATCH 54/78] lxqt.lxqt-panel: 2.1.2 -> 2.1.3 Diff: https://github.com/lxqt/lxqt-panel/compare/2.1.2...2.1.3 --- pkgs/desktops/lxqt/lxqt-panel/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/desktops/lxqt/lxqt-panel/default.nix b/pkgs/desktops/lxqt/lxqt-panel/default.nix index 8fe19bc3254c..bee22f7405f5 100644 --- a/pkgs/desktops/lxqt/lxqt-panel/default.nix +++ b/pkgs/desktops/lxqt/lxqt-panel/default.nix @@ -35,13 +35,13 @@ stdenv.mkDerivation rec { pname = "lxqt-panel"; - version = "2.1.2"; + version = "2.1.3"; src = fetchFromGitHub { owner = "lxqt"; repo = pname; rev = version; - hash = "sha256-090zBmRUFac9Tt10l20Vxvn234Br5YPpdLR6+KreEFM="; + hash = "sha256-p4Y+0Ym0NNG4yrlfrLqCQyMYVjyogNb1V1QAooQ82LE="; }; nativeBuildInputs = [ From efbd47f639b7c330768169d90d261f714032f6d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Romildo?= Date: Mon, 2 Dec 2024 18:49:11 -0300 Subject: [PATCH 55/78] lxqt.lxqt-notificationd: 2.1.0 -> 2.1.1 Diff: https://github.com/lxqt/lxqt-notificationd/compare/2.1.0...2.1.1 --- pkgs/desktops/lxqt/lxqt-notificationd/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/desktops/lxqt/lxqt-notificationd/default.nix b/pkgs/desktops/lxqt/lxqt-notificationd/default.nix index a88f3317b3ea..4f01a9d7114a 100644 --- a/pkgs/desktops/lxqt/lxqt-notificationd/default.nix +++ b/pkgs/desktops/lxqt/lxqt-notificationd/default.nix @@ -17,13 +17,13 @@ stdenv.mkDerivation rec { pname = "lxqt-notificationd"; - version = "2.1.0"; + version = "2.1.1"; src = fetchFromGitHub { owner = "lxqt"; repo = pname; rev = version; - hash = "sha256-Iu3U036ebh9lMxCn1gt29nCQxs7N2ZLQHfg9HP3FSm0="; + hash = "sha256-L2eXab1V2t7+h51wEh0p2Uzdc3ImR4euoiXVVJ8BguA="; }; nativeBuildInputs = [ From 8f72df359887a3bafb44dad279fc0360c7db7e3e Mon Sep 17 00:00:00 2001 From: dish Date: Tue, 3 Dec 2024 01:41:41 -0500 Subject: [PATCH 56/78] nodePackages.kaput-cli: drop --- .../node-packages/main-programs.nix | 1 - .../node-packages/node-packages.json | 1 - .../node-packages/node-packages.nix | 365 ------------------ 3 files changed, 367 deletions(-) diff --git a/pkgs/development/node-packages/main-programs.nix b/pkgs/development/node-packages/main-programs.nix index 38eb64559fd5..d18dd10b11e2 100644 --- a/pkgs/development/node-packages/main-programs.nix +++ b/pkgs/development/node-packages/main-programs.nix @@ -32,7 +32,6 @@ graphql-language-service-cli = "graphql-lsp"; grunt-cli = "grunt"; gulp-cli = "gulp"; - kaput-cli = "kaput"; less = "lessc"; localtunnel = "lt"; lua-fmt = "luafmt"; diff --git a/pkgs/development/node-packages/node-packages.json b/pkgs/development/node-packages/node-packages.json index 24a13c554103..6c57b171efe2 100644 --- a/pkgs/development/node-packages/node-packages.json +++ b/pkgs/development/node-packages/node-packages.json @@ -119,7 +119,6 @@ , "json-server" , "jsonlint" , "jsonplaceholder" -, "kaput-cli" , "katex" , "keyoxide" , "lcov-result-merger" diff --git a/pkgs/development/node-packages/node-packages.nix b/pkgs/development/node-packages/node-packages.nix index e7c23ab18931..9d2c50812251 100644 --- a/pkgs/development/node-packages/node-packages.nix +++ b/pkgs/development/node-packages/node-packages.nix @@ -7249,42 +7249,6 @@ let sha512 = "/2WZxKCNjeHlQogCs1VBtJWlPXjwWke/9gMrwsVsrUt00g2V6LUBvwgwrxhrXepjOmq4IZ5QeNbpDMEOUlx/JA=="; }; }; - "@oclif/command-1.8.36" = { - name = "_at_oclif_slash_command"; - packageName = "@oclif/command"; - version = "1.8.36"; - src = fetchurl { - url = "https://registry.npmjs.org/@oclif/command/-/command-1.8.36.tgz"; - sha512 = "/zACSgaYGtAQRzc7HjzrlIs14FuEYAZrMOEwicRoUnZVyRunG4+t5iSEeQu0Xy2bgbCD0U1SP/EdeNZSTXRwjQ=="; - }; - }; - "@oclif/config-1.18.16" = { - name = "_at_oclif_slash_config"; - packageName = "@oclif/config"; - version = "1.18.16"; - src = fetchurl { - url = "https://registry.npmjs.org/@oclif/config/-/config-1.18.16.tgz"; - sha512 = "VskIxVcN22qJzxRUq+raalq6Q3HUde7sokB7/xk5TqRZGEKRVbFeqdQBxDWwQeudiJEgcNiMvIFbMQ43dY37FA=="; - }; - }; - "@oclif/config-1.18.17" = { - name = "_at_oclif_slash_config"; - packageName = "@oclif/config"; - version = "1.18.17"; - src = fetchurl { - url = "https://registry.npmjs.org/@oclif/config/-/config-1.18.17.tgz"; - sha512 = "k77qyeUvjU8qAJ3XK3fr/QVAqsZO8QOBuESnfeM5HHtPNLSyfVcwiMM2zveSW5xRdLSG3MfV8QnLVkuyCL2ENg=="; - }; - }; - "@oclif/config-1.18.2" = { - name = "_at_oclif_slash_config"; - packageName = "@oclif/config"; - version = "1.18.2"; - src = fetchurl { - url = "https://registry.npmjs.org/@oclif/config/-/config-1.18.2.tgz"; - sha512 = "cE3qfHWv8hGRCP31j7fIS7BfCflm/BNZ2HNqHexH+fDrdF2f1D5S8VmXWLC77ffv3oDvWyvE9AZeR0RfmHCCaA=="; - }; - }; "@oclif/core-1.26.2" = { name = "_at_oclif_slash_core"; packageName = "@oclif/core"; @@ -7312,15 +7276,6 @@ let sha512 = "aXM2O4g7f+kPNzhhOfqGOVRVYDxTVrH7Y720MuH0Twq5WHMxI4XwntnyBaRscoCPG6FWhItZLtiZxsvaUdupGg=="; }; }; - "@oclif/errors-1.3.5" = { - name = "_at_oclif_slash_errors"; - packageName = "@oclif/errors"; - version = "1.3.5"; - src = fetchurl { - url = "https://registry.npmjs.org/@oclif/errors/-/errors-1.3.5.tgz"; - sha512 = "OivucXPH/eLLlOT7FkCMoZXiaVYf8I/w1eTAM1+gKzfhALwWTusxEx7wBmW0uzvkSg/9ovWLycPaBgJbM3LOCQ=="; - }; - }; "@oclif/errors-1.3.6" = { name = "_at_oclif_slash_errors"; packageName = "@oclif/errors"; @@ -7330,15 +7285,6 @@ let sha512 = "fYaU4aDceETd89KXP+3cLyg9EHZsLD3RxF2IU9yxahhBpspWjkWi3Dy3bTgcwZ3V47BgxQaGapzJWDM33XIVDQ=="; }; }; - "@oclif/help-1.0.15" = { - name = "_at_oclif_slash_help"; - packageName = "@oclif/help"; - version = "1.0.15"; - src = fetchurl { - url = "https://registry.npmjs.org/@oclif/help/-/help-1.0.15.tgz"; - sha512 = "Yt8UHoetk/XqohYX76DfdrUYLsPKMc5pgkzsZVHDyBSkLiGRzujVaGZdjr32ckVZU9q3a47IjhWxhip7Dz5W/g=="; - }; - }; "@oclif/linewrap-1.0.0" = { name = "_at_oclif_slash_linewrap"; packageName = "@oclif/linewrap"; @@ -7348,15 +7294,6 @@ let sha512 = "Ups2dShK52xXa8w6iBWLgcjPJWjais6KPJQq3gQ/88AY6BXoTX+MIGFPrWQO1KLMiQfoTpcLnUwloN4brrVUHw=="; }; }; - "@oclif/parser-3.8.17" = { - name = "_at_oclif_slash_parser"; - packageName = "@oclif/parser"; - version = "3.8.17"; - src = fetchurl { - url = "https://registry.npmjs.org/@oclif/parser/-/parser-3.8.17.tgz"; - sha512 = "l04iSd0xoh/16TGVpXb81Gg3z7tlQGrEup16BrVLsZBK6SEYpYHRJZnM32BwZrHI97ZSFfuSwVlzoo6HdsaK8A=="; - }; - }; "@oclif/plugin-autocomplete-2.3.10" = { name = "_at_oclif_slash_plugin-autocomplete"; packageName = "@oclif/plugin-autocomplete"; @@ -7366,15 +7303,6 @@ let sha512 = "Ow1AR8WtjzlyCtiWWPgzMyT8SbcDJFr47009riLioHa+MHX2BCDtVn2DVnN/E6b9JlPV5ptQpjefoRSNWBesmg=="; }; }; - "@oclif/plugin-help-3.3.1" = { - name = "_at_oclif_slash_plugin-help"; - packageName = "@oclif/plugin-help"; - version = "3.3.1"; - src = fetchurl { - url = "https://registry.npmjs.org/@oclif/plugin-help/-/plugin-help-3.3.1.tgz"; - sha512 = "QuSiseNRJygaqAdABYFWn/H1CwIZCp9zp/PLid6yXvy6VcQV7OenEFF5XuYaCvSARe2Tg9r8Jqls5+fw1A9CbQ=="; - }; - }; "@oclif/plugin-help-5.2.20" = { name = "_at_oclif_slash_plugin-help"; packageName = "@oclif/plugin-help"; @@ -8482,15 +8410,6 @@ let sha512 = "ioXoq9gPxkss4MYhD+SFaU9p1IHFUX0ILAWFPyjGaBdjLsYAlZw6j1iLA0N/m12uVHLFDfSYNF7EQccjinIMDA=="; }; }; - "@putdotio/api-client-8.49.0" = { - name = "_at_putdotio_slash_api-client"; - packageName = "@putdotio/api-client"; - version = "8.49.0"; - src = fetchurl { - url = "https://registry.npmjs.org/@putdotio/api-client/-/api-client-8.49.0.tgz"; - sha512 = "PqD4exebcZIZ7lZj9tYg5NAFqU/wjSiZNM1ak0uXpff3qG+Pw6T/pW5gt0lrWwq132Ct68/Q4by18jg9uXBqMw=="; - }; - }; "@react-native/normalize-color-2.1.0" = { name = "_at_react-native_slash_normalize-color"; packageName = "@react-native/normalize-color"; @@ -17230,15 +17149,6 @@ let sha512 = "/1owvF0SZ5Gn54cgrikJ0QskgTzeg30HGjkmjFoaHDJzAqFpuX1DBpFR8aLvsE1J5s9MgeYRENQK4BFwOag5VA=="; }; }; - "cli-ux-5.6.7" = { - name = "cli-ux"; - packageName = "cli-ux"; - version = "5.6.7"; - src = fetchurl { - url = "https://registry.npmjs.org/cli-ux/-/cli-ux-5.6.7.tgz"; - sha512 = "dsKAurMNyFDnO6X1TiiRNiVbL90XReLKcvIq4H777NMqXGBxBws23ag8ubCJE97vVZEgWG2eSUhsyLf63Jv8+g=="; - }; - }; "cli-width-2.2.1" = { name = "cli-width"; packageName = "cli-width"; @@ -18211,15 +18121,6 @@ let sha512 = "fIWyWUXrJ45cHCIQX+Ck1hrZDIf/9DR0P0Zewn3uNht28hbt5OfGUq8rRWsxi96pZWPyBEd0eY9ama01JTaknA=="; }; }; - "conf-6.2.4" = { - name = "conf"; - packageName = "conf"; - version = "6.2.4"; - src = fetchurl { - url = "https://registry.npmjs.org/conf/-/conf-6.2.4.tgz"; - sha512 = "GjgyPRLo1qK1LR9RWAdUagqo+DP18f5HWCFk4va7GS+wpxQTOzfuKTwKOvGW2c01/YXNicAyyoyuSddmdkBzZQ=="; - }; - }; "config-chain-1.1.13" = { name = "config-chain"; packageName = "config-chain"; @@ -20390,15 +20291,6 @@ let sha512 = "XRRe6Glud4rd/ZGQfiV1ruXSfbvfJedlV9Y6zOlP+2K04vBYiJEte6stfFkCP03aMnY5tsipamumUjL14fofug=="; }; }; - "debounce-fn-3.0.1" = { - name = "debounce-fn"; - packageName = "debounce-fn"; - version = "3.0.1"; - src = fetchurl { - url = "https://registry.npmjs.org/debounce-fn/-/debounce-fn-3.0.1.tgz"; - sha512 = "aBoJh5AhpqlRoHZjHmOzZlRx+wz2xVwGL9rjs+Kj0EWUrL4/h4K7OD176thl2Tdoqui/AaA4xhHrNArGLAaI3Q=="; - }; - }; "debounce-fn-5.1.2" = { name = "debounce-fn"; packageName = "debounce-fn"; @@ -24162,15 +24054,6 @@ let sha512 = "M5Ge0JIrn12EtIVpje2G+hI5X78hmX4UDzynZ7Vnp1MiPSqleEonmgr2Rh59eygEEgq3YJ1GDP96rnM8tnVg/Q=="; }; }; - "extract-stack-2.0.0" = { - name = "extract-stack"; - packageName = "extract-stack"; - version = "2.0.0"; - src = fetchurl { - url = "https://registry.npmjs.org/extract-stack/-/extract-stack-2.0.0.tgz"; - sha512 = "AEo4zm+TenK7zQorGK1f9mJ8L14hnTDi2ZQPR+Mub1NX8zimka1mXpV5LpH8x9HoUmFSHZCfLHqWvp0Y4FxxzQ=="; - }; - }; "extract-zip-2.0.1" = { name = "extract-zip"; packageName = "extract-zip"; @@ -31309,15 +31192,6 @@ let sha512 = "TcezgCLdtAn0dTzPXK6kdnzJV4k9vcdTVxt7l0teFV+7LGWRCFIhs49U3WeNKU0aereFtlR5N5qrSWBGqPlw0A=="; }; }; - "js-base64-2.6.3" = { - name = "js-base64"; - packageName = "js-base64"; - version = "2.6.3"; - src = fetchurl { - url = "https://registry.npmjs.org/js-base64/-/js-base64-2.6.3.tgz"; - sha512 = "fiUvdfCaAXoQTHdKMgTvg6IkecXDcVz6V5rlftUTclF9IKBjMizvSdQaCl/z/6TApDeby5NL+axYou3i0mu1Pg=="; - }; - }; "js-base64-3.7.7" = { name = "js-base64"; packageName = "js-base64"; @@ -31696,15 +31570,6 @@ let sha512 = "NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="; }; }; - "json-schema-typed-7.0.3" = { - name = "json-schema-typed"; - packageName = "json-schema-typed"; - version = "7.0.3"; - src = fetchurl { - url = "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-7.0.3.tgz"; - sha512 = "7DE8mpG+/fVw+dTpjbxnx47TaMnDfOI1jwft9g1VybltZCduyRQPJPvc+zzKY9WPHxhPWczyFuYa6I8Mw4iU5A=="; - }; - }; "json-schema-typed-8.0.1" = { name = "json-schema-typed"; packageName = "json-schema-typed"; @@ -37620,15 +37485,6 @@ let sha512 = "M85rnSC7WQ7wnfQTARPT4LrK7nwCHLdDFOCcItZMhTQjyCebJH8GciKqYJNgaOFZs9nFmTmd/VMyi3OW5jA47w=="; }; }; - "node-downloader-helper-1.0.19" = { - name = "node-downloader-helper"; - packageName = "node-downloader-helper"; - version = "1.0.19"; - src = fetchurl { - url = "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-1.0.19.tgz"; - sha512 = "Bwp8WWDDP5ftg+FmAKU08a9+oiUTPoYzMvXgUqZZPQ7VMo1qKBzW3XdTXHeYnqjGLfkTZ2GPibgAWpApfpeS2g=="; - }; - }; "node-emoji-1.10.0" = { name = "node-emoji"; packageName = "node-emoji"; @@ -71837,227 +71693,6 @@ in bypassCache = true; reconstructLock = true; }; - kaput-cli = nodeEnv.buildNodePackage { - name = "kaput-cli"; - packageName = "kaput-cli"; - version = "1.1.1"; - src = fetchurl { - url = "https://registry.npmjs.org/kaput-cli/-/kaput-cli-1.1.1.tgz"; - sha512 = "JbF3PiO1wNawJnOQlb4nJZ2rvDfDWnD7ROCdzqgFXhqi3XTNXBqjpoC7PT3ylfNR6BGibjchndH27VieqyMA0A=="; - }; - dependencies = [ - sources."@nodelib/fs.scandir-2.1.5" - sources."@nodelib/fs.stat-2.0.5" - sources."@nodelib/fs.walk-1.2.8" - sources."@oclif/command-1.8.36" - sources."@oclif/config-1.18.17" - sources."@oclif/errors-1.3.6" - (sources."@oclif/help-1.0.15" // { - dependencies = [ - sources."@oclif/config-1.18.16" - sources."wrap-ansi-6.2.0" - ]; - }) - sources."@oclif/linewrap-1.0.0" - sources."@oclif/parser-3.8.17" - (sources."@oclif/plugin-help-3.3.1" // { - dependencies = [ - sources."@oclif/config-1.18.2" - (sources."@oclif/errors-1.3.5" // { - dependencies = [ - sources."wrap-ansi-7.0.0" - ]; - }) - sources."wrap-ansi-6.2.0" - ]; - }) - sources."@oclif/screen-1.0.4" - (sources."@putdotio/api-client-8.49.0" // { - dependencies = [ - sources."axios-0.21.4" - ]; - }) - sources."ajv-6.12.6" - sources."ansi-escapes-4.3.2" - sources."ansi-regex-5.0.1" - sources."ansi-styles-4.3.0" - sources."ansicolors-0.3.2" - sources."argparse-1.0.10" - sources."array-union-2.1.0" - (sources."axios-0.19.2" // { - dependencies = [ - sources."debug-3.1.0" - sources."follow-redirects-1.5.10" - sources."ms-2.0.0" - ]; - }) - sources."braces-3.0.3" - sources."call-bind-1.0.7" - sources."cardinal-2.1.1" - sources."chalk-4.1.2" - sources."chardet-0.7.0" - sources."clean-stack-3.0.1" - sources."cli-cursor-3.1.0" - sources."cli-progress-3.12.0" - (sources."cli-ux-5.6.7" // { - dependencies = [ - sources."supports-color-8.1.1" - ]; - }) - sources."cli-width-3.0.0" - sources."color-convert-2.0.1" - sources."color-name-1.1.4" - (sources."conf-6.2.4" // { - dependencies = [ - sources."semver-6.3.1" - ]; - }) - sources."cross-spawn-7.0.3" - sources."d-1.0.2" - sources."debounce-fn-3.0.1" - sources."debug-4.3.7" - sources."define-data-property-1.1.4" - sources."dir-glob-3.0.1" - sources."dot-prop-5.3.0" - sources."dotenv-8.6.0" - sources."emoji-regex-8.0.0" - sources."env-paths-2.2.1" - sources."es-define-property-1.0.0" - sources."es-errors-1.3.0" - sources."es5-ext-0.10.64" - sources."es6-iterator-2.0.3" - sources."es6-symbol-3.1.4" - sources."escape-string-regexp-4.0.0" - sources."esniff-2.0.1" - sources."esprima-4.0.1" - sources."event-emitter-0.3.5" - sources."ext-1.7.0" - sources."external-editor-3.1.0" - sources."extract-stack-2.0.0" - sources."fast-deep-equal-3.1.3" - sources."fast-glob-3.3.2" - sources."fast-json-stable-stringify-2.1.0" - sources."fastq-1.17.1" - (sources."figures-3.2.0" // { - dependencies = [ - sources."escape-string-regexp-1.0.5" - ]; - }) - sources."fill-range-7.1.1" - sources."find-up-3.0.0" - sources."follow-redirects-1.15.9" - sources."fs-extra-8.1.0" - sources."function-bind-1.1.2" - sources."get-intrinsic-1.2.4" - sources."glob-parent-5.1.2" - sources."globby-11.1.0" - sources."gopd-1.0.1" - sources."graceful-fs-4.2.11" - sources."has-flag-4.0.0" - sources."has-property-descriptors-1.0.2" - sources."has-proto-1.0.3" - sources."has-symbols-1.0.3" - sources."hasown-2.0.2" - sources."hyperlinker-1.0.0" - sources."iconv-lite-0.4.24" - sources."ignore-5.3.2" - sources."imurmurhash-0.1.4" - sources."indent-string-4.0.0" - sources."inquirer-7.3.3" - sources."is-docker-2.2.1" - sources."is-extglob-2.1.1" - sources."is-fullwidth-code-point-3.0.0" - sources."is-glob-4.0.3" - sources."is-number-7.0.0" - sources."is-obj-2.0.0" - sources."is-typedarray-1.0.0" - sources."is-wsl-2.2.0" - sources."isexe-2.0.0" - sources."js-base64-2.6.3" - sources."js-yaml-3.14.1" - sources."json-schema-traverse-0.4.1" - sources."json-schema-typed-7.0.3" - sources."jsonfile-4.0.0" - sources."locate-path-3.0.0" - sources."lodash-4.17.21" - (sources."make-dir-3.1.0" // { - dependencies = [ - sources."semver-6.3.1" - ]; - }) - sources."merge2-1.4.1" - sources."micromatch-4.0.8" - sources."mimic-fn-2.1.0" - sources."moment-2.30.1" - sources."ms-2.1.3" - sources."mute-stream-0.0.8" - sources."natural-orderby-2.0.3" - sources."next-tick-1.1.0" - sources."node-downloader-helper-1.0.19" - sources."object-inspect-1.13.2" - sources."object-treeify-1.1.33" - sources."onetime-5.1.2" - sources."os-tmpdir-1.0.2" - sources."p-limit-2.3.0" - sources."p-locate-3.0.0" - sources."p-try-2.2.0" - sources."password-prompt-1.1.3" - sources."path-exists-3.0.0" - sources."path-key-3.1.1" - sources."path-type-4.0.0" - sources."picomatch-2.3.1" - sources."pkg-up-3.1.0" - sources."punycode-2.3.1" - sources."qs-6.13.0" - sources."queue-microtask-1.2.3" - sources."redeyed-2.1.1" - sources."restore-cursor-3.1.0" - sources."reusify-1.0.4" - sources."run-async-2.4.1" - sources."run-parallel-1.2.0" - (sources."rxjs-6.6.7" // { - dependencies = [ - sources."tslib-1.14.1" - ]; - }) - sources."safer-buffer-2.1.2" - sources."semver-7.6.3" - sources."set-function-length-1.2.2" - sources."shebang-command-2.0.0" - sources."shebang-regex-3.0.0" - sources."side-channel-1.0.6" - sources."signal-exit-3.0.7" - sources."slash-3.0.0" - sources."sprintf-js-1.0.3" - sources."string-width-4.2.3" - sources."strip-ansi-6.0.1" - sources."supports-color-7.2.0" - sources."supports-hyperlinks-2.3.0" - sources."through-2.3.8" - sources."tmp-0.0.33" - sources."to-regex-range-5.0.1" - sources."tslib-2.7.0" - sources."type-2.7.3" - sources."type-fest-0.21.3" - sources."typedarray-to-buffer-3.1.5" - sources."universalify-0.1.2" - sources."uri-js-4.4.1" - sources."urijs-1.19.11" - sources."which-2.0.2" - sources."widest-line-3.1.0" - sources."wrap-ansi-7.0.0" - sources."write-file-atomic-3.0.3" - ]; - buildInputs = globalBuildInputs; - meta = { - description = "CLI tools for Put.io"; - homepage = "https://github.com/davidchalifoux/kaput-cli"; - license = "MIT"; - }; - production = true; - bypassCache = true; - reconstructLock = true; - }; katex = nodeEnv.buildNodePackage { name = "katex"; packageName = "katex"; From e94d44a63dcd465fdaa9cf03834c965ff83862bf Mon Sep 17 00:00:00 2001 From: dish Date: Tue, 3 Dec 2024 01:41:49 -0500 Subject: [PATCH 57/78] kaput-cli: init at 2.5.0 --- pkgs/by-name/ka/kaput-cli/package.nix | 45 ++++++++++++++++++++++ pkgs/development/node-packages/aliases.nix | 1 + 2 files changed, 46 insertions(+) create mode 100644 pkgs/by-name/ka/kaput-cli/package.nix diff --git a/pkgs/by-name/ka/kaput-cli/package.nix b/pkgs/by-name/ka/kaput-cli/package.nix new file mode 100644 index 000000000000..a03f37ff97bf --- /dev/null +++ b/pkgs/by-name/ka/kaput-cli/package.nix @@ -0,0 +1,45 @@ +{ + lib, + fetchFromGitHub, + rustPlatform, + pkg-config, + openssl, + nix-update-script, +}: + +rustPlatform.buildRustPackage rec { + pname = "kaput-cli"; + version = "2.5.0"; + + src = fetchFromGitHub { + owner = "davidchalifoux"; + repo = "kaput-cli"; + rev = "refs/tags/v${version}"; + hash = "sha256-sy8k9L9rmiRFzvhLc+hYl9OqmmP8INLxMNRjAx7/V8g="; + }; + + cargoHash = "sha256-fnTcCVppI5wIzwG2fA8LjpC1+gTAhdRkWv/EQXUok0c="; + + env = { + OPENSSL_NO_VENDOR = 1; + }; + + nativeBuildInputs = [ + pkg-config + ]; + + buildInputs = [ + openssl + ]; + + passthru.updateScript = nix-update-script { }; + + meta = { + changelog = "https://github.com/davidchalifoux/kaput-cli/releases/tag/v${version}"; + description = "unofficial CLI client for Put.io"; + homepage = "https://kaput.sh/"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ pyrox0 ]; + mainProgram = "kaput"; + }; +} diff --git a/pkgs/development/node-packages/aliases.nix b/pkgs/development/node-packages/aliases.nix index a65be6e33ae7..a38d11119a23 100644 --- a/pkgs/development/node-packages/aliases.nix +++ b/pkgs/development/node-packages/aliases.nix @@ -124,6 +124,7 @@ mapAliases { ionic = throw "ionic was replaced by @ionic/cli"; # added 2023-08-19 inherit (pkgs) jake; # added 2023-08-19 inherit (pkgs) javascript-typescript-langserver; # added 2023-08-19 + inherit (pkgs) kaput-cli; # added 2024-12-03 karma = pkgs.karma-runner; # added 2023-07-29 leetcode-cli = self.vsc-leetcode-cli; # added 2023-08-31 inherit (pkgs) lv_font_conv; # added 2024-06-28 From ea3e278f5bda90d8f81803da451e49769a1dbc08 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Tue, 3 Dec 2024 09:36:14 +0000 Subject: [PATCH 58/78] python312Packages.recipe-scrapers: 15.2.1 -> 15.3.2 --- pkgs/development/python-modules/recipe-scrapers/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/recipe-scrapers/default.nix b/pkgs/development/python-modules/recipe-scrapers/default.nix index eba4cd32df88..de3f12e7680b 100644 --- a/pkgs/development/python-modules/recipe-scrapers/default.nix +++ b/pkgs/development/python-modules/recipe-scrapers/default.nix @@ -16,7 +16,7 @@ buildPythonPackage rec { pname = "recipe-scrapers"; - version = "15.2.1"; + version = "15.3.2"; pyproject = true; disabled = pythonOlder "3.9"; @@ -25,7 +25,7 @@ buildPythonPackage rec { owner = "hhursev"; repo = "recipe-scrapers"; rev = "refs/tags/${version}"; - hash = "sha256-2rwy7tfTKaUmPJv59WMVGAOUP+vGWquJbF/3BbS3kkA="; + hash = "sha256-Q3pbvLbSpV1DOUGSGydgmGOCGrlfGKVENneC6iuVNxs="; }; nativeBuildInputs = [ setuptools ]; From c8ecb7f23376faf8d482b77c7be98797d3fb476d Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Tue, 3 Dec 2024 09:47:21 +0000 Subject: [PATCH 59/78] python312Packages.voip-utils: 0.2.0 -> 0.2.1 --- pkgs/development/python-modules/voip-utils/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/voip-utils/default.nix b/pkgs/development/python-modules/voip-utils/default.nix index 04e0eb2ede82..b6c105ad9924 100644 --- a/pkgs/development/python-modules/voip-utils/default.nix +++ b/pkgs/development/python-modules/voip-utils/default.nix @@ -9,14 +9,14 @@ buildPythonPackage rec { pname = "voip-utils"; - version = "0.2.0"; + version = "0.2.1"; pyproject = true; src = fetchFromGitHub { owner = "home-assistant-libs"; repo = "voip-utils"; - rev = "refs/tags/${version}"; - hash = "sha256-/DBBTL+t0VQJL43HgXGdhMHMKr8uBEkJ7Mf7Xp1mwhU="; + rev = "refs/tags/v${version}"; + hash = "sha256-ONvbkYP1hGMYOVZs5gvs/cr+Tde9gT3jloP6veqE/Ac="; }; postPatch = '' From 69b1c90060b23d929349bec30aedb0b1de8d809d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Romildo?= Date: Sun, 1 Dec 2024 23:33:56 -0300 Subject: [PATCH 60/78] python312Packages.latex2pydata: 0.4.0 -> 0.4.1 --- pkgs/development/python-modules/latex2pydata/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/latex2pydata/default.nix b/pkgs/development/python-modules/latex2pydata/default.nix index 80005a173af0..865af9756e09 100644 --- a/pkgs/development/python-modules/latex2pydata/default.nix +++ b/pkgs/development/python-modules/latex2pydata/default.nix @@ -7,12 +7,12 @@ buildPythonPackage rec { pname = "latex2pydata"; - version = "0.4.0"; + version = "0.4.1"; pyproject = true; src = fetchPypi { inherit pname version; - hash = "sha256-Ega1cHSP187njyelb0yiCdpk08QZyObelRa2S79AE1E="; + hash = "sha256-9MsAwzP8j52mh+azaEkfkaLmDOBWZoVnr9LnGSvK3mk="; }; build-system = [ From 15ca3525fe63abcb0cd5f77e303dbffeab98d879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Romildo?= Date: Tue, 3 Dec 2024 06:51:31 -0300 Subject: [PATCH 61/78] python312packages.latex2pydata: format with nixfmt --- .../python-modules/latex2pydata/default.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkgs/development/python-modules/latex2pydata/default.nix b/pkgs/development/python-modules/latex2pydata/default.nix index 865af9756e09..e4bbd47c0eb0 100644 --- a/pkgs/development/python-modules/latex2pydata/default.nix +++ b/pkgs/development/python-modules/latex2pydata/default.nix @@ -1,8 +1,9 @@ -{ lib -, fetchPypi -, buildPythonPackage -, setuptools -, pytestCheckHook +{ + lib, + fetchPypi, + buildPythonPackage, + setuptools, + pytestCheckHook, }: buildPythonPackage rec { From 48011bb5515dbaf85bcca5ce14a75e95ecebd354 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Tue, 3 Dec 2024 10:41:52 +0000 Subject: [PATCH 62/78] nixos/doc/rl-2505: document retroarch refactoring changes --- nixos/doc/manual/release-notes/rl-2505.section.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2505.section.md b/nixos/doc/manual/release-notes/rl-2505.section.md index a82932ce6626..6dd69941c270 100644 --- a/nixos/doc/manual/release-notes/rl-2505.section.md +++ b/nixos/doc/manual/release-notes/rl-2505.section.md @@ -53,6 +53,8 @@ [official website](https://www.nerdfonts.com/font-downloads) as the titles in preview images, with the "Nerd Font" suffix and any whitespaces trimmed. +- `retroarch` has been refactored and the older `retroarch.override { cores = [ ... ]; }` to create a RetroArch derivation with custom cores doesn't work anymore, use `retroarch.withCores (cores: [ ... ])` instead. If you need more customization (e.g.: custom settings), use `wrapRetroArch` instead. + - `gkraken` software and `hardware.gkraken.enable` option have been removed, use `coolercontrol` via `programs.coolercontrol.enable` option instead. - `containerd` has been updated to v2, which contains breaking changes. See the [containerd From debea81ba7f3b06c34edfeb5b2bec0907d912d66 Mon Sep 17 00:00:00 2001 From: r-vdp Date: Tue, 3 Dec 2024 12:19:10 +0000 Subject: [PATCH 63/78] nixos-rebuild-ng: don't try to register the profile when doing build or test --- .../nixos-rebuild-ng/src/nixos_rebuild/__init__.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 043aa9a3e157..d41a0fcbd1bf 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -322,12 +322,13 @@ def execute(argv: list[str]) -> None: from_host=build_host, **copy_flags, ) - nix.set_profile( - profile, - path_to_config, - target_host=target_host, - sudo=args.sudo, - ) + if action in (Action.SWITCH, Action.BOOT): + nix.set_profile( + profile, + path_to_config, + target_host=target_host, + sudo=args.sudo, + ) if action in (Action.SWITCH, Action.BOOT, Action.TEST, Action.DRY_ACTIVATE): nix.switch_to_configuration( path_to_config, From c6e9bd02ca1197289185ef9df5fc9b295352965f Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Tue, 3 Dec 2024 12:54:13 +0000 Subject: [PATCH 64/78] nixos-rebuild-ng: add test to `nixos-rebuild build` --- .../nixos-rebuild-ng/src/tests/test_main.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py index 9337fda44dc0..913103985f66 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py @@ -445,6 +445,35 @@ def test_execute_switch_rollback(mock_run: Any, tmp_path: Path) -> None: ) +@patch(get_qualified_name(nr.process.subprocess.run), autospec=True) +def test_execute_build(mock_run: Any, tmp_path: Path) -> None: + config_path = tmp_path / "test" + config_path.touch() + mock_run.side_effect = [ + # nixos_build_flake + CompletedProcess([], 0, str(config_path)), + ] + + nr.execute(["nixos-rebuild", "build", "--no-flake", "--fast"]) + + assert mock_run.call_count == 1 + mock_run.assert_has_calls( + [ + call( + [ + "nix-build", + "", + "--attr", + "config.system.build.toplevel", + ], + check=True, + stdout=PIPE, + **DEFAULT_RUN_KWARGS, + ) + ] + ) + + @patch(get_qualified_name(nr.process.subprocess.run), autospec=True) @patch(get_qualified_name(nr.nix.Path.exists, nr.nix), autospec=True, return_value=True) @patch(get_qualified_name(nr.nix.Path.mkdir, nr.nix), autospec=True) From 98e9372c1c7f3cf9fc76715ea6904bd797dc33d1 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Tue, 3 Dec 2024 12:53:04 +0000 Subject: [PATCH 65/78] nixos-rebuild: refactor if-else in match --- .../src/nixos_rebuild/__init__.py | 86 ++++++++++--------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index d41a0fcbd1bf..c7dbf8acb350 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -263,11 +263,12 @@ def execute(argv: list[str]) -> None: attr = "config.system.build.toplevel" dry_run = action == Action.DRY_BUILD no_link = action in (Action.SWITCH, Action.BOOT) + rollback = bool(args.rollback) - if args.rollback: - if action in (Action.SWITCH, Action.BOOT): + match (action, rollback, build_host, flake): + case (Action.SWITCH | Action.BOOT, True, _, _): path_to_config = nix.rollback(profile, target_host, sudo=args.sudo) - elif action in (Action.TEST, Action.BUILD): + case (Action.TEST | Action.BUILD, True, _, _): maybe_path_to_config = nix.rollback_temporary_profile( profile, target_host, @@ -277,45 +278,48 @@ def execute(argv: list[str]) -> None: path_to_config = maybe_path_to_config else: raise NRError("could not find previous generation") - else: + case (_, True, _, _): raise NRError(f"--rollback is incompatible with '{action}'") - else: - if flake: - if build_host: - path_to_config = nix.remote_build_flake( - attr, - flake, - build_host, - flake_build_flags=flake_build_flags, - copy_flags=copy_flags, - build_flags=build_flags, - ) - else: - path_to_config = nix.build_flake( - attr, - flake, - no_link=no_link, - dry_run=dry_run, - **flake_build_flags, - ) - else: - if build_host: - path_to_config = nix.remote_build( - attr, - build_attr, - build_host, - instantiate_flags=common_flags, - copy_flags=copy_flags, - build_flags=build_flags, - ) - else: - path_to_config = nix.build( - attr, - build_attr, - no_out_link=no_link, - dry_run=dry_run, - **build_flags, - ) + case (_, False, Remote(_), Flake(_)): + path_to_config = nix.remote_build_flake( + attr, + flake, + build_host, + flake_build_flags=flake_build_flags, + copy_flags=copy_flags, + build_flags=build_flags, + ) + case (_, False, None, Flake(_)): + path_to_config = nix.build_flake( + attr, + flake, + no_link=no_link, + dry_run=dry_run, + **flake_build_flags, + ) + case (_, False, Remote(_), None): + path_to_config = nix.remote_build( + attr, + build_attr, + build_host, + instantiate_flags=common_flags, + copy_flags=copy_flags, + build_flags=build_flags, + ) + case (_, False, None, None): + path_to_config = nix.build( + attr, + build_attr, + no_out_link=no_link, + dry_run=dry_run, + **build_flags, + ) + case m: + # should never happen, but mypy is not smart enough to + # handle this with assert_never + raise NRError(f"invalid match for build: {m}") + + if not rollback: nix.copy_closure( path_to_config, to_host=target_host, From c496490561595887b8b237ee5d45a7a2e1680137 Mon Sep 17 00:00:00 2001 From: Karolis Stasaitis Date: Tue, 3 Dec 2024 18:02:51 +0100 Subject: [PATCH 66/78] uhubctl: fix darwin build --- pkgs/by-name/uh/uhubctl/package.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkgs/by-name/uh/uhubctl/package.nix b/pkgs/by-name/uh/uhubctl/package.nix index c4ea65eebd97..8feda34f2ad3 100644 --- a/pkgs/by-name/uh/uhubctl/package.nix +++ b/pkgs/by-name/uh/uhubctl/package.nix @@ -1,6 +1,8 @@ { lib , stdenv , fetchFromGitHub +, which +, pkg-config , libusb1 }: @@ -15,6 +17,8 @@ stdenv.mkDerivation rec { sha256 = "sha256-mpeDePHLsa4sGe2+8X9KQ8AYn7wtybDnaZzxnf4oETQ="; }; + nativeBuildInputs = [ which pkg-config ]; + buildInputs = [ libusb1 ]; installFlags = [ "prefix=${placeholder "out"}" ]; @@ -23,7 +27,7 @@ stdenv.mkDerivation rec { homepage = "https://github.com/mvp/uhubctl"; description = "Utility to control USB power per-port on smart USB hubs"; license = licenses.gpl2Only; - maintainers = with maintainers; [ prusnak ]; + maintainers = with maintainers; [ prusnak carlossless ]; platforms = with platforms; linux ++ darwin; mainProgram = "uhubctl"; }; From 9e7ccb4d80efc11c4174817cc2bb52b59a9cabd6 Mon Sep 17 00:00:00 2001 From: aucub <78630225+aucub@users.noreply.github.com> Date: Wed, 4 Dec 2024 02:07:21 +0800 Subject: [PATCH 67/78] brave: fix darwin build --- pkgs/by-name/br/brave/make-brave.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/by-name/br/brave/make-brave.nix b/pkgs/by-name/br/brave/make-brave.nix index 6f7f5d0687d8..11f77869d0a4 100644 --- a/pkgs/by-name/br/brave/make-brave.nix +++ b/pkgs/by-name/br/brave/make-brave.nix @@ -242,7 +242,7 @@ stdenv.mkDerivation { mkdir -p $out/{Applications,bin} - cp -r "Brave Browser.app" $out/Applications/ + cp -r . "$out/Applications/Brave Browser.app" makeWrapper "$out/Applications/Brave Browser.app/Contents/MacOS/Brave Browser" $out/bin/brave From eb48ff562196c485af75879bc81ef99bf52cd231 Mon Sep 17 00:00:00 2001 From: emilylange Date: Tue, 3 Dec 2024 20:11:33 +0100 Subject: [PATCH 68/78] chromium,chromedriver: 131.0.6778.85 -> 131.0.6778.108 https://chromereleases.googleblog.com/2024/12/stable-channel-update-for-desktop.html This update includes 4 security fixes. CVEs: CVE-2024-12053 --- .../networking/browsers/chromium/info.json | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkgs/applications/networking/browsers/chromium/info.json b/pkgs/applications/networking/browsers/chromium/info.json index b1f96ef6c89b..618730284074 100644 --- a/pkgs/applications/networking/browsers/chromium/info.json +++ b/pkgs/applications/networking/browsers/chromium/info.json @@ -1,9 +1,9 @@ { "chromium": { - "version": "131.0.6778.85", + "version": "131.0.6778.108", "chromedriver": { - "hash_darwin": "sha256-li9wQkcgh5ORPu2kjdyfwvvuykRz1gTL3b8UVQs+MY4=", - "hash_darwin_aarch64": "sha256-r/C6QpmkVzovLJKfMTjUome9nL5WoEvPGRFB+AJJFqk=" + "hash_darwin": "sha256-X0kzihCQsICn5SRdU+THdthov0EPxsmMcrm6YJ6hMhs=", + "hash_darwin_aarch64": "sha256-3Qdj44cZD4wQTzkBx47ZCfqHE2HckkLuqYKpJxxfESk=" }, "deps": { "depot_tools": { @@ -19,8 +19,8 @@ "DEPS": { "src": { "url": "https://chromium.googlesource.com/chromium/src.git", - "rev": "3d81e41b6f3ac8bcae63b32e8145c9eb0cd60a2d", - "hash": "sha256-fREToEHVbTD0IVGx/sn7csSju4BYajWZ+LDCiKWV4cI=", + "rev": "3b014839fbc4fb688b2f5af512d6ce312ad208b1", + "hash": "sha256-ypzu3LveMFcOFm7+JlaERjzs3SK/n9+sfm5wOKB8/zw=", "recompress": true }, "src/third_party/clang-format/script": { @@ -120,8 +120,8 @@ }, "src/third_party/dawn": { "url": "https://dawn.googlesource.com/dawn.git", - "rev": "7e742cac42c29a14ab7f54b134b2f17592711267", - "hash": "sha256-K2gwKNwonzCIu4hnlYuOaYyKaRV11hwDzF4oykiKsl0=" + "rev": "740d2502dbbd719a76c5a8d3fb4dac1b5363f42e", + "hash": "sha256-R41YVv4uWCU6SsACXPRppeCDguTs+/NVJckvMGGTgJE=" }, "src/third_party/dawn/third_party/glfw": { "url": "https://chromium.googlesource.com/external/github.com/glfw/glfw", @@ -600,8 +600,8 @@ }, "src/third_party/skia": { "url": "https://skia.googlesource.com/skia.git", - "rev": "94631d9b9a10697325589e1642af63a0137cac94", - "hash": "sha256-SKKLOxjimQWt8W+Q3wlCJaUC/lxw6EIZDFBuVQKmnVY=" + "rev": "f14f6b1ab7cf544c0190074488d17821281cfa4d", + "hash": "sha256-0p57otDuIShl6MngYs22XA1QYxptDVa3vCwJsH59H34=" }, "src/third_party/smhasher/src": { "url": "https://chromium.googlesource.com/external/smhasher.git", @@ -725,8 +725,8 @@ }, "src/third_party/webrtc": { "url": "https://webrtc.googlesource.com/src.git", - "rev": "8445abdf8069cadcbd134369b70d0ebd436ef477", - "hash": "sha256-EitEjXNtm0gB9wtAwIYHBHkU7paHg5zvsTz171hRmK4=" + "rev": "79aff54b0fa9238ce3518dd9eaf9610cd6f22e82", + "hash": "sha256-xkMnUduSG88EWiwq6PITN0KgAKjFd4QOis3dgxedK30=" }, "src/third_party/wuffs/src": { "url": "https://skia.googlesource.com/external/github.com/google/wuffs-mirror-release-c.git", @@ -760,8 +760,8 @@ }, "src/v8": { "url": "https://chromium.googlesource.com/v8/v8.git", - "rev": "bd2671b973062afc614b852ec190524b80aaef8a", - "hash": "sha256-uq0CE7Chqzy2d+iifC3hV9RTnDVinpwjl1pOzyNGbSo=" + "rev": "e38771cb283b9689683c5ac0b5831dd89f8ec690", + "hash": "sha256-csSDnepYxil0R3PD/LVxW7JBcasOKG4l6q6vj8zHV/I=" } } }, From 2f2926f5938977608be8b66a3d0634d1a8a8d6d0 Mon Sep 17 00:00:00 2001 From: Jeremy Baxter Date: Fri, 25 Oct 2024 19:13:27 +1300 Subject: [PATCH 69/78] dmd: patch test needspkgmod.d Closes #350898 --- pkgs/by-name/dm/dmd/generic.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkgs/by-name/dm/dmd/generic.nix b/pkgs/by-name/dm/dmd/generic.nix index cf8cae7b4f5a..1c68817ead2e 100644 --- a/pkgs/by-name/dm/dmd/generic.nix +++ b/pkgs/by-name/dm/dmd/generic.nix @@ -82,6 +82,13 @@ stdenv.mkDerivation (finalAttrs: { extraPrefix = "dmd/"; hash = "sha256-N21mAPfaTo+zGCip4njejasraV5IsWVqlGR5eOdFZZE="; }) + ] ++ lib.optionals (lib.versionOlder version "2.110.0") [ + (fetchpatch { + url = "https://github.com/dlang/dmd/commit/fdd25893e0ac04893d6eba8652903d499b7b0dfc.patch"; + stripLen = 1; + extraPrefix = "dmd/"; + hash = "sha256-Uccb8rBPBLAEPWbOYWgdR5xN3wJoIkKKhLGu58IK1sM="; + }) ]; postPatch = '' From 384f6f592b99f7ed384da7282d6aa6166eae075b Mon Sep 17 00:00:00 2001 From: Thomas Gerbet Date: Tue, 3 Dec 2024 22:45:06 +0100 Subject: [PATCH 70/78] authentik: flag with `knownVulnerabilities` The bump to 2024.10.x is currently stalled in #345940. The fix for CVE-2024-52289 involves a backward incompatible API and DB change so it is also not great for a backport even if we cherry-picks the security fixes. Given no NixOS module is available in nixpkgs marking the package with `knownVulnerabilities` should have a limited impact. --- pkgs/by-name/au/authentik/package.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkgs/by-name/au/authentik/package.nix b/pkgs/by-name/au/authentik/package.nix index ed1da779fb23..4915cc4a1fbc 100644 --- a/pkgs/by-name/au/authentik/package.nix +++ b/pkgs/by-name/au/authentik/package.nix @@ -30,6 +30,12 @@ let license = licenses.mit; platforms = platforms.linux; maintainers = with maintainers; [ jvanbruegge risson ]; + knownVulnerabilities = [ + "CVE-2024-52307" + "CVE-2024-52287" + "CVE-2024-52289" + "Authentik 2024.6.x is end-of-life, consider using https://github.com/nix-community/authentik-nix for an up-to-date alternative" + ]; }; website = buildNpmPackage { From 29ba792f88bcbb974b57cffd74f616583ffef981 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Wed, 4 Dec 2024 00:32:14 +0000 Subject: [PATCH 71/78] silx: 2.1.1 -> 2.1.2 --- pkgs/by-name/si/silx/package.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/si/silx/package.nix b/pkgs/by-name/si/silx/package.nix index 3fa944d5ae53..6b80d173da79 100644 --- a/pkgs/by-name/si/silx/package.nix +++ b/pkgs/by-name/si/silx/package.nix @@ -6,13 +6,13 @@ python3Packages.buildPythonApplication rec { pname = "silx"; - version = "2.1.1"; + version = "2.1.2"; pyproject = true; src = fetchPypi { inherit pname version; - hash = "sha256-LfCRWkUrqQb7zxiFTPhy/g9FWhNMXTRbhEgek4tZb5I="; + hash = "sha256-Uep/BkH3ngGDbPMVptab64SKBbOGqa0qazUoT47idqU="; }; build-system = with python3Packages; [ From b6047c81a60f18c54faeb9e7a56a42097835b3cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Tue, 3 Dec 2024 16:47:17 -0800 Subject: [PATCH 72/78] python312Packages.voip-utils: run tests --- pkgs/development/python-modules/voip-utils/default.nix | 3 --- pkgs/servers/home-assistant/default.nix | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/voip-utils/default.nix b/pkgs/development/python-modules/voip-utils/default.nix index b6c105ad9924..6cf7ebb9375d 100644 --- a/pkgs/development/python-modules/voip-utils/default.nix +++ b/pkgs/development/python-modules/voip-utils/default.nix @@ -34,9 +34,6 @@ buildPythonPackage rec { nativeCheckInputs = [ pytestCheckHook ]; - # no tests as of v0.1.0 - doCheck = false; - meta = { changelog = "https://github.com/home-assistant-libs/voip-utils/blob/${src.rev}/CHANGELOG.md"; description = "Voice over IP Utilities"; diff --git a/pkgs/servers/home-assistant/default.nix b/pkgs/servers/home-assistant/default.nix index 818b834349af..f1fa3e4920cc 100644 --- a/pkgs/servers/home-assistant/default.nix +++ b/pkgs/servers/home-assistant/default.nix @@ -360,6 +360,8 @@ let rev = "refs/tags/v${version}"; hash = "sha256-PG4L6KphH9JIZO76cCN8eClFE2CneEIExlXS+x79k3U="; }; + # tests were not implemented until version 0.2.0 + doCheck = false; }); vulcan-api = super.vulcan-api.overridePythonAttrs (oldAttrs: rec { From c2e85f96e629fd4f22e23f709e6c09157bc10ccc Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Wed, 4 Dec 2024 06:13:57 +0000 Subject: [PATCH 73/78] kubernetes-kcp.tests: fix the eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without the change the eval fails as: $ nix build --no-link -f. kubernetes-kcp.tests error: … while evaluating the attribute 'version' at pkgs/by-name/ku/kubernetes-kcp/package.nix:52:3: 51| 52| passthru.tests.version = testers.testVersion { | ^ 53| command = "kcp --version"; … from call site at pkgs/by-name/ku/kubernetes-kcp/package.nix:52:28: 51| 52| passthru.tests.version = testers.testVersion { | ^ 53| command = "kcp --version"; error: function 'testVersion' called without required argument 'package' at pkgs/build-support/testers/default.nix:66:5: 65| testVersion = 66| { package, | ^ 67| command ? "${package.meta.mainProgram or package.pname or package.name} --version", --- pkgs/by-name/ku/kubernetes-kcp/package.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/by-name/ku/kubernetes-kcp/package.nix b/pkgs/by-name/ku/kubernetes-kcp/package.nix index 7a5ff9afea4b..fa73327a3c70 100644 --- a/pkgs/by-name/ku/kubernetes-kcp/package.nix +++ b/pkgs/by-name/ku/kubernetes-kcp/package.nix @@ -5,6 +5,7 @@ fetchFromGitHub, installShellFiles, testers, + kubernetes-kcp, }: buildGoModule rec { @@ -50,6 +51,7 @@ buildGoModule rec { ''; passthru.tests.version = testers.testVersion { + package = kubernetes-kcp; command = "kcp --version"; # NOTE: Once the go.mod version is pulled in, the version info here needs # to be also updated. From f04f58809dab6715e460973eaae2a085474c776d Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Wed, 4 Dec 2024 07:32:13 +0000 Subject: [PATCH 74/78] python312Packages.lottie: 0.7.0 -> 0.7.1 --- pkgs/development/python-modules/lottie/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/lottie/default.nix b/pkgs/development/python-modules/lottie/default.nix index a1ced236e8c8..91f375d664e3 100644 --- a/pkgs/development/python-modules/lottie/default.nix +++ b/pkgs/development/python-modules/lottie/default.nix @@ -7,12 +7,12 @@ }: buildPythonPackage rec { pname = "lottie"; - version = "0.7.0"; + version = "0.7.1"; pyproject = true; src = fetchPypi { inherit pname version; - hash = "sha256-oyQvi6NwUfvddQPs0WggOgjkrybxe+LsygimSvHn08E="; + hash = "sha256-If05yOaLsfBDvVxmnDxgxwqc3lkCjTW8YV3e+S9CU2o="; }; build-system = [ setuptools ]; From cbba67d309432e65213ac7bf6bf7bddfd44ec58c Mon Sep 17 00:00:00 2001 From: Peter Hoeg Date: Mon, 2 Dec 2024 12:19:19 +0100 Subject: [PATCH 75/78] bombsquad: use the wayback machine for stable links --- pkgs/by-name/bo/bombsquad/package.nix | 52 +++++++++++++++------------ 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/pkgs/by-name/bo/bombsquad/package.nix b/pkgs/by-name/bo/bombsquad/package.nix index 55afb4eb444c..054f1b11d491 100644 --- a/pkgs/by-name/bo/bombsquad/package.nix +++ b/pkgs/by-name/bo/bombsquad/package.nix @@ -9,7 +9,7 @@ curl, gnugrep, libgcc, - makeWrapper, + makeBinaryWrapper, makeDesktopItem, autoPatchelfHook, copyDesktopItems, @@ -22,38 +22,44 @@ let { x86_64-linux = { name = "BombSquad_Linux_x86_64"; - hash = "sha256-aujLYzFcKaW0ff7sRdyJ6SvSQowafWVbmwycQfDQUYY="; + hash = "sha256-ICjaNZSCUbslB5pELbI4e+1zXWrZzkCkv69jLRx4dr0="; }; aarch-64-linux = { name = "BombSquad_Linux_Arm64"; - hash = "sha256-pPP7QZzToTOQtSxzF7Q3ZzlDjUjQWMBM/y79d6Yf38I="; + hash = "sha256-/m0SOQbHssk0CqZJPRLK9YKphup3dtMqkbWGzqcF0+g="; }; } .${stdenv.targetPlatform.system} or (throw "${stdenv.targetPlatform.system} is unsupported."); -in -stdenv.mkDerivation (finalAttrs: { - pname = "bombsquad"; - version = "1.7.37"; - sourceRoot = "."; - src = fetchurl { - url = "https://files.ballistica.net/bombsquad/builds/${archive.name}_${finalAttrs.version}.tar.gz"; - inherit (archive) hash; - }; bombsquadIcon = fetchurl { url = "https://files.ballistica.net/bombsquad/promo/BombSquadIcon.png"; hash = "sha256-MfOvjVmjhLejrJmdLo/goAM9DTGubnYGhlN6uF2GugA="; }; - nativeBuildInputs = [ - python312 +in +stdenv.mkDerivation (finalAttrs: { + pname = "bombsquad"; + version = "1.7.37"; + + src = fetchurl { + url = "https://web.archive.org/web/20240825230506if_/https://files.ballistica.net/bombsquad/builds/${archive.name}_${finalAttrs.version}.tar.gz"; + inherit (archive) hash; + }; + + sourceRoot = "${archive.name}_${finalAttrs.version}"; + + buildInputs = [ SDL2 + libgcc libvorbis openal - libgcc - makeWrapper + python312 + ]; + + nativeBuildInputs = [ autoPatchelfHook copyDesktopItems + makeBinaryWrapper ]; desktopItems = [ @@ -61,6 +67,7 @@ stdenv.mkDerivation (finalAttrs: { name = "bombsquad"; genericName = "bombsquad"; desktopName = "BombSquad"; + icon = "bombsquad"; exec = "bombsquad"; comment = "An explosive arcade-style party game."; @@ -71,17 +78,16 @@ stdenv.mkDerivation (finalAttrs: { installPhase = '' runHook preInstall - base=${archive.name}_${finalAttrs.version} + mkdir -p $out/bin $out/libexec $out/share/bombsquad/ba_data - install -m755 -D $base/bombsquad $out/bin/bombsquad - install -dm755 $base/ba_data $out/usr/share/bombsquad/ba_data - cp -r $base/ba_data $out/usr/share/bombsquad/ + install -Dm555 -t $out/libexec ${finalAttrs.meta.mainProgram} + cp -r ba_data $out/share/bombsquad - wrapProgram "$out/bin/bombsquad" \ + makeWrapper "$out/libexec/${finalAttrs.meta.mainProgram}" "$out/bin/${finalAttrs.meta.mainProgram}" \ --add-flags ${lib.escapeShellArg commandLineArgs} \ - --add-flags "-d $out/usr/share/bombsquad" + --add-flags "-d $out/share/bombsquad" - install -Dm755 ${finalAttrs.bombsquadIcon} $out/usr/share/icons/hicolor/32x32/apps/bombsquad.png + install -Dm755 ${bombsquadIcon} $out/share/icons/hicolor/1024x1024/apps/bombsquad.png runHook postInstall ''; From c3c2032ecb578651d98f8f474e053f825c30ca25 Mon Sep 17 00:00:00 2001 From: Aleksana Date: Wed, 4 Dec 2024 16:58:19 +0800 Subject: [PATCH 76/78] pkgs/README.md: minor refactor to patch section to improve readability (#361688) --- pkgs/README.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pkgs/README.md b/pkgs/README.md index f907d79ef348..49c31a76ed43 100644 --- a/pkgs/README.md +++ b/pkgs/README.md @@ -492,7 +492,9 @@ Examples going from bad to best practices: Sometimes, changes are needed to the source to allow building a derivation in nixpkgs, or to get earlier access to an upstream fix or improvement. When using the `patches` parameter to `mkDerivation`, make sure the patch name clearly describes the reason for the patch, or add a comment. -Patches already merged upstream or published elsewhere should be retrieved using `fetchpatch`. +### Fetching patches + +In the interest of keeping our maintenance burden and the size of Nixpkgs to a minimum, patches already merged upstream or published elsewhere _should_ be retrieved using `fetchpatch`: ```nix { @@ -506,15 +508,22 @@ Patches already merged upstream or published elsewhere should be retrieved using } ``` -Otherwise, you can add a `.patch` file to the `nixpkgs` repository. -In the interest of keeping our maintenance burden and the size of nixpkgs to a minimum, only do this for patches that are unique to `nixpkgs` or that have been proposed upstream but are not merged yet, cannot be easily fetched or have a high chance to disappear in the future due to unstable or unreliable URLs. -The latter avoids link rot when the upstream abandons, squashes or rebases their change, in which case the commit may get garbage-collected. - If a patch is available online but does not cleanly apply, it can be modified in some fixed ways by using additional optional arguments for `fetchpatch`. Check [the `fetchpatch` reference](https://nixos.org/manual/nixpkgs/unstable/#fetchpatch) for details. +### Vendoring patches + +In the following cases, a `.patch` file _should_ be added to Nixpkgs repository, instead of retrieved: + +- solves problems unique to packaging in Nixpkgs +- is already proposed upstream but not merged yet +- cannot be fetched easily +- has a high chance to disappear in the future due to unstable or unreliable URLs + +The latter avoids link rot when the upstream abandons, squashes or rebases their change, in which case the commit may get garbage-collected. + ```nix { - patches = [ ./0001-changes.patch ]; + patches = [ ./0001-add-missing-include.patch ]; } ``` From a128993b4373fc2231ebd1190989bc8a42272dc4 Mon Sep 17 00:00:00 2001 From: Jules Aguillon Date: Tue, 3 Dec 2024 11:04:14 +0100 Subject: [PATCH 77/78] ocamlformat_0_27_0: init --- pkgs/development/ocaml-modules/ocamlformat/generic.nix | 3 ++- pkgs/top-level/all-packages.nix | 2 +- pkgs/top-level/ocaml-packages.nix | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkgs/development/ocaml-modules/ocamlformat/generic.nix b/pkgs/development/ocaml-modules/ocamlformat/generic.nix index d9dfd6dc1b1e..6e6f39630239 100644 --- a/pkgs/development/ocaml-modules/ocamlformat/generic.nix +++ b/pkgs/development/ocaml-modules/ocamlformat/generic.nix @@ -1,4 +1,4 @@ -{ lib, fetchurl, version ? "0.26.2", astring, base, camlp-streams, cmdliner_1_0 +{ lib, fetchurl, version ? "0.27.0", astring, base, camlp-streams, cmdliner_1_0 , cmdliner_1_1, csexp, dune-build-info, either, fix, fpath, menhirLib, menhirSdk , ocaml-version, ocp-indent, odoc-parser, result, stdio, uuseg, uutf , janeStreet_0_15, ... }: @@ -26,6 +26,7 @@ rec { "0.26.0" = "sha256-AxSUq3cM7xCo9qocvrVmDkbDqmwM1FexEP7IWadeh30="; "0.26.1" = "sha256-2gBuQn8VuexhL7gI1EZZm9m3w+4lq+s9VVdHpw10xtc="; "0.26.2" = "sha256-Lk9Za/eqNnqET+g7oPawvxSyplF53cCCNj/peT0DdcU="; + "0.27.0" = "sha256-3b9ITAdtCPmUAO6Et5DsIx9cj8vV0zJKZADVOI6EbRU="; }."${version}"; }; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7ea6186120b2..f7c3a0a6b7b0 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -6787,7 +6787,7 @@ with pkgs; inherit (ocamlPackages) ocamlformat # latest version - ocamlformat_0_26_2; + ocamlformat_0_26_2 ocamlformat_0_27_0; inherit (ocamlPackages) odig; diff --git a/pkgs/top-level/ocaml-packages.nix b/pkgs/top-level/ocaml-packages.nix index 35cb14aff535..5691515d8ba5 100644 --- a/pkgs/top-level/ocaml-packages.nix +++ b/pkgs/top-level/ocaml-packages.nix @@ -1339,6 +1339,7 @@ let ocamlformat_0_26_0 = ocamlformat.override { version = "0.26.0"; }; ocamlformat_0_26_1 = ocamlformat.override { version = "0.26.1"; }; ocamlformat_0_26_2 = ocamlformat.override { version = "0.26.2"; }; + ocamlformat_0_27_0 = ocamlformat.override { version = "0.27.0"; }; ocamlformat = callPackage ../development/ocaml-modules/ocamlformat/ocamlformat.nix {}; From f0fa6364de59cde2e6a13131fd9916c2a46ba65f Mon Sep 17 00:00:00 2001 From: Talha Can Havadar Date: Tue, 5 Nov 2024 15:47:22 +0100 Subject: [PATCH 78/78] python3Packages.adafruit-board-toolkit: init at 1.1.1 Apply suggestions from code review Co-authored-by: Arne Keller <2012gdwu+github@posteo.de> --- .../adafruit-board-toolkit/default.nix | 42 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 44 insertions(+) create mode 100644 pkgs/development/python-modules/adafruit-board-toolkit/default.nix diff --git a/pkgs/development/python-modules/adafruit-board-toolkit/default.nix b/pkgs/development/python-modules/adafruit-board-toolkit/default.nix new file mode 100644 index 000000000000..a719881b9d13 --- /dev/null +++ b/pkgs/development/python-modules/adafruit-board-toolkit/default.nix @@ -0,0 +1,42 @@ +{ + lib, + buildPythonPackage, + fetchPypi, + pythonOlder, + # build-system + setuptools-scm, + # dependencies + pyserial, +}: + +buildPythonPackage rec { + pname = "adafruit-board-toolkit"; + version = "1.1.1"; + pyproject = true; + + disabled = pythonOlder "3.5"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-k8TwmztGnCqS6F5ZrXSBnefZzc2kvuUqYxGAejRAdsI="; + }; + + # Project has not published tests yet + doCheck = false; + + build-system = [ + setuptools-scm + ]; + + dependencies = [ + pyserial + ]; + pythonImportsCheck = [ "adafruit_board_toolkit" ]; + meta = { + description = "CircuitPython board identification and information"; + homepage = "https://github.com/adafruit/Adafruit_Board_Toolkit"; + changelog = "https://github.com/adafruit/Adafruit_Board_Toolkit/releases/tag/${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ talhaHavadar ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 6582ff759efb..2455540b3061 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -55,6 +55,8 @@ self: super: with self; { adafruit-io = callPackage ../development/python-modules/adafruit-io { }; + adafruit-board-toolkit = callPackage ../development/python-modules/adafruit-board-toolkit { }; + adafruit-platformdetect = callPackage ../development/python-modules/adafruit-platformdetect { }; adafruit-pureio = callPackage ../development/python-modules/adafruit-pureio { };