From 2187d1970e627656bf3a2ed7d23d579bc66874be Mon Sep 17 00:00:00 2001 From: r-vdp Date: Wed, 11 Dec 2024 14:37:48 +0100 Subject: [PATCH 1/2] nixos/etc-overlay: make the etc overlay compatible with nixos-enter and nixos-install When using nixos-enter (and so also nixos-install) on a system with etc-overlay enabled, he activation script gets called directly, and there is no systemd running. This violates a couple of assumptions in the etc-overlay activation script which assumed that it only ever ran when switching into a new generation and that the very first /etc would always have been set up by the systemd initrd. As more and more things are being moved into systemd components (initrd services, mount units, tmpfiles, etc), I think that it is going to become increasingly difficult to stay compatible with these tools, but at least for now there is no real alternative and so we probably want to be able to install systems with etc-overlay enabled. --- nixos/modules/system/etc/etc-activation.nix | 1 + nixos/modules/system/etc/etc.nix | 122 ++++++++++++-------- pkgs/by-name/ni/nixos-enter/nixos-enter.sh | 5 +- 3 files changed, 76 insertions(+), 52 deletions(-) diff --git a/nixos/modules/system/etc/etc-activation.nix b/nixos/modules/system/etc/etc-activation.nix index aed58af780b7..acf8c99eab91 100644 --- a/nixos/modules/system/etc/etc-activation.nix +++ b/nixos/modules/system/etc/etc-activation.nix @@ -15,6 +15,7 @@ system.activationScripts.etc = lib.stringAfter [ "users" "groups" + "specialfs" ] config.system.build.etcActivationCommands; } diff --git a/nixos/modules/system/etc/etc.nix b/nixos/modules/system/etc/etc.nix index 5f2acd7bacb9..062e414d5d6b 100644 --- a/nixos/modules/system/etc/etc.nix +++ b/nixos/modules/system/etc/etc.nix @@ -250,69 +250,91 @@ in ); in if config.system.etc.overlay.enable then + #bash '' - # This script atomically remounts /etc when switching configuration. On a (re-)boot - # this should not run because /etc is mounted via a systemd mount unit - # instead. To a large extent this mimics what composefs does. Because + # This script atomically remounts /etc when switching configuration. + # On a (re-)boot this should not run because /etc is mounted via a + # systemd mount unit instead. + # The activation script can also be called in cases where we didn't have + # an initrd though, like for instance when using nixos-enter, + # so we cannot assume that /etc has already been mounted. + # + # To a large extent this mimics what composefs does. Because # it's relatively simple, however, we avoid the composefs dependency. # Since this script is not idempotent, it should not run when etc hasn't # changed. if [[ ! $IN_NIXOS_SYSTEMD_STAGE1 ]] && [[ "${config.system.build.etc}/etc" != "$(readlink -f /run/current-system/etc)" ]]; then echo "remounting /etc..." - tmpMetadataMount=$(mktemp --directory -t nixos-etc-metadata.XXXXXXXXXX) + ${lib.optionalString config.system.etc.overlay.mutable '' + # These directories are usually created in initrd, + # but we need to create them here when we didn't we're called directly, + # for instance by nixos-enter + mkdir --parents /.rw-etc/upper /.rw-etc/work + chmod --recursive 0755 /.rw-etc + ''} + + tmpMetadataMount=$(TMPDIR="" mktemp --tmpdir=/tmp --directory -t nixos-etc-metadata.XXXXXXXXXX) mount --type erofs -o ro ${config.system.build.etcMetadataImage} $tmpMetadataMount - # Mount the new /etc overlay to a temporary private mount. - # This needs the indirection via a private bind mount because you - # cannot move shared mounts. - tmpEtcMount=$(mktemp --directory -t nixos-etc.XXXXXXXXXX) - mount --bind --make-private $tmpEtcMount $tmpEtcMount - mount --type overlay overlay \ - --options lowerdir=$tmpMetadataMount::${config.system.build.etcBasedir},${etcOverlayOptions} \ - $tmpEtcMount + # There was no previous /etc mounted. This happens when we're called + # directly without an initrd, like with nixos-enter. + if ! mountpoint -q /etc; then + mount --type overlay overlay \ + --options lowerdir=$tmpMetadataMount::${config.system.build.etcBasedir},${etcOverlayOptions} \ + /etc + else + # Mount the new /etc overlay to a temporary private mount. + # This needs the indirection via a private bind mount because you + # cannot move shared mounts. + tmpEtcMount=$(TMPDIR="" mktemp --tmpdir=/tmp --directory -t nixos-etc.XXXXXXXXXX) + mount --bind --make-private $tmpEtcMount $tmpEtcMount + mount --type overlay overlay \ + --options lowerdir=$tmpMetadataMount::${config.system.build.etcBasedir},${etcOverlayOptions} \ + $tmpEtcMount - # Before moving the new /etc overlay under the old /etc, we have to - # move mounts on top of /etc to the new /etc mountpoint. - findmnt /etc --submounts --list --noheading --kernel --output TARGET | while read -r mountPoint; do - if [[ "$mountPoint" = "/etc" ]]; then - continue - fi + # Before moving the new /etc overlay under the old /etc, we have to + # move mounts on top of /etc to the new /etc mountpoint. + findmnt /etc --submounts --list --noheading --kernel --output TARGET | while read -r mountPoint; do + if [[ "$mountPoint" = "/etc" ]]; then + continue + fi - tmpMountPoint="$tmpEtcMount/''${mountPoint:5}" - ${ - if config.system.etc.overlay.mutable then - '' - if [[ -f "$mountPoint" ]]; then - touch "$tmpMountPoint" - elif [[ -d "$mountPoint" ]]; then - mkdir -p "$tmpMountPoint" - fi - '' - else - '' - if [[ ! -e "$tmpMountPoint" ]]; then - echo "Skipping undeclared mountpoint in environment.etc: $mountPoint" - continue - fi - '' - } - mount --bind "$mountPoint" "$tmpMountPoint" - done + tmpMountPoint="$tmpEtcMount/''${mountPoint:5}" + ${ + if config.system.etc.overlay.mutable then + '' + if [[ -f "$mountPoint" ]]; then + touch "$tmpMountPoint" + elif [[ -d "$mountPoint" ]]; then + mkdir -p "$tmpMountPoint" + fi + '' + else + '' + if [[ ! -e "$tmpMountPoint" ]]; then + echo "Skipping undeclared mountpoint in environment.etc: $mountPoint" + continue + fi + '' + } + mount --bind "$mountPoint" "$tmpMountPoint" + done - # Move the new temporary /etc mount underneath the current /etc mount. - # - # This should eventually use util-linux to perform this move beneath, - # however, this functionality is not yet in util-linux. See this - # tracking issue: https://github.com/util-linux/util-linux/issues/2604 - ${pkgs.move-mount-beneath}/bin/move-mount --move --beneath $tmpEtcMount /etc + # Move the new temporary /etc mount underneath the current /etc mount. + # + # This should eventually use util-linux to perform this move beneath, + # however, this functionality is not yet in util-linux. See this + # tracking issue: https://github.com/util-linux/util-linux/issues/2604 + ${pkgs.move-mount-beneath}/bin/move-mount --move --beneath $tmpEtcMount /etc - # Unmount the top /etc mount to atomically reveal the new mount. - umount --lazy --recursive /etc + # Unmount the top /etc mount to atomically reveal the new mount. + umount --lazy --recursive /etc - # Unmount the temporary mount - umount --lazy "$tmpEtcMount" - rmdir "$tmpEtcMount" + # Unmount the temporary mount + umount --lazy "$tmpEtcMount" + rmdir "$tmpEtcMount" + fi # Unmount old metadata mounts # For some reason, `findmnt /tmp --submounts` does not show the nested @@ -321,7 +343,7 @@ in findmnt --type erofs --list --kernel --output TARGET | while read -r mountPoint; do if [[ "$mountPoint" =~ ^/tmp/nixos-etc-metadata\..{10}$ && "$mountPoint" != "$tmpMetadataMount" ]]; then - umount --lazy $mountPoint + umount --lazy "$mountPoint" rmdir "$mountPoint" fi done diff --git a/pkgs/by-name/ni/nixos-enter/nixos-enter.sh b/pkgs/by-name/ni/nixos-enter/nixos-enter.sh index 214ab7dfbda4..906c5b0e0b90 100755 --- a/pkgs/by-name/ni/nixos-enter/nixos-enter.sh +++ b/pkgs/by-name/ni/nixos-enter/nixos-enter.sh @@ -58,10 +58,11 @@ if [[ ! -e $mountPoint/etc/NIXOS ]]; then exit 126 fi -mkdir -p "$mountPoint/dev" "$mountPoint/sys" -chmod 0755 "$mountPoint/dev" "$mountPoint/sys" +mkdir -p "$mountPoint/dev" "$mountPoint/sys" "$mountPoint/proc" +chmod 0755 "$mountPoint/dev" "$mountPoint/sys" "$mountPoint/proc" mount --rbind /dev "$mountPoint/dev" mount --rbind /sys "$mountPoint/sys" +mount --rbind /proc "$mountPoint/proc" # modified from https://github.com/archlinux/arch-install-scripts/blob/bb04ab435a5a89cd5e5ee821783477bc80db797f/arch-chroot.in#L26-L52 chroot_add_resolv_conf() { From df7c405f3201216f7be5301848f5af91c6136988 Mon Sep 17 00:00:00 2001 From: r-vdp Date: Thu, 12 Dec 2024 10:14:23 +0100 Subject: [PATCH 2/2] nixos/etc-overlay: always create the metadata mountpoints in /run This avoids the dependence on the presence of /tmp, which causes issues with nixos-install --- nixos/modules/system/etc/etc-activation.nix | 6 +++--- nixos/modules/system/etc/etc.nix | 6 +++--- nixos/tests/activation/etc-overlay-immutable.nix | 16 ++++++++++------ nixos/tests/activation/etc-overlay-mutable.nix | 16 ++++++++++------ 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/nixos/modules/system/etc/etc-activation.nix b/nixos/modules/system/etc/etc-activation.nix index acf8c99eab91..59fa2faa2b80 100644 --- a/nixos/modules/system/etc/etc-activation.nix +++ b/nixos/modules/system/etc/etc-activation.nix @@ -47,7 +47,7 @@ boot.initrd.systemd = { mounts = [ { - where = "/run/etc-metadata"; + where = "/run/nixos-etc-metadata"; what = "/etc-metadata-image"; type = "erofs"; options = "loop,ro"; @@ -82,7 +82,7 @@ "relatime" "redirect_dir=on" "metacopy=on" - "lowerdir=/run/etc-metadata::/etc-basedir" + "lowerdir=/run/nixos-etc-metadata::/etc-basedir" ] ++ lib.optionals config.system.etc.overlay.mutable [ "rw" @@ -112,7 +112,7 @@ unitConfig = { RequiresMountsFor = [ "/sysroot/nix/store" - "/run/etc-metadata" + "/run/nixos-etc-metadata" ]; DefaultDependencies = false; }; diff --git a/nixos/modules/system/etc/etc.nix b/nixos/modules/system/etc/etc.nix index 062e414d5d6b..7c7db061359c 100644 --- a/nixos/modules/system/etc/etc.nix +++ b/nixos/modules/system/etc/etc.nix @@ -274,7 +274,7 @@ in chmod --recursive 0755 /.rw-etc ''} - tmpMetadataMount=$(TMPDIR="" mktemp --tmpdir=/tmp --directory -t nixos-etc-metadata.XXXXXXXXXX) + tmpMetadataMount=$(TMPDIR="/run" mktemp --directory -t nixos-etc-metadata.XXXXXXXXXX) mount --type erofs -o ro ${config.system.build.etcMetadataImage} $tmpMetadataMount # There was no previous /etc mounted. This happens when we're called @@ -287,7 +287,7 @@ in # Mount the new /etc overlay to a temporary private mount. # This needs the indirection via a private bind mount because you # cannot move shared mounts. - tmpEtcMount=$(TMPDIR="" mktemp --tmpdir=/tmp --directory -t nixos-etc.XXXXXXXXXX) + tmpEtcMount=$(TMPDIR="/run" mktemp --directory -t nixos-etc.XXXXXXXXXX) mount --bind --make-private $tmpEtcMount $tmpEtcMount mount --type overlay overlay \ --options lowerdir=$tmpMetadataMount::${config.system.build.etcBasedir},${etcOverlayOptions} \ @@ -341,7 +341,7 @@ in # mounts. So we'll just find all mounts of type erofs and filter on the # name of the mountpoint. findmnt --type erofs --list --kernel --output TARGET | while read -r mountPoint; do - if [[ "$mountPoint" =~ ^/tmp/nixos-etc-metadata\..{10}$ && + if [[ ("$mountPoint" =~ ^/run/nixos-etc-metadata\..{10}$ || "$mountPoint" =~ ^/run/nixos-etc-metadata$ ) && "$mountPoint" != "$tmpMetadataMount" ]]; then umount --lazy "$mountPoint" rmdir "$mountPoint" diff --git a/nixos/tests/activation/etc-overlay-immutable.nix b/nixos/tests/activation/etc-overlay-immutable.nix index da1ea317440c..2adb91114014 100644 --- a/nixos/tests/activation/etc-overlay-immutable.nix +++ b/nixos/tests/activation/etc-overlay-immutable.nix @@ -39,8 +39,8 @@ '' newergen = machine.succeed("realpath /run/current-system/specialisation/newer-generation/bin/switch-to-configuration").rstrip() - with subtest("/run/etc-metadata/ is mounted"): - print(machine.succeed("mountpoint /run/etc-metadata")) + with subtest("/run/nixos-etc-metadata/ is mounted"): + print(machine.succeed("mountpoint /run/nixos-etc-metadata")) with subtest("No temporary files leaked into stage 2"): machine.succeed("[ ! -e /etc-metadata-image ]") @@ -91,10 +91,14 @@ machine.succeed(f"{newergen} switch") - tmpMounts = machine.succeed("find /tmp -maxdepth 1 -type d -regex '/tmp/nixos-etc\\..*' | wc -l").rstrip() - metaMounts = machine.succeed("find /tmp -maxdepth 1 -type d -regex '/tmp/nixos-etc-metadata\\..*' | wc -l").rstrip() + tmpMounts = machine.succeed("find /run -maxdepth 1 -type d -regex '/run/nixos-etc\\..*'").rstrip() + print(tmpMounts) + metaMounts = machine.succeed("find /run -maxdepth 1 -type d -regex '/run/nixos-etc-metadata.*'").rstrip() + print(metaMounts) - assert tmpMounts == "0", f"Found {tmpMounts} remaining tmpmounts" - assert metaMounts == "1", f"Found {metaMounts} remaining metamounts" + numOfTmpMounts = len(tmpMounts.splitlines()) + numOfMetaMounts = len(metaMounts.splitlines()) + assert numOfTmpMounts == 0, f"Found {numOfTmpMounts} remaining tmpmounts" + assert numOfMetaMounts == 1, f"Found {numOfMetaMounts} remaining metamounts" ''; } diff --git a/nixos/tests/activation/etc-overlay-mutable.nix b/nixos/tests/activation/etc-overlay-mutable.nix index be87e1120894..5dc27dedb30b 100644 --- a/nixos/tests/activation/etc-overlay-mutable.nix +++ b/nixos/tests/activation/etc-overlay-mutable.nix @@ -27,8 +27,8 @@ '' newergen = machine.succeed("realpath /run/current-system/specialisation/newer-generation/bin/switch-to-configuration").rstrip() - with subtest("/run/etc-metadata/ is mounted"): - print(machine.succeed("mountpoint /run/etc-metadata")) + with subtest("/run/nixos-etc-metadata/ is mounted"): + print(machine.succeed("mountpoint /run/nixos-etc-metadata")) with subtest("No temporary files leaked into stage 2"): machine.succeed("[ ! -e /etc-metadata-image ]") @@ -68,10 +68,14 @@ machine.succeed(f"{newergen} switch") assert machine.succeed("cat /etc/newergen") == "newergen" - tmpMounts = machine.succeed("find /tmp -maxdepth 1 -type d -regex '/tmp/nixos-etc\\..*' | wc -l").rstrip() - metaMounts = machine.succeed("find /tmp -maxdepth 1 -type d -regex '/tmp/nixos-etc-metadata\\..*' | wc -l").rstrip() + tmpMounts = machine.succeed("find /run -maxdepth 1 -type d -regex '/run/nixos-etc\\..*'").rstrip() + print(tmpMounts) + metaMounts = machine.succeed("find /run -maxdepth 1 -type d -regex '/run/nixos-etc-metadata.*'").rstrip() + print(metaMounts) - assert tmpMounts == "0", f"Found {tmpMounts} remaining tmpmounts" - assert metaMounts == "1", f"Found {metaMounts} remaining metamounts" + numOfTmpMounts = len(tmpMounts.splitlines()) + numOfMetaMounts = len(metaMounts.splitlines()) + assert numOfTmpMounts == 0, f"Found {numOfTmpMounts} remaining tmpmounts" + assert numOfMetaMounts == 1, f"Found {numOfMetaMounts} remaining metamounts" ''; }