From 96a7448a4ea5fd45d9ae1070dd5bce3e2b55113a Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Mon, 9 Sep 2024 14:06:50 +0200 Subject: [PATCH 1/4] nixos/overlayfs: add a switch to disable prefixing with `/sysroot` for initrd mounts --- nixos/modules/tasks/filesystems/overlayfs.nix | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/nixos/modules/tasks/filesystems/overlayfs.nix b/nixos/modules/tasks/filesystems/overlayfs.nix index 3f1f0bc15b63..bc1840f9a96a 100644 --- a/nixos/modules/tasks/filesystems/overlayfs.nix +++ b/nixos/modules/tasks/filesystems/overlayfs.nix @@ -4,7 +4,7 @@ let # The scripted initrd contains some magic to add the prefix to the # paths just in time, so we don't add it here. sysrootPrefix = fs: - if config.boot.initrd.systemd.enable && (utils.fsNeededForBoot fs) then + if config.boot.initrd.systemd.enable && fs.overlay.useStage1BaseDirectories && (utils.fsNeededForBoot fs) then "/sysroot" else ""; @@ -47,11 +47,11 @@ let description = '' The list of path(s) to the lowerdir(s). - To create a writable overlay, you MUST provide an upperdir and a - workdir. + To create a writable overlay, you MUST provide an `upperdir` and a + `workdir`. You can create a read-only overlay when you provide multiple (at - least 2!) lowerdirs and neither an upperdir nor a workdir. + least 2!) lowerdirs and neither an `upperdir` nor a `workdir`. ''; }; @@ -63,6 +63,9 @@ let If this is null, a read-only overlay is created using the lowerdir. + If the filesystem is `neededForBoot`, this will be prefixed with `/sysroot`, + unless `useStage1BaseDirectories` is set to `true`. + If you set this to some value you MUST also set `workdir`. ''; }; @@ -73,10 +76,24 @@ let description = '' The path to the workdir. + If the filesystem is `neededForBoot`, this will be prefixed with `/sysroot`, + unless `useStage1BaseDirectories` is set to `true`. + This MUST be set if you set `upperdir`. ''; }; + useStage1BaseDirectories = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + If enabled, `lowerdir`, `upperdir` and `workdir` will be prefixed with `/sysroot`. + + Disabling this can be useful to create an overlay over directories which aren't on the real root. + + Disabling this does not work with the scripted (i.e. non-systemd) initrd. + ''; + }; }; config = lib.mkIf (config.overlay.lowerdir != null) { @@ -140,6 +157,13 @@ in -> (lib.length fs.overlay.lowerdir) >= 2; message = "A read-only overlay (without an `upperdir`) requires at least 2 `lowerdir`s: ${fs.mountPoint}"; } + { + assertion = !fs.overlay.useStage1BaseDirectories -> config.boot.initrd.systemd.enable; + message = '' + Stage 1 overlay file system ${fs.mountPoint} has 'useStage1BaseDirectories' set to false, + which is not supported with scripted initrd. Please enable 'boot.initrd.systemd.enable'. + ''; + } ]) overlayFileSystems ) ++ lib.mapAttrsToList (_: fs: { From cba277a1dcd82e872c75794e4b593dea84520575 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:26:02 +0100 Subject: [PATCH 2/4] nixos/overlayfs: format --- nixos/modules/tasks/filesystems/overlayfs.nix | 211 ++++++++++-------- 1 file changed, 114 insertions(+), 97 deletions(-) diff --git a/nixos/modules/tasks/filesystems/overlayfs.nix b/nixos/modules/tasks/filesystems/overlayfs.nix index bc1840f9a96a..7a2ddab96dca 100644 --- a/nixos/modules/tasks/filesystems/overlayfs.nix +++ b/nixos/modules/tasks/filesystems/overlayfs.nix @@ -1,17 +1,29 @@ -{ config, lib, pkgs, utils, ... }: +{ + config, + lib, + pkgs, + utils, + ... +}: let # The scripted initrd contains some magic to add the prefix to the # paths just in time, so we don't add it here. - sysrootPrefix = fs: - if config.boot.initrd.systemd.enable && fs.overlay.useStage1BaseDirectories && (utils.fsNeededForBoot fs) then + sysrootPrefix = + fs: + if + config.boot.initrd.systemd.enable + && fs.overlay.useStage1BaseDirectories + && (utils.fsNeededForBoot fs) + then "/sysroot" else ""; # Returns a service that creates the required directories before the mount is # created. - preMountService = _name: fs: + preMountService = + _name: fs: let prefix = sysrootPrefix fs; @@ -21,106 +33,111 @@ let upperdir = prefix + fs.overlay.upperdir; workdir = prefix + fs.overlay.workdir; in - lib.mkIf (fs.overlay.upperdir != null) - { - "rw-${escapedMountpoint}" = { - requiredBy = [ mountUnit ]; - before = [ mountUnit ]; - unitConfig = { - DefaultDependencies = false; - RequiresMountsFor = "${upperdir} ${workdir}"; - }; - serviceConfig = { - Type = "oneshot"; - ExecStart = "${pkgs.coreutils}/bin/mkdir -p -m 0755 ${upperdir} ${workdir}"; - }; + lib.mkIf (fs.overlay.upperdir != null) { + "rw-${escapedMountpoint}" = { + requiredBy = [ mountUnit ]; + before = [ mountUnit ]; + unitConfig = { + DefaultDependencies = false; + RequiresMountsFor = "${upperdir} ${workdir}"; + }; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${pkgs.coreutils}/bin/mkdir -p -m 0755 ${upperdir} ${workdir}"; + }; + }; + }; + + overlayOpts = + { + config, + ... + }: + { + options.overlay = { + lowerdir = lib.mkOption { + type = with lib.types; nullOr (nonEmptyListOf (either str pathInStore)); + default = null; + description = '' + The list of path(s) to the lowerdir(s). + + To create a writable overlay, you MUST provide an `upperdir` and a + `workdir`. + + You can create a read-only overlay when you provide multiple (at + least 2!) lowerdirs and neither an `upperdir` nor a `workdir`. + ''; + }; + + upperdir = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + The path to the upperdir. + + If this is null, a read-only overlay is created using the lowerdir. + + If the filesystem is `neededForBoot`, this will be prefixed with `/sysroot`, + unless `useStage1BaseDirectories` is set to `true`. + + If you set this to some value you MUST also set `workdir`. + ''; + }; + + workdir = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + The path to the workdir. + + If the filesystem is `neededForBoot`, this will be prefixed with `/sysroot`, + unless `useStage1BaseDirectories` is set to `true`. + + This MUST be set if you set `upperdir`. + ''; + }; + + useStage1BaseDirectories = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + If enabled, `lowerdir`, `upperdir` and `workdir` will be prefixed with `/sysroot`. + + Disabling this can be useful to create an overlay over directories which aren't on the real root. + + Disabling this does not work with the scripted (i.e. non-systemd) initrd. + ''; }; }; - overlayOpts = { config, ... }: { + config = lib.mkIf (config.overlay.lowerdir != null) { + fsType = "overlay"; + device = lib.mkDefault "overlay"; + depends = map (x: "${x}") ( + config.overlay.lowerdir + ++ lib.optionals (config.overlay.upperdir != null) [ + config.overlay.upperdir + config.overlay.workdir + ] + ); - options.overlay = { + options = + let + prefix = sysrootPrefix config; - lowerdir = lib.mkOption { - type = with lib.types; nullOr (nonEmptyListOf (either str pathInStore)); - default = null; - description = '' - The list of path(s) to the lowerdir(s). - - To create a writable overlay, you MUST provide an `upperdir` and a - `workdir`. - - You can create a read-only overlay when you provide multiple (at - least 2!) lowerdirs and neither an `upperdir` nor a `workdir`. - ''; - }; - - upperdir = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - description = '' - The path to the upperdir. - - If this is null, a read-only overlay is created using the lowerdir. - - If the filesystem is `neededForBoot`, this will be prefixed with `/sysroot`, - unless `useStage1BaseDirectories` is set to `true`. - - If you set this to some value you MUST also set `workdir`. - ''; - }; - - workdir = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - description = '' - The path to the workdir. - - If the filesystem is `neededForBoot`, this will be prefixed with `/sysroot`, - unless `useStage1BaseDirectories` is set to `true`. - - This MUST be set if you set `upperdir`. - ''; - }; - - useStage1BaseDirectories = lib.mkOption { - type = lib.types.bool; - default = true; - description = '' - If enabled, `lowerdir`, `upperdir` and `workdir` will be prefixed with `/sysroot`. - - Disabling this can be useful to create an overlay over directories which aren't on the real root. - - Disabling this does not work with the scripted (i.e. non-systemd) initrd. - ''; + lowerdir = map (s: prefix + s) config.overlay.lowerdir; + upperdir = prefix + config.overlay.upperdir; + workdir = prefix + config.overlay.workdir; + in + [ + "lowerdir=${lib.concatStringsSep ":" lowerdir}" + ] + ++ lib.optionals (config.overlay.upperdir != null) [ + "upperdir=${upperdir}" + "workdir=${workdir}" + ]; }; }; - - config = lib.mkIf (config.overlay.lowerdir != null) { - fsType = "overlay"; - device = lib.mkDefault "overlay"; - depends = map (x: "${x}") (config.overlay.lowerdir ++ lib.optionals (config.overlay.upperdir != null) [ - config.overlay.upperdir - config.overlay.workdir - ]); - - options = - let - prefix = sysrootPrefix config; - - lowerdir = map (s: prefix + s) config.overlay.lowerdir; - upperdir = prefix + config.overlay.upperdir; - workdir = prefix + config.overlay.workdir; - in - [ - "lowerdir=${lib.concatStringsSep ":" lowerdir}" - ] ++ lib.optionals (config.overlay.upperdir != null) [ - "upperdir=${upperdir}" - "workdir=${workdir}" - ]; - }; - - }; in { From fbaa0f529b53e62f56d5e406994b332f4d3ac272 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:26:48 +0100 Subject: [PATCH 3/4] nixos/tests/filesystems-overlayfs: format --- nixos/tests/filesystems-overlayfs.nix | 60 ++++++++++++++------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/nixos/tests/filesystems-overlayfs.nix b/nixos/tests/filesystems-overlayfs.nix index faac9078a520..79c75af81f1b 100644 --- a/nixos/tests/filesystems-overlayfs.nix +++ b/nixos/tests/filesystems-overlayfs.nix @@ -24,40 +24,42 @@ in meta.maintainers = with lib.maintainers; [ nikstur ]; - nodes.machine = { config, pkgs, ... }: { - boot.initrd.systemd.enable = true; + nodes.machine = + { config, pkgs, ... }: + { + boot.initrd.systemd.enable = true; - virtualisation.fileSystems = { - "/initrd-overlay" = { - overlay = { - lowerdir = [ initrdLowerdir ]; - upperdir = "/.rw-initrd-overlay/upper"; - workdir = "/.rw-initrd-overlay/work"; + virtualisation.fileSystems = { + "/initrd-overlay" = { + overlay = { + lowerdir = [ initrdLowerdir ]; + upperdir = "/.rw-initrd-overlay/upper"; + workdir = "/.rw-initrd-overlay/work"; + }; + neededForBoot = true; }; - neededForBoot = true; - }; - "/userspace-overlay" = { - overlay = { - lowerdir = [ userspaceLowerdir ]; - upperdir = "/.rw-userspace-overlay/upper"; - workdir = "/.rw-userspace-overlay/work"; + "/userspace-overlay" = { + overlay = { + lowerdir = [ userspaceLowerdir ]; + upperdir = "/.rw-userspace-overlay/upper"; + workdir = "/.rw-userspace-overlay/work"; + }; + }; + "/ro-initrd-overlay" = { + overlay.lowerdir = [ + initrdLowerdir + initrdLowerdir2 + ]; + neededForBoot = true; + }; + "/ro-userspace-overlay" = { + overlay.lowerdir = [ + userspaceLowerdir + userspaceLowerdir2 + ]; }; - }; - "/ro-initrd-overlay" = { - overlay.lowerdir = [ - initrdLowerdir - initrdLowerdir2 - ]; - neededForBoot = true; - }; - "/ro-userspace-overlay" = { - overlay.lowerdir = [ - userspaceLowerdir - userspaceLowerdir2 - ]; }; }; - }; testScript = '' machine.wait_for_unit("default.target") From 341179bfa2da981051fb9bffe6f4ed90f878eff7 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:27:19 +0100 Subject: [PATCH 4/4] nixos/tests/filesystems-overlayfs: add test for initrd -> userspace overlays --- nixos/tests/filesystems-overlayfs.nix | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nixos/tests/filesystems-overlayfs.nix b/nixos/tests/filesystems-overlayfs.nix index 79c75af81f1b..f79f45187be9 100644 --- a/nixos/tests/filesystems-overlayfs.nix +++ b/nixos/tests/filesystems-overlayfs.nix @@ -38,6 +38,14 @@ in }; neededForBoot = true; }; + "/initrd-real-root-overlay" = { + overlay = { + lowerdir = [ userspaceLowerdir ]; + upperdir = "/run/upper"; # from initrd + workdir = "/run/work"; # from initrd + useStage1BaseDirectories = false; + }; + }; "/userspace-overlay" = { overlay = { lowerdir = [ userspaceLowerdir ]; @@ -69,6 +77,11 @@ in machine.succeed("touch /initrd-overlay/writable.txt") machine.succeed("findmnt --kernel --types overlay /initrd-overlay") + with subtest("Userspace overlay with upper/workdir in initrd"): + machine.wait_for_file("/initrd-real-root-overlay/userspace.txt", 5) + machine.succeed("touch /initrd-real-root-overlay/writable.txt") + machine.succeed("findmnt --kernel --types overlay /initrd-real-root-overlay") + with subtest("Userspace overlay"): machine.wait_for_file("/userspace-overlay/userspace.txt", 5) machine.succeed("touch /userspace-overlay/writable.txt")