From 54674e9063317a41b498b40ce856614bd5b7050f Mon Sep 17 00:00:00 2001 From: nikstur Date: Thu, 22 Feb 2024 00:12:04 +0100 Subject: [PATCH 1/6] nixos/qemu-vm: remove duplication between scripted and systemd initrd --- nixos/modules/virtualisation/qemu-vm.nix | 26 +++++++----------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index d1dc6404d4f5..f8a9602697a4 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -1046,26 +1046,14 @@ in boot.loader.supportsInitrdSecrets = mkIf (!cfg.useBootLoader) (mkVMOverride false); - boot.initrd.postMountCommands = lib.mkIf (!config.boot.initrd.systemd.enable) - '' - # Mark this as a NixOS machine. - mkdir -p $targetRoot/etc - echo -n > $targetRoot/etc/NIXOS + boot.initrd.postMountCommands = lib.mkIf (!config.boot.initrd.systemd.enable && cfg.writableStore) '' + echo "mounting overlay filesystem on /nix/store..." + mkdir -p -m 0755 $targetRoot/nix/.rw-store/store $targetRoot/nix/.rw-store/work $targetRoot/nix/store + mount -t overlay overlay $targetRoot/nix/store \ + -o lowerdir=$targetRoot/nix/.ro-store,upperdir=$targetRoot/nix/.rw-store/store,workdir=$targetRoot/nix/.rw-store/work || fail + ''; - # Fix the permissions on /tmp. - chmod 1777 $targetRoot/tmp - - mkdir -p $targetRoot/boot - - ${optionalString cfg.writableStore '' - echo "mounting overlay filesystem on /nix/store..." - mkdir -p -m 0755 $targetRoot/nix/.rw-store/store $targetRoot/nix/.rw-store/work $targetRoot/nix/store - mount -t overlay overlay $targetRoot/nix/store \ - -o lowerdir=$targetRoot/nix/.ro-store,upperdir=$targetRoot/nix/.rw-store/store,workdir=$targetRoot/nix/.rw-store/work || fail - ''} - ''; - - systemd.tmpfiles.settings."10-qemu-vm" = lib.mkIf config.boot.initrd.systemd.enable { + systemd.tmpfiles.settings."10-qemu-vm" = { "/etc/NIXOS".f = { mode = "0644"; user = "root"; From ded6d983d69cde4eadbfdc2d4725953464b0850c Mon Sep 17 00:00:00 2001 From: nikstur Date: Fri, 23 Feb 2024 00:00:25 +0100 Subject: [PATCH 2/6] nixos/qemu-vm: use new overlayfs API --- nixos/modules/virtualisation/qemu-vm.nix | 57 +++++++----------------- 1 file changed, 16 insertions(+), 41 deletions(-) diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index f8a9602697a4..215e5f2b9cf5 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -1046,13 +1046,6 @@ in boot.loader.supportsInitrdSecrets = mkIf (!cfg.useBootLoader) (mkVMOverride false); - boot.initrd.postMountCommands = lib.mkIf (!config.boot.initrd.systemd.enable && cfg.writableStore) '' - echo "mounting overlay filesystem on /nix/store..." - mkdir -p -m 0755 $targetRoot/nix/.rw-store/store $targetRoot/nix/.rw-store/work $targetRoot/nix/store - mount -t overlay overlay $targetRoot/nix/store \ - -o lowerdir=$targetRoot/nix/.ro-store,upperdir=$targetRoot/nix/.rw-store/store,workdir=$targetRoot/nix/.rw-store/work || fail - ''; - systemd.tmpfiles.settings."10-qemu-vm" = { "/etc/NIXOS".f = { mode = "0644"; @@ -1081,8 +1074,7 @@ in ''; boot.initrd.availableKernelModules = - optional cfg.writableStore "overlay" - ++ optional (cfg.qemu.diskInterface == "scsi") "sym53c8xx" + optional (cfg.qemu.diskInterface == "scsi") "sym53c8xx" ++ optional (cfg.tpm.enable) "tpm_tis"; virtualisation.additionalPaths = [ config.system.build.toplevel ]; @@ -1090,7 +1082,9 @@ in virtualisation.sharedDirectories = { nix-store = mkIf cfg.mountHostNixStore { source = builtins.storeDir; - target = "/nix/store"; + # Always mount this to /nix/.ro-store because we never want to actually + # write to the host Nix Store. + target = "/nix/.ro-store"; securityModel = "none"; }; xchg = { @@ -1200,10 +1194,7 @@ in virtualisation.fileSystems = let mkSharedDir = tag: share: { - name = - if tag == "nix-store" && cfg.writableStore - then "/nix/.ro-store" - else share.target; + name = share.target; value.device = tag; value.fsType = "9p"; value.neededForBoot = true; @@ -1228,7 +1219,17 @@ in # Sync with systemd's tmp.mount; options = [ "mode=1777" "strictatime" "nosuid" "nodev" "size=${toString config.boot.tmp.tmpfsSize}" ]; }; - "/nix/${if cfg.writableStore then ".ro-store" else "store"}" = lib.mkIf cfg.useNixStoreImage { + "/nix/store" = lib.mkIf (cfg.useNixStoreImage || cfg.mountHostNixStore) (if cfg.writableStore then { + overlay = { + lowerdir = [ "/nix/.ro-store" ]; + upperdir = "/nix/.rw-store/upper"; + workdir = "/nix/.rw-store/work"; + }; + } else { + device = "/nix/.ro-store"; + options = [ "bind" ]; + }); + "/nix/.ro-store" = lib.mkIf cfg.useNixStoreImage { device = "/dev/disk/by-label/${nixStoreFilesystemLabel}"; neededForBoot = true; options = [ "ro" ]; @@ -1246,32 +1247,6 @@ in } ]; - boot.initrd.systemd = lib.mkIf (config.boot.initrd.systemd.enable && cfg.writableStore) { - mounts = [{ - where = "/sysroot/nix/store"; - what = "overlay"; - type = "overlay"; - options = "lowerdir=/sysroot/nix/.ro-store,upperdir=/sysroot/nix/.rw-store/store,workdir=/sysroot/nix/.rw-store/work"; - wantedBy = ["initrd-fs.target"]; - before = ["initrd-fs.target"]; - requires = ["rw-store.service"]; - after = ["rw-store.service"]; - unitConfig.RequiresMountsFor = "/sysroot/nix/.ro-store"; - }]; - services.rw-store = { - before = [ "shutdown.target" ]; - conflicts = [ "shutdown.target" ]; - unitConfig = { - DefaultDependencies = false; - RequiresMountsFor = "/sysroot/nix/.rw-store"; - }; - serviceConfig = { - Type = "oneshot"; - ExecStart = "/bin/mkdir -p -m 0755 /sysroot/nix/.rw-store/store /sysroot/nix/.rw-store/work /sysroot/nix/store"; - }; - }; - }; - swapDevices = (if cfg.useDefaultFilesystems then mkVMOverride else mkDefault) [ ]; boot.initrd.luks.devices = (if cfg.useDefaultFilesystems then mkVMOverride else mkDefault) {}; From 0e5f92985160e5a9064861882179b6d4878f2220 Mon Sep 17 00:00:00 2001 From: nikstur Date: Fri, 23 Feb 2024 00:00:37 +0100 Subject: [PATCH 3/6] nixos/tests/qemu-vm-store: init --- nixos/tests/all-tests.nix | 1 + nixos/tests/qemu-vm-store.nix | 71 +++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 nixos/tests/qemu-vm-store.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 29eb36ab1f28..6503aa5bda7f 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -799,6 +799,7 @@ in { qemu-vm-restrictnetwork = handleTest ./qemu-vm-restrictnetwork.nix {}; qemu-vm-volatile-root = runTest ./qemu-vm-volatile-root.nix; qemu-vm-external-disk-image = runTest ./qemu-vm-external-disk-image.nix; + qemu-vm-store = runTest ./qemu-vm-store.nix; qgis = handleTest ./qgis.nix { qgisPackage = pkgs.qgis; }; qgis-ltr = handleTest ./qgis.nix { qgisPackage = pkgs.qgis-ltr; }; qownnotes = handleTest ./qownnotes.nix {}; diff --git a/nixos/tests/qemu-vm-store.nix b/nixos/tests/qemu-vm-store.nix new file mode 100644 index 000000000000..9fb9f4baaafc --- /dev/null +++ b/nixos/tests/qemu-vm-store.nix @@ -0,0 +1,71 @@ +{ lib, ... }: { + + name = "qemu-vm-store"; + + meta.maintainers = with lib.maintainers; [ nikstur ]; + + nodes = { + sharedWritable = { + virtualisation.writableStore = true; + }; + + sharedReadOnly = { + virtualisation.writableStore = false; + }; + + imageWritable = { + virtualisation.useNixStoreImage = true; + virtualisation.writableStore = true; + }; + + imageReadOnly = { + virtualisation.useNixStoreImage = true; + virtualisation.writableStore = false; + }; + + fullDisk = { + virtualisation.useBootLoader = true; + }; + }; + + testScript = '' + build_derivation = """ + nix-build --option substitute false -E 'derivation { + name = "t"; + builder = "/bin/sh"; + args = ["-c" "echo something > $out"]; + system = builtins.currentSystem; + preferLocalBuild = true; + }' + """ + + start_all() + + with subtest("Nix Store is writable"): + sharedWritable.succeed(build_derivation) + imageWritable.succeed(build_derivation) + fullDisk.succeed(build_derivation) + + with subtest("Nix Store is read only"): + sharedReadOnly.fail(build_derivation) + imageReadOnly.fail(build_derivation) + + # Checking whether the fs type is 9P is just a proxy to test whether the + # Nix Store is shared. If we switch to a different technology (e.g. + # virtiofs) for sharing, we need to adjust these tests. + + with subtest("Nix store is shared from the host via 9P"): + sharedWritable.succeed("findmnt --kernel --type 9P /nix/.ro-store") + sharedReadOnly.succeed("findmnt --kernel --type 9P /nix/.ro-store") + + with subtest("Nix store is not shared via 9P"): + imageWritable.fail("findmnt --kernel --type 9P /nix/.ro-store") + imageReadOnly.fail("findmnt --kernel --type 9P /nix/.ro-store") + + with subtest("Nix store is not mounted separately"): + rootDevice = fullDisk.succeed("stat -c %d /") + nixStoreDevice = fullDisk.succeed("stat -c %d /nix/store") + assert rootDevice == nixStoreDevice, "Nix store is mounted separately from the root fs" + ''; + +} From b7b90d04551c5dd2fd99b924bc0bde860a4cb72f Mon Sep 17 00:00:00 2001 From: nikstur Date: Fri, 23 Feb 2024 00:21:01 +0100 Subject: [PATCH 4/6] nixos/qemu-vm: fsck ESP The ESP is a writable vfat filesystem that can be fscked like any other fs thus we should stick to the default. --- nixos/modules/virtualisation/qemu-vm.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index 215e5f2b9cf5..0c63b640c389 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -1242,7 +1242,6 @@ in "/boot" = lib.mkIf (cfg.useBootLoader && cfg.bootPartition != null) { device = cfg.bootPartition; fsType = "vfat"; - noCheck = true; # fsck fails on a r/o filesystem }; } ]; From cb5a876e8243a6f09ad9883fe5db58f1f9a80b74 Mon Sep 17 00:00:00 2001 From: nikstur Date: Fri, 23 Feb 2024 00:56:13 +0100 Subject: [PATCH 5/6] nixos/qemu-vm: consistently use efiSysMountPoint Remove the explicit creation of efiSysMountPoint. This is not needed in the scripted or the systemd initrd. --- nixos/modules/virtualisation/qemu-vm.nix | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index 0c63b640c389..32a1d47b59fe 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -393,7 +393,7 @@ in The path (inside the VM) to the device containing the EFI System Partition (ESP). If you are *not* booting from a UEFI firmware, this value is, by - default, `null`. The ESP is mounted under `/boot`. + default, `null`. The ESP is mounted to `boot.loader.efi.efiSysMountpoint`. ''; }; @@ -1052,11 +1052,6 @@ in user = "root"; group = "root"; }; - "${config.boot.loader.efi.efiSysMountPoint}".d = { - mode = "0644"; - user = "root"; - group = "root"; - }; }; # After booting, register the closure of the paths in @@ -1239,7 +1234,7 @@ in options = [ "mode=0755" ]; neededForBoot = true; }; - "/boot" = lib.mkIf (cfg.useBootLoader && cfg.bootPartition != null) { + "${config.boot.loader.efi.efiSysMountPoint}" = lib.mkIf (cfg.useBootLoader && cfg.bootPartition != null) { device = cfg.bootPartition; fsType = "vfat"; }; From b6e354f052a79bec05432acec5f4810c1239b05f Mon Sep 17 00:00:00 2001 From: nikstur Date: Sat, 2 Mar 2024 00:06:26 +0100 Subject: [PATCH 6/6] nixos/qemu-vm: remove creation of /etc/nixos /etc/nixos is already created by stage-2-init.sh --- nixos/modules/virtualisation/qemu-vm.nix | 8 -------- 1 file changed, 8 deletions(-) diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index 32a1d47b59fe..bdce2b62a15a 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -1046,14 +1046,6 @@ in boot.loader.supportsInitrdSecrets = mkIf (!cfg.useBootLoader) (mkVMOverride false); - systemd.tmpfiles.settings."10-qemu-vm" = { - "/etc/NIXOS".f = { - mode = "0644"; - user = "root"; - group = "root"; - }; - }; - # After booting, register the closure of the paths in # `virtualisation.additionalPaths' in the Nix database in the VM. This # allows Nix operations to work in the VM. The path to the