From 9b660dff6d7771bae9e620ffa342794c4ff79842 Mon Sep 17 00:00:00 2001 From: "Adam C. Stephens" Date: Tue, 14 Jan 2025 19:34:48 -0500 Subject: [PATCH 1/4] incus: fix instance shutdown when softDaemonRestart enabled --- nixos/modules/virtualisation/incus.nix | 1 + nixos/tests/incus/incus-tests.nix | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/nixos/modules/virtualisation/incus.nix b/nixos/modules/virtualisation/incus.nix index 1a29e6ee104e..ead0ebb1badb 100644 --- a/nixos/modules/virtualisation/incus.nix +++ b/nixos/modules/virtualisation/incus.nix @@ -401,6 +401,7 @@ in "incus.socket" ]; requires = [ "incus.socket" ]; + wantedBy = config.systemd.services.incus.wantedBy; serviceConfig = { ExecStart = "${incus-startup} start"; diff --git a/nixos/tests/incus/incus-tests.nix b/nixos/tests/incus/incus-tests.nix index ee20139c21d8..05c5761b62b2 100644 --- a/nixos/tests/incus/incus-tests.nix +++ b/nixos/tests/incus/incus-tests.nix @@ -190,7 +190,7 @@ import ../make-test-python.nix ( def cleanup(): # avoid conflict between preseed and cleanup operations - machine.wait_for_unit("incus-preseed.service") + machine.execute("systemctl kill incus-preseed.service") instances = json.loads(machine.succeed("incus list --format json --all-projects")) with subtest("Stopping all running instances"): @@ -301,6 +301,13 @@ import ../make-test-python.nix ( machine.succeed(f"ps {pid}") machine.succeed("systemctl start incus") + with subtest("Instances stop with incus-startup.service"): + pid = machine.succeed(f"incus info container-{variant}1 | grep 'PID'").split(":")[1].strip() + machine.succeed(f"ps {pid}") + machine.succeed("systemctl stop incus-startup.service") + machine.wait_until_fails(f"ps {pid}", timeout=120) + machine.succeed("systemctl start incus-startup.service") + cleanup() '' @@ -360,6 +367,14 @@ import ../make-test-python.nix ( machine.succeed("systemctl start incus") + with subtest("Instances stop with incus-startup.service"): + pid = machine.succeed(f"incus info vm-{variant}1 | grep 'PID'").split(":")[1].strip() + machine.succeed(f"ps {pid}") + machine.succeed("systemctl stop incus-startup.service") + machine.wait_until_fails(f"ps {pid}", timeout=120) + machine.succeed("systemctl start incus-startup.service") + + cleanup() '' ) "" initVariants) From 274e0fd934df69ce630f40522ee4f7e9fee6d8ca Mon Sep 17 00:00:00 2001 From: "Adam C. Stephens" Date: Tue, 14 Jan 2025 20:03:50 -0500 Subject: [PATCH 2/4] incus: support per-instance lxcfs This was added in 6.4 and backported to LTS 6.0.2 --- nixos/modules/virtualisation/incus.nix | 1 + nixos/tests/incus/incus-tests.nix | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/nixos/modules/virtualisation/incus.nix b/nixos/modules/virtualisation/incus.nix index ead0ebb1badb..73d52c162fe6 100644 --- a/nixos/modules/virtualisation/incus.nix +++ b/nixos/modules/virtualisation/incus.nix @@ -38,6 +38,7 @@ let libnvidia-container libxfs lvm2 + lxcfs minio minio-client nftables diff --git a/nixos/tests/incus/incus-tests.nix b/nixos/tests/incus/incus-tests.nix index 05c5761b62b2..f41ae58fff5c 100644 --- a/nixos/tests/incus/incus-tests.nix +++ b/nixos/tests/incus/incus-tests.nix @@ -293,6 +293,16 @@ import ../make-test-python.nix ( check_sysctl(f"container-{variant}2") + with subtest("supports per-instance lxcfs"): + machine.succeed(f"incus stop container-{variant}1") + machine.fail(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'") + + machine.succeed("incus config set instances.lxcfs.per_instance=true") + + machine.succeed(f"incus start container-{variant}1") + wait_for_instance(f"container-{variant}1") + machine.succeed(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'") + with subtest("Instance remains running when softDaemonRestart is enabled and service is stopped"): pid = machine.succeed(f"incus info container-{variant}1 | grep 'PID'").split(":")[1].strip() From f3a75905ab2f3ea3cd67f14fc5b132ff2df7f1e5 Mon Sep 17 00:00:00 2001 From: "Adam C. Stephens" Date: Tue, 14 Jan 2025 20:16:26 -0500 Subject: [PATCH 3/4] nixos/tests/incus: fix subtest names and add reboot check --- nixos/tests/incus/incus-tests.nix | 76 ++++++++++++++++++------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/nixos/tests/incus/incus-tests.nix b/nixos/tests/incus/incus-tests.nix index f41ae58fff5c..5f648f4c04dc 100644 --- a/nixos/tests/incus/incus-tests.nix +++ b/nixos/tests/incus/incus-tests.nix @@ -228,39 +228,41 @@ import ../make-test-python.nix ( alias = "nixos/container/${variant}" variant = "${variant}" - with subtest("Container image can be imported"): + with subtest("container image can be imported"): machine.succeed(f"incus image import {metadata} {rootfs} --alias {alias}") - with subtest("Container can be launched and managed"): + with subtest("container can be launched and managed"): machine.succeed(f"incus launch {alias} container-{variant}1") wait_for_instance(f"container-{variant}1") - with subtest("Container mounts lxcfs overlays"): + with subtest("container mounts lxcfs overlays"): machine.succeed(f"incus exec container-{variant}1 mount | grep 'lxcfs on /proc/cpuinfo type fuse.lxcfs'") machine.succeed(f"incus exec container-{variant}1 mount | grep 'lxcfs on /proc/meminfo type fuse.lxcfs'") - with subtest("resource limits"): - with subtest("Container CPU limits can be managed"): - set_config(f"container-{variant}1", "limits.cpu 1", restart=True) - wait_incus_exec_success(f"container-{variant}1", "nproc | grep '^1$'", timeout=15) - - with subtest("Container CPU limits can be hotplug changed"): - set_config(f"container-{variant}1", "limits.cpu 2") - wait_incus_exec_success(f"container-{variant}1", "nproc | grep '^2$'", timeout=15) - - with subtest("Container memory limits can be managed"): - set_config(f"container-{variant}1", "limits.memory 128MB", restart=True) - wait_incus_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*125000 kB' /proc/meminfo", timeout=15) - - with subtest("Container memory limits can be hotplug changed"): - set_config(f"container-{variant}1", "limits.memory 256MB") - wait_incus_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*250000 kB' /proc/meminfo", timeout=15) + with subtest("container CPU limits can be managed"): + set_config(f"container-{variant}1", "limits.cpu 1", restart=True) + wait_incus_exec_success(f"container-{variant}1", "nproc | grep '^1$'", timeout=15) - with subtest("virtual tpm can be configured"): + with subtest("container CPU limits can be hotplug changed"): + set_config(f"container-{variant}1", "limits.cpu 2") + wait_incus_exec_success(f"container-{variant}1", "nproc | grep '^2$'", timeout=15) + + + with subtest("container memory limits can be managed"): + set_config(f"container-{variant}1", "limits.memory 128MB", restart=True) + wait_incus_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*125000 kB' /proc/meminfo", timeout=15) + + + with subtest("container memory limits can be hotplug changed"): + set_config(f"container-{variant}1", "limits.memory 256MB") + wait_incus_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*250000 kB' /proc/meminfo", timeout=15) + + + with subtest("container software tpm can be configured"): machine.succeed(f"incus config device add container-{variant}1 vtpm tpm path=/dev/tpm0 pathrm=/dev/tpmrm0") machine.succeed(f"incus exec container-{variant}1 -- test -e /dev/tpm0") machine.succeed(f"incus exec container-{variant}1 -- test -e /dev/tpmrm0") @@ -268,7 +270,7 @@ import ../make-test-python.nix ( machine.fail(f"incus exec container-{variant}1 -- test -e /dev/tpm0") - with subtest("lxc-generator"): + with subtest("container lxc-generator compatibility"): with subtest("lxc-container generator configures plain container"): # default container is plain machine.succeed(f"incus exec container-{variant}1 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf") @@ -293,7 +295,7 @@ import ../make-test-python.nix ( check_sysctl(f"container-{variant}2") - with subtest("supports per-instance lxcfs"): + with subtest("container supports per-instance lxcfs"): machine.succeed(f"incus stop container-{variant}1") machine.fail(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'") @@ -304,14 +306,19 @@ import ../make-test-python.nix ( machine.succeed(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'") - with subtest("Instance remains running when softDaemonRestart is enabled and service is stopped"): + with subtest("container can successfully restart"): + machine.succeed(f"incus restart container-{variant}1") + wait_for_instance(f"container-{variant}1") + + + with subtest("container remains running when softDaemonRestart is enabled and service is stopped"): pid = machine.succeed(f"incus info container-{variant}1 | grep 'PID'").split(":")[1].strip() machine.succeed(f"ps {pid}") machine.succeed("systemctl stop incus") machine.succeed(f"ps {pid}") machine.succeed("systemctl start incus") - with subtest("Instances stop with incus-startup.service"): + with subtest("containers stop with incus-startup.service"): pid = machine.succeed(f"incus info container-{variant}1 | grep 'PID'").split(":")[1].strip() machine.succeed(f"ps {pid}") machine.succeed("systemctl stop incus-startup.service") @@ -342,7 +349,7 @@ import ../make-test-python.nix ( machine.succeed(f"incus create {alias} vm-{variant}1 --vm --config limits.memory=512MB --config security.secureboot=false") - with subtest("virtual tpm can be configured"): + with subtest("virtual-machine software tpm can be configured"): machine.succeed(f"incus config device add vm-{variant}1 vtpm tpm path=/dev/tpm0") @@ -351,25 +358,30 @@ import ../make-test-python.nix ( wait_for_instance(f"vm-{variant}1") - with subtest("incus-agent is started"): + with subtest("virtual-machine incus-agent is started"): machine.succeed(f"incus exec vm-{variant}1 systemctl is-active incus-agent") - with subtest("incus-agent has a valid path"): + with subtest("virtual-machine incus-agent has a valid path"): machine.succeed(f"incus exec vm-{variant}1 -- bash -c 'true'") - with subtest("Container CPU limits can be managed"): + with subtest("virtual-machine CPU limits can be managed"): set_config(f"vm-{variant}1", "limits.cpu 1", restart=True) wait_incus_exec_success(f"vm-{variant}1", "nproc | grep '^1$'", timeout=90) - with subtest("Container CPU limits can be hotplug changed"): + with subtest("virtual-machine CPU limits can be hotplug changed"): set_config(f"vm-{variant}1", "limits.cpu 2") wait_incus_exec_success(f"vm-{variant}1", "nproc | grep '^2$'", timeout=15) - with subtest("Instance remains running when softDaemonRestart is enabled and service is stopped"): + with subtest("virtual-machine can successfully restart"): + machine.succeed(f"incus restart vm-{variant}1") + wait_for_instance(f"vm-{variant}1") + + + with subtest("virtual-machine remains running when softDaemonRestart is enabled and service is stopped"): pid = machine.succeed(f"incus info vm-{variant}1 | grep 'PID'").split(":")[1].strip() machine.succeed(f"ps {pid}") machine.succeed("systemctl stop incus") @@ -377,7 +389,7 @@ import ../make-test-python.nix ( machine.succeed("systemctl start incus") - with subtest("Instances stop with incus-startup.service"): + with subtest("virtual-machines stop with incus-startup.service"): pid = machine.succeed(f"incus info vm-{variant}1 | grep 'PID'").split(":")[1].strip() machine.succeed(f"ps {pid}") machine.succeed("systemctl stop incus-startup.service") @@ -391,7 +403,7 @@ import ../make-test-python.nix ( + # python '' - with subtest("Can launch CSM virtual machine"): + with subtest("virtual-machine can launch CSM (BIOS)"): machine.succeed("incus init csm --vm --empty -c security.csm=true -c security.secureboot=false") machine.succeed("incus start csm") From 80e73d690ab2c68a0e82eb9f2cd939a2fc2a23d1 Mon Sep 17 00:00:00 2001 From: "Adam C. Stephens" Date: Fri, 17 Jan 2025 08:45:54 -0500 Subject: [PATCH 4/4] nixos/incus: add lxc hook path to service env Fixes nvidia.runtime container passing --- nixos/modules/virtualisation/incus.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nixos/modules/virtualisation/incus.nix b/nixos/modules/virtualisation/incus.nix index 73d52c162fe6..065cd1213b9a 100644 --- a/nixos/modules/virtualisation/incus.nix +++ b/nixos/modules/virtualisation/incus.nix @@ -122,6 +122,7 @@ let environment = lib.mkMerge [ { INCUS_EDK2_PATH = ovmf; + INCUS_LXC_HOOK = "${cfg.lxcPackage}/share/lxc/hooks"; INCUS_LXC_TEMPLATE_CONFIG = "${pkgs.lxcfs}/share/lxc/config"; INCUS_USBIDS_PATH = "${pkgs.hwdata}/share/hwdata/usb.ids"; PATH = lib.mkForce serverBinPath;