nixos/test-driver: add support for nspawn containers

Co-authored-by: Jeremy Fleischman <jeremyfleischman@gmail.com>

nixos/lib/test-driver/default.nix

@@ -19,7 +19,9 @@
   qemu_test,
   setuptools,
   socat,
+  systemd,
   tesseract4,
+  util-linux,
   vde2,
 
   enableOCR ? false,
@@ -51,7 +53,9 @@ buildPythonApplication {
     netpbm
     qemu_pkg
     socat
+    util-linux
     vde2
+    systemd
   ]
   ++ lib.optionals enableOCR [
     imagemagick_light

nixos/lib/test-driver/src/test_driver/__init__.py

@@ -86,6 +86,22 @@ def main() -> None:
         nargs="*",
         help="start scripts for participating virtual machines",
     )
+    arg_parser.add_argument(
+        "--container-names",
+        metavar="CONTAINER-NAME",
+        action=EnvDefault,
+        envvar="containerNames",
+        nargs="*",
+        help="names of participating containers",
+    )
+    arg_parser.add_argument(
+        "--container-start-scripts",
+        metavar="CONTAINER-START-SCRIPT",
+        action=EnvDefault,
+        envvar="containerStartScripts",
+        nargs="*",
+        help="start scripts for participating containers",
+    )
     arg_parser.add_argument(
         "--vlans",
         metavar="VLAN",
@@ -150,10 +166,16 @@ def main() -> None:
         assert len(args.vm_names) == len(args.vm_start_scripts), (
             f"the number of vm names and vm start scripts must be the same: {args.vm_names} vs. {args.vm_start_scripts}"
         )
+    if args.container_names is not None and args.container_start_scripts is not None:
+        assert len(args.container_names) == len(args.container_start_scripts), (
+            f"the number of container names and container start scripts must be the same: {args.container_names} vs. {args.container_start_scripts}"
+        )
 
     with Driver(
         vm_names=args.vm_names,
         vm_start_scripts=args.vm_start_scripts or [],
+        container_names=args.container_names,
+        container_start_scripts=args.container_start_scripts or [],
         vlans=args.vlans,
         tests=args.testscript.read_text(),
         out_dir=output_directory,
@@ -187,6 +209,8 @@ def generate_driver_symbols() -> None:
     d = Driver(
         vm_names=[],
         vm_start_scripts=[],
+        container_names=[],
+        container_start_scripts=[],
         vlans=[],
         tests="",
         out_dir=Path(),

nixos/lib/test-driver/src/test_driver/driver.py

@@ -1,6 +1,7 @@
 import os
 import re
 import signal
+import subprocess
 import sys
 import tempfile
 import threading
@@ -16,7 +17,12 @@ from colorama import Style
 from test_driver.debug import DebugAbstract, DebugNop
 from test_driver.errors import MachineError, RequestedAssertionFailed
 from test_driver.logger import AbstractLogger
-from test_driver.machine import BaseMachine, QemuMachine, retry
+from test_driver.machine import (
+    BaseMachine,
+    NspawnMachine,
+    QemuMachine,
+    retry,
+)
 from test_driver.polling_condition import PollingCondition
 from test_driver.vlan import VLan
 
@@ -64,6 +70,7 @@ class Driver:
     tests: str
     vlans: list[VLan]
     vm_machines: list[QemuMachine]
+    container_machines: list[NspawnMachine]
     polling_conditions: list[PollingCondition]
     global_timeout: int
     race_timer: threading.Timer
@@ -74,6 +81,8 @@ class Driver:
         self,
         vm_names: list[str] | None,
         vm_start_scripts: list[str],
+        container_names: list[str] | None,
+        container_start_scripts: list[str],
         vlans: list[int],
         tests: str,
         out_dir: Path,
@@ -112,10 +121,75 @@ class Driver:
             )
         ]
 
+        if len(container_start_scripts) > 0:
+            self._init_nspawn_environment()
+
+        self.container_machines = [
+            NspawnMachine(
+                name=name,
+                start_command=container_start_script,
+                tmp_dir=tmp_dir,
+                logger=self.logger,
+                keep_vm_state=keep_vm_state,
+                callbacks=[self.check_polling_conditions],
+                out_dir=self.out_dir,
+            )
+            for name, container_start_script in zip(
+                container_names or (len(container_start_scripts) * [None]),
+                container_start_scripts,
+            )
+        ]
+
+    def _init_nspawn_environment(self) -> None:
+        assert os.geteuid() == 0, (
+            f"systemd-nspawn requires root to work. You are {os.geteuid()}"
+        )
+
+        # set up prerequisites for systemd-nspawn containers.
+        # these are not guaranteed to be set up in the Nix sandbox.
+        # if running interactively as root, these will already be set up.
+
+        # check if /run is writable by root
+        if not os.access("/run", os.W_OK):
+            Path("/run").mkdir(parents=True, exist_ok=True)
+            subprocess.run(["mount", "-t", "tmpfs", "none", "/run"], check=True)
+            Path("/run/netns").mkdir(parents=True, exist_ok=True)
+
+        # check if /var/run is a symlink to /run
+        if not (os.path.exists("/var/run") and os.path.samefile("/var/run", "/run")):
+            Path("/var").mkdir(parents=True, exist_ok=True)
+            subprocess.run(["ln", "-s", "/run", "/var/run"], check=True)
+
+        # check if /sys/fs/cgroup is mounted as cgroup2
+        with open("/proc/mounts", encoding="utf-8") as mounts:
+            for line in mounts:
+                parts = line.split()
+                if len(parts) >= 3 and parts[1] == "/sys/fs/cgroup":
+                    if parts[2] == "cgroup2":
+                        break
+            else:
+                Path("/sys/fs/cgroup").mkdir(parents=True, exist_ok=True)
+                subprocess.run(
+                    ["mount", "-t", "cgroup2", "none", "/sys/fs/cgroup"], check=True
+                )
+
+        # ensure /etc/os-release exists
+        if not os.path.isfile("/etc/os-release"):
+            subprocess.run(["touch", "/etc/os-release"], check=True)
+
+        # ensure /etc/machine-id exists and is non-empty
+        if (
+            not os.path.isfile("/etc/machine-id")
+            or os.path.getsize("/etc/machine-id") == 0
+        ):
+            subprocess.run(
+                ["systemd-machine-id-setup"], check=True
+            )  # set up /etc/machine-id
+
     @property
-    def machines(self) -> list[QemuMachine]:
-        machines = self.vm_machines
-        # Sort the machines by name for consistency with `nodes` in <nixos/lib/testing/network.nix>.
+    def machines(self) -> list[QemuMachine | NspawnMachine]:
+        machines = self.vm_machines + self.container_machines
+        # Sort the machines by name for consistency with `nodesAndContainers` in <nixos/lib/testing/network.nix>.
         machines.sort(key=lambda machine: machine.name)
         return machines
 
@@ -155,6 +229,7 @@ class Driver:
             start_all=self.start_all,
             test_script=self.test_script,
             vm_machines=self.vm_machines,
+            container_machines=self.container_machines,
             vlans=self.vlans,
             driver=self,
             log=self.logger,
@@ -286,13 +361,16 @@ class Driver:
         *,
         name: str | None = None,
         keep_vm_state: bool = False,
-    ) -> QemuMachine:
+    ) -> BaseMachine:
+        """
+        Create a `QemuMachine`. This currently only supports qemu "nodes", not containers.
+        """
         tmp_dir = get_tmp_dir()
 
         return QemuMachine(
-            start_command=start_command,
             tmp_dir=tmp_dir,
             out_dir=self.out_dir,
+            start_command=start_command,
             name=name,
             keep_vm_state=keep_vm_state,
             logger=self.logger,

nixos/lib/test-driver/src/test_driver/machine/__init__.py

@@ -1,5 +1,6 @@
 import base64
 import io
+import json
 import os
 import platform
 import queue
@@ -16,6 +17,7 @@ import time
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Generator
 from contextlib import _GeneratorContextManager, contextmanager, nullcontext
+from functools import cached_property
 from pathlib import Path
 from queue import Queue
 from typing import Any
@@ -218,7 +220,6 @@ class BaseMachine(ABC):
     name: str
     callbacks: list[Callable]
     tmp_dir: Path
-
     keep_vm_state: bool
 
     def __repr__(self) -> str:
@@ -239,7 +240,7 @@ class BaseMachine(ABC):
         self.callbacks = callbacks if callbacks is not None else []
         self.tmp_dir = tmp_dir
 
-        # Note: "vm" is a bit of a misnomer here.
+        # Note: "vm" is a bit of a misnomer here as we support both QEMU vms and nspawn containers.
         # Consider renaming to something more generic ("machine"?)
         self.keep_vm_state = keep_vm_state
 
@@ -269,26 +270,13 @@ class BaseMachine(ABC):
         return self.logger.nested(msg, my_attrs)
 
     @abstractmethod
-    def is_up(self) -> bool:
-        """
-        Check whether the machine is running.
-        """
-        pass
+    def is_up(self) -> bool: ...
 
     @abstractmethod
-    def start(self) -> None:
-        """
-        Start the machine.
-        """
-        pass
+    def start(self) -> None: ...
 
     @abstractmethod
-    def wait_for_shutdown(self) -> None:
-        """
-        Wait for the machine to power off. This does *not* initiate a shutdown;
-        that's usually done via `shutdown()`.
-        """
-        pass
+    def wait_for_shutdown(self) -> None: ...
 
     def systemctl(self, q: str, user: str | None = None) -> tuple[int, str]:
         """
@@ -1384,3 +1372,174 @@ class QemuMachine(BaseMachine):
         )
         self.connected = False
         self.connect()
+
+
+class NspawnMachine(BaseMachine):
+    """
+    A handle to a systemd-nspawn container machine with this name, that also
+    knows how to manage the machine lifecycle with the help of a start script / command.
+    """
+
+    start_command: str
+    tmp_dir: Path
+    process: subprocess.Popen | None
+    pid: int | None
+
+    @staticmethod
+    def machine_name_from_start_command(start_command: str) -> str:
+        match = re.search("run-(.+)-nspawn", os.path.basename(start_command))
+        assert match is not None, f"Could not extract node name from {start_command}"
+        return match.group(1)
+
+    def __init__(
+        self,
+        out_dir: Path,
+        name: str | None,
+        start_command: str,
+        tmp_dir: Path,
+        logger: AbstractLogger,
+        callbacks: list[Callable] | None = None,
+        keep_vm_state: bool = False,
+    ):
+        # TODO: don't compute `name` from `start_command` path, instead thread it down explicitly.
+        # See analogous TODO in `QemuStartCommand::machine_name`.
+        super().__init__(
+            out_dir=out_dir,
+            name=name or self.machine_name_from_start_command(start_command),
+            logger=logger,
+            callbacks=callbacks,
+            tmp_dir=tmp_dir,
+            keep_vm_state=keep_vm_state,
+        )
+
+        self.start_command = start_command
+        self.process = None
+        self.pid = None
+
+    def ssh_backdoor_command(self, index: int) -> str:
+        # get IP from `ip addr` inside the container:
+        ip_status, ip_output = self._execute("ip -j addr show")
+        assert ip_status == 0, "Failed to get IP addresses from container"
+        ip_output_data = json.loads(ip_output)
+        ip_addresses = [
+            addr_info.get("local")
+            for iface in ip_output_data
+            if iface.get("ifname") != "lo"
+            for addr_info in iface.get("addr_info", [])
+            if addr_info.get("family") == "inet"
+        ]
+
+        return "\n".join(f"ssh -o User=root {addr}" for addr in ip_addresses)
+
+    def release(self) -> None:
+        if self.pid is None:
+            return
+
+        self.logger.info(f"kill NspawnMachine (pid {self.pid})")
+        assert self.process is not None
+        self.process.terminate()
+        self.process = None
+
+    def is_up(self) -> bool:
+        return self.process is not None
+
+    @cached_property
+    def get_systemd_process(self) -> int:
+        assert self.process is not None, "Machine not started"
+        assert self.process.stdout is not None, "Machine has no stdout"
+
+        systemd_nspawn_pid = None
+        for line_bytes in self.process.stdout:
+            line = line_bytes.decode()
+            print(line, end="")
+
+            systemd_nspawn_pid_prefix = "systemd-nspawn's PID is "
+            if line.startswith(systemd_nspawn_pid_prefix):
+                systemd_nspawn_pid = int(line.removeprefix(systemd_nspawn_pid_prefix))
+
+            if (
+                line.startswith("systemd[1]: Startup finished in")
+                or "Welcome to NixOS" in line
+            ):
+                assert systemd_nspawn_pid is not None, "Must find systemd-nspawn PID"
+                break
+        else:
+            raise RuntimeError(f"Failed to start container {self.name}")
+
+        childs = (
+            Path(f"/proc/{systemd_nspawn_pid}/task/{systemd_nspawn_pid}/children")
+            .read_text()
+            .split()
+        )
+        assert len(childs) == 1, (
+            f"Expected exactly one child process for systemd-nspawn, got {childs}"
+        )
+        (child,) = childs
+
+        try:
+            return int(child)
+        except ValueError as e:
+            raise RuntimeError(f"Failed to parse child process id {child}") from e
+
+    def _execute(
+        self,
+        command: str,
+        check_return: bool = True,
+        check_output: bool = True,
+        timeout: int | None = 900,
+    ) -> tuple[int, str]:
+        self.start()
+
+        container_pid = self.get_systemd_process
+        nsenter = shutil.which("nsenter")
+        assert nsenter is not None
+
+        # Pull in /etc/profile, and some shell sanity.
+        command = f"set -eo pipefail; source /etc/profile; set -xu; {command}"
+        cp = subprocess.run(
+            [
+                nsenter,
+                "--target",
+                str(container_pid),
+                "--mount",
+                "--uts",
+                "--ipc",
+                "--net",
+                "--pid",
+                "--cgroup",
+                "/bin/sh",
+                "-c",
+                command,
+            ],
+            env={},
+            timeout=timeout,
+            stdout=subprocess.PIPE,
+            text=True,
+        )
+        return (cp.returncode, cp.stdout)
+
+    def start(self) -> None:
+        if self.process is not None:
+            return
+
+        self.process = subprocess.Popen(
+            [self.start_command],
+            env={
+                "RUN_NSPAWN_ROOT_DIR": str(self.state_dir),
+                "RUN_NSPAWN_SHARED_DIR": str(self.shared_dir),
+            },
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+
+        self.pid = self.process.pid
+
+        self.log(f"system-nspawn running (pid {self.pid})")
+
+    def wait_for_shutdown(self) -> None:
+        if self.process is None:
+            return
+
+        with self.nested("waiting for the container to power off"):
+            self.process.wait()
+            self.process = None

nixos/lib/test-script-prepend.py

@@ -4,7 +4,7 @@
 from test_driver.debug import DebugAbstract
 from test_driver.driver import Driver
 from test_driver.vlan import VLan
-from test_driver.machine import BaseMachine, QemuMachine
+from test_driver.machine import BaseMachine, NspawnMachine, QemuMachine
 from test_driver.logger import AbstractLogger
 from typing import Callable, Iterator, ContextManager, Optional, List, Dict, Any, Union
 from typing_extensions import Protocol

nixos/lib/testing-python.nix

@@ -56,6 +56,7 @@ pkgs.lib.throwIf (args ? specialArgs)
       {
         machine ? null,
         nodes ? { },
+        containers ? { },
         testScript,
         enableOCR ? false,
         globalTimeout ? (60 * 60),

nixos/lib/testing/driver.nix

@@ -14,12 +14,15 @@ let
     qemu_pkg = config.qemu.package;
     imagemagick_light = hostPkgs.imagemagick_light.override { inherit (hostPkgs) libtiff; };
     tesseract4 = hostPkgs.tesseract4.override { enableLanguages = [ "eng" ]; };
+    # We want `pkgs.systemd`, *not* `python3Packages.system`.
+    systemd = hostPkgs.systemd;
   };
 
   vlans = map (
     m: (m.virtualisation.vlans ++ (lib.mapAttrsToList (_: v: v.vlan) m.virtualisation.interfaces))
-  ) (lib.attrValues config.nodes);
+  ) ((lib.attrValues config.nodes) ++ (lib.attrValues config.containers));
   vms = map (m: m.system.build.vm) (lib.attrValues config.nodes);
+  containers = map (m: m.system.build.nspawn) (lib.attrValues config.containers);
 
   pythonizeName =
     name:
@@ -34,16 +37,28 @@ let
   vlanNames = map (i: "vlan${toString i}: VLan;") uniqueVlans;
 
   vmMachineNames = map (c: c.system.name) (lib.attrValues config.nodes);
+  containerMachineNames = map (c: c.system.name) (lib.attrValues config.containers);
+  allMachineNames =
+    let
+      overlappingNames = lib.intersectLists vmMachineNames containerMachineNames;
+    in
+    assert (
+      lib.asserts.assertMsg (overlappingNames == [ ]) "vm names and container names must not overlap"
+    );
+    vmMachineNames ++ containerMachineNames;
 
   theOnlyMachine =
     let
       exactlyOneMachine = lib.length (lib.attrValues config.nodes) == 1;
     in
-    lib.optional (exactlyOneMachine && !lib.elem "machine" vmMachineNames) "machine";
+    lib.optional (exactlyOneMachine && !lib.elem "machine" allMachineNames) "machine";
 
   pythonizedVmNames = map pythonizeName (vmMachineNames ++ theOnlyMachine);
   vmMachineTypeHints = map (name: "${name}: QemuMachine;") pythonizedVmNames;
 
+  pythonizedContainerNames = map pythonizeName containerMachineNames;
+  containerMachineTypeHints = map (name: "${name}: NspawnMachine;") pythonizedContainerNames;
+
   withChecks = lib.warnIf config.skipLint "Linting is disabled";
 
   driver =
@@ -67,11 +82,14 @@ let
 
         vmNames=(${lib.escapeShellArgs vmMachineNames})
         vmStartScripts=(${lib.escapeShellArgs (map lib.getExe vms)})
+        containerNames=(${lib.escapeShellArgs containerMachineNames})
+        containerStartScripts=(${lib.escapeShellArgs (map lib.getExe containers)})
 
         ${lib.optionalString (!config.skipTypeCheck) ''
           # prepend type hints so the test script can be type checked with mypy
           cat "${../test-script-prepend.py}" >> testScriptWithTypes
           echo "${toString vmMachineTypeHints}" >> testScriptWithTypes
+          echo "${toString containerMachineTypeHints}" >> testScriptWithTypes
           echo "${toString vlanNames}" >> testScriptWithTypes
           echo -n "$testScript" >> testScriptWithTypes
 
@@ -94,7 +112,9 @@ let
           echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipLint"
 
           PYFLAKES_BUILTINS="$(
-            echo -n ${lib.escapeShellArg (lib.concatStringsSep "," pythonizedVmNames)},
+            echo -n ${
+              lib.escapeShellArg (lib.concatStringsSep "," (pythonizedVmNames ++ pythonizedContainerNames))
+            },
             cat ${lib.escapeShellArg "driver-symbols"}
           )" ${hostPkgs.python3Packages.pyflakes}/bin/pyflakes $out/test-script
         ''}
@@ -104,6 +124,8 @@ let
         wrapProgram $out/bin/nixos-test-driver \
           --set vmStartScripts "''${vmStartScripts[*]}" \
           --set vmNames "''${vmNames[*]}" \
+          --set containerStartScripts "''${containerStartScripts[*]}" \
+          --set containerNames "''${containerNames[*]}" \
           --set testScript "$out/test-script" \
           --set globalTimeout "${toString config.globalTimeout}" \
           --set vlans '${toString vlans}' \

nixos/lib/testing/network.nix

@@ -1,11 +1,15 @@
-{ lib, nodes, ... }:
+{
+  containers,
+  nodes,
+  lib,
+  ...
+}:
 
 let
   inherit (lib)
     attrNames
-    concatMap
+    concatMapAttrsStringSep
     concatMapStrings
-    flip
     forEach
     head
     listToAttrs
@@ -20,22 +24,22 @@ let
     zipLists
     ;
 
-  nodeNumbers = listToAttrs (zipListsWith nameValuePair (attrNames nodes) (range 1 254));
+  nodesAndContainers =
+    let
+      nodeNames = lib.attrNames nodes;
+      containerNames = lib.attrNames containers;
+      conflictingNames = lib.intersectLists nodeNames containerNames;
+      message = "`nodes` and `containers` must have unique names. Conflicting names: ${lib.concatStringsSep " " conflictingNames}";
+    in
+    lib.throwIfNot (builtins.length conflictingNames == 0) message (nodes // containers);
+
+  nodeNumbers = listToAttrs (zipListsWith nameValuePair (attrNames nodesAndContainers) (range 1 254));
 
   networkModule =
-    {
-      config,
-      nodes,
-      pkgs,
-      ...
-    }:
+    { config, ... }:
     let
-      qemu-common = import ../qemu-common.nix { inherit (pkgs) lib stdenv; };
-
       interfaces = lib.attrValues config.virtualisation.allInterfaces;
 
-      interfacesNumbered = zipLists interfaces (range 1 255);
-
       # Automatically assign IP addresses to requested interfaces.
       assignIPs = lib.filter (i: i.assignIP) interfaces;
       ipInterfaces = forEach assignIPs (
@@ -56,17 +60,6 @@ let
         }
       );
 
-      qemuOptions = lib.flatten (
-        forEach interfacesNumbered (
-          { fst, snd }: qemu-common.qemuNICFlags snd fst.vlan config.virtualisation.test.nodeNumber
-        )
-      );
-      udevRules = forEach interfaces (
-        interface:
-        # MAC Addresses for QEMU network devices are lowercase, and udev string comparison is case-sensitive.
-        ''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac interface.vlan config.virtualisation.test.nodeNumber)}",NAME="${interface.name}"''
-      );
-
       networkConfig = {
         networking.hostName = mkDefault config.virtualisation.test.nodeName;
 
@@ -85,10 +78,9 @@ let
         # interfaces, use the IP address corresponding to
         # the first interface (i.e. the first network in its
         # virtualisation.vlans option).
-        networking.extraHosts = flip concatMapStrings (attrNames nodes) (
-          m':
+        networking.extraHosts = concatMapAttrsStringSep "" (
+          m': config:
           let
-            config = nodes.${m'};
             hostnames =
               optionalString (
                 config.networking.domain != null
@@ -101,10 +93,7 @@ let
           + optionalString (
             config.networking.primaryIPv6Address != ""
           ) "${config.networking.primaryIPv6Address} ${hostnames}"
-        );
-
-        virtualisation.qemu.options = qemuOptions;
-        boot.initrd.services.udev.rules = concatMapStrings (x: x + "\n") udevRules;
+        ) nodesAndContainers;
       };
 
     in
@@ -117,6 +106,31 @@ let
       };
     };
 
+  qemuNetworkModule =
+    { config, pkgs, ... }:
+    let
+      qemu-common = import ../qemu-common.nix { inherit (pkgs) lib stdenv; };
+
+      interfaces = lib.attrValues config.virtualisation.allInterfaces;
+
+      interfacesNumbered = zipLists interfaces (range 1 255);
+
+      qemuOptions = lib.flatten (
+        forEach interfacesNumbered (
+          { fst, snd }: qemu-common.qemuNICFlags snd fst.vlan config.virtualisation.test.nodeNumber
+        )
+      );
+      udevRules = map (
+        interface:
+        # MAC Addresses for QEMU network devices are lowercase, and udev string comparison is case-sensitive.
+        ''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac interface.vlan config.virtualisation.test.nodeNumber)}",NAME="${interface.name}"''
+      ) interfaces;
+    in
+    {
+      virtualisation.qemu.options = qemuOptions;
+      boot.initrd.services.udev.rules = concatMapStrings (x: x + "\n") udevRules;
+    };
+
   nodeNumberModule = (
     regular@{ config, name, ... }:
     {
@@ -127,7 +141,7 @@ let
           # We need to force this in specialisations, otherwise it'd be
           # readOnly = true;
           description = ''
-            The `name` in `nodes.<name>`; stable across `specialisations`.
+            The `name` in `nodes.<name>` and `containers.<name>`; stable across `specialisations`.
           '';
         };
         virtualisation.test.nodeNumber = mkOption {
@@ -136,7 +150,7 @@ let
           readOnly = true;
           default = nodeNumbers.${config.virtualisation.test.nodeName};
           description = ''
-            A unique number assigned for each node in `nodes`.
+            A unique number assigned for each machine in `nodes` and `containers`.
           '';
         };
 
@@ -172,5 +186,10 @@ in
         nodeNumberModule
       ];
     };
+    extraBaseNodeModules = {
+      imports = [
+        qemuNetworkModule
+      ];
+    };
   };
 }

nixos/lib/testing/nixos-test-base.nix

@@ -7,7 +7,6 @@ let
 in
 {
   imports = [
-    ../../modules/virtualisation/qemu-vm.nix
     ../../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs
     {
       key = "no-manual";
@@ -32,7 +31,9 @@ in
         # This is mostly a Hydra optimization, so we don't rebuild all the tests every time switch-to-configuration-ng changes.
         key = "no-switch-to-configuration";
         system.switch.enable = mkDefault (
-          config.isSpecialisation || config.specialisation != { } || config.virtualisation.installBootLoader
+          config.isSpecialisation
+          || config.specialisation != { }
+          || (!config.boot.isContainer && config.virtualisation.installBootLoader)
         );
       }
     )

nixos/lib/testing/nodes.nix

@@ -2,7 +2,6 @@ testModuleArgs@{
   config,
   lib,
   hostPkgs,
-  nodes,
   options,
   ...
 }:
@@ -12,12 +11,9 @@ let
     literalExpression
     literalMD
     mapAttrs
-    mkDefault
     mkIf
     mkMerge
     mkOption
-    mkForce
-    optional
     optionalAttrs
     types
     ;
@@ -51,13 +47,6 @@ let
         key = "nodes";
         _module.args.nodes = config.nodesCompat;
       }
-      (
-        { config, ... }:
-        {
-          virtualisation.qemu.package = testModuleArgs.config.qemu.package;
-          virtualisation.host.pkgs = hostPkgs;
-        }
-      )
       (
         { options, ... }:
         {
@@ -73,6 +62,44 @@ let
       testModuleArgs.config.extraBaseModules
     ];
   };
+  baseQemuOS = baseOS.extendModules {
+    modules = [
+      ../../modules/virtualisation/qemu-vm.nix
+      config.nodeDefaults
+      {
+        key = "base-qemu";
+        virtualisation.qemu.package = testModuleArgs.config.qemu.package;
+        virtualisation.host.pkgs = hostPkgs;
+      }
+      testModuleArgs.config.extraBaseNodeModules
+    ];
+  };
+  baseNspawnOS = baseOS.extendModules {
+    modules = [
+      ../../modules/virtualisation/nspawn-container
+      config.containerDefaults
+      (
+        { pkgs, ... }:
+        {
+          key = "base-nspawn";
+
+          # PAM requires setuid and doesn't work in the build sandbox.
+          # https://github.com/NixOS/nix/blob/959c244a1265f4048390f3ad21679219d7b27a99/src/libstore/unix/build/linux-derivation-builder.cc#L63
+          services.openssh.settings.UsePAM = false;
+
+          # Gross, insecure hack to make login work. See above.
+          security.pam.services.login = {
+            text = ''
+              auth sufficient ${pkgs.linux-pam}/lib/security/pam_permit.so
+              account sufficient ${pkgs.linux-pam}/lib/security/pam_permit.so
+              password sufficient ${pkgs.linux-pam}/lib/security/pam_permit.so
+              session sufficient ${pkgs.linux-pam}/lib/security/pam_permit.so
+            '';
+          };
+        }
+      )
+    ];
+  };
 
   # TODO (lib): Dedup with run.nix, add to lib/options.nix
   mkOneUp = opt: f: lib.mkOverride (opt.highestPrio - 1) (f opt.value);
@@ -109,15 +136,37 @@ in
 
     node.type = mkOption {
       type = types.raw;
-      default = baseOS.type;
+      default = baseQemuOS.type;
       internal = true;
     };
 
     nodes = mkOption {
       type = types.lazyAttrsOf config.node.type;
+      default = { };
       visible = "shallow";
       description = ''
-        An attribute set of NixOS configuration modules.
+        An attribute set of NixOS configuration modules representing QEMU vms that can be started during a test.
+
+        The configurations are augmented by the [`defaults`](#test-opt-defaults) option.
+
+        They are assigned network addresses according to the `nixos/lib/testing/network.nix` module.
+
+        A few special options are available, that aren't in a plain NixOS configuration. See [Configuring the nodes](#sec-nixos-test-nodes)
+      '';
+    };
+
+    container.type = mkOption {
+      type = types.raw;
+      default = baseNspawnOS.type;
+      internal = true;
+    };
+
+    containers = mkOption {
+      type = types.lazyAttrsOf config.container.type;
+      default = { };
+      visible = "shallow";
+      description = ''
+        An attribute set of NixOS configuration modules representing systemd-nspawn containers that can be started during a test.
 
         The configurations are augmented by the [`defaults`](#test-opt-defaults) option.
 
@@ -128,6 +177,14 @@ in
     };
 
     defaults = mkOption {
+      description = ''
+        NixOS configuration that is applied to all [{option}`nodes`](#test-opt-nodes) and [{option}`containers`](#test-opt-containers).
+      '';
+      type = types.deferredModule;
+      default = { };
+    };
+
+    nodeDefaults = mkOption {
       description = ''
         NixOS configuration that is applied to all [{option}`nodes`](#test-opt-nodes).
       '';
@@ -135,7 +192,23 @@ in
       default = { };
     };
 
+    containerDefaults = mkOption {
+      description = ''
+        NixOS configuration that is applied to all [{option}`containers`](#test-opt-containers).
+      '';
+      type = types.deferredModule;
+      default = { };
+    };
+
     extraBaseModules = mkOption {
+      description = ''
+        NixOS configuration that, like [{option}`defaults`](#test-opt-defaults), is applied to all [{option}`nodes`](#test-opt-nodes) and [{option}`containers`](#test-opt-containers) and can not be undone with [`specialisation.<name>.inheritParentConfig`](https://search.nixos.org/options?show=specialisation.%3Cname%3E.inheritParentConfig&from=0&size=50&sort=relevance&type=packages&query=specialisation).
+      '';
+      type = types.deferredModule;
+      default = { };
+    };
+
+    extraBaseNodeModules = mkOption {
       description = ''
         NixOS configuration that, like [{option}`defaults`](#test-opt-defaults), is applied to all [{option}`nodes`](#test-opt-nodes) and can not be undone with [`specialisation.<name>.inheritParentConfig`](https://search.nixos.org/options?show=specialisation.%3Cname%3E.inheritParentConfig&from=0&size=50&sort=relevance&type=packages&query=specialisation).
       '';
@@ -145,7 +218,7 @@ in
 
     node.pkgs = mkOption {
       description = ''
-        The Nixpkgs to use for the nodes.
+        The Nixpkgs to use for the nodes and containers.
 
         Setting this will make the `nixpkgs.*` options read-only, to avoid mistakenly testing with a Nixpkgs configuration that diverges from regular use.
       '';
@@ -160,7 +233,7 @@ in
       description = ''
         Whether to make the `nixpkgs.*` options read-only. This is only relevant when [`node.pkgs`](#test-opt-node.pkgs) is set.
 
-        Set this to `false` when any of the [`nodes`](#test-opt-nodes) needs to configure any of the `nixpkgs.*` options. This will slow down evaluation of your test a bit.
+        Set this to `false` when any of the [`nodes`](#test-opt-nodes) or [{option}`containers`](#test-opt-containers) need to configure any of the `nixpkgs.*` options. This will slow down evaluation of your test a bit.
       '';
       type = types.bool;
       default = config.node.pkgs != null;
@@ -188,6 +261,7 @@ in
   };
 
   config = {
+    _module.args.containers = config.containers;
     _module.args.nodes = config.nodesCompat;
     nodesCompat = mapAttrs (
       name: config:
@@ -201,6 +275,7 @@ in
     ) config.nodes;
 
     passthru.nodes = config.nodesCompat;
+    passthru.containers = config.containers;
 
     extraDriverArgs = mkIf config.sshBackdoor.enable [
       "--dump-vsocks=${toString config.sshBackdoor.vsockOffset}"
@@ -211,33 +286,35 @@ in
         nixpkgs.pkgs = config.node.pkgs;
         imports = [ ../../modules/misc/nixpkgs/read-only.nix ];
       })
-      (mkIf config.sshBackdoor.enable (
-        let
-          inherit (config.sshBackdoor) vsockOffset;
-        in
-        { config, ... }:
-        {
-          services.openssh = {
-            enable = true;
-            settings = {
-              PermitRootLogin = "yes";
-              PermitEmptyPasswords = "yes";
-            };
+      (mkIf config.sshBackdoor.enable {
+        services.openssh = {
+          enable = true;
+          settings = {
+            PermitRootLogin = "yes";
+            PermitEmptyPasswords = "yes";
           };
+        };
 
-          security.pam.services.sshd = {
-            allowNullPassword = true;
-          };
-
-          virtualisation.qemu.options = [
-            "-device vhost-vsock-pci,guest-cid=${
-              toString (config.virtualisation.test.nodeNumber + vsockOffset)
-            }"
-          ];
-        }
-      ))
+        security.pam.services.sshd = {
+          allowNullPassword = true;
+        };
+      })
     ];
 
+    nodeDefaults = mkIf config.sshBackdoor.enable (
+      let
+        inherit (config.sshBackdoor) vsockOffset;
+      in
+      { config, ... }:
+      {
+        virtualisation.qemu.options = [
+          "-device vhost-vsock-pci,guest-cid=${
+            toString (config.virtualisation.test.nodeNumber + vsockOffset)
+          }"
+        ];
+      }
+    );
+
     # Docs: nixos/doc/manual/development/writing-nixos-tests.section.md
     /**
       See https://nixos.org/manual/nixos/unstable#sec-override-nixos-test

nixos/lib/testing/run.nix

@@ -2,6 +2,7 @@
   config,
   hostPkgs,
   lib,
+  containers,
   options,
   ...
 }:
@@ -96,6 +97,8 @@ in
         requiredSystemFeatures = [
           "nixos-test"
         ]
+        # Containers use systemd-nspawn, which requires pid 0 inside of the sandbox.
+        ++ lib.optional (builtins.length (lib.attrNames containers) > 0) "uid-range"
         ++ lib.optional isLinux "kvm"
         ++ lib.optional isDarwin "apple-virt";
 

nixos/lib/testing/testScript.nix

@@ -56,11 +56,12 @@ in
               # reuse memoized config
               v
           ) config.nodesCompat;
+          containers = config.containers;
         }
       else
         config.testScript;
 
-    defaults =
+    nodeDefaults =
       { config, name, ... }:
       {
         # Make sure all derivations referenced by the test

nixos/modules/testing/test-instrumentation.nix

@@ -86,7 +86,8 @@ in
 
   options.testing = {
     backdoor = lib.mkEnableOption "backdoor service in stage 2" // {
-      default = true;
+      # See assertion below for why the backdoor doesn't work with containers.
+      default = !config.boot.isContainer;
     };
 
     initrdBackdoor = lib.mkEnableOption ''
@@ -105,7 +106,20 @@ in
       {
         assertion = cfg.initrdBackdoor -> config.boot.initrd.systemd.enable;
         message = ''
-          testing.initrdBackdoor requires boot.initrd.systemd.enable to be enabled.
+          `testing.initrdBackdoor` requires `boot.initrd.systemd.enable` to be enabled.
+        '';
+      }
+      {
+        assertion = config.boot.isContainer -> !cfg.backdoor;
+        message = ''
+          `testing.backdoor` uses virtio console, which does not work with
+          containers (we use `nsenter` instead).
+        '';
+      }
+      {
+        assertion = config.boot.isContainer -> !cfg.initrdBackdoor;
+        message = ''
+          `testing.initrdBackdoor` does not work with containers as there is no initrd.
         '';
       }
     ];