diff --git a/nixos/lib/test-driver/default.nix b/nixos/lib/test-driver/default.nix index ed8eb2c8c771..13158d65de6b 100644 --- a/nixos/lib/test-driver/default.nix +++ b/nixos/lib/test-driver/default.nix @@ -19,7 +19,9 @@ qemu_test, setuptools, socat, + systemd, tesseract4, + util-linux, vde2, enableOCR ? false, @@ -51,7 +53,9 @@ buildPythonApplication { netpbm qemu_pkg socat + util-linux vde2 + systemd ] ++ lib.optionals enableOCR [ imagemagick_light diff --git a/nixos/lib/test-driver/src/test_driver/__init__.py b/nixos/lib/test-driver/src/test_driver/__init__.py index 35c5e3b11fb6..9c465af778c7 100755 --- a/nixos/lib/test-driver/src/test_driver/__init__.py +++ b/nixos/lib/test-driver/src/test_driver/__init__.py @@ -86,6 +86,22 @@ def main() -> None: nargs="*", help="start scripts for participating virtual machines", ) + arg_parser.add_argument( + "--container-names", + metavar="CONTAINER-NAME", + action=EnvDefault, + envvar="containerNames", + nargs="*", + help="names of participating containers", + ) + arg_parser.add_argument( + "--container-start-scripts", + metavar="CONTAINER-START-SCRIPT", + action=EnvDefault, + envvar="containerStartScripts", + nargs="*", + help="start scripts for participating containers", + ) arg_parser.add_argument( "--vlans", metavar="VLAN", @@ -150,10 +166,16 @@ def main() -> None: assert len(args.vm_names) == len(args.vm_start_scripts), ( f"the number of vm names and vm start scripts must be the same: {args.vm_names} vs. {args.vm_start_scripts}" ) + if args.container_names is not None and args.container_start_scripts is not None: + assert len(args.container_names) == len(args.container_start_scripts), ( + f"the number of container names and container start scripts must be the same: {args.container_names} vs. {args.container_start_scripts}" + ) with Driver( vm_names=args.vm_names, vm_start_scripts=args.vm_start_scripts or [], + container_names=args.container_names, + container_start_scripts=args.container_start_scripts or [], vlans=args.vlans, tests=args.testscript.read_text(), out_dir=output_directory, @@ -187,6 +209,8 @@ def generate_driver_symbols() -> None: d = Driver( vm_names=[], vm_start_scripts=[], + container_names=[], + container_start_scripts=[], vlans=[], tests="", out_dir=Path(), diff --git a/nixos/lib/test-driver/src/test_driver/driver.py b/nixos/lib/test-driver/src/test_driver/driver.py index c8694d2ea09a..cc04d6b5b707 100644 --- a/nixos/lib/test-driver/src/test_driver/driver.py +++ b/nixos/lib/test-driver/src/test_driver/driver.py @@ -1,6 +1,7 @@ import os import re import signal +import subprocess import sys import tempfile import threading @@ -16,7 +17,12 @@ from colorama import Style from test_driver.debug import DebugAbstract, DebugNop from test_driver.errors import MachineError, RequestedAssertionFailed from test_driver.logger import AbstractLogger -from test_driver.machine import BaseMachine, QemuMachine, retry +from test_driver.machine import ( + BaseMachine, + NspawnMachine, + QemuMachine, + retry, +) from test_driver.polling_condition import PollingCondition from test_driver.vlan import VLan @@ -64,6 +70,7 @@ class Driver: tests: str vlans: list[VLan] vm_machines: list[QemuMachine] + container_machines: list[NspawnMachine] polling_conditions: list[PollingCondition] global_timeout: int race_timer: threading.Timer @@ -74,6 +81,8 @@ class Driver: self, vm_names: list[str] | None, vm_start_scripts: list[str], + container_names: list[str] | None, + container_start_scripts: list[str], vlans: list[int], tests: str, out_dir: Path, @@ -112,10 +121,75 @@ class Driver: ) ] + if len(container_start_scripts) > 0: + self._init_nspawn_environment() + + self.container_machines = [ + NspawnMachine( + name=name, + start_command=container_start_script, + tmp_dir=tmp_dir, + logger=self.logger, + keep_vm_state=keep_vm_state, + callbacks=[self.check_polling_conditions], + out_dir=self.out_dir, + ) + for name, container_start_script in zip( + container_names or (len(container_start_scripts) * [None]), + container_start_scripts, + ) + ] + + def _init_nspawn_environment(self) -> None: + assert os.geteuid() == 0, ( + f"systemd-nspawn requires root to work. You are {os.geteuid()}" + ) + + # set up prerequisites for systemd-nspawn containers. + # these are not guaranteed to be set up in the Nix sandbox. + # if running interactively as root, these will already be set up. + + # check if /run is writable by root + if not os.access("/run", os.W_OK): + Path("/run").mkdir(parents=True, exist_ok=True) + subprocess.run(["mount", "-t", "tmpfs", "none", "/run"], check=True) + Path("/run/netns").mkdir(parents=True, exist_ok=True) + + # check if /var/run is a symlink to /run + if not (os.path.exists("/var/run") and os.path.samefile("/var/run", "/run")): + Path("/var").mkdir(parents=True, exist_ok=True) + subprocess.run(["ln", "-s", "/run", "/var/run"], check=True) + + # check if /sys/fs/cgroup is mounted as cgroup2 + with open("/proc/mounts", encoding="utf-8") as mounts: + for line in mounts: + parts = line.split() + if len(parts) >= 3 and parts[1] == "/sys/fs/cgroup": + if parts[2] == "cgroup2": + break + else: + Path("/sys/fs/cgroup").mkdir(parents=True, exist_ok=True) + subprocess.run( + ["mount", "-t", "cgroup2", "none", "/sys/fs/cgroup"], check=True + ) + + # ensure /etc/os-release exists + if not os.path.isfile("/etc/os-release"): + subprocess.run(["touch", "/etc/os-release"], check=True) + + # ensure /etc/machine-id exists and is non-empty + if ( + not os.path.isfile("/etc/machine-id") + or os.path.getsize("/etc/machine-id") == 0 + ): + subprocess.run( + ["systemd-machine-id-setup"], check=True + ) # set up /etc/machine-id + @property - def machines(self) -> list[QemuMachine]: - machines = self.vm_machines - # Sort the machines by name for consistency with `nodes` in . + def machines(self) -> list[QemuMachine | NspawnMachine]: + machines = self.vm_machines + self.container_machines + # Sort the machines by name for consistency with `nodesAndContainers` in . machines.sort(key=lambda machine: machine.name) return machines @@ -155,6 +229,7 @@ class Driver: start_all=self.start_all, test_script=self.test_script, vm_machines=self.vm_machines, + container_machines=self.container_machines, vlans=self.vlans, driver=self, log=self.logger, @@ -286,13 +361,16 @@ class Driver: *, name: str | None = None, keep_vm_state: bool = False, - ) -> QemuMachine: + ) -> BaseMachine: + """ + Create a `QemuMachine`. This currently only supports qemu "nodes", not containers. + """ tmp_dir = get_tmp_dir() return QemuMachine( - start_command=start_command, tmp_dir=tmp_dir, out_dir=self.out_dir, + start_command=start_command, name=name, keep_vm_state=keep_vm_state, logger=self.logger, diff --git a/nixos/lib/test-driver/src/test_driver/machine/__init__.py b/nixos/lib/test-driver/src/test_driver/machine/__init__.py index fecdeda493b5..32d5bc79166e 100644 --- a/nixos/lib/test-driver/src/test_driver/machine/__init__.py +++ b/nixos/lib/test-driver/src/test_driver/machine/__init__.py @@ -1,5 +1,6 @@ import base64 import io +import json import os import platform import queue @@ -16,6 +17,7 @@ import time from abc import ABC, abstractmethod from collections.abc import Callable, Generator from contextlib import _GeneratorContextManager, contextmanager, nullcontext +from functools import cached_property from pathlib import Path from queue import Queue from typing import Any @@ -218,7 +220,6 @@ class BaseMachine(ABC): name: str callbacks: list[Callable] tmp_dir: Path - keep_vm_state: bool def __repr__(self) -> str: @@ -239,7 +240,7 @@ class BaseMachine(ABC): self.callbacks = callbacks if callbacks is not None else [] self.tmp_dir = tmp_dir - # Note: "vm" is a bit of a misnomer here. + # Note: "vm" is a bit of a misnomer here as we support both QEMU vms and nspawn containers. # Consider renaming to something more generic ("machine"?) self.keep_vm_state = keep_vm_state @@ -269,26 +270,13 @@ class BaseMachine(ABC): return self.logger.nested(msg, my_attrs) @abstractmethod - def is_up(self) -> bool: - """ - Check whether the machine is running. - """ - pass + def is_up(self) -> bool: ... @abstractmethod - def start(self) -> None: - """ - Start the machine. - """ - pass + def start(self) -> None: ... @abstractmethod - def wait_for_shutdown(self) -> None: - """ - Wait for the machine to power off. This does *not* initiate a shutdown; - that's usually done via `shutdown()`. - """ - pass + def wait_for_shutdown(self) -> None: ... def systemctl(self, q: str, user: str | None = None) -> tuple[int, str]: """ @@ -1384,3 +1372,174 @@ class QemuMachine(BaseMachine): ) self.connected = False self.connect() + + +class NspawnMachine(BaseMachine): + """ + A handle to a systemd-nspawn container machine with this name, that also + knows how to manage the machine lifecycle with the help of a start script / command. + """ + + start_command: str + tmp_dir: Path + process: subprocess.Popen | None + pid: int | None + + @staticmethod + def machine_name_from_start_command(start_command: str) -> str: + match = re.search("run-(.+)-nspawn", os.path.basename(start_command)) + assert match is not None, f"Could not extract node name from {start_command}" + return match.group(1) + + def __init__( + self, + out_dir: Path, + name: str | None, + start_command: str, + tmp_dir: Path, + logger: AbstractLogger, + callbacks: list[Callable] | None = None, + keep_vm_state: bool = False, + ): + # TODO: don't compute `name` from `start_command` path, instead thread it down explicitly. + # See analogous TODO in `QemuStartCommand::machine_name`. + super().__init__( + out_dir=out_dir, + name=name or self.machine_name_from_start_command(start_command), + logger=logger, + callbacks=callbacks, + tmp_dir=tmp_dir, + keep_vm_state=keep_vm_state, + ) + + self.start_command = start_command + self.process = None + self.pid = None + + def ssh_backdoor_command(self, index: int) -> str: + # get IP from `ip addr` inside the container: + ip_status, ip_output = self._execute("ip -j addr show") + assert ip_status == 0, "Failed to get IP addresses from container" + ip_output_data = json.loads(ip_output) + ip_addresses = [ + addr_info.get("local") + for iface in ip_output_data + if iface.get("ifname") != "lo" + for addr_info in iface.get("addr_info", []) + if addr_info.get("family") == "inet" + ] + + return "\n".join(f"ssh -o User=root {addr}" for addr in ip_addresses) + + def release(self) -> None: + if self.pid is None: + return + + self.logger.info(f"kill NspawnMachine (pid {self.pid})") + assert self.process is not None + self.process.terminate() + self.process = None + + def is_up(self) -> bool: + return self.process is not None + + @cached_property + def get_systemd_process(self) -> int: + assert self.process is not None, "Machine not started" + assert self.process.stdout is not None, "Machine has no stdout" + + systemd_nspawn_pid = None + for line_bytes in self.process.stdout: + line = line_bytes.decode() + print(line, end="") + + systemd_nspawn_pid_prefix = "systemd-nspawn's PID is " + if line.startswith(systemd_nspawn_pid_prefix): + systemd_nspawn_pid = int(line.removeprefix(systemd_nspawn_pid_prefix)) + + if ( + line.startswith("systemd[1]: Startup finished in") + or "Welcome to NixOS" in line + ): + assert systemd_nspawn_pid is not None, "Must find systemd-nspawn PID" + break + else: + raise RuntimeError(f"Failed to start container {self.name}") + + childs = ( + Path(f"/proc/{systemd_nspawn_pid}/task/{systemd_nspawn_pid}/children") + .read_text() + .split() + ) + assert len(childs) == 1, ( + f"Expected exactly one child process for systemd-nspawn, got {childs}" + ) + (child,) = childs + + try: + return int(child) + except ValueError as e: + raise RuntimeError(f"Failed to parse child process id {child}") from e + + def _execute( + self, + command: str, + check_return: bool = True, + check_output: bool = True, + timeout: int | None = 900, + ) -> tuple[int, str]: + self.start() + + container_pid = self.get_systemd_process + nsenter = shutil.which("nsenter") + assert nsenter is not None + + # Pull in /etc/profile, and some shell sanity. + command = f"set -eo pipefail; source /etc/profile; set -xu; {command}" + cp = subprocess.run( + [ + nsenter, + "--target", + str(container_pid), + "--mount", + "--uts", + "--ipc", + "--net", + "--pid", + "--cgroup", + "/bin/sh", + "-c", + command, + ], + env={}, + timeout=timeout, + stdout=subprocess.PIPE, + text=True, + ) + return (cp.returncode, cp.stdout) + + def start(self) -> None: + if self.process is not None: + return + + self.process = subprocess.Popen( + [self.start_command], + env={ + "RUN_NSPAWN_ROOT_DIR": str(self.state_dir), + "RUN_NSPAWN_SHARED_DIR": str(self.shared_dir), + }, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + self.pid = self.process.pid + + self.log(f"system-nspawn running (pid {self.pid})") + + def wait_for_shutdown(self) -> None: + if self.process is None: + return + + with self.nested("waiting for the container to power off"): + self.process.wait() + self.process = None diff --git a/nixos/lib/test-script-prepend.py b/nixos/lib/test-script-prepend.py index 84a689f93d16..e06a210d6dd7 100644 --- a/nixos/lib/test-script-prepend.py +++ b/nixos/lib/test-script-prepend.py @@ -4,7 +4,7 @@ from test_driver.debug import DebugAbstract from test_driver.driver import Driver from test_driver.vlan import VLan -from test_driver.machine import BaseMachine, QemuMachine +from test_driver.machine import BaseMachine, NspawnMachine, QemuMachine from test_driver.logger import AbstractLogger from typing import Callable, Iterator, ContextManager, Optional, List, Dict, Any, Union from typing_extensions import Protocol diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index 878f9669321a..1d2777ecb033 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -56,6 +56,7 @@ pkgs.lib.throwIf (args ? specialArgs) { machine ? null, nodes ? { }, + containers ? { }, testScript, enableOCR ? false, globalTimeout ? (60 * 60), diff --git a/nixos/lib/testing/driver.nix b/nixos/lib/testing/driver.nix index 5623c63c4dc5..db56f55307c8 100644 --- a/nixos/lib/testing/driver.nix +++ b/nixos/lib/testing/driver.nix @@ -14,12 +14,15 @@ let qemu_pkg = config.qemu.package; imagemagick_light = hostPkgs.imagemagick_light.override { inherit (hostPkgs) libtiff; }; tesseract4 = hostPkgs.tesseract4.override { enableLanguages = [ "eng" ]; }; + # We want `pkgs.systemd`, *not* `python3Packages.system`. + systemd = hostPkgs.systemd; }; vlans = map ( m: (m.virtualisation.vlans ++ (lib.mapAttrsToList (_: v: v.vlan) m.virtualisation.interfaces)) - ) (lib.attrValues config.nodes); + ) ((lib.attrValues config.nodes) ++ (lib.attrValues config.containers)); vms = map (m: m.system.build.vm) (lib.attrValues config.nodes); + containers = map (m: m.system.build.nspawn) (lib.attrValues config.containers); pythonizeName = name: @@ -34,16 +37,28 @@ let vlanNames = map (i: "vlan${toString i}: VLan;") uniqueVlans; vmMachineNames = map (c: c.system.name) (lib.attrValues config.nodes); + containerMachineNames = map (c: c.system.name) (lib.attrValues config.containers); + allMachineNames = + let + overlappingNames = lib.intersectLists vmMachineNames containerMachineNames; + in + assert ( + lib.asserts.assertMsg (overlappingNames == [ ]) "vm names and container names must not overlap" + ); + vmMachineNames ++ containerMachineNames; theOnlyMachine = let exactlyOneMachine = lib.length (lib.attrValues config.nodes) == 1; in - lib.optional (exactlyOneMachine && !lib.elem "machine" vmMachineNames) "machine"; + lib.optional (exactlyOneMachine && !lib.elem "machine" allMachineNames) "machine"; pythonizedVmNames = map pythonizeName (vmMachineNames ++ theOnlyMachine); vmMachineTypeHints = map (name: "${name}: QemuMachine;") pythonizedVmNames; + pythonizedContainerNames = map pythonizeName containerMachineNames; + containerMachineTypeHints = map (name: "${name}: NspawnMachine;") pythonizedContainerNames; + withChecks = lib.warnIf config.skipLint "Linting is disabled"; driver = @@ -67,11 +82,14 @@ let vmNames=(${lib.escapeShellArgs vmMachineNames}) vmStartScripts=(${lib.escapeShellArgs (map lib.getExe vms)}) + containerNames=(${lib.escapeShellArgs containerMachineNames}) + containerStartScripts=(${lib.escapeShellArgs (map lib.getExe containers)}) ${lib.optionalString (!config.skipTypeCheck) '' # prepend type hints so the test script can be type checked with mypy cat "${../test-script-prepend.py}" >> testScriptWithTypes echo "${toString vmMachineTypeHints}" >> testScriptWithTypes + echo "${toString containerMachineTypeHints}" >> testScriptWithTypes echo "${toString vlanNames}" >> testScriptWithTypes echo -n "$testScript" >> testScriptWithTypes @@ -94,7 +112,9 @@ let echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipLint" PYFLAKES_BUILTINS="$( - echo -n ${lib.escapeShellArg (lib.concatStringsSep "," pythonizedVmNames)}, + echo -n ${ + lib.escapeShellArg (lib.concatStringsSep "," (pythonizedVmNames ++ pythonizedContainerNames)) + }, cat ${lib.escapeShellArg "driver-symbols"} )" ${hostPkgs.python3Packages.pyflakes}/bin/pyflakes $out/test-script ''} @@ -104,6 +124,8 @@ let wrapProgram $out/bin/nixos-test-driver \ --set vmStartScripts "''${vmStartScripts[*]}" \ --set vmNames "''${vmNames[*]}" \ + --set containerStartScripts "''${containerStartScripts[*]}" \ + --set containerNames "''${containerNames[*]}" \ --set testScript "$out/test-script" \ --set globalTimeout "${toString config.globalTimeout}" \ --set vlans '${toString vlans}' \ diff --git a/nixos/lib/testing/network.nix b/nixos/lib/testing/network.nix index 9a5facfc2433..99edf16676ae 100644 --- a/nixos/lib/testing/network.nix +++ b/nixos/lib/testing/network.nix @@ -1,11 +1,15 @@ -{ lib, nodes, ... }: +{ + containers, + nodes, + lib, + ... +}: let inherit (lib) attrNames - concatMap + concatMapAttrsStringSep concatMapStrings - flip forEach head listToAttrs @@ -20,22 +24,22 @@ let zipLists ; - nodeNumbers = listToAttrs (zipListsWith nameValuePair (attrNames nodes) (range 1 254)); + nodesAndContainers = + let + nodeNames = lib.attrNames nodes; + containerNames = lib.attrNames containers; + conflictingNames = lib.intersectLists nodeNames containerNames; + message = "`nodes` and `containers` must have unique names. Conflicting names: ${lib.concatStringsSep " " conflictingNames}"; + in + lib.throwIfNot (builtins.length conflictingNames == 0) message (nodes // containers); + + nodeNumbers = listToAttrs (zipListsWith nameValuePair (attrNames nodesAndContainers) (range 1 254)); networkModule = - { - config, - nodes, - pkgs, - ... - }: + { config, ... }: let - qemu-common = import ../qemu-common.nix { inherit (pkgs) lib stdenv; }; - interfaces = lib.attrValues config.virtualisation.allInterfaces; - interfacesNumbered = zipLists interfaces (range 1 255); - # Automatically assign IP addresses to requested interfaces. assignIPs = lib.filter (i: i.assignIP) interfaces; ipInterfaces = forEach assignIPs ( @@ -56,17 +60,6 @@ let } ); - qemuOptions = lib.flatten ( - forEach interfacesNumbered ( - { fst, snd }: qemu-common.qemuNICFlags snd fst.vlan config.virtualisation.test.nodeNumber - ) - ); - udevRules = forEach interfaces ( - interface: - # MAC Addresses for QEMU network devices are lowercase, and udev string comparison is case-sensitive. - ''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac interface.vlan config.virtualisation.test.nodeNumber)}",NAME="${interface.name}"'' - ); - networkConfig = { networking.hostName = mkDefault config.virtualisation.test.nodeName; @@ -85,10 +78,9 @@ let # interfaces, use the IP address corresponding to # the first interface (i.e. the first network in its # virtualisation.vlans option). - networking.extraHosts = flip concatMapStrings (attrNames nodes) ( - m': + networking.extraHosts = concatMapAttrsStringSep "" ( + m': config: let - config = nodes.${m'}; hostnames = optionalString ( config.networking.domain != null @@ -101,10 +93,7 @@ let + optionalString ( config.networking.primaryIPv6Address != "" ) "${config.networking.primaryIPv6Address} ${hostnames}" - ); - - virtualisation.qemu.options = qemuOptions; - boot.initrd.services.udev.rules = concatMapStrings (x: x + "\n") udevRules; + ) nodesAndContainers; }; in @@ -117,6 +106,31 @@ let }; }; + qemuNetworkModule = + { config, pkgs, ... }: + let + qemu-common = import ../qemu-common.nix { inherit (pkgs) lib stdenv; }; + + interfaces = lib.attrValues config.virtualisation.allInterfaces; + + interfacesNumbered = zipLists interfaces (range 1 255); + + qemuOptions = lib.flatten ( + forEach interfacesNumbered ( + { fst, snd }: qemu-common.qemuNICFlags snd fst.vlan config.virtualisation.test.nodeNumber + ) + ); + udevRules = map ( + interface: + # MAC Addresses for QEMU network devices are lowercase, and udev string comparison is case-sensitive. + ''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac interface.vlan config.virtualisation.test.nodeNumber)}",NAME="${interface.name}"'' + ) interfaces; + in + { + virtualisation.qemu.options = qemuOptions; + boot.initrd.services.udev.rules = concatMapStrings (x: x + "\n") udevRules; + }; + nodeNumberModule = ( regular@{ config, name, ... }: { @@ -127,7 +141,7 @@ let # We need to force this in specialisations, otherwise it'd be # readOnly = true; description = '' - The `name` in `nodes.`; stable across `specialisations`. + The `name` in `nodes.` and `containers.`; stable across `specialisations`. ''; }; virtualisation.test.nodeNumber = mkOption { @@ -136,7 +150,7 @@ let readOnly = true; default = nodeNumbers.${config.virtualisation.test.nodeName}; description = '' - A unique number assigned for each node in `nodes`. + A unique number assigned for each machine in `nodes` and `containers`. ''; }; @@ -172,5 +186,10 @@ in nodeNumberModule ]; }; + extraBaseNodeModules = { + imports = [ + qemuNetworkModule + ]; + }; }; } diff --git a/nixos/lib/testing/nixos-test-base.nix b/nixos/lib/testing/nixos-test-base.nix index 23358f2185b1..6b518e39ac11 100644 --- a/nixos/lib/testing/nixos-test-base.nix +++ b/nixos/lib/testing/nixos-test-base.nix @@ -7,7 +7,6 @@ let in { imports = [ - ../../modules/virtualisation/qemu-vm.nix ../../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs { key = "no-manual"; @@ -32,7 +31,9 @@ in # This is mostly a Hydra optimization, so we don't rebuild all the tests every time switch-to-configuration-ng changes. key = "no-switch-to-configuration"; system.switch.enable = mkDefault ( - config.isSpecialisation || config.specialisation != { } || config.virtualisation.installBootLoader + config.isSpecialisation + || config.specialisation != { } + || (!config.boot.isContainer && config.virtualisation.installBootLoader) ); } ) diff --git a/nixos/lib/testing/nodes.nix b/nixos/lib/testing/nodes.nix index 721a3c88b369..b2ee6420d186 100644 --- a/nixos/lib/testing/nodes.nix +++ b/nixos/lib/testing/nodes.nix @@ -2,7 +2,6 @@ testModuleArgs@{ config, lib, hostPkgs, - nodes, options, ... }: @@ -12,12 +11,9 @@ let literalExpression literalMD mapAttrs - mkDefault mkIf mkMerge mkOption - mkForce - optional optionalAttrs types ; @@ -51,13 +47,6 @@ let key = "nodes"; _module.args.nodes = config.nodesCompat; } - ( - { config, ... }: - { - virtualisation.qemu.package = testModuleArgs.config.qemu.package; - virtualisation.host.pkgs = hostPkgs; - } - ) ( { options, ... }: { @@ -73,6 +62,44 @@ let testModuleArgs.config.extraBaseModules ]; }; + baseQemuOS = baseOS.extendModules { + modules = [ + ../../modules/virtualisation/qemu-vm.nix + config.nodeDefaults + { + key = "base-qemu"; + virtualisation.qemu.package = testModuleArgs.config.qemu.package; + virtualisation.host.pkgs = hostPkgs; + } + testModuleArgs.config.extraBaseNodeModules + ]; + }; + baseNspawnOS = baseOS.extendModules { + modules = [ + ../../modules/virtualisation/nspawn-container + config.containerDefaults + ( + { pkgs, ... }: + { + key = "base-nspawn"; + + # PAM requires setuid and doesn't work in the build sandbox. + # https://github.com/NixOS/nix/blob/959c244a1265f4048390f3ad21679219d7b27a99/src/libstore/unix/build/linux-derivation-builder.cc#L63 + services.openssh.settings.UsePAM = false; + + # Gross, insecure hack to make login work. See above. + security.pam.services.login = { + text = '' + auth sufficient ${pkgs.linux-pam}/lib/security/pam_permit.so + account sufficient ${pkgs.linux-pam}/lib/security/pam_permit.so + password sufficient ${pkgs.linux-pam}/lib/security/pam_permit.so + session sufficient ${pkgs.linux-pam}/lib/security/pam_permit.so + ''; + }; + } + ) + ]; + }; # TODO (lib): Dedup with run.nix, add to lib/options.nix mkOneUp = opt: f: lib.mkOverride (opt.highestPrio - 1) (f opt.value); @@ -109,15 +136,37 @@ in node.type = mkOption { type = types.raw; - default = baseOS.type; + default = baseQemuOS.type; internal = true; }; nodes = mkOption { type = types.lazyAttrsOf config.node.type; + default = { }; visible = "shallow"; description = '' - An attribute set of NixOS configuration modules. + An attribute set of NixOS configuration modules representing QEMU vms that can be started during a test. + + The configurations are augmented by the [`defaults`](#test-opt-defaults) option. + + They are assigned network addresses according to the `nixos/lib/testing/network.nix` module. + + A few special options are available, that aren't in a plain NixOS configuration. See [Configuring the nodes](#sec-nixos-test-nodes) + ''; + }; + + container.type = mkOption { + type = types.raw; + default = baseNspawnOS.type; + internal = true; + }; + + containers = mkOption { + type = types.lazyAttrsOf config.container.type; + default = { }; + visible = "shallow"; + description = '' + An attribute set of NixOS configuration modules representing systemd-nspawn containers that can be started during a test. The configurations are augmented by the [`defaults`](#test-opt-defaults) option. @@ -128,6 +177,14 @@ in }; defaults = mkOption { + description = '' + NixOS configuration that is applied to all [{option}`nodes`](#test-opt-nodes) and [{option}`containers`](#test-opt-containers). + ''; + type = types.deferredModule; + default = { }; + }; + + nodeDefaults = mkOption { description = '' NixOS configuration that is applied to all [{option}`nodes`](#test-opt-nodes). ''; @@ -135,7 +192,23 @@ in default = { }; }; + containerDefaults = mkOption { + description = '' + NixOS configuration that is applied to all [{option}`containers`](#test-opt-containers). + ''; + type = types.deferredModule; + default = { }; + }; + extraBaseModules = mkOption { + description = '' + NixOS configuration that, like [{option}`defaults`](#test-opt-defaults), is applied to all [{option}`nodes`](#test-opt-nodes) and [{option}`containers`](#test-opt-containers) and can not be undone with [`specialisation..inheritParentConfig`](https://search.nixos.org/options?show=specialisation.%3Cname%3E.inheritParentConfig&from=0&size=50&sort=relevance&type=packages&query=specialisation). + ''; + type = types.deferredModule; + default = { }; + }; + + extraBaseNodeModules = mkOption { description = '' NixOS configuration that, like [{option}`defaults`](#test-opt-defaults), is applied to all [{option}`nodes`](#test-opt-nodes) and can not be undone with [`specialisation..inheritParentConfig`](https://search.nixos.org/options?show=specialisation.%3Cname%3E.inheritParentConfig&from=0&size=50&sort=relevance&type=packages&query=specialisation). ''; @@ -145,7 +218,7 @@ in node.pkgs = mkOption { description = '' - The Nixpkgs to use for the nodes. + The Nixpkgs to use for the nodes and containers. Setting this will make the `nixpkgs.*` options read-only, to avoid mistakenly testing with a Nixpkgs configuration that diverges from regular use. ''; @@ -160,7 +233,7 @@ in description = '' Whether to make the `nixpkgs.*` options read-only. This is only relevant when [`node.pkgs`](#test-opt-node.pkgs) is set. - Set this to `false` when any of the [`nodes`](#test-opt-nodes) needs to configure any of the `nixpkgs.*` options. This will slow down evaluation of your test a bit. + Set this to `false` when any of the [`nodes`](#test-opt-nodes) or [{option}`containers`](#test-opt-containers) need to configure any of the `nixpkgs.*` options. This will slow down evaluation of your test a bit. ''; type = types.bool; default = config.node.pkgs != null; @@ -188,6 +261,7 @@ in }; config = { + _module.args.containers = config.containers; _module.args.nodes = config.nodesCompat; nodesCompat = mapAttrs ( name: config: @@ -201,6 +275,7 @@ in ) config.nodes; passthru.nodes = config.nodesCompat; + passthru.containers = config.containers; extraDriverArgs = mkIf config.sshBackdoor.enable [ "--dump-vsocks=${toString config.sshBackdoor.vsockOffset}" @@ -211,33 +286,35 @@ in nixpkgs.pkgs = config.node.pkgs; imports = [ ../../modules/misc/nixpkgs/read-only.nix ]; }) - (mkIf config.sshBackdoor.enable ( - let - inherit (config.sshBackdoor) vsockOffset; - in - { config, ... }: - { - services.openssh = { - enable = true; - settings = { - PermitRootLogin = "yes"; - PermitEmptyPasswords = "yes"; - }; + (mkIf config.sshBackdoor.enable { + services.openssh = { + enable = true; + settings = { + PermitRootLogin = "yes"; + PermitEmptyPasswords = "yes"; }; + }; - security.pam.services.sshd = { - allowNullPassword = true; - }; - - virtualisation.qemu.options = [ - "-device vhost-vsock-pci,guest-cid=${ - toString (config.virtualisation.test.nodeNumber + vsockOffset) - }" - ]; - } - )) + security.pam.services.sshd = { + allowNullPassword = true; + }; + }) ]; + nodeDefaults = mkIf config.sshBackdoor.enable ( + let + inherit (config.sshBackdoor) vsockOffset; + in + { config, ... }: + { + virtualisation.qemu.options = [ + "-device vhost-vsock-pci,guest-cid=${ + toString (config.virtualisation.test.nodeNumber + vsockOffset) + }" + ]; + } + ); + # Docs: nixos/doc/manual/development/writing-nixos-tests.section.md /** See https://nixos.org/manual/nixos/unstable#sec-override-nixos-test diff --git a/nixos/lib/testing/run.nix b/nixos/lib/testing/run.nix index e34e585241be..0eefb1b0e38d 100644 --- a/nixos/lib/testing/run.nix +++ b/nixos/lib/testing/run.nix @@ -2,6 +2,7 @@ config, hostPkgs, lib, + containers, options, ... }: @@ -96,6 +97,8 @@ in requiredSystemFeatures = [ "nixos-test" ] + # Containers use systemd-nspawn, which requires pid 0 inside of the sandbox. + ++ lib.optional (builtins.length (lib.attrNames containers) > 0) "uid-range" ++ lib.optional isLinux "kvm" ++ lib.optional isDarwin "apple-virt"; diff --git a/nixos/lib/testing/testScript.nix b/nixos/lib/testing/testScript.nix index bde7b78607b4..4ce368c0b8db 100644 --- a/nixos/lib/testing/testScript.nix +++ b/nixos/lib/testing/testScript.nix @@ -56,11 +56,12 @@ in # reuse memoized config v ) config.nodesCompat; + containers = config.containers; } else config.testScript; - defaults = + nodeDefaults = { config, name, ... }: { # Make sure all derivations referenced by the test diff --git a/nixos/modules/testing/test-instrumentation.nix b/nixos/modules/testing/test-instrumentation.nix index dce59a0a93b7..81cb578bf017 100644 --- a/nixos/modules/testing/test-instrumentation.nix +++ b/nixos/modules/testing/test-instrumentation.nix @@ -86,7 +86,8 @@ in options.testing = { backdoor = lib.mkEnableOption "backdoor service in stage 2" // { - default = true; + # See assertion below for why the backdoor doesn't work with containers. + default = !config.boot.isContainer; }; initrdBackdoor = lib.mkEnableOption '' @@ -105,7 +106,20 @@ in { assertion = cfg.initrdBackdoor -> config.boot.initrd.systemd.enable; message = '' - testing.initrdBackdoor requires boot.initrd.systemd.enable to be enabled. + `testing.initrdBackdoor` requires `boot.initrd.systemd.enable` to be enabled. + ''; + } + { + assertion = config.boot.isContainer -> !cfg.backdoor; + message = '' + `testing.backdoor` uses virtio console, which does not work with + containers (we use `nsenter` instead). + ''; + } + { + assertion = config.boot.isContainer -> !cfg.initrdBackdoor; + message = '' + `testing.initrdBackdoor` does not work with containers as there is no initrd. ''; } ];