diff --git a/nixos/lib/testing/network.nix b/nixos/lib/testing/network.nix index e65767e7fb99..9a5facfc2433 100644 --- a/nixos/lib/testing/network.nix +++ b/nixos/lib/testing/network.nix @@ -32,14 +32,8 @@ let let qemu-common = import ../qemu-common.nix { inherit (pkgs) lib stdenv; }; - # Convert legacy VLANs to named interfaces and merge with explicit interfaces. - vlansNumbered = forEach (zipLists config.virtualisation.vlans (range 1 255)) (v: { - name = "eth${toString v.snd}"; - vlan = v.fst; - assignIP = true; - }); - explicitInterfaces = lib.mapAttrsToList (n: v: v // { name = n; }) config.virtualisation.interfaces; - interfaces = vlansNumbered ++ explicitInterfaces; + interfaces = lib.attrValues config.virtualisation.allInterfaces; + interfacesNumbered = zipLists interfaces (range 1 255); # Automatically assign IP addresses to requested interfaces. @@ -67,10 +61,10 @@ let { fst, snd }: qemu-common.qemuNICFlags snd fst.vlan config.virtualisation.test.nodeNumber ) ); - udevRules = forEach interfacesNumbered ( - { fst, snd }: + udevRules = forEach interfaces ( + interface: # MAC Addresses for QEMU network devices are lowercase, and udev string comparison is case-sensitive. - ''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac fst.vlan config.virtualisation.test.nodeNumber)}",NAME="${fst.name}"'' + ''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac interface.vlan config.virtualisation.test.nodeNumber)}",NAME="${interface.name}"'' ); networkConfig = { diff --git a/nixos/modules/virtualisation/nspawn-container/default.nix b/nixos/modules/virtualisation/nspawn-container/default.nix new file mode 100644 index 000000000000..cdcc68f4b98c --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/default.nix @@ -0,0 +1,253 @@ +# This module creates a lightweight "container" from the NixOS configuration. +# Building the `config.system.build.nspawn` attribute gives you a command +# that starts a systemd-nspawn container running the NixOS configuration +# defined in `config`. By default, the Nix store is shared read-only with the +# host, which makes (re)building very efficient. +# This shares a lot in common with +# `nixos/modules/virtualisation/nixos-containers.nix`, but doesn't use systemd +# units. +# The networking options here match the options in +# `nixos/modules/virtualisation/nixos-containers.nix` which allows using these +# lightweight containers for nixos integration tests. + +{ + config, + pkgs, + lib, + ... +}: +let + inherit (lib) types; + cfg = config.virtualisation; + + interfaceType = types.submodule ( + { name, ... }: + { + options = { + name = lib.mkOption { + type = types.str; + default = name; + description = '' + Interface name + ''; + }; + + vlan = lib.mkOption { + type = types.ints.unsigned; + description = '' + VLAN to which the network interface is connected. + ''; + }; + + assignIP = lib.mkOption { + type = types.bool; + default = false; + description = '' + Automatically assign an IP address to the network interface using the same scheme as + virtualisation.vlans. + ''; + }; + }; + } + ); + + # Convert legacy VLANs to named interfaces. + vlansNumbered = lib.listToAttrs ( + lib.forEach (lib.zipLists cfg.vlans (lib.range 1 255)) ( + v: + let + name = "eth${toString v.snd}"; + in + lib.nameValuePair name { + inherit name; + vlan = v.fst; + assignIP = true; + } + ) + ); +in +{ + options = { + networking.primaryIPAddress = lib.mkOption { + type = types.str; + default = ""; + internal = true; + description = "Primary IP address used in /etc/hosts."; + }; + + networking.primaryIPv6Address = lib.mkOption { + type = types.str; + default = ""; + internal = true; + description = "Primary IPv6 address used in /etc/hosts."; + }; + + virtualisation.vlans = lib.mkOption { + type = types.listOf types.ints.unsigned; + default = if cfg.interfaces == { } then [ 1 ] else [ ]; + defaultText = lib.literalExpression ''if cfg.interfaces == {} then [ 1 ] else [ ]''; + example = [ + 1 + 2 + ]; + description = '' + Virtual networks to which the container is connected. Each number «N» in + this list causes the container to have a virtual Ethernet interface + attached to a separate virtual network on which it will be assigned IP + address `192.168.«N».«M»`, where «M» is the index of this container in + the list of containers. + ''; + }; + + virtualisation.interfaces = lib.mkOption { + default = { }; + example = { + enp1s0.vlan = 1; + }; + description = '' + Network interfaces to add to the container. + ''; + type = types.attrsOf interfaceType; + }; + + virtualisation.allInterfaces = lib.mkOption { + type = types.attrsOf interfaceType; + readOnly = true; + description = '' + All network interfaces for the VM. Combines + {option}`virtualisation.vlans` and {option}`virtualisation.interfaces`. + ''; + default = vlansNumbered // cfg.interfaces; + }; + + virtualisation.initArgs = lib.mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ + "systemd.unit=rescue.target" + "systemd.log_level=debug" + "systemd.log_target=console" + ]; + description = '' + Command line arguments to pass to the init process (likely systemd). + Useful for debugging. + ''; + }; + + virtualisation.rootDir = lib.mkOption { + type = types.str; + default = "./${config.system.name}-root"; + defaultText = lib.literalExpression ''"./''${config.system.name}-root"''; + description = '' + Path to a directory for the root filesystem for the container. + The directory will be created on startup if it does not + exist. + ''; + }; + + virtualisation.systemd-nspawn = { + package = lib.mkPackageOption pkgs "systemd" { }; + options = lib.mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "--bind=/home:/home" ]; + description = '' + Options passed to systemd-nspawn. + See [systemd-nspawn docs](https://www.freedesktop.org/software/systemd/man/latest/systemd-nspawn.html) for a complete list. + ''; + }; + }; + }; + + config = { + assertions = [ + ( + let + conflictingKeys = lib.intersectAttrs vlansNumbered cfg.interfaces; + in + { + assertion = conflictingKeys == { }; + message = '' + `virtualisation.vlans` and `virtualisation.interfaces` have conflicting keys: ${lib.concatStringsSep "," (lib.attrNames conflictingKeys)} + ''; + } + ) + ( + let + allInterfaceNames = + (lib.mapAttrsToList (k: i: i.name) vlansNumbered) + ++ (lib.mapAttrsToList (k: i: i.name) cfg.interfaces); + in + { + assertion = lib.allUnique allInterfaceNames; + message = '' + `virtualisation.vlans` and `virtualisation.interfaces` have conflicting interface names: ${lib.concatStringsSep "," allInterfaceNames} + ''; + } + ) + ]; + + boot.isNspawnContainer = true; + + # Needed since nixpkgs 7fb2f407c01b017737eafc26b065d7f56434a992 removed the getty unit by default. + console.enable = true; + + virtualisation.systemd-nspawn.options = [ + "--private-network" + "--machine=${config.system.name}" + "--bind-ro=/nix/store:/nix/store" + + # systemd-nspawn does some cleverness to mount a procfs and sysfs in an + # unprivileged container, see + # . + # Unfortunately, this doesn't work in the Nix build sandbox as we do not + # have permission to mount filesystems of type `sysfs` nor `procfs`. + # Fortunately, the build sandbox does provide a `/proc` and `/sys` that + # we can just forward onto the container. + "--private-users=no" + "--bind=/proc:/run/host/proc" + "--bind=/sys:/run/host/sys" + + # From `man systemd-nspawn`: + # > Use --keep-unit and --register=no in combination to disable any + # > kind of unit allocation or registration with systemd-machined. + "--keep-unit" + "--register=no" + ]; + + system.build.nspawn = + let + toPythonStr = + s: + assert builtins.typeOf s == "string"; + # Hack: JSON strings are (AFAIK) valid Python expressions that + # represent the exact same string. + builtins.toJSON s; + + toPythonExpression = data: /* python */ "json.loads(${toPythonStr (builtins.toJSON data)})"; + in + pkgs.writers.writePython3Bin "run-${config.system.name}-nspawn" + { + libraries = ps: [ + (pkgs.callPackage ./run-nspawn { python3Packages = ps; }) + ]; + } + '' + import json + import os + import sys + from pathlib import Path + + import run_nspawn + + run_nspawn.run( + container_name=${toPythonStr config.system.name}, # noqa + root_dir_str=Path(os.environ.get("RUN_NSPAWN_ROOT_DIR", ${toPythonStr cfg.rootDir})), # noqa + interfaces=${toPythonExpression (lib.attrValues cfg.allInterfaces)}, # noqa + nspawn_options=${toPythonExpression config.virtualisation.systemd-nspawn.options} + sys.argv[1:], # noqa + init=${toPythonStr "${config.system.build.toplevel}/init"}, # noqa + init_args=${toPythonExpression cfg.initArgs}, # noqa + ) + ''; + }; +} diff --git a/nixos/modules/virtualisation/nspawn-container/run-nspawn/default.nix b/nixos/modules/virtualisation/nspawn-container/run-nspawn/default.nix new file mode 100644 index 000000000000..477f8179d34e --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/run-nspawn/default.nix @@ -0,0 +1,6 @@ +{ python3Packages, systemd }: + +python3Packages.callPackage ./package.nix { + # We want `pkgs.systemd`, *not* `python3Packages.system`. + inherit systemd; +} diff --git a/nixos/modules/virtualisation/nspawn-container/run-nspawn/package.nix b/nixos/modules/virtualisation/nspawn-container/run-nspawn/package.nix new file mode 100644 index 000000000000..08b07c6b8cfa --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/run-nspawn/package.nix @@ -0,0 +1,50 @@ +{ + buildPythonPackage, + e2fsprogs, + iproute2, + lib, + mypy, + ruff, + setuptools, + systemd, +}: + +buildPythonPackage { + pname = "run-nspawn"; + version = "1.0"; + pyproject = true; + + src = ./src; + + postPatch = '' + substituteInPlace run_nspawn/__init__.py \ + --replace-fail "@ip@" "${lib.getExe' iproute2 "ip"}" \ + --replace-fail "@systemd-nspawn@" "${lib.getExe' systemd "systemd-nspawn"}" \ + --replace-fail "@chattr@" "${lib.getExe' e2fsprogs "chattr"}" + ''; + + build-system = [ + setuptools + ]; + + propagatedBuildInputs = [ + systemd + iproute2 + ]; + + doCheck = true; + + nativeCheckInputs = [ + mypy + ruff + ]; + + checkPhase = '' + echo -e "\x1b[32m## run mypy\x1b[0m" + mypy run_nspawn + echo -e "\x1b[32m## run ruff check\x1b[0m" + ruff check . + echo -e "\x1b[32m## run ruff format\x1b[0m" + ruff format --check --diff . + ''; +} diff --git a/nixos/modules/virtualisation/nspawn-container/run-nspawn/shell.nix b/nixos/modules/virtualisation/nspawn-container/run-nspawn/shell.nix new file mode 100644 index 000000000000..ee8c77543238 --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/run-nspawn/shell.nix @@ -0,0 +1,4 @@ +{ + pkgs ? import ../../../../.. { }, +}: +pkgs.callPackage ./default.nix { } diff --git a/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/pyproject.toml b/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/pyproject.toml new file mode 100644 index 000000000000..e67bb76c3c24 --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "run-nspawn" +version = "0.0.0" + +[tool.setuptools.packages] +find = {} diff --git a/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/run_nspawn/__init__.py b/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/run_nspawn/__init__.py new file mode 100644 index 000000000000..76b1d414f732 --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/run_nspawn/__init__.py @@ -0,0 +1,206 @@ +import contextlib +import dataclasses +import fcntl +import logging +import os +import signal +import subprocess +import sys +import typing +from pathlib import Path + +logger = logging.getLogger() + + +# Install a SIGTERM handler so all our context managers get a chance to +# clean up. +signal.signal(signal.SIGTERM, lambda _signum, _frame: sys.exit(0)) + + +@dataclasses.dataclass +class Netns: + name: str + path: Path + + +def run_ip(*args: str) -> None: + subprocess.run( + ["@ip@", *args], + check=True, + ) + + +@contextlib.contextmanager +def mk_netns(name: str) -> typing.Generator[Netns, None, None]: + logger.info("creating netns %s", name) + run_ip("netns", "add", name) + try: + yield Netns(name=name, path=Path("/run/netns") / name) + finally: + logger.info("deleting netns %s", name) + run_ip("netns", "delete", name) + + +@contextlib.contextmanager +def vlan_lock(vlan: int) -> typing.Generator[None, None, None]: + lockfile = Path(f"/run/nixos-nspawn/vlan-{vlan}.lock") + lockfile.parent.mkdir(parents=True, exist_ok=True) + with lockfile.open("w") as f: + # Grab an exclusive lock. + fcntl.flock(f, fcntl.LOCK_EX) + try: + yield + finally: + # Release the exclusive lock. + fcntl.flock(f, fcntl.LOCK_UN) + + +@contextlib.contextmanager +def ensure_vlan_bridge(vlan: int) -> typing.Generator[str, None, None]: + """ + Ensure a bridge for the given vlan exists, and create one if it does not. + + Note that this bridge may get used by other containers, so we're careful + to not delete it unless we're sure nobody else is using it. + """ + # These IP addresses correspond to the static IP assignment logic in + # . + ipv4_addr = f"192.168.{vlan}.254/24" + ipv6_addr = f"2001:db8:{vlan}::fe/64" + + bridge_name = f"br{vlan}" + bridge_path = Path("/sys/class/net") / bridge_name + try: + # To avoid racing against other nspawn containers that also + # need this vlan, grab an exclusive lock. + with vlan_lock(vlan): + if not bridge_path.exists(): + logger.info("creating bridge %s", bridge_name) + run_ip("link", "add", bridge_name, "type", "bridge") + run_ip("link", "set", bridge_name, "up") + run_ip("addr", "add", ipv4_addr, "dev", bridge_name) + run_ip("addr", "add", ipv6_addr, "dev", bridge_name) + + yield bridge_name + finally: + # To avoid racing against other nspawn containers that also + # releasing this vlan, grab an exclusive lock. + with vlan_lock(vlan): + if bridge_path.exists(): + child_intf_count = len(list((bridge_path / "brif").iterdir())) + if child_intf_count == 0: + logger.info("deleting bridge %s", bridge_name) + run_ip("link", "delete", bridge_name) + + +@contextlib.contextmanager +def mk_veth( + container_name: str, + netns: Netns, + container_intf_name: str, + vlan: int, +) -> typing.Generator[None, None, None]: + host_intf_name = f"{container_name}-{container_intf_name}" + with ensure_vlan_bridge(vlan) as bridge_name: + logger.info("creating interface %s", host_intf_name) + run_ip( + "link", + "add", + host_intf_name, + "type", + "veth", + "peer", + "name", + container_intf_name, + "netns", + netns.name, + ) + try: + run_ip("link", "set", host_intf_name, "master", bridge_name) + run_ip("link", "set", host_intf_name, "up") + yield + finally: + logger.info("deleting interface %s", host_intf_name) + run_ip("link", "delete", host_intf_name) + + +def run( + container_name: str, + root_dir_str: str, + interfaces: dict, + nspawn_options: list[str], + init: str, + init_args: list[str], +) -> None: + logging.basicConfig( + format=f"nixos-nspawn({container_name}): %(message)s", + level=logging.WARNING, + ) + + assert os.geteuid() == 0, ( + f"systemd-nspawn requires root to work. You are {os.geteuid()}" + ) + + root_dir = Path(root_dir_str) + + root_dir.mkdir(parents=True, exist_ok=True) + root_dir.chmod(0o755) + + with ( + mk_netns(f"nixos-nspawn-{container_name}") as netns, + contextlib.ExitStack() as stack, + ): + for interface in interfaces: + stack.enter_context( + mk_veth( + container_name=container_name, + netns=netns, + container_intf_name=interface["name"], + vlan=interface["vlan"], + ) + ) + + def print_pid() -> None: + print( + f"systemd-nspawn's PID is {os.getpid()}", + # Need to flush stdout before systemd-nspawn gets exec-ed. + flush=True, + ) + + cp = subprocess.Popen( + [ + "@systemd-nspawn@", + *nspawn_options, + f"--directory={root_dir}", + f"--network-namespace-path={netns.path}", + init, + *init_args, + ], + preexec_fn=print_pid, + ) + + try: + exit_code = cp.wait() + finally: + # If we get interrupted for any reason (most likely a SIGTERM), + # be sure to kill our child process. + cp.terminate() + + # NixOS creates `/var/empty` with the immutable filesystem attribute [0]. + # This makes it difficult to clean up the machine's root directory + # (which the test driver does to ensure tests start fresh). + # We unset that bit here so others are less likely to run into this issue. + # Note: this may cause issues on filesystems that don't support + # "Linux file attributes". Please improve this if you run into this! + # + # [0]: https://github.com/NixOS/nixpkgs/blob/d1eff395720e1fe15838263642a2bc1dca1eea32/nixos/modules/system/activation/activation-script.nix#L281 + subprocess.run( + [ + "@chattr@", + "-i", + root_dir / "var/empty", + ], + check=True, + ) + + sys.exit(exit_code) diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index 3be38c02085d..a83f825de444 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -360,6 +360,53 @@ let copyChannel = false; OVMF = cfg.efi.OVMF; }; + + interfaceType = types.submodule ( + { name, ... }: + { + options = { + name = mkOption { + type = types.str; + default = name; + description = '' + Interface name + ''; + }; + + vlan = mkOption { + type = types.ints.unsigned; + description = '' + VLAN to which the network interface is connected. + ''; + }; + + assignIP = mkOption { + type = types.bool; + default = false; + description = '' + Automatically assign an IP address to the network interface using the same scheme as + virtualisation.vlans. + ''; + }; + }; + } + ); + + # Convert legacy VLANs to named interfaces. + vlansNumbered = lib.listToAttrs ( + lib.forEach (lib.zipLists cfg.vlans (lib.range 1 255)) ( + v: + let + name = "eth${toString v.snd}"; + in + lib.nameValuePair name { + inherit name; + vlan = v.fst; + assignIP = true; + } + ) + ); + in { imports = [ @@ -705,29 +752,20 @@ in enp1s0.vlan = 1; }; description = '' - Network interfaces to add to the VM. + Extra network interfaces to add to the VM in addition to the ones + created by {option}`virtualisation.vlans`. ''; - type = - with types; - attrsOf (submodule { - options = { - vlan = mkOption { - type = types.ints.unsigned; - description = '' - VLAN to which the network interface is connected. - ''; - }; + type = types.attrsOf interfaceType; + }; - assignIP = mkOption { - type = types.bool; - default = false; - description = '' - Automatically assign an IP address to the network interface using the same scheme as - virtualisation.vlans. - ''; - }; - }; - }); + virtualisation.allInterfaces = mkOption { + type = types.attrsOf interfaceType; + readOnly = true; + description = '' + All network interfaces for the VM. Combines + {option}`virtualisation.vlans` and {option}`virtualisation.interfaces`. + ''; + default = vlansNumbered // cfg.interfaces; }; virtualisation.writableStore = mkOption { @@ -1248,6 +1286,30 @@ in ) ) ++ [ + ( + let + conflictingKeys = lib.intersectAttrs vlansNumbered cfg.interfaces; + in + { + assertion = conflictingKeys == { }; + message = '' + `virtualisation.vlans` and `virtualisation.interfaces` have conflicting keys: ${lib.concatStringsSep "," (lib.attrNames conflictingKeys)} + ''; + } + ) + ( + let + allInterfaceNames = + (lib.mapAttrsToList (k: i: i.name) vlansNumbered) + ++ (lib.mapAttrsToList (k: i: i.name) cfg.interfaces); + in + { + assertion = lib.allUnique allInterfaceNames; + message = '' + `virtualisation.vlans` and `virtualisation.interfaces` have conflicting interface names: ${lib.concatStringsSep "," allInterfaceNames} + ''; + } + ) { assertion = pkgs.stdenv.hostPlatform.is32bit -> cfg.memorySize < 2047; message = ''