diff --git a/nixos/lib/testing/network.nix b/nixos/lib/testing/network.nix index e65767e7fb99..9a5facfc2433 100644 --- a/nixos/lib/testing/network.nix +++ b/nixos/lib/testing/network.nix @@ -32,14 +32,8 @@ let let qemu-common = import ../qemu-common.nix { inherit (pkgs) lib stdenv; }; - # Convert legacy VLANs to named interfaces and merge with explicit interfaces. - vlansNumbered = forEach (zipLists config.virtualisation.vlans (range 1 255)) (v: { - name = "eth${toString v.snd}"; - vlan = v.fst; - assignIP = true; - }); - explicitInterfaces = lib.mapAttrsToList (n: v: v // { name = n; }) config.virtualisation.interfaces; - interfaces = vlansNumbered ++ explicitInterfaces; + interfaces = lib.attrValues config.virtualisation.allInterfaces; + interfacesNumbered = zipLists interfaces (range 1 255); # Automatically assign IP addresses to requested interfaces. @@ -67,10 +61,10 @@ let { fst, snd }: qemu-common.qemuNICFlags snd fst.vlan config.virtualisation.test.nodeNumber ) ); - udevRules = forEach interfacesNumbered ( - { fst, snd }: + udevRules = forEach interfaces ( + interface: # MAC Addresses for QEMU network devices are lowercase, and udev string comparison is case-sensitive. - ''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac fst.vlan config.virtualisation.test.nodeNumber)}",NAME="${fst.name}"'' + ''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac interface.vlan config.virtualisation.test.nodeNumber)}",NAME="${interface.name}"'' ); networkConfig = { diff --git a/nixos/modules/virtualisation/guest-networking-options.nix b/nixos/modules/virtualisation/guest-networking-options.nix new file mode 100644 index 000000000000..fb450cfa09ba --- /dev/null +++ b/nixos/modules/virtualisation/guest-networking-options.nix @@ -0,0 +1,138 @@ +# This module defines networking options for virtual machines and containers. +# It is intended to be used with systemd-nspawn containers and QEMU virtual machines. +{ config, lib, ... }: +let + inherit (lib) types; + + interfaceType = types.submodule ( + { name, ... }: + { + options = { + name = lib.mkOption { + type = types.str; + default = name; + description = '' + Interface name + ''; + }; + + vlan = lib.mkOption { + type = types.ints.unsigned; + description = '' + VLAN to which the network interface is connected. + ''; + }; + + assignIP = lib.mkOption { + type = types.bool; + default = false; + description = '' + Automatically assign an IP address to the network interface using the same scheme as + virtualisation.vlans. + ''; + }; + }; + } + ); + + cfg = config.virtualisation; + + # Convert legacy VLANs to named interfaces. + vlansNumbered = lib.listToAttrs ( + lib.forEach (lib.zipLists cfg.vlans (lib.range 1 255)) ( + v: + let + name = "eth${toString v.snd}"; + in + lib.nameValuePair name { + inherit name; + vlan = v.fst; + assignIP = true; + } + ) + ); +in +{ + options = { + networking.primaryIPAddress = lib.mkOption { + type = types.str; + default = ""; + internal = true; + description = "Primary IP address used in /etc/hosts."; + }; + + networking.primaryIPv6Address = lib.mkOption { + type = types.str; + default = ""; + internal = true; + description = "Primary IPv6 address used in /etc/hosts."; + }; + + virtualisation.vlans = lib.mkOption { + type = types.listOf types.ints.unsigned; + default = if cfg.interfaces == { } then [ 1 ] else [ ]; + defaultText = lib.literalExpression ''if cfg.interfaces == {} then [ 1 ] else [ ]''; + example = [ + 1 + 2 + ]; + description = '' + Virtual networks to which the container or VM is connected. Each number «N» in + this list causes the container to have a virtual Ethernet interface + attached to a separate virtual network on which it will be assigned IP + address `192.168.«N».«M»`, where «M» is the index of this container in + the list of containers. + ''; + }; + + virtualisation.interfaces = lib.mkOption { + default = { }; + example = { + enp1s0.vlan = 1; + }; + description = '' + Extra network interfaces to add to the container or VM in addition to the ones + created by {option}`virtualisation.vlans`. + ''; + type = types.attrsOf interfaceType; + }; + + virtualisation.allInterfaces = lib.mkOption { + type = types.attrsOf interfaceType; + readOnly = true; + description = '' + All network interfaces for the container or VM. Combines + {option}`virtualisation.vlans` and {option}`virtualisation.interfaces`. + ''; + default = vlansNumbered // cfg.interfaces; + }; + }; + config = { + assertions = [ + ( + let + conflictingKeys = lib.intersectAttrs vlansNumbered cfg.interfaces; + in + { + assertion = conflictingKeys == { }; + message = '' + `virtualisation.vlans` and `virtualisation.interfaces` have conflicting keys: ${lib.concatStringsSep "," (lib.attrNames conflictingKeys)} + ''; + } + ) + ( + let + allInterfaceNames = + (lib.mapAttrsToList (k: i: i.name) vlansNumbered) + ++ (lib.mapAttrsToList (k: i: i.name) cfg.interfaces); + in + { + assertion = lib.allUnique allInterfaceNames; + message = '' + `virtualisation.vlans` and `virtualisation.interfaces` have conflicting interface names: ${lib.concatStringsSep "," allInterfaceNames} + ''; + } + ) + ]; + }; +} diff --git a/nixos/modules/virtualisation/nspawn-container/default.nix b/nixos/modules/virtualisation/nspawn-container/default.nix new file mode 100644 index 000000000000..6f303eec16df --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/default.nix @@ -0,0 +1,114 @@ +# This module creates a lightweight "container" from the NixOS configuration. +# Building the `config.system.build.nspawn` attribute gives you a command +# that starts a systemd-nspawn container running the NixOS configuration +# defined in `config`. By default, the Nix store is shared read-only with the +# host, which makes (re)building very efficient. +# This shares a lot in common with +# `nixos/modules/virtualisation/nixos-containers.nix`, but doesn't use systemd +# units. +# The networking options here match the options in +# `nixos/modules/virtualisation/nixos-containers.nix` which allows using these +# lightweight containers for nixos integration tests. + +{ + config, + pkgs, + lib, + ... +}: +let + inherit (lib) types; + cfg = config.virtualisation; +in +{ + imports = [ ../guest-networking-options.nix ]; + + options = { + + virtualisation.cmdline = lib.mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ + "systemd.unit=rescue.target" + "systemd.log_level=debug" + "systemd.log_target=console" + ]; + description = '' + Command line arguments to pass to the init process (likely systemd). + Useful for debugging. + ''; + }; + + virtualisation.rootDir = lib.mkOption { + type = types.str; + default = "./${config.system.name}-root"; + defaultText = lib.literalExpression ''"./''${config.system.name}-root"''; + description = '' + Path to a directory for the root filesystem for the container. + The directory will be created on startup if it does not + exist. + ''; + }; + + virtualisation.systemd-nspawn = { + + package = lib.mkPackageOption pkgs "systemd" { }; + + options = lib.mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "--bind=/home:/home" ]; + description = '' + Options passed to systemd-nspawn. + See [systemd-nspawn docs](https://www.freedesktop.org/software/systemd/man/latest/systemd-nspawn.html) for a complete list. + ''; + }; + + }; + }; + + config = { + boot.isNspawnContainer = true; + + # TODO(arianvp): Remove after https://github.com/NixOS/nixpkgs/pull/480686 is merged + console.enable = true; + + virtualisation.systemd-nspawn.options = [ + "--private-network" + "--machine=${config.system.name}" + "--bind-ro=/nix/store:/nix/store" + + # systemd-nspawn does some cleverness to mount a procfs and sysfs in an + # unprivileged container, see + # . + # Unfortunately, this doesn't work in the Nix build sandbox as we do not + # have permission to mount filesystems of type `sysfs` nor `procfs`. + # Fortunately, the build sandbox does provide a `/proc` and `/sys` that + # we can just forward onto the container. + "--private-users=no" + "--bind=/proc:/run/host/proc" + "--bind=/sys:/run/host/sys" + + # From `man systemd-nspawn`: + # > Use --keep-unit and --register=no in combination to disable any + # > kind of unit allocation or registration with systemd-machined. + "--keep-unit" + "--register=no" + ]; + + system.build.nspawn = + let + run-nspawn = pkgs.callPackage ./run-nspawn { }; + commandLineOptions = lib.cli.toCommandLineShellGNU { } { + container-name = config.system.name; + root-dir = cfg.rootDir; + interfaces-json = builtins.toJSON (lib.attrValues cfg.allInterfaces); + init = "${config.system.build.toplevel}/init"; + cmdline-json = builtins.toJSON cfg.cmdline; + }; + in + pkgs.writers.writeDashBin "run-${config.system.name}-nspawn" '' + exec ${lib.getExe run-nspawn} ${commandLineOptions} ${lib.escapeShellArgs config.virtualisation.systemd-nspawn.options} "$@" + ''; + }; +} diff --git a/nixos/modules/virtualisation/nspawn-container/run-nspawn/default.nix b/nixos/modules/virtualisation/nspawn-container/run-nspawn/default.nix new file mode 100644 index 000000000000..3301e9a4d512 --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/run-nspawn/default.nix @@ -0,0 +1,6 @@ +{ python3Packages, systemd }: + +python3Packages.callPackage ./package.nix { + # We want `pkgs.systemd`, *not* `python3Packages.systemd`. + inherit systemd; +} diff --git a/nixos/modules/virtualisation/nspawn-container/run-nspawn/package.nix b/nixos/modules/virtualisation/nspawn-container/run-nspawn/package.nix new file mode 100644 index 000000000000..4f7a0e68748b --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/run-nspawn/package.nix @@ -0,0 +1,54 @@ +{ + buildPythonApplication, + e2fsprogs, + iproute2, + lib, + mypy, + ruff, + setuptools, + systemd, +}: + +buildPythonApplication { + pname = "run-nspawn"; + version = "1.0"; + pyproject = true; + + src = ./src; + + postPatch = '' + substituteInPlace run_nspawn/__init__.py \ + --replace-fail "@ip@" "${lib.getExe' iproute2 "ip"}" \ + --replace-fail "@systemd-nspawn@" "${lib.getExe' systemd "systemd-nspawn"}" \ + --replace-fail "@chattr@" "${lib.getExe' e2fsprogs "chattr"}" + ''; + + build-system = [ + setuptools + ]; + + propagatedBuildInputs = [ + systemd + iproute2 + ]; + + doCheck = true; + + nativeCheckInputs = [ + mypy + ruff + ]; + + checkPhase = '' + echo -e "\x1b[32m## run mypy\x1b[0m" + mypy run_nspawn + echo -e "\x1b[32m## run ruff check\x1b[0m" + ruff check . + echo -e "\x1b[32m## run ruff format\x1b[0m" + ruff format --check --diff . + ''; + + meta = { + mainProgram = "run-nspawn"; + }; +} diff --git a/nixos/modules/virtualisation/nspawn-container/run-nspawn/shell.nix b/nixos/modules/virtualisation/nspawn-container/run-nspawn/shell.nix new file mode 100644 index 000000000000..ee8c77543238 --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/run-nspawn/shell.nix @@ -0,0 +1,4 @@ +{ + pkgs ? import ../../../../.. { }, +}: +pkgs.callPackage ./default.nix { } diff --git a/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/pyproject.toml b/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/pyproject.toml new file mode 100644 index 000000000000..6fb4fe4336dc --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "run-nspawn" +version = "1.0" + +[project.scripts] +run-nspawn = "run_nspawn:main" + +[tool.setuptools.packages] +find = {} diff --git a/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/run_nspawn/__init__.py b/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/run_nspawn/__init__.py new file mode 100644 index 000000000000..99f50038fd7c --- /dev/null +++ b/nixos/modules/virtualisation/nspawn-container/run-nspawn/src/run_nspawn/__init__.py @@ -0,0 +1,250 @@ +import contextlib +import dataclasses +import fcntl +import logging +import os +import signal +import subprocess +import sys +import typing +from pathlib import Path + +logger = logging.getLogger() + +# Install a SIGTERM handler so all our context managers get a chance to +# clean up. +signal.signal(signal.SIGTERM, lambda _signum, _frame: sys.exit(0)) + + +@dataclasses.dataclass +class Netns: + name: str + path: Path + + +def run_ip(*args: str) -> None: + subprocess.run( + ["@ip@", *args], + check=True, + ) + + +@contextlib.contextmanager +def mk_netns(name: str) -> typing.Generator[Netns, None, None]: + logger.info("creating netns %s", name) + run_ip("netns", "add", name) + try: + yield Netns(name=name, path=Path("/run/netns") / name) + finally: + logger.info("deleting netns %s", name) + run_ip("netns", "delete", name) + + +@contextlib.contextmanager +def vlan_lock(vlan: int) -> typing.Generator[None, None, None]: + lockfile = Path(f"/run/nixos-nspawn/vlan-{vlan}.lock") + lockfile.parent.mkdir(parents=True, exist_ok=True) + with lockfile.open("w") as f: + # Grab an exclusive lock. + fcntl.flock(f, fcntl.LOCK_EX) + try: + yield + finally: + # Release the exclusive lock. + fcntl.flock(f, fcntl.LOCK_UN) + + +@contextlib.contextmanager +def ensure_vlan_bridge(vlan: int) -> typing.Generator[str, None, None]: + """ + Ensure a bridge for the given vlan exists, and create one if it does not. + + Note that this bridge may get used by other containers, so we're careful + to not delete it unless we're sure nobody else is using it. + """ + # These IP addresses correspond to the static IP assignment logic in + # . + ipv4_addr = f"192.168.{vlan}.254/24" + ipv6_addr = f"2001:db8:{vlan}::fe/64" + + bridge_name = f"br{vlan}" + bridge_path = Path("/sys/class/net") / bridge_name + try: + # To avoid racing against other nspawn containers that also + # need this vlan, grab an exclusive lock. + with vlan_lock(vlan): + if not bridge_path.exists(): + logger.info("creating bridge %s", bridge_name) + run_ip("link", "add", bridge_name, "type", "bridge") + run_ip("link", "set", bridge_name, "up") + run_ip("addr", "add", ipv4_addr, "dev", bridge_name) + run_ip("addr", "add", ipv6_addr, "dev", bridge_name) + + yield bridge_name + finally: + # To avoid racing against other nspawn containers that also + # releasing this vlan, grab an exclusive lock. + with vlan_lock(vlan): + if bridge_path.exists(): + child_intf_count = len(list((bridge_path / "brif").iterdir())) + if child_intf_count == 0: + logger.info("deleting bridge %s", bridge_name) + run_ip("link", "delete", bridge_name) + + +@contextlib.contextmanager +def mk_veth( + container_name: str, + netns: Netns, + container_intf_name: str, + vlan: int, +) -> typing.Generator[None, None, None]: + host_intf_name = f"{container_name}-{container_intf_name}" + with ensure_vlan_bridge(vlan) as bridge_name: + logger.info("creating interface %s", host_intf_name) + run_ip( + "link", + "add", + host_intf_name, + "type", + "veth", + "peer", + "name", + container_intf_name, + "netns", + netns.name, + ) + try: + run_ip("link", "set", host_intf_name, "master", bridge_name) + run_ip("link", "set", host_intf_name, "up") + yield + finally: + logger.info("deleting interface %s", host_intf_name) + run_ip("link", "delete", host_intf_name) + + +def run( + container_name: str, + root_dir_str: str, + interfaces: dict, + nspawn_options: list[str], + init: str, + cmdline: list[str], +) -> None: + logging.basicConfig( + format=f"nixos-nspawn({container_name}): %(message)s", + level=logging.WARNING, + ) + + assert os.geteuid() == 0, ( + f"systemd-nspawn requires root to work. You are {os.geteuid()}" + ) + + root_dir = Path(root_dir_str) + + root_dir.mkdir(parents=True, exist_ok=True) + root_dir.chmod(0o755) + + with ( + mk_netns(f"nixos-nspawn-{container_name}") as netns, + contextlib.ExitStack() as stack, + ): + for interface in interfaces: + stack.enter_context( + mk_veth( + container_name=container_name, + netns=netns, + container_intf_name=interface["name"], + vlan=interface["vlan"], + ) + ) + + def print_pid() -> None: + print( + f"systemd-nspawn's PID is {os.getpid()}", + # Need to flush stdout before systemd-nspawn gets exec-ed. + flush=True, + ) + + cp = subprocess.Popen( + [ + "@systemd-nspawn@", + *nspawn_options, + f"--directory={root_dir}", + f"--network-namespace-path={netns.path}", + init, + *cmdline, + ], + preexec_fn=print_pid, + ) + + try: + exit_code = cp.wait() + finally: + # If we get interrupted for any reason (most likely a SIGTERM), + # be sure to kill our child process. + cp.terminate() + + # NixOS creates `/var/empty` with the immutable filesystem attribute [0]. + # This makes it difficult to clean up the machine's root directory + # (which the test driver does to ensure tests start fresh). + # We unset that bit here so others are less likely to run into this issue. + # Note: this may cause issues on filesystems that don't support + # "Linux file attributes". Please improve this if you run into this! + # + # [0]: https://github.com/NixOS/nixpkgs/blob/d1eff395720e1fe15838263642a2bc1dca1eea32/nixos/modules/system/activation/activation-script.nix#L281 + subprocess.run( + [ + "@chattr@", + "-i", + root_dir / "var/empty", + ], + check=True, + ) + + sys.exit(exit_code) + + +def main(): + import argparse + import json + + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--container-name", required=True, help="Name of the container" + ) + arg_parser.add_argument( + "--root-dir", + required=True, + help="Path to container root directory (overridable with RUN_NSPAWN_ROOT_DIR)", + ) + arg_parser.add_argument( + "--interfaces-json", + dest="interfaces", + type=json.loads, + required=True, + help="JSON-encoded list of interfaces, each with 'name' and 'vlan' fields", + ) + arg_parser.add_argument("--init", required=True, help="Path to init binary") + arg_parser.add_argument( + "--cmdline-json", + dest="cmdline", + type=json.loads, + default=[], + help="JSON-encoded list of command line arguments to pass to init", + ) + # Parse only known args to allow for extra args to be passed to systemd-nspawn. + args, nspawn_options = arg_parser.parse_known_args() + + run( + container_name=args.container_name, + root_dir_str=os.getenv("RUN_NSPAWN_ROOT_DIR", default=args.root_dir), + interfaces=args.interfaces, + nspawn_options=nspawn_options, + init=args.init, + cmdline=args.cmdline, + ) + + +if __name__ == "__main__": + main() diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index 3be38c02085d..762349a39753 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -365,6 +365,7 @@ in imports = [ ../profiles/qemu-guest.nix ./disk-size-option.nix + ./guest-networking-options.nix (mkRenamedOptionModule [ "virtualisation" @@ -679,57 +680,6 @@ in ''; }; - virtualisation.vlans = mkOption { - type = types.listOf types.ints.unsigned; - default = if config.virtualisation.interfaces == { } then [ 1 ] else [ ]; - defaultText = lib.literalExpression ''if config.virtualisation.interfaces == {} then [ 1 ] else [ ]''; - example = [ - 1 - 2 - ]; - description = '' - Virtual networks to which the VM is connected. Each - number «N» in this list causes - the VM to have a virtual Ethernet interface attached to a - separate virtual network on which it will be assigned IP - address - `192.168.«N».«M»`, - where «M» is the index of this VM - in the list of VMs. - ''; - }; - - virtualisation.interfaces = mkOption { - default = { }; - example = { - enp1s0.vlan = 1; - }; - description = '' - Network interfaces to add to the VM. - ''; - type = - with types; - attrsOf (submodule { - options = { - vlan = mkOption { - type = types.ints.unsigned; - description = '' - VLAN to which the network interface is connected. - ''; - }; - - assignIP = mkOption { - type = types.bool; - default = false; - description = '' - Automatically assign an IP address to the network interface using the same scheme as - virtualisation.vlans. - ''; - }; - }; - }); - }; - virtualisation.writableStore = mkOption { type = types.bool; default = cfg.mountHostNixStore; @@ -752,20 +702,6 @@ in ''; }; - networking.primaryIPAddress = mkOption { - type = types.str; - default = ""; - internal = true; - description = "Primary IP address used in /etc/hosts."; - }; - - networking.primaryIPv6Address = mkOption { - type = types.str; - default = ""; - internal = true; - description = "Primary IPv6 address used in /etc/hosts."; - }; - virtualisation.host.pkgs = mkOption { type = options.nixpkgs.pkgs.type; default = pkgs;