nixos/nspawn-container: init a new nspawn-container profile (#470248)

This commit is contained in:
Arian van Putten
2026-01-20 12:01:44 +00:00
committed by GitHub
9 changed files with 585 additions and 76 deletions
+5 -11
View File
@@ -32,14 +32,8 @@ let
let
qemu-common = import ../qemu-common.nix { inherit (pkgs) lib stdenv; };
# Convert legacy VLANs to named interfaces and merge with explicit interfaces.
vlansNumbered = forEach (zipLists config.virtualisation.vlans (range 1 255)) (v: {
name = "eth${toString v.snd}";
vlan = v.fst;
assignIP = true;
});
explicitInterfaces = lib.mapAttrsToList (n: v: v // { name = n; }) config.virtualisation.interfaces;
interfaces = vlansNumbered ++ explicitInterfaces;
interfaces = lib.attrValues config.virtualisation.allInterfaces;
interfacesNumbered = zipLists interfaces (range 1 255);
# Automatically assign IP addresses to requested interfaces.
@@ -67,10 +61,10 @@ let
{ fst, snd }: qemu-common.qemuNICFlags snd fst.vlan config.virtualisation.test.nodeNumber
)
);
udevRules = forEach interfacesNumbered (
{ fst, snd }:
udevRules = forEach interfaces (
interface:
# MAC Addresses for QEMU network devices are lowercase, and udev string comparison is case-sensitive.
''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac fst.vlan config.virtualisation.test.nodeNumber)}",NAME="${fst.name}"''
''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac interface.vlan config.virtualisation.test.nodeNumber)}",NAME="${interface.name}"''
);
networkConfig = {
@@ -0,0 +1,138 @@
# This module defines networking options for virtual machines and containers.
# It is intended to be used with systemd-nspawn containers and QEMU virtual machines.
{ config, lib, ... }:
let
inherit (lib) types;
interfaceType = types.submodule (
{ name, ... }:
{
options = {
name = lib.mkOption {
type = types.str;
default = name;
description = ''
Interface name
'';
};
vlan = lib.mkOption {
type = types.ints.unsigned;
description = ''
VLAN to which the network interface is connected.
'';
};
assignIP = lib.mkOption {
type = types.bool;
default = false;
description = ''
Automatically assign an IP address to the network interface using the same scheme as
virtualisation.vlans.
'';
};
};
}
);
cfg = config.virtualisation;
# Convert legacy VLANs to named interfaces.
vlansNumbered = lib.listToAttrs (
lib.forEach (lib.zipLists cfg.vlans (lib.range 1 255)) (
v:
let
name = "eth${toString v.snd}";
in
lib.nameValuePair name {
inherit name;
vlan = v.fst;
assignIP = true;
}
)
);
in
{
options = {
networking.primaryIPAddress = lib.mkOption {
type = types.str;
default = "";
internal = true;
description = "Primary IP address used in /etc/hosts.";
};
networking.primaryIPv6Address = lib.mkOption {
type = types.str;
default = "";
internal = true;
description = "Primary IPv6 address used in /etc/hosts.";
};
virtualisation.vlans = lib.mkOption {
type = types.listOf types.ints.unsigned;
default = if cfg.interfaces == { } then [ 1 ] else [ ];
defaultText = lib.literalExpression ''if cfg.interfaces == {} then [ 1 ] else [ ]'';
example = [
1
2
];
description = ''
Virtual networks to which the container or VM is connected. Each number «N» in
this list causes the container to have a virtual Ethernet interface
attached to a separate virtual network on which it will be assigned IP
address `192.168.«N».«M»`, where «M» is the index of this container in
the list of containers.
'';
};
virtualisation.interfaces = lib.mkOption {
default = { };
example = {
enp1s0.vlan = 1;
};
description = ''
Extra network interfaces to add to the container or VM in addition to the ones
created by {option}`virtualisation.vlans`.
'';
type = types.attrsOf interfaceType;
};
virtualisation.allInterfaces = lib.mkOption {
type = types.attrsOf interfaceType;
readOnly = true;
description = ''
All network interfaces for the container or VM. Combines
{option}`virtualisation.vlans` and {option}`virtualisation.interfaces`.
'';
default = vlansNumbered // cfg.interfaces;
};
};
config = {
assertions = [
(
let
conflictingKeys = lib.intersectAttrs vlansNumbered cfg.interfaces;
in
{
assertion = conflictingKeys == { };
message = ''
`virtualisation.vlans` and `virtualisation.interfaces` have conflicting keys: ${lib.concatStringsSep "," (lib.attrNames conflictingKeys)}
'';
}
)
(
let
allInterfaceNames =
(lib.mapAttrsToList (k: i: i.name) vlansNumbered)
++ (lib.mapAttrsToList (k: i: i.name) cfg.interfaces);
in
{
assertion = lib.allUnique allInterfaceNames;
message = ''
`virtualisation.vlans` and `virtualisation.interfaces` have conflicting interface names: ${lib.concatStringsSep "," allInterfaceNames}
'';
}
)
];
};
}
@@ -0,0 +1,114 @@
# This module creates a lightweight "container" from the NixOS configuration.
# Building the `config.system.build.nspawn` attribute gives you a command
# that starts a systemd-nspawn container running the NixOS configuration
# defined in `config`. By default, the Nix store is shared read-only with the
# host, which makes (re)building very efficient.
# This shares a lot in common with
# `nixos/modules/virtualisation/nixos-containers.nix`, but doesn't use systemd
# units.
# The networking options here match the options in
# `nixos/modules/virtualisation/nixos-containers.nix` which allows using these
# lightweight containers for nixos integration tests.
{
config,
pkgs,
lib,
...
}:
let
inherit (lib) types;
cfg = config.virtualisation;
in
{
imports = [ ../guest-networking-options.nix ];
options = {
virtualisation.cmdline = lib.mkOption {
type = types.listOf types.str;
default = [ ];
example = [
"systemd.unit=rescue.target"
"systemd.log_level=debug"
"systemd.log_target=console"
];
description = ''
Command line arguments to pass to the init process (likely systemd).
Useful for debugging.
'';
};
virtualisation.rootDir = lib.mkOption {
type = types.str;
default = "./${config.system.name}-root";
defaultText = lib.literalExpression ''"./''${config.system.name}-root"'';
description = ''
Path to a directory for the root filesystem for the container.
The directory will be created on startup if it does not
exist.
'';
};
virtualisation.systemd-nspawn = {
package = lib.mkPackageOption pkgs "systemd" { };
options = lib.mkOption {
type = types.listOf types.str;
default = [ ];
example = [ "--bind=/home:/home" ];
description = ''
Options passed to systemd-nspawn.
See [systemd-nspawn docs](https://www.freedesktop.org/software/systemd/man/latest/systemd-nspawn.html) for a complete list.
'';
};
};
};
config = {
boot.isNspawnContainer = true;
# TODO(arianvp): Remove after https://github.com/NixOS/nixpkgs/pull/480686 is merged
console.enable = true;
virtualisation.systemd-nspawn.options = [
"--private-network"
"--machine=${config.system.name}"
"--bind-ro=/nix/store:/nix/store"
# systemd-nspawn does some cleverness to mount a procfs and sysfs in an
# unprivileged container, see
# <https://github.com/systemd/systemd/blob/v258.2/src/nspawn/nspawn.c#L4341-L4349>.
# Unfortunately, this doesn't work in the Nix build sandbox as we do not
# have permission to mount filesystems of type `sysfs` nor `procfs`.
# Fortunately, the build sandbox does provide a `/proc` and `/sys` that
# we can just forward onto the container.
"--private-users=no"
"--bind=/proc:/run/host/proc"
"--bind=/sys:/run/host/sys"
# From `man systemd-nspawn`:
# > Use --keep-unit and --register=no in combination to disable any
# > kind of unit allocation or registration with systemd-machined.
"--keep-unit"
"--register=no"
];
system.build.nspawn =
let
run-nspawn = pkgs.callPackage ./run-nspawn { };
commandLineOptions = lib.cli.toCommandLineShellGNU { } {
container-name = config.system.name;
root-dir = cfg.rootDir;
interfaces-json = builtins.toJSON (lib.attrValues cfg.allInterfaces);
init = "${config.system.build.toplevel}/init";
cmdline-json = builtins.toJSON cfg.cmdline;
};
in
pkgs.writers.writeDashBin "run-${config.system.name}-nspawn" ''
exec ${lib.getExe run-nspawn} ${commandLineOptions} ${lib.escapeShellArgs config.virtualisation.systemd-nspawn.options} "$@"
'';
};
}
@@ -0,0 +1,6 @@
{ python3Packages, systemd }:
python3Packages.callPackage ./package.nix {
# We want `pkgs.systemd`, *not* `python3Packages.systemd`.
inherit systemd;
}
@@ -0,0 +1,54 @@
{
buildPythonApplication,
e2fsprogs,
iproute2,
lib,
mypy,
ruff,
setuptools,
systemd,
}:
buildPythonApplication {
pname = "run-nspawn";
version = "1.0";
pyproject = true;
src = ./src;
postPatch = ''
substituteInPlace run_nspawn/__init__.py \
--replace-fail "@ip@" "${lib.getExe' iproute2 "ip"}" \
--replace-fail "@systemd-nspawn@" "${lib.getExe' systemd "systemd-nspawn"}" \
--replace-fail "@chattr@" "${lib.getExe' e2fsprogs "chattr"}"
'';
build-system = [
setuptools
];
propagatedBuildInputs = [
systemd
iproute2
];
doCheck = true;
nativeCheckInputs = [
mypy
ruff
];
checkPhase = ''
echo -e "\x1b[32m## run mypy\x1b[0m"
mypy run_nspawn
echo -e "\x1b[32m## run ruff check\x1b[0m"
ruff check .
echo -e "\x1b[32m## run ruff format\x1b[0m"
ruff format --check --diff .
'';
meta = {
mainProgram = "run-nspawn";
};
}
@@ -0,0 +1,4 @@
{
pkgs ? import ../../../../.. { },
}:
pkgs.callPackage ./default.nix { }
@@ -0,0 +1,13 @@
[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"
[project]
name = "run-nspawn"
version = "1.0"
[project.scripts]
run-nspawn = "run_nspawn:main"
[tool.setuptools.packages]
find = {}
@@ -0,0 +1,250 @@
import contextlib
import dataclasses
import fcntl
import logging
import os
import signal
import subprocess
import sys
import typing
from pathlib import Path
logger = logging.getLogger()
# Install a SIGTERM handler so all our context managers get a chance to
# clean up.
signal.signal(signal.SIGTERM, lambda _signum, _frame: sys.exit(0))
@dataclasses.dataclass
class Netns:
name: str
path: Path
def run_ip(*args: str) -> None:
subprocess.run(
["@ip@", *args],
check=True,
)
@contextlib.contextmanager
def mk_netns(name: str) -> typing.Generator[Netns, None, None]:
logger.info("creating netns %s", name)
run_ip("netns", "add", name)
try:
yield Netns(name=name, path=Path("/run/netns") / name)
finally:
logger.info("deleting netns %s", name)
run_ip("netns", "delete", name)
@contextlib.contextmanager
def vlan_lock(vlan: int) -> typing.Generator[None, None, None]:
lockfile = Path(f"/run/nixos-nspawn/vlan-{vlan}.lock")
lockfile.parent.mkdir(parents=True, exist_ok=True)
with lockfile.open("w") as f:
# Grab an exclusive lock.
fcntl.flock(f, fcntl.LOCK_EX)
try:
yield
finally:
# Release the exclusive lock.
fcntl.flock(f, fcntl.LOCK_UN)
@contextlib.contextmanager
def ensure_vlan_bridge(vlan: int) -> typing.Generator[str, None, None]:
"""
Ensure a bridge for the given vlan exists, and create one if it does not.
Note that this bridge may get used by other containers, so we're careful
to not delete it unless we're sure nobody else is using it.
"""
# These IP addresses correspond to the static IP assignment logic in
# <nixos/lib/testing/network.nix>.
ipv4_addr = f"192.168.{vlan}.254/24"
ipv6_addr = f"2001:db8:{vlan}::fe/64"
bridge_name = f"br{vlan}"
bridge_path = Path("/sys/class/net") / bridge_name
try:
# To avoid racing against other nspawn containers that also
# need this vlan, grab an exclusive lock.
with vlan_lock(vlan):
if not bridge_path.exists():
logger.info("creating bridge %s", bridge_name)
run_ip("link", "add", bridge_name, "type", "bridge")
run_ip("link", "set", bridge_name, "up")
run_ip("addr", "add", ipv4_addr, "dev", bridge_name)
run_ip("addr", "add", ipv6_addr, "dev", bridge_name)
yield bridge_name
finally:
# To avoid racing against other nspawn containers that also
# releasing this vlan, grab an exclusive lock.
with vlan_lock(vlan):
if bridge_path.exists():
child_intf_count = len(list((bridge_path / "brif").iterdir()))
if child_intf_count == 0:
logger.info("deleting bridge %s", bridge_name)
run_ip("link", "delete", bridge_name)
@contextlib.contextmanager
def mk_veth(
container_name: str,
netns: Netns,
container_intf_name: str,
vlan: int,
) -> typing.Generator[None, None, None]:
host_intf_name = f"{container_name}-{container_intf_name}"
with ensure_vlan_bridge(vlan) as bridge_name:
logger.info("creating interface %s", host_intf_name)
run_ip(
"link",
"add",
host_intf_name,
"type",
"veth",
"peer",
"name",
container_intf_name,
"netns",
netns.name,
)
try:
run_ip("link", "set", host_intf_name, "master", bridge_name)
run_ip("link", "set", host_intf_name, "up")
yield
finally:
logger.info("deleting interface %s", host_intf_name)
run_ip("link", "delete", host_intf_name)
def run(
container_name: str,
root_dir_str: str,
interfaces: dict,
nspawn_options: list[str],
init: str,
cmdline: list[str],
) -> None:
logging.basicConfig(
format=f"nixos-nspawn({container_name}): %(message)s",
level=logging.WARNING,
)
assert os.geteuid() == 0, (
f"systemd-nspawn requires root to work. You are {os.geteuid()}"
)
root_dir = Path(root_dir_str)
root_dir.mkdir(parents=True, exist_ok=True)
root_dir.chmod(0o755)
with (
mk_netns(f"nixos-nspawn-{container_name}") as netns,
contextlib.ExitStack() as stack,
):
for interface in interfaces:
stack.enter_context(
mk_veth(
container_name=container_name,
netns=netns,
container_intf_name=interface["name"],
vlan=interface["vlan"],
)
)
def print_pid() -> None:
print(
f"systemd-nspawn's PID is {os.getpid()}",
# Need to flush stdout before systemd-nspawn gets exec-ed.
flush=True,
)
cp = subprocess.Popen(
[
"@systemd-nspawn@",
*nspawn_options,
f"--directory={root_dir}",
f"--network-namespace-path={netns.path}",
init,
*cmdline,
],
preexec_fn=print_pid,
)
try:
exit_code = cp.wait()
finally:
# If we get interrupted for any reason (most likely a SIGTERM),
# be sure to kill our child process.
cp.terminate()
# NixOS creates `/var/empty` with the immutable filesystem attribute [0].
# This makes it difficult to clean up the machine's root directory
# (which the test driver does to ensure tests start fresh).
# We unset that bit here so others are less likely to run into this issue.
# Note: this may cause issues on filesystems that don't support
# "Linux file attributes". Please improve this if you run into this!
#
# [0]: https://github.com/NixOS/nixpkgs/blob/d1eff395720e1fe15838263642a2bc1dca1eea32/nixos/modules/system/activation/activation-script.nix#L281
subprocess.run(
[
"@chattr@",
"-i",
root_dir / "var/empty",
],
check=True,
)
sys.exit(exit_code)
def main():
import argparse
import json
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"--container-name", required=True, help="Name of the container"
)
arg_parser.add_argument(
"--root-dir",
required=True,
help="Path to container root directory (overridable with RUN_NSPAWN_ROOT_DIR)",
)
arg_parser.add_argument(
"--interfaces-json",
dest="interfaces",
type=json.loads,
required=True,
help="JSON-encoded list of interfaces, each with 'name' and 'vlan' fields",
)
arg_parser.add_argument("--init", required=True, help="Path to init binary")
arg_parser.add_argument(
"--cmdline-json",
dest="cmdline",
type=json.loads,
default=[],
help="JSON-encoded list of command line arguments to pass to init",
)
# Parse only known args to allow for extra args to be passed to systemd-nspawn.
args, nspawn_options = arg_parser.parse_known_args()
run(
container_name=args.container_name,
root_dir_str=os.getenv("RUN_NSPAWN_ROOT_DIR", default=args.root_dir),
interfaces=args.interfaces,
nspawn_options=nspawn_options,
init=args.init,
cmdline=args.cmdline,
)
if __name__ == "__main__":
main()
+1 -65
View File
@@ -365,6 +365,7 @@ in
imports = [
../profiles/qemu-guest.nix
./disk-size-option.nix
./guest-networking-options.nix
(mkRenamedOptionModule
[
"virtualisation"
@@ -679,57 +680,6 @@ in
'';
};
virtualisation.vlans = mkOption {
type = types.listOf types.ints.unsigned;
default = if config.virtualisation.interfaces == { } then [ 1 ] else [ ];
defaultText = lib.literalExpression ''if config.virtualisation.interfaces == {} then [ 1 ] else [ ]'';
example = [
1
2
];
description = ''
Virtual networks to which the VM is connected. Each
number «N» in this list causes
the VM to have a virtual Ethernet interface attached to a
separate virtual network on which it will be assigned IP
address
`192.168.«N».«M»`,
where «M» is the index of this VM
in the list of VMs.
'';
};
virtualisation.interfaces = mkOption {
default = { };
example = {
enp1s0.vlan = 1;
};
description = ''
Network interfaces to add to the VM.
'';
type =
with types;
attrsOf (submodule {
options = {
vlan = mkOption {
type = types.ints.unsigned;
description = ''
VLAN to which the network interface is connected.
'';
};
assignIP = mkOption {
type = types.bool;
default = false;
description = ''
Automatically assign an IP address to the network interface using the same scheme as
virtualisation.vlans.
'';
};
};
});
};
virtualisation.writableStore = mkOption {
type = types.bool;
default = cfg.mountHostNixStore;
@@ -752,20 +702,6 @@ in
'';
};
networking.primaryIPAddress = mkOption {
type = types.str;
default = "";
internal = true;
description = "Primary IP address used in /etc/hosts.";
};
networking.primaryIPv6Address = mkOption {
type = types.str;
default = "";
internal = true;
description = "Primary IPv6 address used in /etc/hosts.";
};
virtualisation.host.pkgs = mkOption {
type = options.nixpkgs.pkgs.type;
default = pkgs;