gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/nspawn-container: init a new nspawn-container profile

Browse Source 
This shares a lot in common with the
<nixos/modules/virtualisation/nixos-containers.nix> infrastructure, but
is designed to behave like our `qemu-vm.nix` profile (provides a lot of
the same `virtualisation.*` options, produces a simple script you can
run).

This lays the groundwork to be able to rework the nixos test
infrastructure to allow for containers as well as qemu nodes. That work
isn't quite done yet, but if you want more context, you can see the
followup work in <https://github.com/applicative-systems/nixpkgs/compare/nspawn-container-profile...applicative-systems:nixpkgs:nixos-test-containers>.

Credit due to the [Clan.lol](https://clan.lol/) team for first
implementing this. I'm just cleaning it up and making it play nicely
with upstream.

To try it out, create a `demo.nix`:

```nix
let
  pkgs = import ./. { };
  mkContainer =
    {
      nodeNumber,
      vlans,
    }:
    pkgs.nixos (
      {
        config,
        modulesPath,
        pkgs,
        lib,
        ...
      }:
      let
        interfaces = lib.attrValues config.virtualisation.allInterfaces;

        # Automatically assign IP addresses to requested interfaces.
        assignIPs = lib.filter (i: i.assignIP) interfaces;
        ipInterfaces = lib.forEach assignIPs (
          i:
          lib.nameValuePair i.name {
            ipv4.addresses = [
              {
                address = "192.168.${toString i.vlan}.${toString nodeNumber}";
                prefixLength = 24;
              }
            ];
          }
        );
      in
      {
        imports = [ "${modulesPath}/virtualisation/nspawn-container" ];
        users.users.root.password = "";

        networking.hostName = "c${toString nodeNumber}";
        virtualisation.vlans = vlans;
        networking.interfaces = lib.listToAttrs ipInterfaces;

        environment.systemPackages = [ pkgs.neovim ];
        system.stateVersion = lib.trivial.release;
      }
    );
in
{
  container1 = mkContainer {
    nodeNumber = 1;
    vlans = [ 1 ];
  };
  container2 = mkContainer {
    nodeNumber = 2;
    vlans = [ 2 ];
  };
  container12 = mkContainer {
    nodeNumber = 12;
    vlans = [
      1
      2
    ];
  };
}
```

Build and run the machines in separate terminals (unfortunately,
`systemd-nspawn` requires `sudo`):

```console
$ sudo $(nix-build ./demo.nix -A container1.config.system.build.nspawn)/bin/run-c1-nspawn
$ sudo $(nix-build ./demo.nix -A container2.config.system.build.nspawn)/bin/run-c2-nspawn
$ sudo $(nix-build ./demo.nix -A container12.config.system.build.nspawn)/bin/run-c12-nspawn
```

You can log into this machines as `root`, and verify they can ping each
other:

`c1` can ping `c12`:

```
[root@c1:~]# ping 192.168.1.12 -c 1
PING 192.168.1.12 (192.168.1.12) 56(84) bytes of data.
64 bytes from 192.168.1.12: icmp_seq=1 ttl=64 time=0.164 ms
...
```

So can `c2`:

```
[root@c2:~]# ping 192.168.2.12
PING 192.168.2.12 (192.168.2.12) 56(84) bytes of data.
64 bytes from 192.168.2.12: icmp_seq=1 ttl=64 time=0.127 ms
```
This commit is contained in:
Jeremy Fleischman
2025-12-05 17:04:01 -08:00
committed by Kierán Meinhardt
parent 9a9938e5fe
commit 4bd5482aa6
8 changed files with 617 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/lib/testing/network.nix
+5 -11
View File
@@ -32,14 +32,8 @@ let
let
qemu-common = import ../qemu-common.nix { inherit (pkgs) lib stdenv; };
# Convert legacy VLANs to named interfaces and merge with explicit interfaces.
vlansNumbered = forEach (zipLists config.virtualisation.vlans (range 1 255)) (v: {
name = "eth${toString v.snd}";
vlan = v.fst;
assignIP = true;
});
explicitInterfaces = lib.mapAttrsToList (n: v: v // { name = n; }) config.virtualisation.interfaces;
interfaces = vlansNumbered ++ explicitInterfaces;
interfaces = lib.attrValues config.virtualisation.allInterfaces;
interfacesNumbered = zipLists interfaces (range 1 255);
# Automatically assign IP addresses to requested interfaces.
@@ -67,10 +61,10 @@ let
{ fst, snd }: qemu-common.qemuNICFlags snd fst.vlan config.virtualisation.test.nodeNumber
)
);
udevRules = forEach interfacesNumbered (
{ fst, snd }:
udevRules = forEach interfaces (
interface:
# MAC Addresses for QEMU network devices are lowercase, and udev string comparison is case-sensitive.
''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac fst.vlan config.virtualisation.test.nodeNumber)}",NAME="${fst.name}"''
''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower (qemu-common.qemuNicMac interface.vlan config.virtualisation.test.nodeNumber)}",NAME="${interface.name}"''
);
networkConfig = {
nixos/modules/virtualisation/nspawn-container/default.nix
+253
View File
@@ -0,0 +1,253 @@
# This module creates a lightweight "container" from the NixOS configuration.
# Building the `config.system.build.nspawn` attribute gives you a command
# that starts a systemd-nspawn container running the NixOS configuration
# defined in `config`. By default, the Nix store is shared read-only with the
# host, which makes (re)building very efficient.
# This shares a lot in common with
# `nixos/modules/virtualisation/nixos-containers.nix`, but doesn't use systemd
# units.
# The networking options here match the options in
# `nixos/modules/virtualisation/nixos-containers.nix` which allows using these
# lightweight containers for nixos integration tests.
{
config,
pkgs,
lib,
...
}:
let
inherit (lib) types;
cfg = config.virtualisation;
interfaceType = types.submodule (
{ name, ... }:
{
options = {
name = lib.mkOption {
type = types.str;
default = name;
description = ''
Interface name
'';
};
vlan = lib.mkOption {
type = types.ints.unsigned;
description = ''
VLAN to which the network interface is connected.
'';
};
assignIP = lib.mkOption {
type = types.bool;
default = false;
description = ''
Automatically assign an IP address to the network interface using the same scheme as
virtualisation.vlans.
'';
};
};
}
);
# Convert legacy VLANs to named interfaces.
vlansNumbered = lib.listToAttrs (
lib.forEach (lib.zipLists cfg.vlans (lib.range 1 255)) (
v:
let
name = "eth${toString v.snd}";
in
lib.nameValuePair name {
inherit name;
vlan = v.fst;
assignIP = true;
}
)
);
in
{
options = {
networking.primaryIPAddress = lib.mkOption {
type = types.str;
default = "";
internal = true;
description = "Primary IP address used in /etc/hosts.";
};
networking.primaryIPv6Address = lib.mkOption {
type = types.str;
default = "";
internal = true;
description = "Primary IPv6 address used in /etc/hosts.";
};
virtualisation.vlans = lib.mkOption {
type = types.listOf types.ints.unsigned;
default = if cfg.interfaces == { } then [ 1 ] else [ ];
defaultText = lib.literalExpression ''if cfg.interfaces == {} then [ 1 ] else [ ]'';
example = [
1
2
];
description = ''
Virtual networks to which the container is connected. Each number «N» in
this list causes the container to have a virtual Ethernet interface
attached to a separate virtual network on which it will be assigned IP
address `192.168.«N».«M»`, where «M» is the index of this container in
the list of containers.
'';
};
virtualisation.interfaces = lib.mkOption {
default = { };
example = {
enp1s0.vlan = 1;
};
description = ''
Network interfaces to add to the container.
'';
type = types.attrsOf interfaceType;
};
virtualisation.allInterfaces = lib.mkOption {
type = types.attrsOf interfaceType;
readOnly = true;
description = ''
All network interfaces for the VM. Combines
{option}`virtualisation.vlans` and {option}`virtualisation.interfaces`.
'';
default = vlansNumbered // cfg.interfaces;
};
virtualisation.initArgs = lib.mkOption {
type = types.listOf types.str;
default = [ ];
example = [
"systemd.unit=rescue.target"
"systemd.log_level=debug"
"systemd.log_target=console"
];
description = ''
Command line arguments to pass to the init process (likely systemd).
Useful for debugging.
'';
};
virtualisation.rootDir = lib.mkOption {
type = types.str;
default = "./${config.system.name}-root";
defaultText = lib.literalExpression ''"./''${config.system.name}-root"'';
description = ''
Path to a directory for the root filesystem for the container.
The directory will be created on startup if it does not
exist.
'';
};
virtualisation.systemd-nspawn = {
package = lib.mkPackageOption pkgs "systemd" { };
options = lib.mkOption {
type = types.listOf types.str;
default = [ ];
example = [ "--bind=/home:/home" ];
description = ''
Options passed to systemd-nspawn.
See [systemd-nspawn docs](https://www.freedesktop.org/software/systemd/man/latest/systemd-nspawn.html) for a complete list.
'';
};
};
};
config = {
assertions = [
(
let
conflictingKeys = lib.intersectAttrs vlansNumbered cfg.interfaces;
in
{
assertion = conflictingKeys == { };
message = ''
`virtualisation.vlans` and `virtualisation.interfaces` have conflicting keys: ${lib.concatStringsSep "," (lib.attrNames conflictingKeys)}
'';
}
)
(
let
allInterfaceNames =
(lib.mapAttrsToList (k: i: i.name) vlansNumbered)
++ (lib.mapAttrsToList (k: i: i.name) cfg.interfaces);
in
{
assertion = lib.allUnique allInterfaceNames;
message = ''
`virtualisation.vlans` and `virtualisation.interfaces` have conflicting interface names: ${lib.concatStringsSep "," allInterfaceNames}
'';
}
)
];
boot.isNspawnContainer = true;
# Needed since nixpkgs 7fb2f407c01b017737eafc26b065d7f56434a992 removed the getty unit by default.
console.enable = true;
virtualisation.systemd-nspawn.options = [
"--private-network"
"--machine=${config.system.name}"
"--bind-ro=/nix/store:/nix/store"
# systemd-nspawn does some cleverness to mount a procfs and sysfs in an
# unprivileged container, see
# <https://github.com/systemd/systemd/blob/v258.2/src/nspawn/nspawn.c#L4341-L4349>.
# Unfortunately, this doesn't work in the Nix build sandbox as we do not
# have permission to mount filesystems of type `sysfs` nor `procfs`.
# Fortunately, the build sandbox does provide a `/proc` and `/sys` that
# we can just forward onto the container.
"--private-users=no"
"--bind=/proc:/run/host/proc"
"--bind=/sys:/run/host/sys"
# From `man systemd-nspawn`:
# > Use --keep-unit and --register=no in combination to disable any
# > kind of unit allocation or registration with systemd-machined.
"--keep-unit"
"--register=no"
];
system.build.nspawn =
let
toPythonStr =
s:
assert builtins.typeOf s == "string";
# Hack: JSON strings are (AFAIK) valid Python expressions that
# represent the exact same string.
builtins.toJSON s;
toPythonExpression = data: /* python */ "json.loads(${toPythonStr (builtins.toJSON data)})";
in
pkgs.writers.writePython3Bin "run-${config.system.name}-nspawn"
{
libraries = ps: [
(pkgs.callPackage ./run-nspawn { python3Packages = ps; })
];
}
''
import json
import os
import sys
from pathlib import Path
import run_nspawn
run_nspawn.run(
container_name=${toPythonStr config.system.name}, # noqa
root_dir_str=Path(os.environ.get("RUN_NSPAWN_ROOT_DIR", ${toPythonStr cfg.rootDir})), # noqa
interfaces=${toPythonExpression (lib.attrValues cfg.allInterfaces)}, # noqa
nspawn_options=${toPythonExpression config.virtualisation.systemd-nspawn.options} + sys.argv[1:], # noqa
init=${toPythonStr "${config.system.build.toplevel}/init"}, # noqa
init_args=${toPythonExpression cfg.initArgs}, # noqa
)
'';
};
}
nixos/modules/virtualisation/nspawn-container/run-nspawn/default.nix
+6
View File
@@ -0,0 +1,6 @@
{ python3Packages, systemd }:
python3Packages.callPackage ./package.nix {
# We want `pkgs.systemd`, *not* `python3Packages.system`.
inherit systemd;
}
nixos/modules/virtualisation/nspawn-container/run-nspawn/package.nix
+50
View File
@@ -0,0 +1,50 @@
{
buildPythonPackage,
e2fsprogs,
iproute2,
lib,
mypy,
ruff,
setuptools,
systemd,
}:
buildPythonPackage {
pname = "run-nspawn";
version = "1.0";
pyproject = true;
src = ./src;
postPatch = ''
substituteInPlace run_nspawn/__init__.py \
--replace-fail "@ip@" "${lib.getExe' iproute2 "ip"}" \
--replace-fail "@systemd-nspawn@" "${lib.getExe' systemd "systemd-nspawn"}" \
--replace-fail "@chattr@" "${lib.getExe' e2fsprogs "chattr"}"
'';
build-system = [
setuptools
];
propagatedBuildInputs = [
systemd
iproute2
];
doCheck = true;
nativeCheckInputs = [
mypy
ruff
];
checkPhase = ''
echo -e "\x1b[32m## run mypy\x1b[0m"
mypy run_nspawn
echo -e "\x1b[32m## run ruff check\x1b[0m"
ruff check .
echo -e "\x1b[32m## run ruff format\x1b[0m"
ruff format --check --diff .
'';
}
nixos/modules/virtualisation/nspawn-container/run-nspawn/shell.nix
+4
View File
@@ -0,0 +1,4 @@
{
pkgs ? import ../../../../.. { },
}:
pkgs.callPackage ./default.nix { }
nixos/modules/virtualisation/nspawn-container/run-nspawn/src/pyproject.toml
+10
View File
@@ -0,0 +1,10 @@
[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"
[project]
name = "run-nspawn"
version = "0.0.0"
[tool.setuptools.packages]
find = {}
nixos/modules/virtualisation/nspawn-container/run-nspawn/src/run_nspawn/__init__.py
+206
View File
@@ -0,0 +1,206 @@
import contextlib
import dataclasses
import fcntl
import logging
import os
import signal
import subprocess
import sys
import typing
from pathlib import Path
logger = logging.getLogger()
# Install a SIGTERM handler so all our context managers get a chance to
# clean up.
signal.signal(signal.SIGTERM, lambda _signum, _frame: sys.exit(0))
@dataclasses.dataclass
class Netns:
name: str
path: Path
def run_ip(*args: str) -> None:
subprocess.run(
["@ip@", *args],
check=True,
)
@contextlib.contextmanager
def mk_netns(name: str) -> typing.Generator[Netns, None, None]:
logger.info("creating netns %s", name)
run_ip("netns", "add", name)
try:
yield Netns(name=name, path=Path("/run/netns") / name)
finally:
logger.info("deleting netns %s", name)
run_ip("netns", "delete", name)
@contextlib.contextmanager
def vlan_lock(vlan: int) -> typing.Generator[None, None, None]:
lockfile = Path(f"/run/nixos-nspawn/vlan-{vlan}.lock")
lockfile.parent.mkdir(parents=True, exist_ok=True)
with lockfile.open("w") as f:
# Grab an exclusive lock.
fcntl.flock(f, fcntl.LOCK_EX)
try:
yield
finally:
# Release the exclusive lock.
fcntl.flock(f, fcntl.LOCK_UN)
@contextlib.contextmanager
def ensure_vlan_bridge(vlan: int) -> typing.Generator[str, None, None]:
"""
Ensure a bridge for the given vlan exists, and create one if it does not.
Note that this bridge may get used by other containers, so we're careful
to not delete it unless we're sure nobody else is using it.
"""
# These IP addresses correspond to the static IP assignment logic in
# <nixos/lib/testing/network.nix>.
ipv4_addr = f"192.168.{vlan}.254/24"
ipv6_addr = f"2001:db8:{vlan}::fe/64"
bridge_name = f"br{vlan}"
bridge_path = Path("/sys/class/net") / bridge_name
try:
# To avoid racing against other nspawn containers that also
# need this vlan, grab an exclusive lock.
with vlan_lock(vlan):
if not bridge_path.exists():
logger.info("creating bridge %s", bridge_name)
run_ip("link", "add", bridge_name, "type", "bridge")
run_ip("link", "set", bridge_name, "up")
run_ip("addr", "add", ipv4_addr, "dev", bridge_name)
run_ip("addr", "add", ipv6_addr, "dev", bridge_name)
yield bridge_name
finally:
# To avoid racing against other nspawn containers that also
# releasing this vlan, grab an exclusive lock.
with vlan_lock(vlan):
if bridge_path.exists():
child_intf_count = len(list((bridge_path / "brif").iterdir()))
if child_intf_count == 0:
logger.info("deleting bridge %s", bridge_name)
run_ip("link", "delete", bridge_name)
@contextlib.contextmanager
def mk_veth(
container_name: str,
netns: Netns,
container_intf_name: str,
vlan: int,
) -> typing.Generator[None, None, None]:
host_intf_name = f"{container_name}-{container_intf_name}"
with ensure_vlan_bridge(vlan) as bridge_name:
logger.info("creating interface %s", host_intf_name)
run_ip(
"link",
"add",
host_intf_name,
"type",
"veth",
"peer",
"name",
container_intf_name,
"netns",
netns.name,
)
try:
run_ip("link", "set", host_intf_name, "master", bridge_name)
run_ip("link", "set", host_intf_name, "up")
yield
finally:
logger.info("deleting interface %s", host_intf_name)
run_ip("link", "delete", host_intf_name)
def run(
container_name: str,
root_dir_str: str,
interfaces: dict,
nspawn_options: list[str],
init: str,
init_args: list[str],
) -> None:
logging.basicConfig(
format=f"nixos-nspawn({container_name}): %(message)s",
level=logging.WARNING,
)
assert os.geteuid() == 0, (
f"systemd-nspawn requires root to work. You are {os.geteuid()}"
)
root_dir = Path(root_dir_str)
root_dir.mkdir(parents=True, exist_ok=True)
root_dir.chmod(0o755)
with (
mk_netns(f"nixos-nspawn-{container_name}") as netns,
contextlib.ExitStack() as stack,
):
for interface in interfaces:
stack.enter_context(
mk_veth(
container_name=container_name,
netns=netns,
container_intf_name=interface["name"],
vlan=interface["vlan"],
)
)
def print_pid() -> None:
print(
f"systemd-nspawn's PID is {os.getpid()}",
# Need to flush stdout before systemd-nspawn gets exec-ed.
flush=True,
)
cp = subprocess.Popen(
[
"@systemd-nspawn@",
*nspawn_options,
f"--directory={root_dir}",
f"--network-namespace-path={netns.path}",
init,
*init_args,
],
preexec_fn=print_pid,
)
try:
exit_code = cp.wait()
finally:
# If we get interrupted for any reason (most likely a SIGTERM),
# be sure to kill our child process.
cp.terminate()
# NixOS creates `/var/empty` with the immutable filesystem attribute [0].
# This makes it difficult to clean up the machine's root directory
# (which the test driver does to ensure tests start fresh).
# We unset that bit here so others are less likely to run into this issue.
# Note: this may cause issues on filesystems that don't support
# "Linux file attributes". Please improve this if you run into this!
#
# [0]: https://github.com/NixOS/nixpkgs/blob/d1eff395720e1fe15838263642a2bc1dca1eea32/nixos/modules/system/activation/activation-script.nix#L281
subprocess.run(
[
"@chattr@",
"-i",
root_dir / "var/empty",
],
check=True,
)
sys.exit(exit_code)
nixos/modules/virtualisation/qemu-vm.nix
+83 -21
View File
@@ -360,6 +360,53 @@ let
copyChannel = false;
OVMF = cfg.efi.OVMF;
};
interfaceType = types.submodule (
{ name, ... }:
{
options = {
name = mkOption {
type = types.str;
default = name;
description = ''
Interface name
'';
};
vlan = mkOption {
type = types.ints.unsigned;
description = ''
VLAN to which the network interface is connected.
'';
};
assignIP = mkOption {
type = types.bool;
default = false;
description = ''
Automatically assign an IP address to the network interface using the same scheme as
virtualisation.vlans.
'';
};
};
}
);
# Convert legacy VLANs to named interfaces.
vlansNumbered = lib.listToAttrs (
lib.forEach (lib.zipLists cfg.vlans (lib.range 1 255)) (
v:
let
name = "eth${toString v.snd}";
in
lib.nameValuePair name {
inherit name;
vlan = v.fst;
assignIP = true;
}
)
);
in
{
imports = [
@@ -705,29 +752,20 @@ in
enp1s0.vlan = 1;
};
description = ''
Network interfaces to add to the VM.
Extra network interfaces to add to the VM in addition to the ones
created by {option}`virtualisation.vlans`.
'';
type =
with types;
attrsOf (submodule {
options = {
vlan = mkOption {
type = types.ints.unsigned;
description = ''
VLAN to which the network interface is connected.
'';
};
type = types.attrsOf interfaceType;
};
assignIP = mkOption {
type = types.bool;
default = false;
description = ''
Automatically assign an IP address to the network interface using the same scheme as
virtualisation.vlans.
'';
};
};
});
virtualisation.allInterfaces = mkOption {
type = types.attrsOf interfaceType;
readOnly = true;
description = ''
All network interfaces for the VM. Combines
{option}`virtualisation.vlans` and {option}`virtualisation.interfaces`.
'';
default = vlansNumbered // cfg.interfaces;
};
virtualisation.writableStore = mkOption {
@@ -1248,6 +1286,30 @@ in
)
)
++ [
(
let
conflictingKeys = lib.intersectAttrs vlansNumbered cfg.interfaces;
in
{
assertion = conflictingKeys == { };
message = ''
`virtualisation.vlans` and `virtualisation.interfaces` have conflicting keys: ${lib.concatStringsSep "," (lib.attrNames conflictingKeys)}
'';
}
)
(
let
allInterfaceNames =
(lib.mapAttrsToList (k: i: i.name) vlansNumbered)
++ (lib.mapAttrsToList (k: i: i.name) cfg.interfaces);
in
{
assertion = lib.allUnique allInterfaceNames;
message = ''
`virtualisation.vlans` and `virtualisation.interfaces` have conflicting interface names: ${lib.concatStringsSep "," allInterfaceNames}
'';
}
)
{
assertion = pkgs.stdenv.hostPlatform.is32bit -> cfg.memorySize < 2047;
message = ''