gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/test-driver: use vhost-device-vsock for SSH backdoor (#453305)

Browse Source
This commit is contained in:
Jacek Galowicz
2026-04-15 18:15:52 +00:00
committed by GitHub
parent c17bc0e1d2 1987c483d8
commit ea044acd82
15 changed files with 294 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/doc/manual/development/running-nixos-tests-interactively.section.md
+15 -33
View File
@@ -88,52 +88,34 @@ An SSH-based backdoor to log into machines can be enabled with
}
```
::: {.warning}
Make sure to only enable the backdoor for interactive tests
(i.e. by using `interactive.sshBackdoor.enable`)! This is the only
supported configuration.
Running a test in a sandbox with this will fail because `/dev/vhost-vsock` isn't available
in the sandbox.
:::
This creates a [vsock socket](https://man7.org/linux/man-pages/man7/vsock.7.html)
for each VM to log in with SSH. This configures root login with an empty password.
When the VMs get started interactively with the test-driver, it's possible to
connect to `machine` with
On the host-side a UNIX domain-socket is used with
[vhost-device-vsock](https://github.com/rust-vmm/vhost-device/blob/main/vhost-device-vsock/README.md).
That way, it's not necessary to assign system-wide unique vsock numbers.
```
$ ssh vsock/3 -o User=root
$ ssh vsock-mux//tmp/path/to/host -o User=root
```
The socket numbers correspond to the node number of the test VM, but start
at three instead of one because that's the lowest possible
vsock number. The exact SSH commands are also printed out when starting
`nixos-test-driver`.
The socket paths are printed when starting the test driver:
```
Note: this requires systemd-ssh-proxy(1) to be enabled (default on NixOS 25.05 and newer).
machine: ssh -o User=root vsock-mux//tmp/tmpg1rp9nti/machine_host.socket
```
On non-NixOS systems you'll probably need to enable
the SSH config from {manpage}`systemd-ssh-proxy(1)` yourself.
If starting VM fails with an error like
During a test-run, it's possible to print the SSH commands again by running
```
qemu-system-x86_64: -device vhost-vsock-pci,guest-cid=3: vhost-vsock: unable to set guest cid: Address already in use
```
it means that the vsock numbers for the VMs are already in use. This can happen
if another interactive test with SSH backdoor enabled is running on the machine.
In that case, you need to assign another range of vsock numbers. You can pick another
offset with
```nix
{
sshBackdoor = {
enable = true;
vsockOffset = 23542;
};
}
In [2]: dump_machine_ssh()
SSH backdoor enabled, the machines can be accessed like this:
Note: this requires systemd-ssh-proxy(1) to be enabled (default on NixOS 25.05 and newer).
machine: ssh -o User=root vsock-mux//tmp/tmpg1rp9nti/machine_host.socket
```
## Port forwarding to NixOS test VMs {#sec-nixos-test-port-forwarding}
nixos/doc/manual/development/writing-nixos-tests.section.md
+3 -11
View File
@@ -512,19 +512,11 @@ Once you are in the sandbox shell, you can access the VMs (for example, `machine
with SSH over vsock:
```
bash# ssh -F ./ssh_config vsock/3
bash# ssh -F ./ssh_config -o User=root vsock-mux//tmp/.../machine_host.socket
```
For the AF_VSOCK feature to work, `/dev/vhost-vsock` is needed in the sandbox
which can be done with e.g.
```
nix-build -A nixosTests.foo --option sandbox-paths /dev/vhost-vsock
```
As described in [](#sec-nixos-test-ssh-access), the numbers for vsock start at
`3` instead of `1`. So the first VM in the network (sorted alphabetically) can
be accessed with `vsock/3`.
The socket paths are printed at the beginning of the test. See
[](#sec-nixos-test-ssh-access) for more context.
### SSH access to test containers {#sec-test-container-ssh-access}
nixos/doc/manual/redirects.json
-3
View File
@@ -2174,9 +2174,6 @@
"test-opt-sshBackdoor.enable": [
"index.html#test-opt-sshBackdoor.enable"
],
"test-opt-sshBackdoor.vsockOffset": [
"index.html#test-opt-sshBackdoor.vsockOffset"
],
"test-opt-enableDebugHook": [
"index.html#test-opt-enableDebugHook"
],
nixos/lib/test-driver/default.nix
+2
View File
@@ -14,6 +14,7 @@
ty,
netpbm,
vhost-device-vsock,
nixosTests,
qemu_pkg ? qemu_test,
qemu_test,
@@ -56,6 +57,7 @@ buildPythonApplication {
socat
util-linux
vde2
vhost-device-vsock
]
++ lib.optionals enableNspawn [
systemd
nixos/lib/test-driver/src/test_driver/__init__.py
+5 -4
View File
@@ -148,9 +148,9 @@ def main() -> None:
type=Path,
)
arg_parser.add_argument(
"--dump-vsocks",
"--enable-ssh-backdoor",
help="indicates that the interactive SSH backdoor is active and dumps information about it on start",
type=int,
action="store_true",
)
log_level_map = {level.name.lower(): level for level in LogLevel}
arg_parser.add_argument(
@@ -210,9 +210,10 @@ def main() -> None:
keep_machine_state=args.keep_machine_state,
global_timeout=args.global_timeout,
debug=debugger,
enable_ssh_backdoor=args.enable_ssh_backdoor,
) as driver:
if offset := args.dump_vsocks:
driver.dump_machine_ssh(offset)
if args.enable_ssh_backdoor:
driver.dump_machine_ssh()
if args.interactive:
history_dir = os.getcwd()
history_path = os.path.join(history_dir, ".nixos-test-history")
nixos/lib/test-driver/src/test_driver/driver.py
+118 -27
View File
@@ -8,6 +8,7 @@ import threading
import traceback
from collections.abc import Callable, Iterator
from contextlib import AbstractContextManager, contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from unittest import TestCase
@@ -63,19 +64,64 @@ def pythonize_name(name: str) -> str:
return re.sub(r"^[^A-Za-z_]|[^A-Za-z0-9_]", "_", name)
@dataclass
class VsockPair:
guest: Path
host: Path
cid: int
class VHostDeviceVsock:
def __init__(self, tmp_dir: Path, machines: list[str]):
self.temp_dir_handle = tempfile.TemporaryDirectory(dir=tmp_dir)
self.temp_dir = Path(self.temp_dir_handle.name)
self.sockets = {
machine: VsockPair(
self.temp_dir / f"{machine}_guest.socket",
self.temp_dir / f"{machine}_host.socket",
cid,
)
for cid, machine in enumerate(machines, start=3)
}
self.vhost_proc = subprocess.Popen(
[
"vhost-device-vsock",
*(
arg
for vsock_pair in self.sockets.values()
for arg in (
"--vm",
f"guest-cid={vsock_pair.cid},socket={vsock_pair.guest},uds-path={vsock_pair.host}",
)
),
]
)
def __del__(self) -> None:
self.vhost_proc.kill()
self.temp_dir_handle.cleanup()
class Driver:
"""A handle to the driver that sets up the environment
and runs the tests"""
tests: str
vlans: list[VLan]
machines_qemu: list[QemuMachine]
machines_nspawn: list[NspawnMachine]
vlans: list[VLan] = []
machines_qemu: list[QemuMachine] = []
machines_nspawn: list[NspawnMachine] = []
polling_conditions: list[PollingCondition]
global_timeout: int
race_timer: threading.Timer
vm_start_scripts: dict[str, str]
container_start_scripts: dict[str, str]
vlan_ids: list[int]
keep_machine_state: bool
logger: AbstractLogger
debug: DebugAbstract
vhost_vsock: VHostDeviceVsock | None = None
enable_ssh_backdoor: bool
def __init__(
self,
@@ -90,36 +136,62 @@ class Driver:
keep_machine_state: bool = False,
global_timeout: int = 24 * 60 * 60 * 7,
debug: DebugAbstract = DebugNop(),
enable_ssh_backdoor: bool = False,
):
self.tests = tests
self.out_dir = out_dir
self.global_timeout = global_timeout
self.race_timer = threading.Timer(global_timeout, self.terminate_test)
self.logger = logger
self.debug = debug
self.vlan_ids = list(set(vlans))
self.polling_conditions = []
self.keep_machine_state = keep_machine_state
self.global_timeout = global_timeout
self.vm_start_scripts = dict(zip(vm_names, vm_start_scripts))
self.container_start_scripts = dict(
zip(container_names, container_start_scripts)
)
self.enable_ssh_backdoor = enable_ssh_backdoor
def __enter__(self) -> "Driver":
self.race_timer = threading.Timer(self.global_timeout, self.terminate_test)
tmp_dir = get_tmp_dir()
with self.logger.nested("start all VLans"):
vlans = list(set(vlans))
self.vlans = [VLan(nr, tmp_dir, self.logger) for nr in vlans]
self.vlans = [VLan(nr, tmp_dir, self.logger) for nr in self.vlan_ids]
self.polling_conditions = []
if self.enable_ssh_backdoor and self.vm_start_scripts:
with self.logger.nested("start vhost-device-vsock"):
self.vhost_vsock = VHostDeviceVsock(
tmp_dir, list(self.vm_start_scripts.keys())
)
self.machines_qemu = [
QemuMachine(
name=name,
start_command=vm_start_script,
keep_machine_state=keep_machine_state,
keep_machine_state=self.keep_machine_state,
tmp_dir=tmp_dir,
callbacks=[self.check_polling_conditions],
out_dir=self.out_dir,
logger=self.logger,
vsock_host=(
self.vhost_vsock.sockets[name].host
if self.vhost_vsock is not None
else None
),
vsock_guest=(
self.vhost_vsock.sockets[name].guest
if self.vhost_vsock is not None
else None
),
)
for name, vm_start_script in zip(vm_names, vm_start_scripts)
for name, vm_start_script in self.vm_start_scripts.items()
]
if len(container_start_scripts) > 0:
if len(self.container_start_scripts) > 0:
self._init_nspawn_environment()
self.machines_nspawn = [
@@ -128,16 +200,15 @@ class Driver:
start_command=container_start_script,
tmp_dir=tmp_dir,
logger=self.logger,
keep_machine_state=keep_machine_state,
keep_machine_state=self.keep_machine_state,
callbacks=[self.check_polling_conditions],
out_dir=self.out_dir,
)
for name, container_start_script in zip(
container_names,
container_start_scripts,
)
for name, container_start_script in self.container_start_scripts.items()
]
return self
def _init_nspawn_environment(self) -> None:
assert os.geteuid() == 0, (
f"systemd-nspawn requires root to work. You are {os.geteuid()}"
@@ -193,9 +264,6 @@ class Driver:
machines.sort(key=lambda machine: machine.name)
return machines
def __enter__(self) -> "Driver":
return self
def __exit__(self, *_: Any) -> None:
with self.logger.nested("cleanup"):
self.race_timer.cancel()
@@ -211,6 +279,14 @@ class Driver:
except Exception as e:
self.logger.error(f"Error during cleanup of vlan{vlan.nr}: {e}")
if self.enable_ssh_backdoor:
try:
del self.vhost_vsock
except Exception as e:
self.logger.error(
f"Error during cleanup of vhost-device-vsock process: {e}"
)
def subtest(self, name: str) -> Iterator[None]:
"""Group logs under a given test name"""
with self.logger.subtest(name):
@@ -248,6 +324,7 @@ class Driver:
NspawnMachine=NspawnMachine, # for typing
t=AssertionTester(),
debug=self.debug,
dump_machine_ssh=self.dump_machine_ssh,
)
machine_symbols = {pythonize_name(m.name): m for m in self.machines}
# If there's exactly one machine, make it available under the name
@@ -267,18 +344,28 @@ class Driver:
)
return {**general_symbols, **machine_symbols, **vlan_symbols}
def dump_machine_ssh(self, offset: int) -> None:
print("SSH backdoor enabled, the machines can be accessed like this:")
print(
f"{Style.BRIGHT}Note:{Style.RESET_ALL} vsocks require {Style.BRIGHT}systemd-ssh-proxy(1){Style.RESET_ALL} to be enabled (default on NixOS 25.05 and newer)."
)
longest_name = len(max((machine.name for machine in self.machines), key=len))
for index, machine in enumerate(self.machines, start=offset + 1):
name = machine.name
spaces = " " * (longest_name - len(name) + 2)
def dump_machine_ssh(self) -> None:
if not self.enable_ssh_backdoor:
return
assert self.vhost_vsock is not None
if self.machines:
print("SSH backdoor enabled, the machines can be accessed like this:")
print(
f" {name}:{spaces}{Style.BRIGHT}{machine.ssh_backdoor_command(index)}{Style.RESET_ALL}"
f"{Style.BRIGHT}Note:{Style.RESET_ALL} this requires {Style.BRIGHT}systemd-ssh-proxy(1){Style.RESET_ALL} to be enabled (default on NixOS 25.05 and newer)."
)
longest_name = len(
max((machine.name for machine in self.machines), key=len)
)
for index, machine in enumerate(self.machines):
name = machine.name
spaces = " " * (longest_name - len(name) + 2)
print(
f" {name}:{spaces}{Style.BRIGHT}{machine.ssh_backdoor_command()}{Style.RESET_ALL}"
)
else:
print("SSH backdoor enabled, but no machines defined")
def test_script(self) -> None:
"""Run the test script"""
@@ -396,6 +483,10 @@ class Driver:
"""
tmp_dir = get_tmp_dir()
if self.enable_ssh_backdoor:
self.logger.warning(
f"create_machine({name}): not enabling SSH backdoor, this is not supported for VMs created with create_machine!"
)
return QemuMachine(
tmp_dir=tmp_dir,
out_dir=self.out_dir,
nixos/lib/test-driver/src/test_driver/machine/__init__.py
+25 -4
View File
@@ -149,6 +149,7 @@ class QemuStartCommand:
qmp_socket_path: Path,
shell_socket_path: Path,
allow_reboot: bool = False,
vsock_guest: Path | None = None,
) -> str:
display_opts = ""
@@ -172,6 +173,12 @@ class QemuStartCommand:
if not allow_reboot:
qemu_opts += " -no-reboot"
if vsock_guest is not None:
qemu_opts += (
f" -chardev socket,id=vsock_ssh,path={vsock_guest} "
f"-device vhost-user-vsock-pci,chardev=vsock_ssh "
)
return (
f"{self._cmd}"
f" -qmp unix:{qmp_socket_path},server=on,wait=off"
@@ -205,10 +212,15 @@ class QemuStartCommand:
qmp_socket_path: Path,
shell_socket_path: Path,
allow_reboot: bool,
vsock_guest: Path | None = None,
) -> subprocess.Popen:
return subprocess.Popen(
self.cmd(
monitor_socket_path, qmp_socket_path, shell_socket_path, allow_reboot
monitor_socket_path,
qmp_socket_path,
shell_socket_path,
allow_reboot,
vsock_guest,
),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
@@ -727,6 +739,9 @@ class QemuMachine(BaseMachine):
shell: socket.socket | None
serial_thread: threading.Thread | None
vsock_guest: Path | None
vsock_host: Path | None
booted: bool
connected: bool
# Store last serial console lines for use
@@ -744,6 +759,8 @@ class QemuMachine(BaseMachine):
name: str | None = None,
keep_machine_state: bool = False,
callbacks: list[Callable] | None = None,
vsock_guest: Path | None = None,
vsock_host: Path | None = None,
) -> None:
self.start_command = QemuStartCommand(start_command)
super().__init__(
@@ -756,6 +773,8 @@ class QemuMachine(BaseMachine):
)
self.full_console_log = []
self.vsock_guest = vsock_guest
self.vsock_host = vsock_host
# set up directories
self.monitor_path = self.state_dir / "monitor"
@@ -772,8 +791,9 @@ class QemuMachine(BaseMachine):
self.booted = False
self.connected = False
def ssh_backdoor_command(self, index: int) -> str:
return f"ssh -o User=root vsock/{index}"
def ssh_backdoor_command(self) -> str:
assert self.vsock_host is not None
return f"ssh -o User=root vsock-mux/{self.vsock_host}"
def is_up(self) -> bool:
return self.booted and self.connected
@@ -1227,6 +1247,7 @@ class QemuMachine(BaseMachine):
self.qmp_path,
self.shell_path,
allow_reboot,
self.vsock_guest,
)
def accept_or_fail(sock: socket.socket, name: str) -> socket.socket:
@@ -1458,7 +1479,7 @@ class NspawnMachine(BaseMachine):
self.machine_sock_path = self.tmp_dir / f"{self.name}-nspawn.sock"
def ssh_backdoor_command(self, index: int) -> str:
def ssh_backdoor_command(self) -> str:
# documented in systemd-ssh-generator(8) and https://systemd.io/CONTAINER_INTERFACE/
socket_path = f"/run/systemd/nspawn/unix-export/{self.name}/ssh"
proxy_cmd = f"socat - UNIX-CLIENT:{socket_path}"
nixos/lib/test-script-prepend.py
+1
View File
@@ -58,3 +58,4 @@ serial_stdout_on: Callable[[], None]
polling_condition: PollingConditionProtocol
debug: DebugAbstract
t: TestCase
dump_machine_ssh: Callable[[], None]
nixos/lib/testing/driver.nix
+24
View File
@@ -7,6 +7,8 @@
let
inherit (lib) mkOption types literalMD;
inherit (config) sshBackdoor;
# Reifies and correctly wraps the python test driver for
# the respective qemu version and with or without ocr support
testDriver = config.pythonTestDriverPackage.override {
@@ -254,5 +256,27 @@ in
# make available on the test runner
passthru.driver = config.driver;
nodeDefaults =
{ config, ... }:
{
# This is needed for the SSH backdoor to function.
# Set this to `true` by default to not change essential QEMU flags
# depending on whether debugging is enabled.
#
# If needed, this can still be turned off.
virtualisation.qemu.enableSharedMemory = lib.mkDefault true;
assertions = [
{
assertion = sshBackdoor.enable -> config.virtualisation.qemu.enableSharedMemory;
message = ''
When turning on the SSH backdoor of the NixOS test-framework,
`virtualisation.qemu.enableSharedMemory` MUST be `true`
(affected: ${config.networking.hostName}).
'';
}
];
};
};
}
nixos/lib/testing/nodes.nix
+8 -31
View File
@@ -13,6 +13,7 @@ let
mapAttrs
mkIf
mkMerge
mkRemovedOptionModule
mkOption
optionalAttrs
types
@@ -130,6 +131,12 @@ let
in
{
imports = [
(mkRemovedOptionModule [ "sshBackdoor" "vsockOffset" ] ''
The option `sshBackdoor.vsockOffset` has been removed from the testing framework.
The functionality provided by it is not needed anymore.
'')
];
options = {
sshBackdoor = {
@@ -139,22 +146,6 @@ in
type = types.bool;
description = "Whether to turn on the VSOCK-based access to all VMs. This provides an unauthenticated access intended for debugging.";
};
vsockOffset = mkOption {
default = 2;
type = types.ints.between 2 4294967296;
description = ''
This field is only relevant when multiple users run the (interactive)
driver outside the sandbox and with the SSH backdoor activated.
The typical symptom for this being a problem are error messages like this:
`vhost-vsock: unable to set guest cid: Address already in use`
This option allows to assign an offset to each vsock number to
resolve this.
This is a 32bit number. The lowest possible vsock number is `3`
(i.e. with the lowest node number being `1`, this is 2+1).
'';
};
};
node.type = mkOption {
@@ -320,7 +311,7 @@ in
passthru.containers = config.containers;
extraDriverArgs = mkIf config.sshBackdoor.enable [
"--dump-vsocks=${toString config.sshBackdoor.vsockOffset}"
"--enable-ssh-backdoor"
];
defaults = mkMerge [
@@ -343,20 +334,6 @@ in
})
];
nodeDefaults = mkIf config.sshBackdoor.enable (
let
inherit (config.sshBackdoor) vsockOffset;
in
{ config, ... }:
{
virtualisation.qemu.options = [
"-device vhost-vsock-pci,guest-cid=${
toString (config.virtualisation.test.nodeNumber + vsockOffset)
}"
];
}
);
# Docs: nixos/doc/manual/development/writing-nixos-tests.section.md
/**
See https://nixos.org/manual/nixos/unstable#sec-override-nixos-test
nixos/lib/testing/run.nix
+5 -1
View File
@@ -80,6 +80,10 @@ in
};
};
imports = [
../../modules/misc/assertions.nix
];
config = {
rawTestDerivation = hostPkgs.stdenv.mkDerivation config.rawTestDerivationArg;
rawTestDerivationArg =
@@ -131,7 +135,7 @@ in
};
test = lib.lazyDerivation {
# lazyDerivation improves performance when only passthru items and/or meta are used.
derivation = config.rawTestDerivation;
derivation = lib.asserts.checkAssertWarn config.assertions config.warnings config.rawTestDerivation;
inherit (config) passthru meta;
};
nixos/modules/virtualisation/qemu-vm.nix
+6
View File
@@ -750,6 +750,8 @@ in
};
virtualisation.qemu = {
enableSharedMemory = mkEnableOption "shared memory";
package = mkOption {
type = types.package;
default =
@@ -1383,6 +1385,10 @@ in
"-device usb-kbd"
"-device usb-tablet"
])
(mkIf cfg.qemu.enableSharedMemory [
"-object memory-backend-memfd,id=mem0,size=${toString config.virtualisation.memorySize}M,share=on"
"-machine memory-backend=mem0"
])
(
let
alphaNumericChars = lowerChars ++ upperChars ++ (map toString (range 0 9));
nixos/tests/all-tests.nix
+1
View File
@@ -149,6 +149,7 @@ in
lib-extend = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./nixos-test-driver/lib-extend.nix { };
node-name = runTest ./nixos-test-driver/node-name.nix;
busybox = runTest ./nixos-test-driver/busybox.nix;
ssh-backdoor = runTestOn [ "x86_64-linux" ] ./nixos-test-driver/ssh-backdoor.nix;
console-log = runTest ./nixos-test-driver/console-log.nix;
containers = runTest ./nixos-test-driver/containers.nix;
driver-timeout =
nixos/tests/nixos-test-driver/ssh-backdoor.nix
+36
View File
@@ -0,0 +1,36 @@
{ pkgs, lib, ... }:
{
name = "ssh-backdoor";
sshBackdoor.enable = true;
nodes.machine = { };
testScript = ''
import subprocess
start_all()
machine.wait_for_unit("multi-user.target")
assert driver.vhost_vsock is not None
host_socket = driver.vhost_vsock.sockets["machine"].host
with subtest("ssh from the host via systemd-ssh-proxy"):
subprocess.run(
[
"${lib.getExe pkgs.openssh}",
"-vvv",
f"vsock-mux/{host_socket}",
"-o",
"User=root",
"-F",
# The backdoor feature of the driver copies this into NIX_BUILD_TOP.
# We can't do this here since `enableDebugHook=true;` would halt
# instead of terminating the test execution if it fails.
"${pkgs.systemd}/lib/systemd/ssh_config.d/20-systemd-ssh-proxy.conf",
"--",
"true"
],
check=True
)
'';
}
pkgs/by-name/vh/vhost-device-vsock/package.nix
+45
View File
@@ -0,0 +1,45 @@
{
lib,
fetchFromGitHub,
rustPlatform,
installShellFiles,
}:
rustPlatform.buildRustPackage (finalAttrs: {
pname = "vhost-device-vsock";
version = "0.3.0";
src = fetchFromGitHub {
owner = "rust-vmm";
repo = "vhost-device";
tag = "vhost-device-vsock-v${finalAttrs.version}";
hash = "sha256-g+u6WBJtizIgQwC0kkWdAcTiYCM1zMI4YBLVRU4MOrs=";
};
__structuredAttrs = true;
outputs = [
"out"
"man"
];
cargoBuildFlags = "-p vhost-device-vsock";
cargoTestFlags = "-p vhost-device-vsock";
cargoHash = "sha256-mtORRCY/TNeIEgRCQ1ZbjpsykteRm2FHRveKaQxD/Pw=";
nativeBuildInputs = [ installShellFiles ];
postInstall = ''
installManPage vhost-device-vsock/*.1
'';
meta = {
homepage = "https://github.com/rust-vmm/vhost-device/blob/main/vhost-device-vsock/README.md";
maintainers = with lib.maintainers; [ ma27 ];
license = with lib.licenses; [
asl20
bsd3
];
platforms = lib.platforms.linux;
};
})