diff --git a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md index 2b0f44a04e44..b29f6df5bdaa 100644 --- a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md +++ b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md @@ -87,11 +87,33 @@ $ ssh vsock/3 -o User=root The socket numbers correspond to the node number of the test VM, but start at three instead of one because that's the lowest possible -vsock number. +vsock number. The exact SSH commands are also printed out when starting +`nixos-test-driver`. On non-NixOS systems you'll probably need to enable the SSH config from {manpage}`systemd-ssh-proxy(1)` yourself. +If starting VM fails with an error like + +``` +qemu-system-x86_64: -device vhost-vsock-pci,guest-cid=3: vhost-vsock: unable to set guest cid: Address already in use +``` + +it means that the vsock numbers for the VMs are already in use. This can happen +if another interactive test with SSH backdoor enabled is running on the machine. + +In that case, you need to assign another range of vsock numbers. You can pick another +offset with + +```nix +{ + sshBackdoor = { + enable = true; + vsockOffset = 23542; + }; +} +``` + ## Port forwarding to NixOS test VMs {#sec-nixos-test-port-forwarding} If your test has only a single VM, you may use e.g. diff --git a/nixos/doc/manual/redirects.json b/nixos/doc/manual/redirects.json index 7abcc36257f5..0abbf8970eba 100644 --- a/nixos/doc/manual/redirects.json +++ b/nixos/doc/manual/redirects.json @@ -1832,6 +1832,9 @@ "test-opt-sshBackdoor.enable": [ "index.html#test-opt-sshBackdoor.enable" ], + "test-opt-sshBackdoor.vsockOffset": [ + "index.html#test-opt-sshBackdoor.vsockOffset" + ], "test-opt-defaults": [ "index.html#test-opt-defaults" ], diff --git a/nixos/lib/test-driver/src/test_driver/__init__.py b/nixos/lib/test-driver/src/test_driver/__init__.py index ff6922e5b90e..86e663da9b7d 100755 --- a/nixos/lib/test-driver/src/test_driver/__init__.py +++ b/nixos/lib/test-driver/src/test_driver/__init__.py @@ -112,7 +112,7 @@ def main() -> None: arg_parser.add_argument( "--dump-vsocks", help="indicates that the interactive SSH backdoor is active and dumps information about it on start", - action="store_true", + type=int, ) args = arg_parser.parse_args() @@ -141,8 +141,8 @@ def main() -> None: if args.interactive: history_dir = os.getcwd() history_path = os.path.join(history_dir, ".nixos-test-history") - if args.dump_vsocks: - driver.dump_machine_ssh() + if offset := args.dump_vsocks: + driver.dump_machine_ssh(offset) ptpython.ipython.embed( user_ns=driver.test_symbols(), history_filename=history_path, diff --git a/nixos/lib/test-driver/src/test_driver/driver.py b/nixos/lib/test-driver/src/test_driver/driver.py index e65c6c5ba511..bf3dda06a617 100644 --- a/nixos/lib/test-driver/src/test_driver/driver.py +++ b/nixos/lib/test-driver/src/test_driver/driver.py @@ -178,14 +178,14 @@ class Driver: ) return {**general_symbols, **machine_symbols, **vlan_symbols} - def dump_machine_ssh(self) -> None: + def dump_machine_ssh(self, offset: int) -> None: print("SSH backdoor enabled, the machines can be accessed like this:") print( f"{Style.BRIGHT}Note:{Style.RESET_ALL} this requires {Style.BRIGHT}systemd-ssh-proxy(1){Style.RESET_ALL} to be enabled (default on NixOS 25.05 and newer)." ) names = [machine.name for machine in self.machines] longest_name = len(max(names, key=len)) - for num, name in enumerate(names, start=3): + for num, name in enumerate(names, start=offset + 1): spaces = " " * (longest_name - len(name) + 2) print( f" {name}:{spaces}{Style.BRIGHT}ssh -o User=root vsock/{num}{Style.RESET_ALL}" diff --git a/nixos/lib/testing/nodes.nix b/nixos/lib/testing/nodes.nix index 38b32dbb6701..b2352c478110 100644 --- a/nixos/lib/testing/nodes.nix +++ b/nixos/lib/testing/nodes.nix @@ -84,6 +84,22 @@ in type = types.bool; description = "Whether to turn on the VSOCK-based access to all VMs. This provides an unauthenticated access intended for debugging."; }; + vsockOffset = mkOption { + default = 2; + type = types.ints.between 2 4294967296; + description = '' + This field is only relevant when multiple users run the (interactive) + driver outside the sandbox and with the SSH backdoor activated. + The typical symptom for this being a problem are error messages like this: + `vhost-vsock: unable to set guest cid: Address already in use` + + This option allows to assign an offset to each vsock number to + resolve this. + + This is a 32bit number. The lowest possible vsock number is `3` + (i.e. with the lowest node number being `1`, this is 2+1). + ''; + }; }; node.type = mkOption { @@ -182,7 +198,7 @@ in passthru.nodes = config.nodesCompat; extraDriverArgs = mkIf config.sshBackdoor.enable [ - "--dump-vsocks" + "--dump-vsocks=${toString config.sshBackdoor.vsockOffset}" ]; defaults = mkMerge [ @@ -190,9 +206,31 @@ in nixpkgs.pkgs = config.node.pkgs; imports = [ ../../modules/misc/nixpkgs/read-only.nix ]; }) - (mkIf config.sshBackdoor.enable { - testing.sshBackdoor.enable = true; - }) + (mkIf config.sshBackdoor.enable ( + let + inherit (config.sshBackdoor) vsockOffset; + in + { config, ... }: + { + services.openssh = { + enable = true; + settings = { + PermitRootLogin = "yes"; + PermitEmptyPasswords = "yes"; + }; + }; + + security.pam.services.sshd = { + allowNullPassword = true; + }; + + virtualisation.qemu.options = [ + "-device vhost-vsock-pci,guest-cid=${ + toString (config.virtualisation.test.nodeNumber + vsockOffset) + }" + ]; + } + )) ]; }; diff --git a/nixos/modules/testing/test-instrumentation.nix b/nixos/modules/testing/test-instrumentation.nix index 8ace4ab30abf..80852be51f1d 100644 --- a/nixos/modules/testing/test-instrumentation.nix +++ b/nixos/modules/testing/test-instrumentation.nix @@ -86,11 +86,6 @@ in enables commands to be sent to test and debug stage 1. Use machine.switch_root() to leave stage 1 and proceed to stage 2 ''; - - sshBackdoor = { - enable = mkEnableOption "vsock-based ssh backdoor for the VM"; - }; - }; config = { @@ -104,18 +99,6 @@ in } ]; - services.openssh = mkIf config.testing.sshBackdoor.enable { - enable = true; - settings = { - PermitRootLogin = "yes"; - PermitEmptyPasswords = "yes"; - }; - }; - - security.pam.services.sshd = mkIf config.testing.sshBackdoor.enable { - allowNullPassword = true; - }; - systemd.services.backdoor = lib.mkMerge [ backdoorService { @@ -191,10 +174,6 @@ in # we avoid defining attributes if not possible. # TODO: refactor such that test-instrumentation can import qemu-vm package = lib.mkDefault pkgs.qemu_test; - - options = mkIf config.testing.sshBackdoor.enable [ - "-device vhost-vsock-pci,guest-cid=${toString (config.virtualisation.test.nodeNumber + 2)}" - ]; }; };