diff --git a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md index 2b0f44a04e44..b29f6df5bdaa 100644 --- a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md +++ b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md @@ -87,11 +87,33 @@ $ ssh vsock/3 -o User=root The socket numbers correspond to the node number of the test VM, but start at three instead of one because that's the lowest possible -vsock number. +vsock number. The exact SSH commands are also printed out when starting +`nixos-test-driver`. On non-NixOS systems you'll probably need to enable the SSH config from {manpage}`systemd-ssh-proxy(1)` yourself. +If starting VM fails with an error like + +``` +qemu-system-x86_64: -device vhost-vsock-pci,guest-cid=3: vhost-vsock: unable to set guest cid: Address already in use +``` + +it means that the vsock numbers for the VMs are already in use. This can happen +if another interactive test with SSH backdoor enabled is running on the machine. + +In that case, you need to assign another range of vsock numbers. You can pick another +offset with + +```nix +{ + sshBackdoor = { + enable = true; + vsockOffset = 23542; + }; +} +``` + ## Port forwarding to NixOS test VMs {#sec-nixos-test-port-forwarding} If your test has only a single VM, you may use e.g. diff --git a/nixos/doc/manual/redirects.json b/nixos/doc/manual/redirects.json index 2c9b0242e9e8..f3548fc6ea04 100644 --- a/nixos/doc/manual/redirects.json +++ b/nixos/doc/manual/redirects.json @@ -1826,6 +1826,9 @@ "test-opt-sshBackdoor.enable": [ "index.html#test-opt-sshBackdoor.enable" ], + "test-opt-sshBackdoor.vsockOffset": [ + "index.html#test-opt-sshBackdoor.vsockOffset" + ], "test-opt-defaults": [ "index.html#test-opt-defaults" ], diff --git a/nixos/lib/test-driver/src/test_driver/__init__.py b/nixos/lib/test-driver/src/test_driver/__init__.py index ff6922e5b90e..86e663da9b7d 100755 --- a/nixos/lib/test-driver/src/test_driver/__init__.py +++ b/nixos/lib/test-driver/src/test_driver/__init__.py @@ -112,7 +112,7 @@ def main() -> None: arg_parser.add_argument( "--dump-vsocks", help="indicates that the interactive SSH backdoor is active and dumps information about it on start", - action="store_true", + type=int, ) args = arg_parser.parse_args() @@ -141,8 +141,8 @@ def main() -> None: if args.interactive: history_dir = os.getcwd() history_path = os.path.join(history_dir, ".nixos-test-history") - if args.dump_vsocks: - driver.dump_machine_ssh() + if offset := args.dump_vsocks: + driver.dump_machine_ssh(offset) ptpython.ipython.embed( user_ns=driver.test_symbols(), history_filename=history_path, diff --git a/nixos/lib/test-driver/src/test_driver/driver.py b/nixos/lib/test-driver/src/test_driver/driver.py index e65c6c5ba511..bf3dda06a617 100644 --- a/nixos/lib/test-driver/src/test_driver/driver.py +++ b/nixos/lib/test-driver/src/test_driver/driver.py @@ -178,14 +178,14 @@ class Driver: ) return {**general_symbols, **machine_symbols, **vlan_symbols} - def dump_machine_ssh(self) -> None: + def dump_machine_ssh(self, offset: int) -> None: print("SSH backdoor enabled, the machines can be accessed like this:") print( f"{Style.BRIGHT}Note:{Style.RESET_ALL} this requires {Style.BRIGHT}systemd-ssh-proxy(1){Style.RESET_ALL} to be enabled (default on NixOS 25.05 and newer)." ) names = [machine.name for machine in self.machines] longest_name = len(max(names, key=len)) - for num, name in enumerate(names, start=3): + for num, name in enumerate(names, start=offset + 1): spaces = " " * (longest_name - len(name) + 2) print( f" {name}:{spaces}{Style.BRIGHT}ssh -o User=root vsock/{num}{Style.RESET_ALL}" diff --git a/nixos/lib/testing/nodes.nix b/nixos/lib/testing/nodes.nix index 38b32dbb6701..258888fefa0a 100644 --- a/nixos/lib/testing/nodes.nix +++ b/nixos/lib/testing/nodes.nix @@ -84,6 +84,21 @@ in type = types.bool; description = "Whether to turn on the VSOCK-based access to all VMs. This provides an unauthenticated access intended for debugging."; }; + vsockOffset = mkOption { + default = 2; + type = types.ints.between 2 4294967296; + description = '' + By default this assigns vsock numbers starting at 3 to the nodes. + On e.g. large builders used by multiple people, this would cause conflicts + between multiple users doing interactive debugging. + + This option allows to assign an offset to each vsock number to + resolve this. + + This is a 32bit number. The lowest possible vsock number is `3` + (i.e. with the lowest node number being `1`, this is 2+1). + ''; + }; }; node.type = mkOption { @@ -182,7 +197,7 @@ in passthru.nodes = config.nodesCompat; extraDriverArgs = mkIf config.sshBackdoor.enable [ - "--dump-vsocks" + "--dump-vsocks=${toString config.sshBackdoor.vsockOffset}" ]; defaults = mkMerge [ @@ -191,7 +206,9 @@ in imports = [ ../../modules/misc/nixpkgs/read-only.nix ]; }) (mkIf config.sshBackdoor.enable { - testing.sshBackdoor.enable = true; + testing.sshBackdoor = { + inherit (config.sshBackdoor) enable vsockOffset; + }; }) ]; diff --git a/nixos/modules/testing/test-instrumentation.nix b/nixos/modules/testing/test-instrumentation.nix index 8ace4ab30abf..38784ab1c5fc 100644 --- a/nixos/modules/testing/test-instrumentation.nix +++ b/nixos/modules/testing/test-instrumentation.nix @@ -89,6 +89,21 @@ in sshBackdoor = { enable = mkEnableOption "vsock-based ssh backdoor for the VM"; + vsockOffset = mkOption { + default = 2; + type = types.ints.between 2 4294967296; + description = '' + By default this assigns vsock numbers starting at 3 to the nodes. + On e.g. large builders used by multiple people, this would cause conflicts + between multiple users doing interactive debugging. + + This option allows to assign an offset to each vsock number to + resolve this. + + This is a 32bit number. The lowest possible vsock number is `3` + (i.e. with the lowest node number being `1`, this is 2+1). + ''; + }; }; }; @@ -193,7 +208,9 @@ in package = lib.mkDefault pkgs.qemu_test; options = mkIf config.testing.sshBackdoor.enable [ - "-device vhost-vsock-pci,guest-cid=${toString (config.virtualisation.test.nodeNumber + 2)}" + "-device vhost-vsock-pci,guest-cid=${ + toString (config.virtualisation.test.nodeNumber + config.testing.sshBackdoor.vsockOffset) + }" ]; }; };