nixos-test-driver: Use ty anf ruff instead of mypy and pyflakes

nixos/lib/test-driver/src/pyproject.toml

@@ -8,7 +8,6 @@ version = "0.0.0"
 
 [project.scripts]
 nixos-test-driver = "test_driver:main"
-generate-driver-symbols = "test_driver:generate_driver_symbols"
 
 [tool.setuptools.packages]
 find = {}

nixos/lib/test-driver/src/test_driver/__init__.py

@@ -8,7 +8,7 @@ from pathlib import Path
 import ptpython.ipython
 
 from test_driver.debug import Debug, DebugAbstract, DebugNop
-from test_driver.driver import Driver, DriverConfiguration, load_driver_configuration
+from test_driver.driver import Driver, load_driver_configuration
 from test_driver.logger import (
     CompositeLogger,
     JunitXMLLogger,
@@ -159,30 +159,3 @@ def main() -> None:
             driver.run_tests()
             toc = time.time()
             logger.info(f"test script finished in {(toc - tic):.2f}s")
-
-
-def generate_driver_symbols() -> None:
-    """
-    This generates a file with symbols of the test-driver code that can be used
-    in user's test scripts. That list is then used by pyflakes to lint those
-    scripts.
-    """
-    d = Driver(
-        config=DriverConfiguration(
-            vms=dict(),
-            containers=dict(),
-            vlans=[],
-            global_timeout=0,
-            enable_ssh_backdoor=False,
-            test_script=(
-                Path("testScriptWithTypes")
-                if (Path("testScriptWithTypes").is_file())
-                else Path("testScriptFile")
-            ),
-        ),
-        out_dir=Path(),
-        logger=CompositeLogger([]),
-    )
-    test_symbols = d.test_symbols()
-    with open("driver-symbols", "w") as fp:
-        fp.write(",".join(test_symbols.keys()))

nixos/lib/test-driver/src/test_driver/driver.py

@@ -7,7 +7,7 @@ import sys
 import tempfile
 import threading
 import traceback
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Generator, Iterator
 from contextlib import AbstractContextManager, contextmanager
 from dataclasses import dataclass
 from pathlib import Path
@@ -299,7 +299,7 @@ class Driver:
                         f"Error during cleanup of vhost-device-vsock process: {e}"
                     )
 
-    def subtest(self, name: str) -> Iterator[None]:
+    def subtest(self, name: str) -> Generator[None]:
         """Group logs under a given test name"""
         with self.logger.subtest(name):
             try:

nixos/lib/test-script-prepend.py

@@ -1,19 +1,30 @@
 # This file contains type hints that can be prepended to Nix test scripts so they can be type
 # checked.
 
-from test_driver.debug import DebugAbstract
-from test_driver.driver import Driver
-from test_driver.vlan import VLan
-from test_driver.machine import BaseMachine, NspawnMachine, QemuMachine
-from test_driver.logger import AbstractLogger
-from typing import Callable, Iterator, ContextManager, Optional, List, Dict, Any, Union
-from typing_extensions import Protocol
-from pathlib import Path
+from contextlib import contextmanager
+from typing import Any, Callable, ContextManager, Generator, List, Optional, Union
 from unittest import TestCase
 
+from test_driver.debug import DebugAbstract, DebugNop
+from test_driver.driver import Driver
+from typing_extensions import Protocol
 
-class RetryProtocol(Protocol):
-    def __call__(self, fn: Callable, timeout_seconds: int = 900) -> None:
+from test_driver.machine import BaseMachine, NspawnMachine, QemuMachine
+from test_driver.vlan import VLan
+
+
+# Protocols
+
+
+class CreateMachineProtocol(Protocol):
+    def __call__(
+        self,
+        start_command: str | dict,
+        *,
+        name: Optional[str] = None,
+        keep_machine_state: bool = False,
+        **kwargs: Any,  # to allow usage of deprecated keep_vm_state
+    ) -> QemuMachine:
         raise Exception("This is just type information for the Nix test driver")
 
 
@@ -28,34 +39,94 @@ class PollingConditionProtocol(Protocol):
         raise Exception("This is just type information for the Nix test driver")
 
 
-class CreateMachineProtocol(Protocol):
-    def __call__(
-        self,
-        start_command: str | dict,
-        *,
-        name: Optional[str] = None,
-        keep_machine_state: bool = False,
-        **kwargs: Any, # to allow usage of deprecated keep_vm_state
-    ) -> QemuMachine:
-        raise Exception("This is just type information for the Nix test driver")
+# Classes
 
 
-start_all: Callable[[], None]
-subtest: Callable[[str], ContextManager[None]]
-retry: RetryProtocol
-test_script: Callable[[], None]
-machines: List[BaseMachine]
-machines_qemu: List[QemuMachine]
-machines_nspawn: List[NspawnMachine]
-vlans: List[VLan]
-driver: Driver
-log: AbstractLogger
-create_machine: CreateMachineProtocol
-run_tests: Callable[[], None]
-join_all: Callable[[], None]
-serial_stdout_off: Callable[[], None]
-serial_stdout_on: Callable[[], None]
-polling_condition: PollingConditionProtocol
-debug: DebugAbstract
-t: TestCase
-dump_machine_ssh: Callable[[], None]
+class AssertionTester(TestCase):
+    pass
+
+
+# Global Variables
+
+debug: DebugAbstract = DebugNop()
+machines: List[BaseMachine] = []
+machines_nspawn: List[NspawnMachine] = []
+machines_qemu: List[QemuMachine] = []
+t = AssertionTester()
+vlans: List[VLan] = []
+
+
+def create_fake_driver() -> Driver:
+    raise Exception("fake driver")
+
+
+driver = create_fake_driver()
+
+
+# Functions
+
+
+# these are going to be called by the testScriptWithTypes in driver.nix
+def create_fake_qemu_machine() -> QemuMachine:
+    raise Exception("fake qemu machine")
+
+
+def create_fake_nspawn_machine() -> NspawnMachine:
+    raise Exception("fake nspawn machine")
+
+
+def create_fake_vlan() -> VLan:
+    raise Exception("fake vlan")
+
+
+def create_machine(
+    start_command: str, name: str | None = None, keep_machine_state: bool = False
+) -> QemuMachine:
+    raise Exception("fake machine")
+
+
+def dump_machine_ssh() -> None:
+    return None
+
+
+def join_all() -> None:
+    return None
+
+
+def log(msg: str):
+    pass
+
+
+def polling_condition(
+    fun: Callable | None, seconds_interval: float = 0.0, description: str | None = None
+):
+    pass
+
+
+def retry(fn: Callable, timeout_seconds: int = 900) -> None:
+    pass
+
+
+def run_tests() -> None:
+    return
+
+
+def serial_stdout_off() -> None:
+    return None
+
+
+def serial_stdout_on() -> None:
+    return None
+
+
+def start_all() -> None:
+    return
+
+
+def test_script() -> None:
+    return
+
+
+@contextmanager
+def subtest(str: str) -> Generator[None, None, None]:
+    yield

nixos/lib/testing/driver.nix

@@ -19,34 +19,38 @@ let
     enableNspawn = config.containers != { };
   };
 
-  pythonizeName =
-    name:
+  typeHints =
     let
-      head = lib.substring 0 1 name;
-      tail = lib.substring 1 (-1) name;
+      pythonizeName =
+        name:
+        let
+          head = lib.substring 0 1 name;
+          tail = lib.substring 1 (-1) name;
+        in
+        (if builtins.match "[A-z_]" head == null then "_" else head)
+        + lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail;
+
+      vmMachineNames = lib.attrNames config.driverConfiguration.vms;
+      containerMachineNames = lib.attrNames config.driverConfiguration.containers;
+
+      theOnlyMachine =
+        let
+          exactlyOneMachine = lib.length (lib.attrValues config.nodes) == 1;
+          allMachineNames = map (c: c.system.name) (lib.attrValues config.allMachines);
+        in
+        lib.optional (exactlyOneMachine && !lib.elem "machine" allMachineNames) "machine";
+
+      vmMachineTypeHints = map (name: "${pythonizeName name} = create_fake_qemu_machine()") (
+        vmMachineNames ++ theOnlyMachine
+      );
+      containerMachineTypeHints = map (
+        name: "${pythonizeName name} = create_fake_nspawn_machine()"
+      ) containerMachineNames;
+      vlanTypeHints = map (i: "vlan${toString i} = create_fake_vlan()") config.driverConfiguration.vlans;
     in
-    (if builtins.match "[A-z_]" head == null then "_" else head)
-    + lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail;
-
-  vlanTypeHints = lib.strings.concatMapStringsSep "\n" (
-    i: "vlan${toString i}: VLan"
-  ) config.driverConfiguration.vlans;
-
-  vmMachineNames = lib.attrNames config.driverConfiguration.vms;
-  containerMachineNames = lib.attrNames config.driverConfiguration.containers;
-
-  theOnlyMachine =
-    let
-      exactlyOneMachine = lib.length (lib.attrValues config.nodes) == 1;
-      allMachineNames = map (c: c.system.name) (lib.attrValues config.allMachines);
-    in
-    lib.optional (exactlyOneMachine && !lib.elem "machine" allMachineNames) "machine";
-
-  pythonizedVmNames = map pythonizeName (vmMachineNames ++ theOnlyMachine);
-  vmMachineTypeHints = map (name: "${name}: QemuMachine;") pythonizedVmNames;
-
-  pythonizedContainerNames = map pythonizeName containerMachineNames;
-  containerMachineTypeHints = map (name: "${name}: NspawnMachine;") pythonizedContainerNames;
+    lib.strings.concatStringsSep "\n" (
+      vlanTypeHints ++ vmMachineTypeHints ++ containerMachineTypeHints
+    );
 
   withChecks = lib.warnIf config.skipLint "Linting is disabled";
 
@@ -57,7 +61,8 @@ let
         nativeBuildInputs = [
           hostPkgs.makeWrapper
         ]
-        ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.mypy ];
+        ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.ty ]
+        ++ lib.optionals (!config.skipLint) [ hostPkgs.ruff ];
         buildInputs = [ testDriver ];
         testScript = config.testScriptString;
         preferLocalBuild = true;
@@ -69,43 +74,34 @@ let
       ''
         mkdir -p $out/bin
 
-        ${lib.optionalString (!config.skipTypeCheck) ''
-          # prepend type hints so the test script can be type checked with mypy
-
+        ${lib.optionalString (!config.skipTypeCheck || !config.skipLint) ''
+          # prepend type hints so the test script can be type checked with ty
           cat "${../test-script-prepend.py}" >> testScriptWithTypes
-          echo "${toString vmMachineTypeHints}" >> testScriptWithTypes
-          echo "${toString containerMachineTypeHints}" >> testScriptWithTypes
-          echo "${toString vlanTypeHints}" >> testScriptWithTypes
+          echo "${toString typeHints}" >> testScriptWithTypes
           echo -n "$testScript" >> testScriptWithTypes
+        ''}
 
+        ${lib.optionalString (!config.skipTypeCheck) ''
           echo "Running type check (enable/disable: config.skipTypeCheck)"
           echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipTypeCheck"
 
-          mypy  --no-implicit-optional \
-                --pretty \
-                --no-color-output \
-                testScriptWithTypes
+          ty check --error-on-warning testScriptWithTypes
         ''}
 
-        echo -n "$testScript" > testScriptFile
-
-        cp "${config.driverConfiguration.test_script}" $out/test-script
-
-        ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
-
-        ${testDriver}/bin/generate-driver-symbols
         ${lib.optionalString (!config.skipLint) ''
           echo "Linting test script (enable/disable: config.skipLint)"
           echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipLint"
 
-          PYFLAKES_BUILTINS="$(
-            echo -n ${
-              lib.escapeShellArg (lib.concatStringsSep "," (pythonizedVmNames ++ pythonizedContainerNames))
-            },
-            cat ${lib.escapeShellArg "driver-symbols"}
-          )" ${hostPkgs.python3Packages.pyflakes}/bin/pyflakes $out/test-script
+          # F are the "(py)flakes" checks.
+          # we can't go with the defaults because these include
+          # code style/formatting
+          ruff check --select F testScriptWithTypes
         ''}
 
+        cp "${config.driverConfiguration.test_script}" $out/test-script
+
+        ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
+
         wrapProgram $out/bin/nixos-test-driver \
           --add-flags "--config ${config.driverConfigurationFile}" \
           --add-flags "--log-level ${config.logLevel}" \