diff --git a/nixos/lib/test-driver/src/pyproject.toml b/nixos/lib/test-driver/src/pyproject.toml index c348bf1e4dfb..5dc422479adf 100644 --- a/nixos/lib/test-driver/src/pyproject.toml +++ b/nixos/lib/test-driver/src/pyproject.toml @@ -8,7 +8,6 @@ version = "0.0.0" [project.scripts] nixos-test-driver = "test_driver:main" -generate-driver-symbols = "test_driver:generate_driver_symbols" [tool.setuptools.packages] find = {} diff --git a/nixos/lib/test-driver/src/test_driver/__init__.py b/nixos/lib/test-driver/src/test_driver/__init__.py index 1e5a062b18c8..038edca7ef3c 100644 --- a/nixos/lib/test-driver/src/test_driver/__init__.py +++ b/nixos/lib/test-driver/src/test_driver/__init__.py @@ -8,7 +8,7 @@ from pathlib import Path import ptpython.ipython from test_driver.debug import Debug, DebugAbstract, DebugNop -from test_driver.driver import Driver, DriverConfiguration, load_driver_configuration +from test_driver.driver import Driver, load_driver_configuration from test_driver.logger import ( CompositeLogger, JunitXMLLogger, @@ -159,30 +159,3 @@ def main() -> None: driver.run_tests() toc = time.time() logger.info(f"test script finished in {(toc - tic):.2f}s") - - -def generate_driver_symbols() -> None: - """ - This generates a file with symbols of the test-driver code that can be used - in user's test scripts. That list is then used by pyflakes to lint those - scripts. - """ - d = Driver( - config=DriverConfiguration( - vms=dict(), - containers=dict(), - vlans=[], - global_timeout=0, - enable_ssh_backdoor=False, - test_script=( - Path("testScriptWithTypes") - if (Path("testScriptWithTypes").is_file()) - else Path("testScriptFile") - ), - ), - out_dir=Path(), - logger=CompositeLogger([]), - ) - test_symbols = d.test_symbols() - with open("driver-symbols", "w") as fp: - fp.write(",".join(test_symbols.keys())) diff --git a/nixos/lib/test-driver/src/test_driver/driver.py b/nixos/lib/test-driver/src/test_driver/driver.py index 9ee11ff41ea0..d96255159e8f 100644 --- a/nixos/lib/test-driver/src/test_driver/driver.py +++ b/nixos/lib/test-driver/src/test_driver/driver.py @@ -7,7 +7,7 @@ import sys import tempfile import threading import traceback -from collections.abc import Callable, Iterator +from collections.abc import Callable, Generator, Iterator from contextlib import AbstractContextManager, contextmanager from dataclasses import dataclass from pathlib import Path @@ -299,7 +299,7 @@ class Driver: f"Error during cleanup of vhost-device-vsock process: {e}" ) - def subtest(self, name: str) -> Iterator[None]: + def subtest(self, name: str) -> Generator[None]: """Group logs under a given test name""" with self.logger.subtest(name): try: diff --git a/nixos/lib/test-script-prepend.py b/nixos/lib/test-script-prepend.py index b1ca8e7b1678..7d362479e460 100644 --- a/nixos/lib/test-script-prepend.py +++ b/nixos/lib/test-script-prepend.py @@ -1,19 +1,30 @@ # This file contains type hints that can be prepended to Nix test scripts so they can be type # checked. -from test_driver.debug import DebugAbstract -from test_driver.driver import Driver -from test_driver.vlan import VLan -from test_driver.machine import BaseMachine, NspawnMachine, QemuMachine -from test_driver.logger import AbstractLogger -from typing import Callable, Iterator, ContextManager, Optional, List, Dict, Any, Union -from typing_extensions import Protocol -from pathlib import Path +from contextlib import contextmanager +from typing import Any, Callable, ContextManager, Generator, List, Optional, Union from unittest import TestCase +from test_driver.debug import DebugAbstract, DebugNop +from test_driver.driver import Driver +from typing_extensions import Protocol -class RetryProtocol(Protocol): - def __call__(self, fn: Callable, timeout_seconds: int = 900) -> None: +from test_driver.machine import BaseMachine, NspawnMachine, QemuMachine +from test_driver.vlan import VLan + + +# Protocols + + +class CreateMachineProtocol(Protocol): + def __call__( + self, + start_command: str | dict, + *, + name: Optional[str] = None, + keep_machine_state: bool = False, + **kwargs: Any, # to allow usage of deprecated keep_vm_state + ) -> QemuMachine: raise Exception("This is just type information for the Nix test driver") @@ -28,34 +39,94 @@ class PollingConditionProtocol(Protocol): raise Exception("This is just type information for the Nix test driver") -class CreateMachineProtocol(Protocol): - def __call__( - self, - start_command: str | dict, - *, - name: Optional[str] = None, - keep_machine_state: bool = False, - **kwargs: Any, # to allow usage of deprecated keep_vm_state - ) -> QemuMachine: - raise Exception("This is just type information for the Nix test driver") +# Classes -start_all: Callable[[], None] -subtest: Callable[[str], ContextManager[None]] -retry: RetryProtocol -test_script: Callable[[], None] -machines: List[BaseMachine] -machines_qemu: List[QemuMachine] -machines_nspawn: List[NspawnMachine] -vlans: List[VLan] -driver: Driver -log: AbstractLogger -create_machine: CreateMachineProtocol -run_tests: Callable[[], None] -join_all: Callable[[], None] -serial_stdout_off: Callable[[], None] -serial_stdout_on: Callable[[], None] -polling_condition: PollingConditionProtocol -debug: DebugAbstract -t: TestCase -dump_machine_ssh: Callable[[], None] +class AssertionTester(TestCase): + pass + + +# Global Variables + +debug: DebugAbstract = DebugNop() +machines: List[BaseMachine] = [] +machines_nspawn: List[NspawnMachine] = [] +machines_qemu: List[QemuMachine] = [] +t = AssertionTester() +vlans: List[VLan] = [] + + +def create_fake_driver() -> Driver: + raise Exception("fake driver") + + +driver = create_fake_driver() + + +# Functions + + +# these are going to be called by the testScriptWithTypes in driver.nix +def create_fake_qemu_machine() -> QemuMachine: + raise Exception("fake qemu machine") + + +def create_fake_nspawn_machine() -> NspawnMachine: + raise Exception("fake nspawn machine") + + +def create_fake_vlan() -> VLan: + raise Exception("fake vlan") + + +def create_machine( + start_command: str, name: str | None = None, keep_machine_state: bool = False +) -> QemuMachine: + raise Exception("fake machine") + + +def dump_machine_ssh() -> None: + return None + + +def join_all() -> None: + return None + + +def log(msg: str): + pass + + +def polling_condition( + fun: Callable | None, seconds_interval: float = 0.0, description: str | None = None +): + pass + + +def retry(fn: Callable, timeout_seconds: int = 900) -> None: + pass + + +def run_tests() -> None: + return + + +def serial_stdout_off() -> None: + return None + + +def serial_stdout_on() -> None: + return None + + +def start_all() -> None: + return + + +def test_script() -> None: + return + + +@contextmanager +def subtest(str: str) -> Generator[None, None, None]: + yield diff --git a/nixos/lib/testing/driver.nix b/nixos/lib/testing/driver.nix index 40584b10e00d..3b082140e935 100644 --- a/nixos/lib/testing/driver.nix +++ b/nixos/lib/testing/driver.nix @@ -19,34 +19,38 @@ let enableNspawn = config.containers != { }; }; - pythonizeName = - name: + typeHints = let - head = lib.substring 0 1 name; - tail = lib.substring 1 (-1) name; + pythonizeName = + name: + let + head = lib.substring 0 1 name; + tail = lib.substring 1 (-1) name; + in + (if builtins.match "[A-z_]" head == null then "_" else head) + + lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail; + + vmMachineNames = lib.attrNames config.driverConfiguration.vms; + containerMachineNames = lib.attrNames config.driverConfiguration.containers; + + theOnlyMachine = + let + exactlyOneMachine = lib.length (lib.attrValues config.nodes) == 1; + allMachineNames = map (c: c.system.name) (lib.attrValues config.allMachines); + in + lib.optional (exactlyOneMachine && !lib.elem "machine" allMachineNames) "machine"; + + vmMachineTypeHints = map (name: "${pythonizeName name} = create_fake_qemu_machine()") ( + vmMachineNames ++ theOnlyMachine + ); + containerMachineTypeHints = map ( + name: "${pythonizeName name} = create_fake_nspawn_machine()" + ) containerMachineNames; + vlanTypeHints = map (i: "vlan${toString i} = create_fake_vlan()") config.driverConfiguration.vlans; in - (if builtins.match "[A-z_]" head == null then "_" else head) - + lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail; - - vlanTypeHints = lib.strings.concatMapStringsSep "\n" ( - i: "vlan${toString i}: VLan" - ) config.driverConfiguration.vlans; - - vmMachineNames = lib.attrNames config.driverConfiguration.vms; - containerMachineNames = lib.attrNames config.driverConfiguration.containers; - - theOnlyMachine = - let - exactlyOneMachine = lib.length (lib.attrValues config.nodes) == 1; - allMachineNames = map (c: c.system.name) (lib.attrValues config.allMachines); - in - lib.optional (exactlyOneMachine && !lib.elem "machine" allMachineNames) "machine"; - - pythonizedVmNames = map pythonizeName (vmMachineNames ++ theOnlyMachine); - vmMachineTypeHints = map (name: "${name}: QemuMachine;") pythonizedVmNames; - - pythonizedContainerNames = map pythonizeName containerMachineNames; - containerMachineTypeHints = map (name: "${name}: NspawnMachine;") pythonizedContainerNames; + lib.strings.concatStringsSep "\n" ( + vlanTypeHints ++ vmMachineTypeHints ++ containerMachineTypeHints + ); withChecks = lib.warnIf config.skipLint "Linting is disabled"; @@ -57,7 +61,8 @@ let nativeBuildInputs = [ hostPkgs.makeWrapper ] - ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.mypy ]; + ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.ty ] + ++ lib.optionals (!config.skipLint) [ hostPkgs.ruff ]; buildInputs = [ testDriver ]; testScript = config.testScriptString; preferLocalBuild = true; @@ -69,43 +74,34 @@ let '' mkdir -p $out/bin - ${lib.optionalString (!config.skipTypeCheck) '' - # prepend type hints so the test script can be type checked with mypy - + ${lib.optionalString (!config.skipTypeCheck || !config.skipLint) '' + # prepend type hints so the test script can be type checked with ty cat "${../test-script-prepend.py}" >> testScriptWithTypes - echo "${toString vmMachineTypeHints}" >> testScriptWithTypes - echo "${toString containerMachineTypeHints}" >> testScriptWithTypes - echo "${toString vlanTypeHints}" >> testScriptWithTypes + echo "${toString typeHints}" >> testScriptWithTypes echo -n "$testScript" >> testScriptWithTypes + ''} + ${lib.optionalString (!config.skipTypeCheck) '' echo "Running type check (enable/disable: config.skipTypeCheck)" echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipTypeCheck" - mypy --no-implicit-optional \ - --pretty \ - --no-color-output \ - testScriptWithTypes + ty check --error-on-warning testScriptWithTypes ''} - echo -n "$testScript" > testScriptFile - - cp "${config.driverConfiguration.test_script}" $out/test-script - - ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver - - ${testDriver}/bin/generate-driver-symbols ${lib.optionalString (!config.skipLint) '' echo "Linting test script (enable/disable: config.skipLint)" echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipLint" - PYFLAKES_BUILTINS="$( - echo -n ${ - lib.escapeShellArg (lib.concatStringsSep "," (pythonizedVmNames ++ pythonizedContainerNames)) - }, - cat ${lib.escapeShellArg "driver-symbols"} - )" ${hostPkgs.python3Packages.pyflakes}/bin/pyflakes $out/test-script + # F are the "(py)flakes" checks. + # we can't go with the defaults because these include + # code style/formatting + ruff check --select F testScriptWithTypes ''} + cp "${config.driverConfiguration.test_script}" $out/test-script + + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver + wrapProgram $out/bin/nixos-test-driver \ --add-flags "--config ${config.driverConfigurationFile}" \ --add-flags "--log-level ${config.logLevel}" \