From ba3f159cc8261160511b0c7f42446b7d3a66de1b Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Mon, 20 Feb 2023 16:43:25 +0100 Subject: [PATCH 1/5] nixos/tests/home-assistant: Overhaul and refactor Prepare the tests for a change in dependency handling, by not relying on bespoke files dropped into the package output. Instead we now check the journal log for whether a configured component was setup, once for the initial specialisation another time for the one introducing esphome configuration. Also improve abstractions for getting journal data relative to a cursor and generally make a few things more concise. --- nixos/tests/home-assistant.nix | 91 ++++++++++++++++------------------ 1 file changed, 44 insertions(+), 47 deletions(-) diff --git a/nixos/tests/home-assistant.nix b/nixos/tests/home-assistant.nix index 8d58de75eabc..0c2176c261cf 100644 --- a/nixos/tests/home-assistant.nix +++ b/nixos/tests/home-assistant.nix @@ -22,22 +22,23 @@ in { enable = true; inherit configDir; - # tests loading components by overriding the package + # provide dependencies through package overrides package = (pkgs.home-assistant.override { extraPackages = ps: with ps; [ colorama ]; - extraComponents = [ "zha" ]; - }).overrideAttrs (oldAttrs: { - doInstallCheck = false; + extraComponents = [ + # test char-tty device allow propagation into the service + "zha" + ]; }); - # tests loading components from the module + # provide component dependencies explicitly from the module extraComponents = [ - "wake_on_lan" + "mqtt" ]; - # test extra package passing from the module + # provide package for postgresql support extraPackages = python3Packages: with python3Packages; [ psycopg2 ]; @@ -114,33 +115,29 @@ in { system = nodes.hass.config.system.build.toplevel; in '' - import re import json start_all() - # Parse the package path out of the systemd unit, as we cannot - # access the final package, that is overridden inside the module, - # by any other means. - pattern = re.compile(r"path=(?P[\/a-z0-9-.]+)\/bin\/hass") - response = hass.execute("systemctl show -p ExecStart home-assistant.service")[1] - match = pattern.search(response) - assert match - package = match.group('path') - - def get_journal_cursor(host) -> str: - exit, out = host.execute("journalctl -u home-assistant.service -n1 -o json-pretty --output-fields=__CURSOR") + def get_journal_cursor() -> str: + exit, out = hass.execute("journalctl -u home-assistant.service -n1 -o json-pretty --output-fields=__CURSOR") assert exit == 0 return json.loads(out)["__CURSOR"] - def wait_for_homeassistant(host, cursor): - host.wait_until_succeeds(f"journalctl --after-cursor='{cursor}' -u home-assistant.service | grep -q 'Home Assistant initialized in'") + def get_journal_since(cursor) -> str: + exit, out = hass.execute(f"journalctl --after-cursor='{cursor}' -u home-assistant.service") + assert exit == 0 + return out + + + def wait_for_homeassistant(cursor): + hass.wait_until_succeeds(f"journalctl --after-cursor='{cursor}' -u home-assistant.service | grep -q 'Home Assistant initialized in'") hass.wait_for_unit("home-assistant.service") - cursor = get_journal_cursor(hass) + cursor = get_journal_cursor() with subtest("Check that YAML configuration file is in place"): hass.succeed("test -L ${configDir}/configuration.yaml") @@ -148,19 +145,16 @@ in { with subtest("Check the lovelace config is copied because lovelaceConfigWritable = true"): hass.succeed("test -f ${configDir}/ui-lovelace.yaml") - with subtest("Check extraComponents and extraPackages are considered from the package"): - hass.succeed(f"grep -q 'colorama' {package}/extra_packages") - hass.succeed(f"grep -q 'zha' {package}/extra_components") - - with subtest("Check extraComponents and extraPackages are considered from the module"): - hass.succeed(f"grep -q 'psycopg2' {package}/extra_packages") - hass.succeed(f"grep -q 'wake_on_lan' {package}/extra_components") - with subtest("Check that Home Assistant's web interface and API can be reached"): - wait_for_homeassistant(hass, cursor) + wait_for_homeassistant(cursor) hass.wait_for_open_port(8123) hass.succeed("curl --fail http://localhost:8123/lovelace") + with subtest("Check that declaratively configured components get setup"): + journal = get_journal_since(cursor) + for domain in ["emulated_hue", "wake_on_lan"]: + assert f"Setup of domain {domain} took" in journal, f"{domain} setup missing" + with subtest("Check that capabilities are passed for emulated_hue to bind to port 80"): hass.wait_for_open_port(80) hass.succeed("curl --fail http://localhost:80/description.xml") @@ -169,25 +163,28 @@ in { hass.succeed("systemctl show -p DeviceAllow home-assistant.service | grep -q char-ttyUSB") with subtest("Check service reloads when configuration changes"): - # store the old pid of the process - pid = hass.succeed("systemctl show --property=MainPID home-assistant.service") - cursor = get_journal_cursor(hass) - hass.succeed("${system}/specialisation/differentName/bin/switch-to-configuration test") - new_pid = hass.succeed("systemctl show --property=MainPID home-assistant.service") - assert pid == new_pid, "The PID of the process should not change between process reloads" - wait_for_homeassistant(hass, cursor) + pid = hass.succeed("systemctl show --property=MainPID home-assistant.service") + cursor = get_journal_cursor() + hass.succeed("${system}/specialisation/differentName/bin/switch-to-configuration test") + new_pid = hass.succeed("systemctl show --property=MainPID home-assistant.service") + assert pid == new_pid, "The PID of the process should not change between process reloads" + wait_for_homeassistant(cursor) - with subtest("check service restarts when package changes"): - pid = new_pid - cursor = get_journal_cursor(hass) - hass.succeed("${system}/specialisation/newFeature/bin/switch-to-configuration test") - new_pid = hass.succeed("systemctl show --property=MainPID home-assistant.service") - assert pid != new_pid, "The PID of the process shoudl change when the HA binary changes" - wait_for_homeassistant(hass, cursor) + with subtest("Check service restarts when dependencies change"): + pid = new_pid + cursor = get_journal_cursor() + hass.succeed("${system}/specialisation/newFeature/bin/switch-to-configuration test") + new_pid = hass.succeed("systemctl show --property=MainPID home-assistant.service") + assert pid != new_pid, "The PID of the process should change when its PYTHONPATH changess" + wait_for_homeassistant(cursor) + + with subtest("Check that new components get setup after restart"): + journal = get_journal_since(cursor) + for domain in ["esphome"]: + assert f"Setup of domain {domain} took" in journal, f"{domain} setup missing" with subtest("Check that no errors were logged"): - output_log = hass.succeed("cat ${configDir}/home-assistant.log") - assert "ERROR" not in output_log + hass.fail("journalctl -u home-assistant -o cat | grep -q ERROR") with subtest("Check systemd unit hardening"): hass.log(hass.succeed("systemctl cat home-assistant.service")) From 3fa7dc206a54265f04fc2ea2afd1cf1cdb650c83 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Mon, 20 Feb 2023 16:52:22 +0100 Subject: [PATCH 2/5] home-assistant: Stop exposing component & package files These were only ever used in a test, but for an upcoming change in dependency handling, they mean we need to rebuild home-assistant for each change in used components for no good reason. This would mean building two additional home-assistant derivations for the NixOS test alone. --- pkgs/servers/home-assistant/default.nix | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pkgs/servers/home-assistant/default.nix b/pkgs/servers/home-assistant/default.nix index 4d6e0b6cca62..7ff1473edec3 100644 --- a/pkgs/servers/home-assistant/default.nix +++ b/pkgs/servers/home-assistant/default.nix @@ -265,10 +265,6 @@ let # Ensure that we are using a consistent package set extraBuildInputs = extraPackages python.pkgs; - # Create info about included packages and components - extraComponentsFile = writeText "home-assistant-components" (lib.concatStringsSep "\n" extraComponents); - extraPackagesFile = writeText "home-assistant-packages" (lib.concatMapStringsSep "\n" (pkg: pkg.pname) extraBuildInputs); - # Don't forget to run parse-requirements.py after updating hassVersion = "2023.2.5"; @@ -428,11 +424,6 @@ in python.pkgs.buildPythonApplication rec { export PATH=${inetutils}/bin:$PATH ''; - postInstall = '' - cp -v ${extraComponentsFile} $out/extra_components - cp -v ${extraPackagesFile} $out/extra_packages - ''; - passthru = { inherit availableComponents From e01ccd6245fe1fe34aa49aa48fe3d53e3b013c20 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Mon, 20 Feb 2023 16:54:37 +0100 Subject: [PATCH 3/5] home-assistant: Inject extra dependencies through PYTHONPATH Injecting configuration specific dependencies into the propagatedBuildInputs of the home-assistant package forces alot of rebuilds while setting up home-assistant, which is annoying. By passing optional dependencies into home-assistant via the systemd units PYTHONPATH environment variable, only he concatenation of library paths in the systemd unit requires a rebuild. This also means users can rely heavily on the cached home-assistant package and will rarely have to build from source, if ever. --- nixos/modules/services/home-automation/home-assistant.nix | 1 + pkgs/servers/home-assistant/default.nix | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/home-automation/home-assistant.nix b/nixos/modules/services/home-automation/home-assistant.nix index 9f6f153a1056..6adc58ec58ec 100644 --- a/nixos/modules/services/home-automation/home-assistant.nix +++ b/nixos/modules/services/home-automation/home-assistant.nix @@ -409,6 +409,7 @@ in { (optionalString (cfg.config != null) copyConfig) + (optionalString (cfg.lovelaceConfig != null) copyLovelaceConfig) ; + environment.PYTHONPATH = package.pythonPath; serviceConfig = let # List of capabilities to equip home-assistant with, depending on configured components capabilities = lib.unique ([ diff --git a/pkgs/servers/home-assistant/default.nix b/pkgs/servers/home-assistant/default.nix index 7ff1473edec3..b5e42fca6ae4 100644 --- a/pkgs/servers/home-assistant/default.nix +++ b/pkgs/servers/home-assistant/default.nix @@ -355,7 +355,7 @@ in python.pkgs.buildPythonApplication rec { yarl # Implicit dependency via homeassistant/requirements.py setuptools - ] ++ componentBuildInputs ++ extraBuildInputs; + ]; makeWrapperArgs = lib.optional skipPip "--add-flags --skip-pip"; @@ -431,6 +431,7 @@ in python.pkgs.buildPythonApplication rec { getPackages python supportedComponentsWithTests; + pythonPath = python3.pkgs.makePythonPath (componentBuildInputs ++ extraBuildInputs); intents = python.pkgs.home-assistant-intents; tests = { nixos = nixosTests.home-assistant; From 9bf8744a732fad91482a66c28c5462a683be727c Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Mon, 20 Feb 2023 17:01:02 +0100 Subject: [PATCH 4/5] nixos/tests/home-assistant: Check dependencies arrive in the PYTHONPATH Both extraPackages and derived packages from extraComponents should be present in the units PYTHONPATH environment variable. --- nixos/tests/home-assistant.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nixos/tests/home-assistant.nix b/nixos/tests/home-assistant.nix index 0c2176c261cf..aec4784e9c5f 100644 --- a/nixos/tests/home-assistant.nix +++ b/nixos/tests/home-assistant.nix @@ -132,6 +132,12 @@ in { return out + def get_unit_property(property) -> str: + exit, out = hass.execute(f"systemctl show --property={property} home-assistant.service") + assert exit == 0 + return out + + def wait_for_homeassistant(cursor): hass.wait_until_succeeds(f"journalctl --after-cursor='{cursor}' -u home-assistant.service | grep -q 'Home Assistant initialized in'") @@ -150,6 +156,12 @@ in { hass.wait_for_open_port(8123) hass.succeed("curl --fail http://localhost:8123/lovelace") + with subtest("Check that optional dependencies are in the PYTHONPATH"): + env = get_unit_property("Environment") + python_path = env.split("PYTHONPATH=")[1].split()[0] + for package in ["colorama", "paho-mqtt", "psycopg2"]: + assert package in python_path, f"{package} not in PYTHONPATH" + with subtest("Check that declaratively configured components get setup"): journal = get_journal_since(cursor) for domain in ["emulated_hue", "wake_on_lan"]: From f98462a27d097f1b8397ed07ca545b26009bf6c9 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Mon, 20 Feb 2023 17:59:50 +0100 Subject: [PATCH 5/5] nixos/tests/home-assistant: Resolve deprecation warning > trace: warning: Module argument `nodes.hass.config` is deprecated. Use `nodes.hass` instead. --- nixos/tests/home-assistant.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/tests/home-assistant.nix b/nixos/tests/home-assistant.nix index aec4784e9c5f..8585cb3585fe 100644 --- a/nixos/tests/home-assistant.nix +++ b/nixos/tests/home-assistant.nix @@ -112,7 +112,7 @@ in { }; testScript = { nodes, ... }: let - system = nodes.hass.config.system.build.toplevel; + system = nodes.hass.system.build.toplevel; in '' import json